完善任务审计范围快照与身份关联并同步整改进度

新增三库第13号迁移和历史范围回填,保留已删除资源的任务与审计。补充仓储、调用链、身份来源及迁移失败重试回归,同步任务清单、支持矩阵和运维边界。项目成员授权与生产验收仍待完成。
This commit is contained in:
xuans committed 2026-09-28 11:48:27 +08:00
1 parent 5237e8d810
commit 0ccdb14b26
34 files changed
+989 -48

No files matched your search

@@ -0,0 +1,42 @@
package database
import (
"context"
"database/sql"
"fmt"
)
// 历史范围是不可级联删除的快照,不强行给已删除资源补默认项目或操作者。
// MySQL 的部分 DDL 重试只回填仍未知的记录,不覆盖已经保存的范围。
func backfillHistoryScope(ctx context.Context, db interface {
ExecContext(context.Context, string, ...any) (sql.Result, error)
}) error {
statements := []string{
`UPDATE tasks SET scope_type = 'system' WHERE scope_type = 'unknown'
AND task_type = 'management_renewal' AND domain_id = 'management-certificate'`,
`UPDATE tasks SET scope_type = 'project',
project_id = (SELECT d.project_id FROM domains d WHERE d.id = tasks.domain_id)
WHERE scope_type = 'unknown' AND EXISTS (SELECT 1 FROM domains d WHERE d.id = tasks.domain_id)`,
`UPDATE audit_logs SET scope_type = 'system' WHERE scope_type = 'unknown'
AND (resource_type IN ('auth', 'user', 'super_admin')
OR (resource_type = 'management_certificate' AND resource_id = 'management-certificate'))`,
`UPDATE audit_logs SET task_id = resource_id WHERE resource_type = 'task' AND task_id = ''`,
`UPDATE audit_logs SET scope_type = (SELECT t.scope_type FROM tasks t WHERE t.id = audit_logs.task_id),
project_id = (SELECT t.project_id FROM tasks t WHERE t.id = audit_logs.task_id)
WHERE scope_type = 'unknown' AND task_id <> '' AND EXISTS (SELECT 1 FROM tasks t WHERE t.id = audit_logs.task_id)`,
`UPDATE audit_logs SET scope_type = 'project',
project_id = (SELECT d.project_id FROM domains d WHERE d.id = audit_logs.resource_id)
WHERE scope_type = 'unknown' AND resource_type IN ('domain', 'certificate')
AND EXISTS (SELECT 1 FROM domains d WHERE d.id = audit_logs.resource_id)`,
`UPDATE audit_logs SET scope_type = 'project',
project_id = (SELECT s.project_id FROM servers s WHERE s.id = audit_logs.resource_id)
WHERE scope_type = 'unknown' AND resource_type = 'server'
AND EXISTS (SELECT 1 FROM servers s WHERE s.id = audit_logs.resource_id)`,
}
for _, statement := range statements {
if _, err := db.ExecContext(ctx, statement); err != nil {
return fmt.Errorf("回填任务审计历史范围失败:%w", err)
}
}
return nil
}
@@ -0,0 +1,223 @@
package database
import (
"database/sql"
"strings"
"testing"
)
func seedHistoryScope(t *testing.T, db *sql.DB, driver string) {
t.Helper()
if _, err := db.ExecContext(t.Context(), migrationTableSQL(driver)); err != nil {
t.Fatal(err)
}
for version := 1; version <= 12; version++ {
if err := applyMigration(t.Context(), db, driver, version); err != nil {
t.Fatal(err)
}
}
for _, statement := range []string{
`INSERT INTO domains (id, name, certificate_type, sans_json, status) VALUES ('domain', 'history.example.test', 'single', '[]', 'pending')`,
`INSERT INTO servers (id, name, address, port, username, private_key_ref, host_key_fingerprint, certificate_path, private_key_path, certificate_chain_path, web_server, reload_command_json, validation_command_json, ssh_key_error) VALUES ('server', '历史服务器', '192.0.2.1', 22, 'deploy', 'secret://test', 'test-only', '/test/cert.pem', '/test/key.pem', '/test/chain.pem', 'nginx', '[]', '[]', '')`,
`INSERT INTO tasks (id, domain_id, task_type, status, message, operator_id) VALUES
('business', 'domain', 'certificate_issue', 'success', '保留原记录', 'known-actor'),
('orphan', 'deleted-domain', 'certificate_issue', 'success', '保留原记录', ''),
('management', 'management-certificate', 'management_renewal', 'success', '保留原记录', ''),
('unknown', 'management-certificate', 'certificate_issue', 'success', '保留原记录', '')`,
`INSERT INTO task_attempts (id, task_id, attempt_number, stage, status, message) VALUES
('business-attempt', 'business', 1, 'issue', 'success', '保留原记录'),
('deleted-attempt', 'deleted-task', 1, 'issue', 'success', '保留原记录')`,
`INSERT INTO audit_logs (action, resource_type, resource_id, status, message) VALUES
('business', 'domain', 'domain', 'success', '保留原记录'),
('certificate', 'certificate', 'domain', 'success', '保留原记录'),
('server', 'server', 'server', 'success', '保留原记录'),
('orphan', 'domain', 'deleted-domain', 'success', '保留原记录'),
('task', 'task', 'business', 'success', '保留原记录'),
('deleted-task', 'task', 'deleted-task', 'success', '保留原记录'),
('management', 'management_certificate', 'management-certificate', 'success', '保留原记录'),
('unknown', 'management_certificate', 'different', 'success', '保留原记录'),
('auth', 'auth', 'login', 'success', '保留原记录')`,
} {
if _, err := db.ExecContext(t.Context(), statement); err != nil {
t.Fatal(err)
}
}
}
func assertHistoryScope(t *testing.T, db *sql.DB) {
t.Helper()
rows, err := db.QueryContext(t.Context(), `SELECT id, scope_type, project_id, operator_id, message FROM tasks ORDER BY id`)
if err != nil {
t.Fatal(err)
}
count := 0
for rows.Next() {
var id, scope, project, actor, message string
if err := rows.Scan(&id, &scope, &project, &actor, &message); err != nil {
t.Fatal(err)
}
wantScope, wantProject, wantActor := "unknown", "", ""
if id == "business" {
wantScope, wantProject, wantActor = "project", "default", "known-actor"
}
if id == "management" {
wantScope = "system"
}
if scope != wantScope || project != wantProject || actor != wantActor || message != "保留原记录" {
t.Errorf("任务历史错误:%s %s %s %s", id, scope, project, actor)
}
count++
}
if err := rows.Err(); err != nil {
t.Fatal(err)
}
if err := rows.Close(); err != nil {
t.Fatal(err)
}
if count != 4 {
t.Fatalf("任务历史丢失:%d", count)
}
rows, err = db.QueryContext(t.Context(), `SELECT action, scope_type, project_id, task_id, operator_id, message FROM audit_logs`)
if err != nil {
t.Fatal(err)
}
count = 0
for rows.Next() {
var action, scope, project, task, actor, message string
if err := rows.Scan(&action, &scope, &project, &task, &actor, &message); err != nil {
t.Fatal(err)
}
wantScope, wantProject, wantTask := "unknown", "", ""
switch action {
case "business", "certificate", "server", "task":
wantScope, wantProject = "project", "default"
case "management", "auth":
wantScope = "system"
}
if action == "task" {
wantTask = "business"
}
if action == "deleted-task" {
wantTask = "deleted-task"
}
if scope != wantScope || project != wantProject || task != wantTask || actor != "" || message != "保留原记录" {
t.Errorf("审计历史错误:%s %s %s %s %s", action, scope, project, task, actor)
}
count++
}
if err := rows.Err(); err != nil {
t.Fatal(err)
}
if err := rows.Close(); err != nil {
t.Fatal(err)
}
if count != 9 {
t.Fatalf("审计历史丢失:%d", count)
}
if err := db.QueryRowContext(t.Context(), `SELECT COUNT(*) FROM task_attempts WHERE message = '保留原记录'`).Scan(&count); err != nil || count != 2 {
t.Fatalf("历史尝试被改写或删除:%d,%v", count, err)
}
}
func TestHistoryScopeSQLiteMigration(t *testing.T) {
db, err := Open(t.Context(), sqliteBusinessUpgradeConfig(t))
if err != nil {
t.Fatal(err)
}
defer db.Close()
seedHistoryScope(t, db.DB, "sqlite")
before := snapshotSQLiteHistoricalBusiness(t, db.DB)
if err := Migrate(t.Context(), db.DB, "sqlite"); err != nil {
t.Fatal(err)
}
assertSQLiteHistoricalBusiness(t, db.DB, before)
assertHistoryScope(t, db.DB)
after := snapshotSQLiteHistoricalBusiness(t, db.DB)
if err := Migrate(t.Context(), db.DB, "sqlite"); err != nil {
t.Fatal(err)
}
assertSQLiteHistoricalBusiness(t, db.DB, after)
}
func TestHistoryScopeSQLiteMigrationRollback(t *testing.T) {
for _, tc := range []struct{ name, setup, repair string }{
{"version", `CREATE TRIGGER fail_history_version BEFORE INSERT ON schema_migrations WHEN NEW.version = 13 BEGIN SELECT RAISE(ABORT, 'test-only'); END`, `DROP TRIGGER fail_history_version`},
{"backfill", `CREATE TRIGGER fail_history_backfill BEFORE UPDATE ON audit_logs BEGIN SELECT RAISE(ABORT, 'test-only'); END`, `DROP TRIGGER fail_history_backfill`},
{"index", `CREATE INDEX idx_audit_project_history ON audit_logs(status)`, `DROP INDEX idx_audit_project_history`},
} {
t.Run(tc.name, func(t *testing.T) {
db, err := Open(t.Context(), sqliteBusinessUpgradeConfig(t))
if err != nil {
t.Fatal(err)
}
defer db.Close()
seedHistoryScope(t, db.DB, "sqlite")
if _, err := db.DB.ExecContext(t.Context(), tc.setup); err != nil {
t.Fatal(err)
}
before := snapshotSQLiteHistoricalBusiness(t, db.DB)
schemaQuery := `SELECT type, name, sql FROM sqlite_master ORDER BY name`
schema := snapshotSQLiteUpgradeQuery(t, db.DB, schemaQuery)
if err := Migrate(t.Context(), db.DB, "sqlite"); err == nil {
t.Fatal("故障迁移未拒绝")
}
assertSQLiteHistoricalBusiness(t, db.DB, before)
if snapshotSQLiteUpgradeQuery(t, db.DB, schemaQuery) != schema {
t.Fatal("失败迁移没有完整回滚模式")
}
if version, err := currentMigrationVersion(t.Context(), db.DB); err != nil || version != 12 {
t.Fatalf("失败迁移版本错误:%d,%v", version, err)
}
if _, err := db.DB.ExecContext(t.Context(), tc.repair); err != nil {
t.Fatal(err)
}
for repeat := 0; repeat < 2; repeat++ {
if err := Migrate(t.Context(), db.DB, "sqlite"); err != nil {
t.Fatal(err)
}
}
assertHistoryScope(t, db.DB)
})
}
}
func TestHistoryScopeMySQLMigrationRetry(t *testing.T) {
fixture := startRealMySQL(t)
for _, phase := range []string{"backfill", "version"} {
t.Run(phase, func(t *testing.T) {
db := fixture.database(t)
ctx := t.Context()
seedHistoryScope(t, db.DB, "mysql")
var schema string
if err := db.DB.QueryRowContext(ctx, "SELECT DATABASE()").Scan(&schema); err != nil {
t.Fatal(err)
}
schema = "`" + strings.ReplaceAll(schema, "`", "``") + "`"
target := "BEFORE INSERT ON " + schema + ".schema_migrations"
if phase == "backfill" {
target = "BEFORE UPDATE ON " + schema + ".audit_logs"
}
if _, err := fixture.admin.ExecContext(ctx, `CREATE TRIGGER `+schema+`.fail_history `+target+` FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'test-only'`); err != nil {
t.Fatal(err)
}
if err := Migrate(ctx, db.DB, "mysql"); err == nil {
t.Fatal("故障未阻断迁移")
}
if version, err := currentMigrationVersion(ctx, db.DB); err != nil || version != 12 {
t.Fatalf("失败版本错误:%d,%v", version, err)
}
if _, err := fixture.admin.ExecContext(ctx, "DROP TRIGGER "+schema+".fail_history"); err != nil {
t.Fatal(err)
}
for repeat := 0; repeat < 2; repeat++ {
if err := Migrate(ctx, db.DB, "mysql"); err != nil {
t.Fatal(err)
}
}
assertHistoryScope(t, db.DB)
if version, err := currentMigrationVersion(ctx, db.DB); err != nil || version != 13 {
t.Fatalf("重试版本错误:%d,%v", version, err)
}
})
}
}
+15 -1
View File
@@ -14,6 +14,7 @@ import (
//go:embed migrations/sqlite/0010_projects.sql migrations/mysql/0010_projects.sql migrations/postgres/0010_projects.sql
//go:embed migrations/sqlite/0011_resource_projects.sql migrations/mysql/0011_resource_projects.sql migrations/postgres/0011_resource_projects.sql
//go:embed migrations/sqlite/0012_resource_replacement.sql migrations/mysql/0012_resource_replacement.sql migrations/postgres/0012_resource_replacement.sql
//go:embed migrations/sqlite/0013_history_scope.sql migrations/mysql/0013_history_scope.sql migrations/postgres/0013_history_scope.sql
var migrationFiles embed.FS
func Migrate(ctx context.Context, db *sql.DB, driver string) error {
@@ -38,7 +39,7 @@ func Migrate(ctx context.Context, db *sql.DB, driver string) error {
return nil
}
const latestMigrationVersion = 12
const latestMigrationVersion = 13
// currentMigrationVersion 在执行业务迁移前拒绝新版数据库及不连续的迁移历史。
func currentMigrationVersion(ctx context.Context, db *sql.DB) (int, error) {
@@ -90,6 +91,11 @@ func applyMigration(ctx context.Context, db *sql.DB, driver string, version int)
return fmt.Errorf("执行数据库迁移失败:%w", err)
}
}
if version == 13 {
if err := backfillHistoryScope(ctx, db); err != nil {
return err
}
}
if _, err := db.ExecContext(ctx, "INSERT INTO schema_migrations (version) VALUES (?)", version); err != nil {
return fmt.Errorf("记录迁移版本失败:%w", err)
}
@@ -117,6 +123,12 @@ func applyMigration(ctx context.Context, db *sql.DB, driver string, version int)
return err
}
}
if version == 13 {
if err := backfillHistoryScope(ctx, tx); err != nil {
_ = tx.Rollback()
return err
}
}
migrationVersionInsert := "INSERT INTO schema_migrations (version) VALUES (?)"
if driver == "postgres" {
migrationVersionInsert = "INSERT INTO schema_migrations (version) VALUES ($1)"
@@ -192,6 +204,8 @@ func migrationName(version int) string {
return "resource_projects"
case 12:
return "resource_replacement"
case 13:
return "history_scope"
default:
return "unknown"
}
@@ -0,0 +1,9 @@
ALTER TABLE tasks ADD COLUMN scope_type VARCHAR(16) NOT NULL DEFAULT 'unknown';
ALTER TABLE tasks ADD COLUMN project_id VARCHAR(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '';
ALTER TABLE audit_logs ADD COLUMN scope_type VARCHAR(16) NOT NULL DEFAULT 'unknown';
ALTER TABLE audit_logs ADD COLUMN project_id VARCHAR(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '';
ALTER TABLE audit_logs ADD COLUMN task_id VARCHAR(36) NOT NULL DEFAULT '';
ALTER TABLE audit_logs ADD COLUMN operator_id VARCHAR(36) NOT NULL DEFAULT '';
CREATE INDEX idx_tasks_project_history ON tasks(project_id, created_at);
CREATE INDEX idx_audit_project_history ON audit_logs(project_id, created_at);
CREATE INDEX idx_audit_task_history ON audit_logs(task_id);
@@ -0,0 +1,9 @@
ALTER TABLE tasks ADD COLUMN scope_type TEXT NOT NULL DEFAULT 'unknown';
ALTER TABLE tasks ADD COLUMN project_id TEXT NOT NULL DEFAULT '';
ALTER TABLE audit_logs ADD COLUMN scope_type TEXT NOT NULL DEFAULT 'unknown';
ALTER TABLE audit_logs ADD COLUMN project_id TEXT NOT NULL DEFAULT '';
ALTER TABLE audit_logs ADD COLUMN task_id TEXT NOT NULL DEFAULT '';
ALTER TABLE audit_logs ADD COLUMN operator_id TEXT NOT NULL DEFAULT '';
CREATE INDEX idx_tasks_project_history ON tasks(project_id, created_at);
CREATE INDEX idx_audit_project_history ON audit_logs(project_id, created_at);
CREATE INDEX idx_audit_task_history ON audit_logs(task_id);
@@ -0,0 +1,9 @@
ALTER TABLE tasks ADD COLUMN scope_type TEXT NOT NULL DEFAULT 'unknown';
ALTER TABLE tasks ADD COLUMN project_id TEXT NOT NULL DEFAULT '';
ALTER TABLE audit_logs ADD COLUMN scope_type TEXT NOT NULL DEFAULT 'unknown';
ALTER TABLE audit_logs ADD COLUMN project_id TEXT NOT NULL DEFAULT '';
ALTER TABLE audit_logs ADD COLUMN task_id TEXT NOT NULL DEFAULT '';
ALTER TABLE audit_logs ADD COLUMN operator_id TEXT NOT NULL DEFAULT '';
CREATE INDEX idx_tasks_project_history ON tasks(project_id, created_at);
CREATE INDEX idx_audit_project_history ON audit_logs(project_id, created_at);
CREATE INDEX idx_audit_task_history ON audit_logs(task_id);
@@ -50,6 +50,9 @@ func TestSQLiteBusinessUpgradePreservesDependentObjects(t *testing.T) {
t.Fatal("行号或生成列发生变化")
}
var count, actualForeignKeys int
if err := db.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM trigger_events").Scan(&count); err != nil || count != 2 {
t.Fatalf("范围回填应保留并调用两次历史更新触发器:%d,%v", count, err)
}
if err := db.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM task_links WHERE task_id = 'task'").Scan(&count); err != nil || count != 1 {
t.Fatalf("任务表重建触发了级联删除:%v", err)
}
@@ -59,7 +62,7 @@ func TestSQLiteBusinessUpgradePreservesDependentObjects(t *testing.T) {
if _, err := db.DB.ExecContext(t.Context(), `INSERT INTO a_task_view VALUES ('task', '升级后视图写入'); INSERT INTO settings (key, value) VALUES ('after', '升级后外部触发器')`); err != nil {
t.Fatal(err)
}
if err := db.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM trigger_events").Scan(&count); err != nil || count != 2 {
if err := db.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM trigger_events").Scan(&count); err != nil || count != 4 {
t.Fatalf("触发器执行结果错误:%d,%v", count, err)
}
})