package repository import ( "context" "database/sql" "errors" "maps" "reflect" "testing" "git.pcux.cn/xuans/Scert/backend/internal/deployment" ) // 远端文件是测试替身;事务、关联、占用交接与审计均经过真实数据库和产品执行器。 type reconciliationClient struct { *executorIntegrationClient failClosed, readOnly bool connections, writes int } func (c *reconciliationClient) Connect(context.Context, deployment.Server) (deployment.Client, error) { c.connections++ return c, nil } func (c *reconciliationClient) Close() error { if c.failClosed { if _, exists := c.files["/cert.pem.scert-lock"]; !exists { return errors.New("注入动作完成后的连接关闭失败") } } return nil } func (c *reconciliationClient) writable() error { c.writes++ if c.readOnly { return errors.New("重新核查不得执行远端写操作") } return nil } func (c *reconciliationClient) Remove(ctx context.Context, path string) error { if err := c.writable(); err != nil { return err } return c.executorIntegrationClient.Remove(ctx, path) } func (c *reconciliationClient) Rename(ctx context.Context, from, to string) error { if err := c.writable(); err != nil { return err } return c.executorIntegrationClient.Rename(ctx, from, to) } func (c *reconciliationClient) Unlock(ctx context.Context, path string) error { if err := c.writable(); err != nil { return err } return c.executorIntegrationClient.Unlock(ctx, path) } func (c *reconciliationClient) Run(ctx context.Context, command []string) (deployment.CommandResult, error) { if err := c.writable(); err != nil { return deployment.CommandResult{}, err } return c.executorIntegrationClient.Run(ctx, command) } func seedUncertainReconciliation(t *testing.T, repo *SQLDeploymentTransactionRepository, db *sql.DB, action deployment.RecoveryAction) (*deployment.Service, *reconciliationClient, deployment.Server, deployment.TransactionRecord, deployment.RecoveryDisposition) { t.Helper() _, _, _, remote, server, record := seedRecoveryExecutor(t, repo, db) client := &reconciliationClient{executorIntegrationClient: remote, failClosed: true} service := deployment.NewServiceWithJournal(client, repo) service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error { return nil } resolve := func(context.Context, string) (deployment.Server, error) { return server, nil } confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve) if err != nil { t.Fatal(err) } source, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve) if !errors.Is(err, deployment.ErrRecoveryActionUncertain) || source.Status != deployment.RecoveryResultUncertain { t.Fatalf("未形成真实执行器不确定结果:%+v %v", source, err) } // 比较实际持久化的前后快照,不能把内存纳秒与数据库时间精度差异误判为历史改写。 persisted, err := repo.GetRecoveryDisposition(t.Context(), source.Intent.ID) if err != nil || !persisted.Intent.SameRequest(source.Intent) || persisted.Status != source.Status || persisted.ResultCode != source.ResultCode { t.Fatal("原不确定处置未可靠持久化", err) } source = persisted client.failClosed, client.readOnly = false, true client.writes = 0 return service, client, server, record, source } func assertReconciliationSource(t *testing.T, repo *SQLDeploymentTransactionRepository, source deployment.RecoveryDisposition) { t.Helper() stored, err := repo.GetRecoveryDisposition(t.Context(), source.Intent.ID) if err != nil || !reflect.DeepEqual(stored, source) { t.Fatalf("重新核查改写了原不确定历史:%+v %v", stored, err) } } func runReconciliationSuccess(t *testing.T, repo *SQLDeploymentTransactionRepository, db *sql.DB, action deployment.RecoveryAction) { t.Helper() service, client, server, record, source := seedUncertainReconciliation(t, repo, db, action) resolve := func(context.Context, string) (deployment.Server, error) { return server, nil } files := maps.Clone(client.files) verified := false service.VerifyRecoveryTarget = func(_ context.Context, _ deployment.Server, _ deployment.RecoveryBaseline, committed bool) error { if committed != (action == deployment.RecoveryActionCommit) { t.Fatal("重新核查擅自切换方向") } verified = true return nil } confirmation, token, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve) if err != nil || confirmation.Summary().Validate() != nil { t.Fatal("准备重新核查失败", err) } other, otherToken, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve) if err != nil { t.Fatal(err) } result, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve) if err != nil || result.Status != "succeeded" || result.Intent.SourceDispositionID != source.Intent.ID || !verified { t.Fatalf("重新核查失败:%+v %v", result, err) } if client.writes != 0 || !reflect.DeepEqual(files, client.files) { t.Fatal("只读重新核查修改了远端") } assertReconciliationSource(t, repo, source) recoveryCount(t, db, "recovery_dispositions", 2) recoveryCount(t, db, "recovery_events", 4) recoveryCount(t, db, "recovery_claims", 0) pending, err := repo.ListPendingDeploymentTransactions(t.Context()) if err != nil || len(pending) != 0 { t.Fatal("原事务未原子结束", err) } view, err := service.GetRecoveryDisposition(t.Context(), result.Intent.ID) if err != nil || view.SourceDispositionID != source.Intent.ID || view.Validate() != nil { t.Fatal("查询缺少可信关联", err) } connections := client.connections if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, deployment.ErrRecoveryConfirmation) { t.Fatal("重新核查确认被重放", err) } if _, err := service.ExecuteRecoveryDisposition(t.Context(), other, "unix:0", otherToken, resolve); err == nil { t.Fatal("其他旧确认再次接管已完成的事务") } if client.connections != connections { t.Fatal("完成后仍使用旧确认连接远端") } if record.ID != result.Intent.TransactionID { t.Fatal("结束了其他事务") } } func TestRecoveryReconciliationSQLiteSuccess(t *testing.T) { for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} { t.Run(string(action), func(t *testing.T) { _, db := newTaskRepositoryTest(t) t.Cleanup(func() { _ = db.Close() }) runReconciliationSuccess(t, NewSQLDeploymentTransactionRepository(db), db, action) }) } } func TestRecoveryReconciliationSQLiteRejectsUnsafeCompletion(t *testing.T) { for _, failure := range []string{"file", "backup", "temporary", "lock", "https", "during-https", "close", "changed-after-prepare", "started-audit", "finished-audit", "identity", "phase", "running", "direction"} { t.Run(failure, func(t *testing.T) { _, db := newTaskRepositoryTest(t) t.Cleanup(func() { _ = db.Close() }) repo := NewSQLDeploymentTransactionRepository(db) service, client, server, record, source := seedUncertainReconciliation(t, repo, db, deployment.RecoveryActionCommit) resolve := func(context.Context, string) (deployment.Server, error) { return server, nil } switch failure { case "file": client.files[record.Paths[0]] = "tampered" case "backup": client.files[record.Backups[0].Saved] = "old" case "temporary": client.files[record.Paths[0]+".scert-tmp-"+record.ID] = "unexpected" case "lock": client.files[record.LockPaths[0]] = "" case "https": service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error { return errors.New("严格 HTTPS 未通过") } case "during-https": service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error { client.files[record.Paths[0]] = "changed" return nil } case "identity": server.PasswordRef = "secret://ssh/password/changed" case "phase": if _, err := db.ExecContext(t.Context(), `UPDATE deployment_transactions SET phase = 'reload_attempted'`); err != nil { t.Fatal(err) } case "running": if _, err := db.ExecContext(t.Context(), `UPDATE recovery_dispositions SET status = 'running', result_code = ''`); err != nil { t.Fatal(err) } case "direction": if _, err := db.ExecContext(t.Context(), `UPDATE recovery_dispositions SET action = 'restore_baseline'`); err != nil { t.Fatal(err) } } confirmation, token, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve) if failure == "identity" || failure == "phase" || failure == "running" || failure == "direction" { if err == nil { t.Fatal("危险来源允许重新核查") } recoveryCount(t, db, "recovery_claims", 1) return } if err != nil { t.Fatal("准备失败", err) } if failure == "close" { service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error { client.failClosed = true return nil } } if failure == "changed-after-prepare" { client.files[record.Paths[1]] = "changed" } if failure == "started-audit" || failure == "finished-audit" { stage := "started" if failure == "finished-audit" { stage = "finished" } if _, err := db.ExecContext(t.Context(), `CREATE TRIGGER reject_recheck BEFORE INSERT ON recovery_events WHEN NEW.event_type = '`+stage+`' BEGIN SELECT RAISE(ABORT, 'injected'); END`); err != nil { t.Fatal(err) } } result, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve) if err == nil || result.Status == "succeeded" || client.writes != 0 { t.Fatalf("危险收尾未拒绝或执行了写操作:%+v %v", result, err) } assertReconciliationSource(t, repo, source) var owner string if err := db.QueryRowContext(t.Context(), `SELECT disposition_id FROM recovery_claims WHERE transaction_id = ?`, record.ID).Scan(&owner); err != nil { t.Fatal(err) } expected := source.Intent.ID if failure == "finished-audit" { expected = confirmation.DispositionID() } if owner != expected { t.Fatal("失败后占用归属不正确", owner, expected) } pending, err := repo.ListPendingDeploymentTransactions(t.Context()) if err != nil || len(pending) != 1 || pending[0].Status != deployment.TransactionStatusRecoveryRequired { t.Fatal("失败改变了原事务", err) } }) } }