diff --git a/backend/internal/deployment/recovery_actions.go b/backend/internal/deployment/recovery_actions.go new file mode 100644 index 0000000..4772bf1 --- /dev/null +++ b/backend/internal/deployment/recovery_actions.go @@ -0,0 +1,128 @@ +package deployment + +import "context" + +// validateManualTemporaryFiles 检查完整目标集合，不能因为历史临时路径列表缺项就忽略未知文件。 +func validateManualTemporaryFiles(ctx context.Context, client Client, record TransactionRecord) error { + for _, target := range record.Paths { + path := target + ".scert-tmp-" + record.ID + if exists, err := client.Exists(ctx, path); err != nil { + return err + } else if exists { + if err := checkFileDigest(ctx, client, path, record.Evidence.TemporaryFiles[path]); err != nil { + return err + } + } + } + return ctx.Err() +} + +func validateRecoveryCommit(ctx context.Context, client Client, record TransactionRecord, allowMissingLocks bool) error { + if err := validateTransactionLocks(ctx, client, record, allowMissingLocks); err != nil { + return err + } + for _, path := range record.Paths { + if err := checkFileDigest(ctx, client, path, record.Evidence.Files[path]); err != nil { + return err + } + } + if err := validateManualTemporaryFiles(ctx, client, record); err != nil { + return err + } + for _, backup := range record.Backups { + if _, err := validateRecoveryBackup(ctx, client, backup); err != nil { + return err + } + } + return ctx.Err() +} + +func (s *Service) executeRecoveryCommit(ctx context.Context, record *TransactionRecord, server Server, client Client, mutated *bool) error { + allowMissingLocks := record.Phase == TransactionPhaseCommitVerified + if err := validateRecoveryCommit(ctx, client, *record, allowMissingLocks); err != nil { + return err + } + // 即使历史阶段已核验，也必须再次核验真实新证书，不能仅凭历史状态清理。 + if err := s.VerifyRecoveryTarget(ctx, server, record.Evidence.Baseline, true); err != nil { + return err + } + if err := validateRecoveryCommit(ctx, client, *record, allowMissingLocks); err != nil { + return err + } + *mutated = true + for _, temporary := range record.TemporaryPaths { + if err := client.Remove(ctx, temporary); err != nil { + return err + } + } + for _, backup := range record.Backups { + if err := client.Remove(ctx, backup.Saved); err != nil { + return err + } + } + if err := s.persistRecoveryPhase(ctx, record, TransactionPhaseCommitVerified); err != nil { + return err + } + return releaseTransactionLocks(ctx, client, *record, true, allowMissingLocks) +} + +func validateRecoveredBaseline(ctx context.Context, client Client, record TransactionRecord) error { + for _, backup := range record.Backups { + if backup.OriginalExists { + if err := checkFileDigest(ctx, client, backup.Original, backup.OriginalDigest); err != nil { + return err + } + } else if exists, err := client.Exists(ctx, backup.Original); err != nil || exists { + return ErrRecoveryActionRejected + } + // 还原完成后不允许留下新出现或未消费的临时文件、备份。 + for _, path := range []string{backup.Original + ".scert-tmp-" + record.ID, backup.Saved} { + if exists, err := client.Exists(ctx, path); err != nil || exists { + return ErrRecoveryActionRejected + } + } + } + return ctx.Err() +} + +func (s *Service) executeRecoveryRollback(ctx context.Context, record *TransactionRecord, server Server, client Client, mutated *bool) error { + allowMissingLocks := record.Phase == TransactionPhaseRollbackVerified + if err := validateManualTemporaryFiles(ctx, client, *record); err != nil { + return err + } + if allowMissingLocks { + if err := validateTransactionLocks(ctx, client, *record, true); err != nil { + return err + } + if err := validateRecoveredBaseline(ctx, client, *record); err != nil { + return err + } + } else if err := validateRollbackFiles(ctx, client, *record); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + *mutated = true + if !allowMissingLocks { + if err := restoreTransactionFiles(ctx, client, *record); err != nil { + return err + } + } + for _, command := range [][]string{server.ValidationCommand, server.ReloadCommand} { + response, err := client.Run(ctx, command) + if err != nil || response.ExitCode != 0 { + return ErrRecoveryActionUncertain + } + } + if err := s.VerifyRecoveryTarget(ctx, server, record.Evidence.Baseline, false); err != nil { + return err + } + if err := validateRecoveredBaseline(ctx, client, *record); err != nil { + return err + } + if err := s.persistRecoveryPhase(ctx, record, TransactionPhaseRollbackVerified); err != nil { + return err + } + return releaseTransactionLocks(ctx, client, *record, true, allowMissingLocks) +} diff --git a/backend/internal/deployment/recovery_confirmation.go b/backend/internal/deployment/recovery_confirmation.go new file mode 100644 index 0000000..267b24a --- /dev/null +++ b/backend/internal/deployment/recovery_confirmation.go @@ -0,0 +1,93 @@ +package deployment + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "errors" + "sync/atomic" +) + +var ErrRecoveryConfirmation = errors.New("人工处置确认无效、已使用或已过期，请重新核查") + +// RecoveryConfirmation 的副本共享消费状态，不能通过复制对象重复授权。 +// 本机会话持有对象，终端仅收到短时令牌和脱敏摘要；令牌不写入数据库。 +type RecoveryConfirmation struct { + state *recoveryConfirmationState +} + +type recoveryConfirmationState struct { + review RecoveryReview + token [sha256.Size]byte + intent RecoveryDispositionIntent + used atomic.Bool +} + +func (c RecoveryConfirmation) Summary() RecoveryReviewSummary { + if c.state == nil { + return RecoveryReviewSummary{} + } + return c.state.review.Summary() +} + +func (c RecoveryConfirmation) DispositionID() string { + if c.state == nil { + return "" + } + return c.state.intent.ID +} + +func (s *Service) PrepareRecoveryConfirmation(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryConfirmation, string, error) { + review, err := s.PrepareRecoveryReview(ctx, id, operator, action, resolve) + if err != nil { + return RecoveryConfirmation{}, "", err + } + return s.newRecoveryConfirmation(ctx, review) +} + +func (s *Service) newRecoveryConfirmation(ctx context.Context, review RecoveryReview) (RecoveryConfirmation, string, error) { + if s == nil || review.service != s { + return RecoveryConfirmation{}, "", ErrRecoveryConfirmation + } + var entropy [96]byte + if _, err := rand.Read(entropy[:]); err != nil { + return RecoveryConfirmation{}, "", ErrRecoveryConfirmation + } + token := hex.EncodeToString(entropy[:32]) + intent, err := review.DispositionIntent(hex.EncodeToString(entropy[32:64]), hex.EncodeToString(entropy[64:])) + if err != nil { + return RecoveryConfirmation{}, "", err + } + if err := ctx.Err(); err != nil { + return RecoveryConfirmation{}, "", err + } + return RecoveryConfirmation{state: &recoveryConfirmationState{ + review: review, intent: intent, token: sha256.Sum256([]byte(token)), + }}, token, nil +} + +// consumeRecoveryConfirmation 每个对象只允许一次确认尝试，错误身份、错误令牌和取消均不可重放。 +// 成功消费仍不是远端写授权，执行器必须取得持久化占用并重新检查现场。 +func (s *Service) consumeRecoveryConfirmation(ctx context.Context, c RecoveryConfirmation, operator, token string) (RecoveryReview, RecoveryDispositionIntent, error) { + fail := func(err error) (RecoveryReview, RecoveryDispositionIntent, error) { + return RecoveryReview{}, RecoveryDispositionIntent{}, err + } + state := c.state + if state == nil || !state.used.CompareAndSwap(false, true) { + return fail(ErrRecoveryConfirmation) + } + if s == nil || state.review.service != s || s.now == nil || !validReviewOperator(operator) || state.review.operator != operator || + !s.now().Before(state.review.summary.ExpiresAt) || len(token) != 64 { + return fail(ErrRecoveryConfirmation) + } + digest := sha256.Sum256([]byte(token)) + if subtle.ConstantTimeCompare(digest[:], state.token[:]) != 1 { + return fail(ErrRecoveryConfirmation) + } + if err := ctx.Err(); err != nil { + return fail(err) + } + return state.review, state.intent, nil +} diff --git a/backend/internal/deployment/recovery_confirmation_test.go b/backend/internal/deployment/recovery_confirmation_test.go new file mode 100644 index 0000000..4108fa6 --- /dev/null +++ b/backend/internal/deployment/recovery_confirmation_test.go @@ -0,0 +1,97 @@ +package deployment + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func confirmationFixture(t *testing.T) (*Service, RecoveryConfirmation, string) { + t.Helper() + server, client, record := recoveryPreflightFixture() + service := NewServiceWithJournal(fixedClientFactory{client}, &memoryTransactionJournal{pending: []TransactionRecord{record}}) + confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", RecoveryActionRecord, func(context.Context, string) (Server, error) { return server, nil }) + if err != nil { + t.Fatal(err) + } + return service, confirmation, token +} + +func TestRecoveryConfirmationSingleUseAndPrivateBindings(t *testing.T) { + service, confirmation, token := confirmationFixture(t) + if len(token) != 64 || len(confirmation.DispositionID()) != 64 || token == confirmation.DispositionID() { + t.Fatal("invalid confirmation entropy") + } + encoded, err := json.Marshal(confirmation) + if err != nil || string(encoded) != "{}" { + t.Fatal("private confirmation serialized") + } + copy := confirmation + review, intent, err := service.consumeRecoveryConfirmation(t.Context(), confirmation, "unix:0", token) + if err != nil || review.Summary() != confirmation.Summary() || intent.Validate() != nil || intent.ID != confirmation.DispositionID() { + t.Fatalf("confirmation failed: %v", err) + } + encoded, _ = json.Marshal(intent) + if strings.Contains(string(encoded), token) { + t.Fatal("raw token persisted") + } + if _, _, err := service.consumeRecoveryConfirmation(t.Context(), copy, "unix:0", token); !errors.Is(err, ErrRecoveryConfirmation) { + t.Fatal("copied confirmation replayed") + } +} + +func TestRecoveryConfirmationRejectsAndConsumesInvalidAttempt(t *testing.T) { + for _, scenario := range []string{"identity", "token", "oversized", "expired", "service", "cancelled", "empty"} { + t.Run(scenario, func(t *testing.T) { + service, confirmation, token := confirmationFixture(t) + originalService, originalToken := service, token + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + operator := "unix:0" + switch scenario { + case "identity": + operator = "unix:1" + case "token": + token = strings.Repeat("0", 64) + case "oversized": + token = strings.Repeat("a", 8192) + case "expired": + service.now = func() time.Time { return confirmation.Summary().ExpiresAt } + case "service": + service = NewService(nil) + case "cancelled": + cancel() + case "empty": + confirmation = RecoveryConfirmation{} + } + if _, _, err := service.consumeRecoveryConfirmation(ctx, confirmation, operator, token); err == nil { + t.Fatal("invalid confirmation accepted") + } + if _, _, err := originalService.consumeRecoveryConfirmation(t.Context(), confirmation, "unix:0", originalToken); err == nil { + t.Fatal("failed attempt could be replayed") + } + }) + } +} + +func TestRecoveryConfirmationConcurrentCopiesConsumeOnlyOnce(t *testing.T) { + service, confirmation, token := confirmationFixture(t) + var accepted atomic.Int32 + var group sync.WaitGroup + for range 32 { + group.Go(func() { + if _, _, err := service.consumeRecoveryConfirmation(t.Context(), confirmation, "unix:0", token); err == nil { + accepted.Add(1) + } + }) + } + group.Wait() + if accepted.Load() != 1 { + t.Fatalf("accepted %d confirmations", accepted.Load()) + } +} diff --git a/backend/internal/deployment/recovery_executor.go b/backend/internal/deployment/recovery_executor.go new file mode 100644 index 0000000..c70db41 --- /dev/null +++ b/backend/internal/deployment/recovery_executor.go @@ -0,0 +1,139 @@ +package deployment + +import ( + "context" + "errors" + "time" +) + +var ( + ErrRecoveryActionRejected = errors.New("人工处置前置核验未通过，未执行远端变更") + ErrRecoveryActionUncertain = errors.New("人工处置结果不确定，已保留占用，请查询记录并核查现场") +) + +// ExecuteRecoveryDisposition 仅供受保护本机会话使用，操作者必须来自本机对端身份。 +// 一次性确认、共同操作门、持久化占用和远端锁各自独立，缺一不可。 +func (s *Service) ExecuteRecoveryDisposition(ctx context.Context, confirmation RecoveryConfirmation, operator, token string, resolve func(context.Context, string) (Server, error)) (RecoveryDisposition, error) { + review, intent, err := s.consumeRecoveryConfirmation(ctx, confirmation, operator, token) + if err != nil { + return RecoveryDisposition{}, err + } + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + release, err := s.acquireOperation(ctx) + if err != nil { + return RecoveryDisposition{}, err + } + defer release() + journal, ok := s.journal.(RecoveryDispositionJournal) + if !ok || resolve == nil { + return RecoveryDisposition{}, ErrRecoveryDispositionUnavailable + } + if !s.now().Before(review.summary.ExpiresAt) { + return RecoveryDisposition{}, ErrRecoveryConfirmation + } + item, created, err := journal.BeginRecoveryDisposition(ctx, intent) + if err != nil { + return RecoveryDisposition{}, ErrRecoveryDispositionUnavailable + } + if !created { + // 响应丢失后的既有请求只返回持久结果，绝不重新连接或执行。 + return item, nil + } + finish := func(result string, actionErr error) (RecoveryDisposition, error) { + // 请求取消不能跳过结束审计；保存失败保留原占用且绝不返回成功。 + finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer finishCancel() + finished, finishErr := journal.FinishRecoveryDisposition(finishCtx, intent.ID, result) + if finishErr != nil { + return item, ErrRecoveryDispositionUnavailable + } + return finished, actionErr + } + if err := s.RecheckRecoveryReview(ctx, review, operator, resolve); err != nil { + return finish(RecoveryResultRejected, ErrRecoveryActionRejected) + } + if review.summary.Action == RecoveryActionRecord { + return finish(RecoveryResultObserved, nil) + } + result, actionErr := s.executeRecoveryAction(ctx, review, resolve) + return finish(result, actionErr) +} + +func (s *Service) executeRecoveryAction(ctx context.Context, review RecoveryReview, resolve func(context.Context, string) (Server, error)) (result string, actionErr error) { + result, actionErr = RecoveryResultRejected, ErrRecoveryActionRejected + record, recordDigest, serverDigest, err := s.reviewRecord(ctx, review.summary.TransactionID, resolve) + if err != nil || recordDigest != review.recordDigest || serverDigest != review.serverDigest { + return + } + server, err := resolve(ctx, record.ServerID) + if err != nil { + return + } + server = server.Normalize() + currentServerDigest, err := reviewServerDigest(server) + if err != nil || currentServerDigest != review.serverDigest || validateRecoveryAction(record, review.summary.Action, s.VerifyRecoveryTarget != nil) != nil { + return + } + client, err := s.factory.Connect(ctx, server) + if err != nil { + return + } + mutated := false + defer func() { + // 在结果审计和释放共同操作门之前关闭连接，关闭失败不能伪装成成功。 + if closeErr := client.Close(); closeErr != nil { + if mutated { + result, actionErr = RecoveryResultUncertain, ErrRecoveryActionUncertain + } else { + result, actionErr = RecoveryResultRejected, ErrRecoveryActionRejected + } + } + }() + if err := ctx.Err(); err != nil || !s.now().Before(review.summary.ExpiresAt) { + return + } + switch review.summary.Action { + case RecoveryActionCommit: + err = s.executeRecoveryCommit(ctx, &record, server, client, &mutated) + result = RecoveryResultCommitted + case RecoveryActionRollback: + err = s.executeRecoveryRollback(ctx, &record, server, client, &mutated) + result = RecoveryResultRolledBack + default: + return + } + if err == nil { + return result, nil + } + if mutated { + return RecoveryResultUncertain, ErrRecoveryActionUncertain + } + return RecoveryResultRejected, ErrRecoveryActionRejected +} + +func validateRecoveryAction(record TransactionRecord, action RecoveryAction, verifier bool) error { + if record.Evidence == nil || !verifier || len(record.Backups) != len(record.Paths) { + return ErrRecoveryActionRejected + } + switch action { + case RecoveryActionCommit: + if record.Evidence.Baseline.ExpectedFingerprint == "" { + return ErrRecoveryActionRejected + } + switch record.Phase { + case TransactionPhaseReloadAttempted, TransactionPhaseCommitting, TransactionPhaseCommitVerified: + return nil + } + case RecoveryActionRollback: + if record.Evidence.Baseline.PreviousFingerprint == "" { + return ErrRecoveryActionRejected + } + switch record.Phase { + case TransactionPhaseLocked, TransactionPhaseUploaded, TransactionPhaseReplacing, TransactionPhaseInstalled, + TransactionPhaseReloadAttempted, TransactionPhaseCommitting, TransactionPhaseRollbackVerified: + return nil + } + } + return ErrRecoveryActionRejected +} diff --git a/backend/internal/deployment/recovery_executor_test.go b/backend/internal/deployment/recovery_executor_test.go new file mode 100644 index 0000000..ecfad02 --- /dev/null +++ b/backend/internal/deployment/recovery_executor_test.go @@ -0,0 +1,428 @@ +package deployment + +import ( + "context" + "errors" + "maps" + "reflect" + "testing" + "time" +) + +type executorJournal struct { + memoryTransactionJournal + item RecoveryDisposition + claimed bool + beginCalls, finishCalls int + beginErr, finishErr, updateErr error + finishContextErr error + existing bool + beforeFinish func() +} + +func (j *executorJournal) BeginRecoveryDisposition(_ context.Context, intent RecoveryDispositionIntent) (RecoveryDisposition, bool, error) { + j.beginCalls++ + if j.beginErr != nil { + return RecoveryDisposition{}, false, j.beginErr + } + if j.existing { + return j.item, false, nil + } + if j.claimed { + return RecoveryDisposition{}, false, ErrRecoveryDispositionBusy + } + if !intent.MatchesRecord(j.pending[0]) { + return RecoveryDisposition{}, false, ErrRecoveryReviewChanged + } + j.item = RecoveryDisposition{Intent: intent, Status: "running"} + j.claimed = true + return j.item, true, nil +} + +func (j *executorJournal) FinishRecoveryDisposition(ctx context.Context, id, result string) (RecoveryDisposition, error) { + j.finishCalls++ + j.finishContextErr = ctx.Err() + if j.beforeFinish != nil { + j.beforeFinish() + } + if j.finishErr != nil { + return RecoveryDisposition{}, j.finishErr + } + if id != j.item.Intent.ID || !j.item.Intent.AllowsResult(result) || !j.claimed { + return RecoveryDisposition{}, ErrRecoveryDispositionConflict + } + j.item.Status, j.item.ResultCode = "succeeded", result + if result == RecoveryResultRejected || result == RecoveryResultUncertain { + j.item.Status = result + } + j.claimed = result == RecoveryResultUncertain + return j.item, nil +} + +func (j *executorJournal) GetRecoveryDisposition(context.Context, string) (RecoveryDisposition, error) { + return j.item, nil +} + +func (j *executorJournal) HasRecoveryClaim(context.Context, string) (bool, error) { + return j.claimed, nil +} + +func (j *executorJournal) UpdateDeploymentTransaction(ctx context.Context, record TransactionRecord) error { + if j.updateErr != nil { + return j.updateErr + } + return j.memoryTransactionJournal.UpdateDeploymentTransaction(ctx, record) +} + +type executorClient struct { + *memoryClient + closeErr error + closed int + beforeClose func() +} + +func (c *executorClient) Close() error { + c.closed++ + if c.beforeClose != nil { + c.beforeClose() + } + return c.closeErr +} + +func executorFixture(t *testing.T) (*Service, *executorJournal, *executorClient, Server, TransactionRecord) { + t.Helper() + server, memory, record := recoveryPreflightFixture() + journal := &executorJournal{memoryTransactionJournal: memoryTransactionJournal{pending: []TransactionRecord{record}}} + client := &executorClient{memoryClient: memory} + service := NewServiceWithJournal(fixedClientFactory{client}, journal) + service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { return nil } + return service, journal, client, server, record +} + +func TestRecoveryExecutorThreeExplicitActions(t *testing.T) { + for _, action := range []RecoveryAction{RecoveryActionRecord, RecoveryActionCommit, RecoveryActionRollback} { + t.Run(string(action), func(t *testing.T) { + service, journal, client, server, record := executorFixture(t) + before := maps.Clone(client.files) + resolve := func(context.Context, string) (Server, error) { return server, nil } + confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve) + if err != nil { + t.Fatal(err) + } + var verified []bool + service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, commit bool) error { + verified = append(verified, commit) + return nil + } + item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve) + if err != nil || item.Status != "succeeded" || journal.claimed || journal.beginCalls != 1 || journal.finishCalls != 1 || len(journal.finished) != 0 { + t.Fatalf("execution failed: %+v, %v", item, err) + } + if action == RecoveryActionRecord { + if item.ResultCode != RecoveryResultObserved || len(verified) != 0 || len(journal.updated) != 0 || !reflect.DeepEqual(before, client.files) { + t.Fatal("record-only action changed original state") + } + } else { + expected, phase, result := "new", TransactionPhaseCommitVerified, RecoveryResultCommitted + if action == RecoveryActionRollback { + expected, phase, result = "old", TransactionPhaseRollbackVerified, RecoveryResultRolledBack + } + if !reflect.DeepEqual(verified, []bool{action == RecoveryActionCommit}) || item.ResultCode != result || len(journal.updated) != 1 || journal.updated[0].Phase != phase { + t.Fatal("wrong action, verification or phase") + } + for _, path := range record.Paths { + if string(client.files[path].data) != expected { + t.Fatal("wrong certificate files") + } + } + if len(client.files) != 3 { + t.Fatal("transaction leftovers remain") + } + } + if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, ErrRecoveryConfirmation) || journal.beginCalls != 1 { + t.Fatal("execution replayed") + } + }) + } +} + +func TestRecoveryExecutorFailureBoundaries(t *testing.T) { + for _, scenario := range []string{"changed", "foreign-lock", "legacy", "phase", "missing-baseline", "missing-backup-entry", "begin", "finish", "persist", "cleanup", "reload", "commit-https", "rollback-https", "cancel", "close", "existing"} { + t.Run(scenario, func(t *testing.T) { + service, journal, client, server, record := executorFixture(t) + action := RecoveryActionCommit + if scenario == "reload" || scenario == "rollback-https" || scenario == "missing-backup-entry" { + action = RecoveryActionRollback + } + if scenario == "legacy" { + record.Evidence = nil + action = RecoveryActionRecord + } + if scenario == "phase" { + record.Phase = TransactionPhaseUploaded + } + if scenario == "missing-baseline" { + record.Evidence.Baseline.ExpectedFingerprint = "" + } + if scenario == "missing-backup-entry" { + record.Backups = record.Backups[:2] + } + journal.pending[0] = record + resolve := func(context.Context, string) (Server, error) { return server, nil } + confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + verifyCalls := 0 + service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, commit bool) error { + verifyCalls++ + if commit != (action == RecoveryActionCommit) { + t.Fatal("failed action silently switched direction") + } + if scenario == "commit-https" || scenario == "rollback-https" { + return errors.New("private upstream details") + } + if scenario == "cancel" { + cancel() + } + if scenario == "close" { + client.closeErr = errors.New("private close details") + } + return nil + } + switch scenario { + case "changed": + client.files[record.Paths[0]] = memoryFile{data: []byte("foreign")} + case "foreign-lock": + client.files[record.LockPaths[0]+"/owner"] = memoryFile{data: []byte("foreign")} + case "begin": + journal.beginErr = errors.New("private database error") + case "finish": + journal.finishErr = errors.New("private audit error") + case "persist": + journal.updateErr = errors.New("private update error") + case "cleanup": + client.removeError = ".scert-backup-" + case "reload": + client.runErrors = []error{errors.New("private command error")} + case "existing": + journal.existing = true + journal.item = RecoveryDisposition{Status: "uncertain", ResultCode: RecoveryResultUncertain} + } + before := maps.Clone(client.files) + closedBefore := client.closed + item, err := service.ExecuteRecoveryDisposition(ctx, confirmation, "unix:0", token, resolve) + if scenario == "legacy" { + if err != nil || item.ResultCode != RecoveryResultObserved || len(journal.updated) != 0 || !reflect.DeepEqual(before, client.files) { + t.Fatal("legacy inspection mutated history") + } + return + } + if scenario == "existing" { + if err != nil || item.Status != "uncertain" || verifyCalls != 0 || journal.finishCalls != 0 || client.closed != closedBefore || !reflect.DeepEqual(before, client.files) { + t.Fatal("existing request executed again") + } + return + } + if err == nil || item.Status == "succeeded" || journal.finishContextErr != nil { + t.Fatalf("failure reported success or skipped independent audit: %+v %v", item, err) + } + uncertain := scenario == "finish" || scenario == "persist" || scenario == "cleanup" || scenario == "reload" || scenario == "rollback-https" || scenario == "close" + if journal.claimed != uncertain { + t.Fatalf("wrong claim retention: %v", journal.claimed) + } + if !uncertain && !reflect.DeepEqual(before, client.files) { + t.Fatal("preflight rejection changed remote files") + } + if len(journal.finished) != 0 { + t.Fatal("executor bypassed atomic disposition completion") + } + }) + } +} + +func TestRecoveryExecutorWaitCancellationConsumesConfirmation(t *testing.T) { + service, journal, _, server, record := executorFixture(t) + resolve := func(context.Context, string) (Server, error) { return server, nil } + confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", RecoveryActionRecord, resolve) + if err != nil { + t.Fatal(err) + } + release, err := service.acquireOperation(t.Context()) + if err != nil { + t.Fatal(err) + } + defer release() + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond) + defer cancel() + if _, err := service.ExecuteRecoveryDisposition(ctx, confirmation, "unix:0", token, resolve); !errors.Is(err, context.DeadlineExceeded) || journal.beginCalls != 0 { + t.Fatal("cancelled waiter acquired persistent claim") + } + if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, ErrRecoveryConfirmation) { + t.Fatal("cancelled confirmation replayed") + } +} + +func TestRecoveryExecutorHoldsOperationThroughVerificationCloseAndAudit(t *testing.T) { + service, journal, client, server, record := executorFixture(t) + resolve := func(context.Context, string) (Server, error) { return server, nil } + confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", RecoveryActionCommit, resolve) + if err != nil { + t.Fatal(err) + } + checked := map[string]int{} + checkHeld := func(stage string) { + checked[stage]++ + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Millisecond) + defer cancel() + release, err := service.acquireOperation(ctx) + if err == nil { + release() + t.Fatalf("operation gate released during %s", stage) + } + if !errors.Is(err, context.DeadlineExceeded) || !journal.claimed { + t.Fatalf("missing durable claim or invalid wait during %s", stage) + } + } + service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { + checkHeld("verify") + return nil + } + client.beforeClose = func() { checkHeld("close") } + journal.beforeFinish = func() { checkHeld("audit") } + if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); err != nil { + t.Fatal(err) + } + if checked["verify"] != 1 || checked["close"] != 2 || checked["audit"] != 1 { + t.Fatalf("missing gate coverage: %v", checked) + } + release, err := service.acquireOperation(t.Context()) + if err != nil { + t.Fatal(err) + } + release() +} + +func TestRecoveryExecutorVerifiedHistoryStillRechecksHTTPS(t *testing.T) { + for _, action := range []RecoveryAction{RecoveryActionCommit, RecoveryActionRollback} { + for _, failVerification := range []bool{false, true} { + t.Run(string(action)+map[bool]string{false: "/success", true: "/failure"}[failVerification], func(t *testing.T) { + service, journal, client, server, record := executorFixture(t) + record.Phase = TransactionPhaseCommitVerified + if action == RecoveryActionRollback { + record.Phase = TransactionPhaseRollbackVerified + for _, path := range record.Paths { + client.files[path] = memoryFile{data: []byte("old")} + } + } + for _, path := range record.TemporaryPaths { + delete(client.files, path) + } + for _, backup := range record.Backups { + delete(client.files, backup.Saved) + } + delete(client.files, record.LockPaths[0]) + delete(client.files, record.LockPaths[0]+"/owner") + journal.pending[0] = record + resolve := func(context.Context, string) (Server, error) { return server, nil } + confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve) + if err != nil { + t.Fatal(err) + } + verified := 0 + service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, commit bool) error { + verified++ + if commit != (action == RecoveryActionCommit) { + t.Fatal("wrong HTTPS verification direction") + } + if failVerification { + return errors.New("HTTPS mismatch") + } + return nil + } + item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve) + if verified != 1 || (err != nil) != failVerification || (item.Status == "succeeded") == failVerification { + t.Fatalf("historical phase bypassed new verification: %+v %v", item, err) + } + if !failVerification && len(client.files) != 3 { + t.Fatal("partial unlock history failed to finish safely") + } + if failVerification { + if _, exists := client.files[record.LockPaths[1]]; !exists { + t.Fatal("failed verification removed retained lock") + } + } + }) + } + } +} + +func TestRecoveryExecutorRechecksAfterHTTPSAndAuditsCancellation(t *testing.T) { + for _, scenario := range []string{"target", "backup", "lock", "cancel-after-write"} { + t.Run(scenario, func(t *testing.T) { + service, journal, client, server, record := executorFixture(t) + action := RecoveryActionCommit + if scenario == "cancel-after-write" { + action = RecoveryActionRollback + } + resolve := func(context.Context, string) (Server, error) { return server, nil } + confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { + path := record.Paths[0] + switch scenario { + case "backup": + path = record.Backups[0].Saved + case "lock": + path = record.LockPaths[0] + "/owner" + case "cancel-after-write": + cancel() + return ctx.Err() + } + client.files[path] = memoryFile{data: []byte("foreign-change")} + return nil + } + item, err := service.ExecuteRecoveryDisposition(ctx, confirmation, "unix:0", token, resolve) + if err == nil || item.Status == "succeeded" || journal.finishCalls != 1 || journal.finishContextErr != nil { + t.Fatal("changed or cancelled execution skipped audit or reported success") + } + if journal.claimed != (scenario == "cancel-after-write") { + t.Fatal("wrong claim retention after cancelled write") + } + if _, exists := client.files[record.LockPaths[0]]; !exists { + t.Fatal("changed state unlocked") + } + }) + } +} + +func TestRecoveryExecutorRejectsUnlistedTemporaryFiles(t *testing.T) { + for _, action := range []RecoveryAction{RecoveryActionCommit, RecoveryActionRollback} { + t.Run(string(action), func(t *testing.T) { + service, journal, client, server, record := executorFixture(t) + delete(record.Evidence.TemporaryFiles, record.TemporaryPaths[0]) + record.TemporaryPaths = record.TemporaryPaths[1:] + journal.pending[0] = record + resolve := func(context.Context, string) (Server, error) { return server, nil } + confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve) + if err != nil { + t.Fatal(err) + } + before := maps.Clone(client.files) + service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { + t.Fatal("unknown temporary file reached HTTPS verification") + return nil + } + item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve) + if !errors.Is(err, ErrRecoveryActionRejected) || item.ResultCode != RecoveryResultRejected || journal.claimed || !reflect.DeepEqual(before, client.files) { + t.Fatal("unlisted temporary file ignored or modified") + } + }) + } +} diff --git a/backend/internal/deployment/recovery_result.go b/backend/internal/deployment/recovery_result.go new file mode 100644 index 0000000..dcb88f8 --- /dev/null +++ b/backend/internal/deployment/recovery_result.go @@ -0,0 +1,79 @@ +package deployment + +import ( + "context" + "strings" + "time" +) + +// RecoveryDispositionSummary 是本机查询的明确输出边界，不包含请求键、绑定摘要或操作者凭据。 +type RecoveryDispositionSummary struct { + ID string `json:"id"` + TransactionID string `json:"transaction_id"` + ServerID string `json:"server_id"` + Action RecoveryAction `json:"action"` + Status string `json:"status"` + ResultCode string `json:"result_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (r RecoveryDisposition) Summary() RecoveryDispositionSummary { + return RecoveryDispositionSummary{ + ID: r.Intent.ID, TransactionID: r.Intent.TransactionID, ServerID: r.Intent.ServerID, + Action: r.Intent.Action, Status: r.Status, ResultCode: r.ResultCode, + CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt, + } +} + +func validDispositionID(id string) bool { + return validDigest(id) && id == strings.ToLower(id) && id != strings.Repeat("0", 64) +} + +func (r RecoveryDispositionSummary) Validate() error { + if !validDispositionID(r.ID) || !validInspectionID(r.TransactionID) || !validRecoverySummary(r.ServerID) || + !validRecoveryAction(r.Action) || r.CreatedAt.IsZero() || r.UpdatedAt.IsZero() || r.UpdatedAt.Before(r.CreatedAt) { + return ErrRecoveryDispositionInvalid + } + intent := RecoveryDispositionIntent{Action: r.Action} + switch r.Status { + case "running": + if r.ResultCode == "" { + return nil + } + case "succeeded": + if r.ResultCode != RecoveryResultRejected && r.ResultCode != RecoveryResultUncertain && intent.AllowsResult(r.ResultCode) { + return nil + } + case RecoveryResultRejected, RecoveryResultUncertain: + if r.ResultCode == r.Status { + return nil + } + } + return ErrRecoveryDispositionInvalid +} + +// GetRecoveryDisposition 不取得写操作门，执行中或重启后均可查询；查询本身不会重放动作或释放占用。 +func (s *Service) GetRecoveryDisposition(ctx context.Context, id string) (RecoveryDispositionSummary, error) { + if !validDispositionID(id) { + return RecoveryDispositionSummary{}, ErrRecoveryDispositionInvalid + } + if s == nil { + return RecoveryDispositionSummary{}, ErrRecoveryDispositionUnavailable + } + journal, ok := s.journal.(RecoveryDispositionJournal) + if !ok { + return RecoveryDispositionSummary{}, ErrRecoveryDispositionUnavailable + } + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + item, err := journal.GetRecoveryDisposition(ctx, id) + if err != nil || ctx.Err() != nil || item.Intent.ID != id || item.Intent.Validate() != nil { + return RecoveryDispositionSummary{}, ErrRecoveryDispositionUnavailable + } + summary := item.Summary() + if summary.Validate() != nil { + return RecoveryDispositionSummary{}, ErrRecoveryDispositionUnavailable + } + return summary, nil +} diff --git a/backend/internal/deployment/recovery_result_test.go b/backend/internal/deployment/recovery_result_test.go new file mode 100644 index 0000000..981ba38 --- /dev/null +++ b/backend/internal/deployment/recovery_result_test.go @@ -0,0 +1,67 @@ +package deployment + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +func TestRecoveryDispositionSummaryValidation(t *testing.T) { + now := time.Now().UTC() + for _, action := range []RecoveryAction{RecoveryActionRecord, RecoveryActionCommit, RecoveryActionRollback} { + for _, result := range []string{"", RecoveryResultObserved, RecoveryResultCommitted, RecoveryResultRolledBack, RecoveryResultRejected, RecoveryResultUncertain} { + status := "succeeded" + if result == "" { + status = "running" + } + if result == RecoveryResultRejected || result == RecoveryResultUncertain { + status = result + } + summary := RecoveryDispositionSummary{ID: strings.Repeat("a", 64), TransactionID: "transaction", ServerID: "server", Action: action, Status: status, ResultCode: result, CreatedAt: now, UpdatedAt: now} + want := result == "" || (RecoveryDispositionIntent{Action: action}).AllowsResult(result) + if (summary.Validate() == nil) != want { + t.Fatalf("wrong result validation: %+v", summary) + } + } + } +} + +func TestRecoveryDispositionQueryIsReadOnlyAndPrivate(t *testing.T) { + service, journal, _, _, _ := executorFixture(t) + now := time.Now().UTC() + journal.item = RecoveryDisposition{ + Intent: RecoveryDispositionIntent{ + ID: strings.Repeat("a", 64), RequestKey: strings.Repeat("b", 64), RecordDigest: strings.Repeat("c", 64), + ServerDigest: strings.Repeat("d", 64), ObservationDigest: strings.Repeat("e", 64), + TransactionID: "transaction", ServerID: "server", Operator: "unix:private", Action: RecoveryActionCommit, + }, Status: "running", CreatedAt: now, UpdatedAt: now, + } + journal.claimed = true + release, err := service.acquireOperation(t.Context()) + if err != nil { + t.Fatal(err) + } + defer release() + result, err := service.GetRecoveryDisposition(t.Context(), journal.item.Intent.ID) + if err != nil || result.Status != "running" || !journal.claimed || journal.beginCalls != 0 || journal.finishCalls != 0 { + t.Fatal("query changed state or blocked behind writer", err) + } + encoded, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + for _, private := range []string{journal.item.Intent.RequestKey, journal.item.Intent.RecordDigest, journal.item.Intent.ServerDigest, journal.item.Intent.ObservationDigest, journal.item.Intent.Operator} { + if strings.Contains(string(encoded), private) { + t.Fatal("private binding leaked in result") + } + } + if _, err := service.GetRecoveryDisposition(t.Context(), strings.Repeat("f", 64)); err == nil { + t.Fatal("wrong result identifier accepted") + } + journal.item.Status = "succeeded" + journal.item.ResultCode = RecoveryResultRolledBack + if _, err := service.GetRecoveryDisposition(t.Context(), journal.item.Intent.ID); err == nil { + t.Fatal("inconsistent action result accepted") + } +} diff --git a/backend/internal/repository/recovery_executor_integration_test.go b/backend/internal/repository/recovery_executor_integration_test.go new file mode 100644 index 0000000..61e07c0 --- /dev/null +++ b/backend/internal/repository/recovery_executor_integration_test.go @@ -0,0 +1,236 @@ +package repository + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "maps" + "reflect" + "strings" + "testing" + + "git.pcux.cn/xuans/Scert/backend/internal/deployment" +) + +// executorIntegrationClient 仅模拟远端文件；处置占用、审计和终态使用真实 SQLite。 +// 未列出的远端方法故意保持不可用，避免测试意外放行额外写操作。 +type executorIntegrationClient struct { + deployment.Client + files map[string]string +} + +var _ deployment.EvidenceClient = (*executorIntegrationClient)(nil) + +func (c *executorIntegrationClient) CreateOwnedLock(context.Context, string, string) error { + return errors.New("人工处置不能创建新的部署锁") +} + +func (c *executorIntegrationClient) Connect(context.Context, deployment.Server) (deployment.Client, error) { + return c, nil +} +func (c *executorIntegrationClient) Close() error { return nil } +func (c *executorIntegrationClient) Exists(_ context.Context, path string) (bool, error) { + _, exists := c.files[path] + return exists, nil +} +func (c *executorIntegrationClient) FileDigest(_ context.Context, path string) (string, error) { + value, exists := c.files[path] + if !exists { + return "", errors.New("测试文件不存在") + } + return executorDigest(value), nil +} +func (c *executorIntegrationClient) ReadLockOwner(_ context.Context, path string) (string, error) { + return c.files[path+"/owner"], nil +} +func (c *executorIntegrationClient) Remove(_ context.Context, path string) error { + delete(c.files, path) + return nil +} +func (c *executorIntegrationClient) Rename(_ context.Context, from, to string) error { + value, exists := c.files[from] + if !exists { + return errors.New("测试源文件不存在") + } + c.files[to] = value + delete(c.files, from) + return nil +} +func (c *executorIntegrationClient) Unlock(_ context.Context, path string) error { + delete(c.files, path) + delete(c.files, path+"/owner") + return nil +} +func (c *executorIntegrationClient) Run(context.Context, []string) (deployment.CommandResult, error) { + return deployment.CommandResult{}, nil +} + +func executorDigest(value string) string { + digest := sha256.Sum256([]byte(value)) + return hex.EncodeToString(digest[:]) +} + +func executorIntegrationFixture(t *testing.T) (*SQLDeploymentTransactionRepository, *sql.DB, *deployment.Service, *executorIntegrationClient, deployment.Server, deployment.TransactionRecord) { + t.Helper() + _, db := newTaskRepositoryTest(t) + t.Cleanup(func() { _ = db.Close() }) + repo := NewSQLDeploymentTransactionRepository(db) + server := deployment.Server{ + ID: "executor-server", Name: "executor-server", Address: "192.0.2.10", Port: 22, Username: "deploy", + AuthMethod: deployment.SSHAuthPassword, PasswordRef: "secret://ssh/password/test", HostKeyFingerprint: "SHA256:test", + CertificatePath: "/cert.pem", CertificateChainPath: "/chain.pem", PrivateKeyPath: "/key.pem", WebServer: deployment.WebServerNginx, + }.Normalize() + record := deployment.TransactionRecord{ + ID: "executor-transaction", TaskID: "task", AttemptID: "attempt", AttemptNumber: 1, ServerID: server.ID, ServerName: server.Name, + Paths: []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath}, + Status: deployment.TransactionStatusRecoveryRequired, Phase: deployment.TransactionPhaseReloadAttempted, ReloadAttempted: true, + Evidence: &deployment.RecoveryEvidence{ + Version: 1, LockToken: executorDigest("owner"), Address: server.Address, Port: server.Port, HostKeyFingerprint: server.HostKeyFingerprint, + ValidationCommand: server.ValidationCommand, ReloadCommand: server.ReloadCommand, Files: map[string]string{}, TemporaryFiles: map[string]string{}, + Baseline: deployment.RecoveryBaseline{Endpoint: "https://example.com", Domains: []string{"example.com"}, PreviousFingerprint: "old", ExpectedFingerprint: "new"}, + }, + } + client := &executorIntegrationClient{files: map[string]string{}} + for _, path := range record.Paths { + lock, backup, temporary := path+".scert-lock", path+".scert-backup-"+record.ID, path+".scert-tmp-"+record.ID + record.LockPaths = append(record.LockPaths, lock) + record.TemporaryPaths = append(record.TemporaryPaths, temporary) + record.InstalledPaths = append(record.InstalledPaths, path) + record.Backups = append(record.Backups, deployment.TransactionBackup{Original: path, Saved: backup, OriginalExists: true, OriginalDigest: executorDigest("old")}) + record.Evidence.Files[path], record.Evidence.TemporaryFiles[temporary] = executorDigest("new"), executorDigest("temporary") + client.files[path], client.files[backup], client.files[temporary] = "new", "old", "temporary" + client.files[lock], client.files[lock+"/owner"] = "", record.Evidence.LockToken + } + if err := repo.CreateDeploymentTransaction(t.Context(), record); err != nil { + t.Fatal(err) + } + service := deployment.NewServiceWithJournal(client, repo) + service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error { return nil } + return repo, db, service, client, server, record +} + +func TestRecoveryExecutorSQLiteAtomicLifecycle(t *testing.T) { + for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionRecord, deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} { + t.Run(string(action), func(t *testing.T) { + repo, db, service, client, server, record := executorIntegrationFixture(t) + before, err := repo.ListPendingDeploymentTransactions(t.Context()) + if err != nil { + t.Fatal(err) + } + filesBefore := maps.Clone(client.files) + resolve := func(context.Context, string) (deployment.Server, error) { return server, nil } + confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve) + if err != nil { + t.Fatal(err) + } + item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve) + if err != nil || item.Status != "succeeded" { + t.Fatalf("execution failed: %+v %v", item, err) + } + recoveryCount(t, db, "recovery_dispositions", 1) + recoveryCount(t, db, "recovery_events", 2) + recoveryCount(t, db, "recovery_claims", 0) + stored, err := repo.GetRecoveryDisposition(t.Context(), confirmation.DispositionID()) + if err != nil || !reflect.DeepEqual(stored, item) { + t.Fatal("result query lost completed result", err) + } + view, err := service.GetRecoveryDisposition(t.Context(), confirmation.DispositionID()) + if err != nil || view.Validate() != nil || !reflect.DeepEqual(view, item.Summary()) { + t.Fatal("safe query omitted or changed persisted result", err) + } + var status string + if err := db.QueryRowContext(t.Context(), "SELECT status FROM deployment_transactions WHERE id = ?", record.ID).Scan(&status); err != nil { + t.Fatal(err) + } + switch action { + case deployment.RecoveryActionRecord: + after, err := repo.ListPendingDeploymentTransactions(t.Context()) + if err != nil || !reflect.DeepEqual(before, after) || !reflect.DeepEqual(filesBefore, client.files) { + t.Fatal("record-only changed original state", err) + } + case deployment.RecoveryActionCommit: + if status != deployment.TransactionStatusCommitted || client.files[record.Paths[0]] != "new" { + t.Fatal("commit not atomic with audit") + } + case deployment.RecoveryActionRollback: + if status != deployment.TransactionStatusRolledBack || client.files[record.Paths[0]] != "old" { + t.Fatal("rollback not atomic with audit") + } + } + if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, deployment.ErrRecoveryConfirmation) { + t.Fatal("completed confirmation replayed") + } + }) + } +} + +func TestRecoveryExecutorSQLiteAuditFailuresAndRestart(t *testing.T) { + for _, stage := range []string{"started", "finished"} { + t.Run(stage, func(t *testing.T) { + repo, db, service, client, server, record := executorIntegrationFixture(t) + resolve := func(context.Context, string) (deployment.Server, error) { return server, nil } + confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", deployment.RecoveryActionCommit, resolve) + if err != nil { + t.Fatal(err) + } + query := "CREATE TRIGGER executor_audit_failure BEFORE INSERT ON recovery_events WHEN NEW.event_type = '" + stage + "' BEGIN SELECT RAISE(ABORT, '测试审计失败'); END" + if _, err := db.ExecContext(t.Context(), query); err != nil { + t.Fatal(err) + } + filesBefore := maps.Clone(client.files) + item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve) + if !errors.Is(err, deployment.ErrRecoveryDispositionUnavailable) || item.Status == "succeeded" { + t.Fatal("audit failure reported success") + } + if stage == "started" { + recoveryCount(t, db, "recovery_dispositions", 0) + recoveryCount(t, db, "recovery_claims", 0) + recoveryCount(t, db, "recovery_events", 0) + if !reflect.DeepEqual(filesBefore, client.files) { + t.Fatal("begin audit failure modified remote") + } + return + } + recoveryCount(t, db, "recovery_claims", 1) + recoveryCount(t, db, "recovery_events", 1) + var sequence int + var name, path string + if err := db.QueryRowContext(t.Context(), "PRAGMA database_list").Scan(&sequence, &name, &path); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + reopened, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + repo = NewSQLDeploymentTransactionRepository(reopened) + stored, err := repo.GetRecoveryDisposition(t.Context(), confirmation.DispositionID()) + if err != nil || stored.Status != "running" { + t.Fatal("uncertain result lost after reopening", err) + } + pending, err := repo.ListPendingDeploymentTransactions(t.Context()) + if err != nil || len(pending) != 1 || pending[0].Status != deployment.TransactionStatusRecoveryRequired || pending[0].Phase != deployment.TransactionPhaseCommitVerified { + t.Fatal("failed finish altered original terminal state", err) + } + if strings.Contains(stored.Intent.RecordDigest, token) { + t.Fatal("confirmation token persisted") + } + restarted := deployment.NewServiceWithJournal(nil, repo) + view, err := restarted.GetRecoveryDisposition(t.Context(), confirmation.DispositionID()) + if err != nil || view.Status != "running" || view.ID != confirmation.DispositionID() { + t.Fatal("query after process restart failed", err) + } + if err := restarted.RecoverPending(t.Context(), func(context.Context, string) (deployment.Server, error) { + t.Error("restart re-executed occupied transaction") + return deployment.Server{}, errors.New("unexpected resolver") + }); !errors.Is(err, deployment.ErrRecoveryDispositionBusy) { + t.Fatal("restart did not honor durable claim", err) + } + }) + } +} diff --git a/doc/Linux生产流程测试记录.md b/doc/Linux生产流程测试记录.md index 854f795..c6e0fc1 100644 --- a/doc/Linux生产流程测试记录.md +++ b/doc/Linux生产流程测试记录.md @@ -1,5 +1,17 @@ # Linux 生产流程测试记录 +## 2026年10月3日20:52 一次性确认与执行器回归 + +范围为第一台隔离源码目录 `/var/tmp/scert-local-recovery-20261003-1828`，不替代正式安装包；远端文件和 HTTPS 回调在本批 Go 测试中使用测试实现，SQLite 为真实临时数据库。线上业务数据库未清理、正式服务未升级。 + +- [✓] `scert-executor-20261003-2035.service`：部署全包竞态通过，约 3.254 秒，`EXECUTOR_RACE_EXIT=0`。 +- [✓] `scert-executor-extra-20261003-2041.service` 的部署竞态通过，约 3.510 秒；但同组 SQLite 五场景因测试客户端缺少证据接口方法而失败，整组失败记录保留于 `executor-extra.log`，不能记整组成功。 +- [x] 测试夹具补齐拒绝建锁的方法及编译期证据接口断言，未放宽生产核查边界。 +- [✓] `scert-executor-sqlite-20261003-2045.service`：SQLite 五场景竞态复测通过，仓储约 5.303 秒，`EXECUTOR_SQLITE_FIX_EXIT=0`；完整静态检查和后端全量通过，`EXECUTOR_VET_EXIT=0`、`EXECUTOR_FULL_EXIT=0`。服务已退出，`MainPID=0`、`Result=success`。 +- [~] 最新追加未知临时文件漏项拦截与脱敏查询，八份源码归档 SHA-256：`101eb370619bac98d4b069ea3c242206b3035fca7cd097006b83fb8f4494a83f`，本地与第一台一致。重新执行任务 `scert-executor-final-20261003-2052.service`，日志 `executor-final.log`，当前未记通过。 +- [x] 本轮第二台根分区复核可用 26 GB、使用率 10%，未继续删除文件；第一台 Scert、Nginx 运行，可用约 9.7 GB。 +- [ ] 本机协议/终端、PostgreSQL 等必要实库和正式双机生产闭环仍待完成；生产验收未通过。 + ## 2026年10月3日20:20 后端全量最终结果 - [✓] 第一台 `scert-operation-full-20261003-2012.service` 已终止，`MainPID=0`、`Result=success`、`ActiveState=inactive`；整组约四分五十八秒，`OPERATION_FULL_RUN_EXIT=0`。 diff --git a/doc/Scert开发任务清单.md b/doc/Scert开发任务清单.md index b48b8ac..6fd2122 100644 --- a/doc/Scert开发任务清单.md +++ b/doc/Scert开发任务清单.md @@ -1,10 +1,13 @@ # Scert 开发任务清单 -## 2026年10月3日20:20 回归收口 +## 2026年10月3日20:52 当前开发进度 -- [✓] 完整部署互斥与占用防护批次的原生 Linux 后端全量、部署竞态、补充竞态及完整静态检查通过；代码 `b3f27e8` 已推送。 -- [ ] 下一开发项：一次性本机确认、三动作执行器、受保护协议及终端接入。PostgreSQL 实库及正式生产验收未完成。 -- [~] 本节覆盖下方“全量运行中”状态；自动回归不等于生产验收。 +- [x] 一次性确认、记录/提交/回滚执行器、只读脱敏结果查询已编码；执行器共用部署互斥，绑定持久化意图与原子审计，不在失败后切换动作。 +- [x] 补上未知临时文件完整集合检查、核验后再检查、部分解锁历史的重新 HTTPS 核验，以及请求取消后独立有界结束审计。 +- [✓] 基础两轮部署竞态和 SQLite 五场景通过，静态检查、后端全量通过一轮；SQLite 夹具首次失败已修复，保留原记录。 +- [~] 最新八份源码正在重新执行完整回归，未把运行中的结果提前记为通过。 +- [ ] 下一开发项：受保护本机协议的准备、确认、查询和管理员终端；身份与会话失效、混合载荷、重放、断连测试。 +- [ ] PostgreSQL 等必要实库和正式生产部署验收仍未完成；本节覆盖下方历史状态。 ## 2026年10月3日20:17 当前开发进度 diff --git a/doc/Scert整改任务执行清单.md b/doc/Scert整改任务执行清单.md index b2ee46f..5d34f97 100644 --- a/doc/Scert整改任务执行清单.md +++ b/doc/Scert整改任务执行清单.md @@ -1,10 +1,12 @@ # Scert 整改任务执行清单 -## 2026年10月3日20:20 最新执行结论 +## 2026年10月3日20:52 最新执行结论 -- [✓] 本批完整互斥及人工占用防护通过原生 Linux 后端全量，整组退出码零；竞态及静态检查通过，代码 `b3f27e8` 已推送。 -- [ ] R3 一次性本机确认、动作执行器和终端仍待接入；R4 PostgreSQL 实库与执行器边界、R5 正式包生产门禁仍未完成。 -- [~] 本节覆盖下方全量待结果描述；未据局部开发回归宣布可上线。 +- [x] R3 确认和三动作执行器已编码，并接入 R4 共同操作门、持久占用、原子开始/结束审计；查询结果不触发重放。 +- [✓] 基础部署竞态、SQLite 原子三动作、开始/结束审计失败和数据库重开拦截通过；完整静态检查及后端全量通过一轮。 +- [~] 最新追加的未知临时文件完整集合检查与安全查询正在重新回归；测试任务 `scert-executor-final-20261003-2052.service`，源码归档哈希已比对。 +- [ ] 本机协议和终端尚未接入，因此 R3 整体仍未完成；R4 PostgreSQL 等必要实库与 R5 正式包生产门禁仍未完成。 +- [~] 文档同步；本节覆盖下面历史状态，不将自动回归等同于生产验收。 ## 2026年10月3日20:17 当前执行状态 diff --git a/doc/Scert整改合并优化清单.md b/doc/Scert整改合并优化清单.md index ac1c1ba..f53ca29 100644 --- a/doc/Scert整改合并优化清单.md +++ b/doc/Scert整改合并优化清单.md @@ -1,14 +1,15 @@ # Scert 整改、合并与优化清单 -## 2026年10月3日20:20 最新回归结论 +## 2026年10月3日20:52 最新整改进度 -本节覆盖下方“全量运行中”状态。 +本节覆盖下面历史状态；仅 Linux，生产验收仍未通过。 -- [x] 代码检查点 `b3f27e8` 已推送；部署完整互斥和持久化人工占用防护已接入。 -- [✓] 本批原生 Linux 后端全量回归通过：`OPERATION_FULL_EXIT=0`，整组退出码零，约四分五十八秒；此前部署竞态、补充竞态及完整静态检查亦通过。默认全量未开启外部数据库矩阵，不扩大为三库实机验收。 -- [x] MySQL 审计故障注入已修复，六场景实库复测通过；迁移十二场景通过，原失败记录保留。 -- [ ] 下一项仍为 R3：一次性本机确认、记录/提交/回滚执行器及本机协议和终端接入；随后补充 PostgreSQL 实库、执行器并发重启边界和正式包生产门禁。 -- [~] 六份相关文档同步；第一台 Scert、Nginx 运行，未更新线上正式包、未清理业务数据库，整体生产验收仍未通过。 +- [x] R3 一次性确认已编码：短时令牌、操作者与准备摘要绑定、复制共享消费状态、错误确认及取消后不可重放。 +- [x] 记录、提交收尾、回滚三个明确执行器接入共同操作门和持久化意图/结束审计；失败不切换动作，可能已写入的失败保留占用，历史已核验阶段也须重新核验站点。 +- [x] 新增脱敏结果查询，执行中及重启后可读取持久结果，不重放动作、不释放占用；补上历史临时路径列表漏项时的未知文件拦截。 +- [✓] 执行器基础两轮部署竞态、SQLite 三动作和两类审计故障/数据库重开测试通过，静态检查和后端全量通过一轮。SQLite 首次测试夹具缺少证据接口，修复后五场景通过，原失败日志保留。 +- [~] 最后追加的未知临时文件拦截与安全查询随最新八份源码重新进行 Linux 竞态、SQLite、静态检查和全量回归，当前任务为 `scert-executor-final-20261003-2052.service`。 +- [ ] R3 本机协议和管理员终端接入；R4 PostgreSQL 等必要实库及故障矩阵；R5 正式 Linux 包和完整生产门禁。尚未开放人工写入口、未更新线上正式包、未清理业务数据库。 ## 2026年10月3日20:17 最新执行进度 diff --git a/doc/部署事务人工处置设计.md b/doc/部署事务人工处置设计.md index 3bff70c..4ae5367 100644 --- a/doc/部署事务人工处置设计.md +++ b/doc/部署事务人工处置设计.md @@ -1,10 +1,14 @@ # 部署事务人工处置设计 -## 2026年10月3日20:20 互斥批次复核结果 +## 2026年10月3日20:52 确认和执行器实现边界 -- [✓] 已接入的完整部署互斥、启动恢复占用识别和失败关闭通过最新原生 Linux 后端全量、竞态与静态检查，代码 `b3f27e8` 已推送。 -- [ ] 一次性本机确认和人工执行器仍须接入此共同操作门、持久化意图与原子结束审计；不把现有历史核验阶段当成本次人工动作已执行的证据。 -- [ ] 本机协议和终端、PostgreSQL 实库、动作失败及重启矩阵和正式生产门禁仍待完成；本节覆盖此前全量待结果描述。 +- [x] 确认对象持有不可公开的准备绑定、令牌摘要和持久请求标识，副本共享原子消费状态；随机令牌不入库。有效期沿用一分钟，失败确认亦消费，不能重放。 +- [x] 确认消费后等待共同操作门；持久化开始和占用成功后才重新核查。已有幂等记录只返回结果，不再次连接；准备变化只结束为拒绝，可能发生写入的失败结束为不确定并保留占用。 +- [x] 提交必须重新验证新证书，回滚必须具有旧站点基线并重载、验证旧证书；历史已核验阶段也不能跳过本次验证。不使用按阶段自动选择方向的恢复入口，提交与回滚不互相回退。 +- [x] 连接关闭和独立有界结束审计均在共同操作门内；审计失败不能返回成功。结果查询输出明确的处置标识、事务、服务器、动作、状态和时间，不输出请求键、绑定摘要或确认令牌。 +- [x] 对全部目标推导临时路径，拒绝历史列表缺项掩盖的未知文件；回滚后再次核对基线和残留，避免解锁未知现场。 +- [✓] 基础部署竞态、SQLite 三动作原子结束和两类审计失败/数据库重开通过；静态检查、后端全量通过一轮。 +- [~] 最新安全查询和未知临时文件补充正在回归。本机协议/终端、PostgreSQL 等实库及正式生产门禁仍待完成，不开放半成品写入口。 ## 2026年10月3日20:17 互斥接入状态 diff --git a/doc/部署事务人工核查任务清单.md b/doc/部署事务人工核查任务清单.md index d102fa1..ac70c5a 100644 --- a/doc/部署事务人工核查任务清单.md +++ b/doc/部署事务人工核查任务清单.md @@ -1,5 +1,16 @@ # 部署事务人工核查任务清单 +## 2026年10月3日20:52 本轮执行任务 + +沿用已确认的 Linux 整改范围，不增加公网接口，不处理 Windows。本节优先于下面历史进度。 + +- [x] R3-B/R3-C：短时一次性确认、操作者/动作/准备摘要绑定，副本共享消费状态，失败确认不可重放。 +- [x] R3-D/R3-E：记录、提交、回滚三个明确动作接入共同互斥及持久化开始/结束审计；增加不重放、不释放占用的脱敏查询。 +- [✓] R4-C 部分回归：并发消费、取消等待、核验/连接收尾/审计期间互斥、失败不切换方向、历史核验阶段重新检查，以及 SQLite 审计失败后真实关闭重开数据库保留占用通过。 +- [~] 最新未知临时文件漏项拦截与查询补充正在回归；尚未通过本机终端执行真实服务器动作，不将 R3/R4 整项标完成。 +- [ ] R3 本机协议和管理员终端接入；完成之前不对用户开放写入口。 +- [ ] R4 PostgreSQL 等必要实库与 R5 正式包生产门禁，仍未完成。 + ## 2026年10月3日20:20 最新复核结论 - [✓] 部署和自动恢复共同互斥、持久化占用检查及数据库重开拦截专项通过；本批后端全量、竞态和静态检查通过。