实现人工处置一次性确认和三动作执行及脱敏查询

This commit is contained in:
xuans 2026-10-03 20:59:22 +08:00
parent e36dd3ddc7
commit bf4caebaae
14 changed files with 1319 additions and 19 deletions

View File

@ -0,0 +1,128 @@
package deployment
import "context"
// validateManualTemporaryFiles 检查完整目标集合,不能因为历史临时路径列表缺项就忽略未知文件。
func validateManualTemporaryFiles(ctx context.Context, client Client, record TransactionRecord) error {
for _, target := range record.Paths {
path := target + ".scert-tmp-" + record.ID
if exists, err := client.Exists(ctx, path); err != nil {
return err
} else if exists {
if err := checkFileDigest(ctx, client, path, record.Evidence.TemporaryFiles[path]); err != nil {
return err
}
}
}
return ctx.Err()
}
func validateRecoveryCommit(ctx context.Context, client Client, record TransactionRecord, allowMissingLocks bool) error {
if err := validateTransactionLocks(ctx, client, record, allowMissingLocks); err != nil {
return err
}
for _, path := range record.Paths {
if err := checkFileDigest(ctx, client, path, record.Evidence.Files[path]); err != nil {
return err
}
}
if err := validateManualTemporaryFiles(ctx, client, record); err != nil {
return err
}
for _, backup := range record.Backups {
if _, err := validateRecoveryBackup(ctx, client, backup); err != nil {
return err
}
}
return ctx.Err()
}
func (s *Service) executeRecoveryCommit(ctx context.Context, record *TransactionRecord, server Server, client Client, mutated *bool) error {
allowMissingLocks := record.Phase == TransactionPhaseCommitVerified
if err := validateRecoveryCommit(ctx, client, *record, allowMissingLocks); err != nil {
return err
}
// 即使历史阶段已核验,也必须再次核验真实新证书,不能仅凭历史状态清理。
if err := s.VerifyRecoveryTarget(ctx, server, record.Evidence.Baseline, true); err != nil {
return err
}
if err := validateRecoveryCommit(ctx, client, *record, allowMissingLocks); err != nil {
return err
}
*mutated = true
for _, temporary := range record.TemporaryPaths {
if err := client.Remove(ctx, temporary); err != nil {
return err
}
}
for _, backup := range record.Backups {
if err := client.Remove(ctx, backup.Saved); err != nil {
return err
}
}
if err := s.persistRecoveryPhase(ctx, record, TransactionPhaseCommitVerified); err != nil {
return err
}
return releaseTransactionLocks(ctx, client, *record, true, allowMissingLocks)
}
func validateRecoveredBaseline(ctx context.Context, client Client, record TransactionRecord) error {
for _, backup := range record.Backups {
if backup.OriginalExists {
if err := checkFileDigest(ctx, client, backup.Original, backup.OriginalDigest); err != nil {
return err
}
} else if exists, err := client.Exists(ctx, backup.Original); err != nil || exists {
return ErrRecoveryActionRejected
}
// 还原完成后不允许留下新出现或未消费的临时文件、备份。
for _, path := range []string{backup.Original + ".scert-tmp-" + record.ID, backup.Saved} {
if exists, err := client.Exists(ctx, path); err != nil || exists {
return ErrRecoveryActionRejected
}
}
}
return ctx.Err()
}
func (s *Service) executeRecoveryRollback(ctx context.Context, record *TransactionRecord, server Server, client Client, mutated *bool) error {
allowMissingLocks := record.Phase == TransactionPhaseRollbackVerified
if err := validateManualTemporaryFiles(ctx, client, *record); err != nil {
return err
}
if allowMissingLocks {
if err := validateTransactionLocks(ctx, client, *record, true); err != nil {
return err
}
if err := validateRecoveredBaseline(ctx, client, *record); err != nil {
return err
}
} else if err := validateRollbackFiles(ctx, client, *record); err != nil {
return err
}
if err := ctx.Err(); err != nil {
return err
}
*mutated = true
if !allowMissingLocks {
if err := restoreTransactionFiles(ctx, client, *record); err != nil {
return err
}
}
for _, command := range [][]string{server.ValidationCommand, server.ReloadCommand} {
response, err := client.Run(ctx, command)
if err != nil || response.ExitCode != 0 {
return ErrRecoveryActionUncertain
}
}
if err := s.VerifyRecoveryTarget(ctx, server, record.Evidence.Baseline, false); err != nil {
return err
}
if err := validateRecoveredBaseline(ctx, client, *record); err != nil {
return err
}
if err := s.persistRecoveryPhase(ctx, record, TransactionPhaseRollbackVerified); err != nil {
return err
}
return releaseTransactionLocks(ctx, client, *record, true, allowMissingLocks)
}

View File

@ -0,0 +1,93 @@
package deployment
import (
"context"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"encoding/hex"
"errors"
"sync/atomic"
)
var ErrRecoveryConfirmation = errors.New("人工处置确认无效、已使用或已过期,请重新核查")
// RecoveryConfirmation 的副本共享消费状态,不能通过复制对象重复授权。
// 本机会话持有对象,终端仅收到短时令牌和脱敏摘要;令牌不写入数据库。
type RecoveryConfirmation struct {
state *recoveryConfirmationState
}
type recoveryConfirmationState struct {
review RecoveryReview
token [sha256.Size]byte
intent RecoveryDispositionIntent
used atomic.Bool
}
func (c RecoveryConfirmation) Summary() RecoveryReviewSummary {
if c.state == nil {
return RecoveryReviewSummary{}
}
return c.state.review.Summary()
}
func (c RecoveryConfirmation) DispositionID() string {
if c.state == nil {
return ""
}
return c.state.intent.ID
}
func (s *Service) PrepareRecoveryConfirmation(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryConfirmation, string, error) {
review, err := s.PrepareRecoveryReview(ctx, id, operator, action, resolve)
if err != nil {
return RecoveryConfirmation{}, "", err
}
return s.newRecoveryConfirmation(ctx, review)
}
func (s *Service) newRecoveryConfirmation(ctx context.Context, review RecoveryReview) (RecoveryConfirmation, string, error) {
if s == nil || review.service != s {
return RecoveryConfirmation{}, "", ErrRecoveryConfirmation
}
var entropy [96]byte
if _, err := rand.Read(entropy[:]); err != nil {
return RecoveryConfirmation{}, "", ErrRecoveryConfirmation
}
token := hex.EncodeToString(entropy[:32])
intent, err := review.DispositionIntent(hex.EncodeToString(entropy[32:64]), hex.EncodeToString(entropy[64:]))
if err != nil {
return RecoveryConfirmation{}, "", err
}
if err := ctx.Err(); err != nil {
return RecoveryConfirmation{}, "", err
}
return RecoveryConfirmation{state: &recoveryConfirmationState{
review: review, intent: intent, token: sha256.Sum256([]byte(token)),
}}, token, nil
}
// consumeRecoveryConfirmation 每个对象只允许一次确认尝试,错误身份、错误令牌和取消均不可重放。
// 成功消费仍不是远端写授权,执行器必须取得持久化占用并重新检查现场。
func (s *Service) consumeRecoveryConfirmation(ctx context.Context, c RecoveryConfirmation, operator, token string) (RecoveryReview, RecoveryDispositionIntent, error) {
fail := func(err error) (RecoveryReview, RecoveryDispositionIntent, error) {
return RecoveryReview{}, RecoveryDispositionIntent{}, err
}
state := c.state
if state == nil || !state.used.CompareAndSwap(false, true) {
return fail(ErrRecoveryConfirmation)
}
if s == nil || state.review.service != s || s.now == nil || !validReviewOperator(operator) || state.review.operator != operator ||
!s.now().Before(state.review.summary.ExpiresAt) || len(token) != 64 {
return fail(ErrRecoveryConfirmation)
}
digest := sha256.Sum256([]byte(token))
if subtle.ConstantTimeCompare(digest[:], state.token[:]) != 1 {
return fail(ErrRecoveryConfirmation)
}
if err := ctx.Err(); err != nil {
return fail(err)
}
return state.review, state.intent, nil
}

View File

@ -0,0 +1,97 @@
package deployment
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
func confirmationFixture(t *testing.T) (*Service, RecoveryConfirmation, string) {
t.Helper()
server, client, record := recoveryPreflightFixture()
service := NewServiceWithJournal(fixedClientFactory{client}, &memoryTransactionJournal{pending: []TransactionRecord{record}})
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", RecoveryActionRecord, func(context.Context, string) (Server, error) { return server, nil })
if err != nil {
t.Fatal(err)
}
return service, confirmation, token
}
func TestRecoveryConfirmationSingleUseAndPrivateBindings(t *testing.T) {
service, confirmation, token := confirmationFixture(t)
if len(token) != 64 || len(confirmation.DispositionID()) != 64 || token == confirmation.DispositionID() {
t.Fatal("invalid confirmation entropy")
}
encoded, err := json.Marshal(confirmation)
if err != nil || string(encoded) != "{}" {
t.Fatal("private confirmation serialized")
}
copy := confirmation
review, intent, err := service.consumeRecoveryConfirmation(t.Context(), confirmation, "unix:0", token)
if err != nil || review.Summary() != confirmation.Summary() || intent.Validate() != nil || intent.ID != confirmation.DispositionID() {
t.Fatalf("confirmation failed: %v", err)
}
encoded, _ = json.Marshal(intent)
if strings.Contains(string(encoded), token) {
t.Fatal("raw token persisted")
}
if _, _, err := service.consumeRecoveryConfirmation(t.Context(), copy, "unix:0", token); !errors.Is(err, ErrRecoveryConfirmation) {
t.Fatal("copied confirmation replayed")
}
}
func TestRecoveryConfirmationRejectsAndConsumesInvalidAttempt(t *testing.T) {
for _, scenario := range []string{"identity", "token", "oversized", "expired", "service", "cancelled", "empty"} {
t.Run(scenario, func(t *testing.T) {
service, confirmation, token := confirmationFixture(t)
originalService, originalToken := service, token
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
operator := "unix:0"
switch scenario {
case "identity":
operator = "unix:1"
case "token":
token = strings.Repeat("0", 64)
case "oversized":
token = strings.Repeat("a", 8192)
case "expired":
service.now = func() time.Time { return confirmation.Summary().ExpiresAt }
case "service":
service = NewService(nil)
case "cancelled":
cancel()
case "empty":
confirmation = RecoveryConfirmation{}
}
if _, _, err := service.consumeRecoveryConfirmation(ctx, confirmation, operator, token); err == nil {
t.Fatal("invalid confirmation accepted")
}
if _, _, err := originalService.consumeRecoveryConfirmation(t.Context(), confirmation, "unix:0", originalToken); err == nil {
t.Fatal("failed attempt could be replayed")
}
})
}
}
func TestRecoveryConfirmationConcurrentCopiesConsumeOnlyOnce(t *testing.T) {
service, confirmation, token := confirmationFixture(t)
var accepted atomic.Int32
var group sync.WaitGroup
for range 32 {
group.Go(func() {
if _, _, err := service.consumeRecoveryConfirmation(t.Context(), confirmation, "unix:0", token); err == nil {
accepted.Add(1)
}
})
}
group.Wait()
if accepted.Load() != 1 {
t.Fatalf("accepted %d confirmations", accepted.Load())
}
}

View File

@ -0,0 +1,139 @@
package deployment
import (
"context"
"errors"
"time"
)
var (
ErrRecoveryActionRejected = errors.New("人工处置前置核验未通过,未执行远端变更")
ErrRecoveryActionUncertain = errors.New("人工处置结果不确定,已保留占用,请查询记录并核查现场")
)
// ExecuteRecoveryDisposition 仅供受保护本机会话使用,操作者必须来自本机对端身份。
// 一次性确认、共同操作门、持久化占用和远端锁各自独立,缺一不可。
func (s *Service) ExecuteRecoveryDisposition(ctx context.Context, confirmation RecoveryConfirmation, operator, token string, resolve func(context.Context, string) (Server, error)) (RecoveryDisposition, error) {
review, intent, err := s.consumeRecoveryConfirmation(ctx, confirmation, operator, token)
if err != nil {
return RecoveryDisposition{}, err
}
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
release, err := s.acquireOperation(ctx)
if err != nil {
return RecoveryDisposition{}, err
}
defer release()
journal, ok := s.journal.(RecoveryDispositionJournal)
if !ok || resolve == nil {
return RecoveryDisposition{}, ErrRecoveryDispositionUnavailable
}
if !s.now().Before(review.summary.ExpiresAt) {
return RecoveryDisposition{}, ErrRecoveryConfirmation
}
item, created, err := journal.BeginRecoveryDisposition(ctx, intent)
if err != nil {
return RecoveryDisposition{}, ErrRecoveryDispositionUnavailable
}
if !created {
// 响应丢失后的既有请求只返回持久结果,绝不重新连接或执行。
return item, nil
}
finish := func(result string, actionErr error) (RecoveryDisposition, error) {
// 请求取消不能跳过结束审计;保存失败保留原占用且绝不返回成功。
finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer finishCancel()
finished, finishErr := journal.FinishRecoveryDisposition(finishCtx, intent.ID, result)
if finishErr != nil {
return item, ErrRecoveryDispositionUnavailable
}
return finished, actionErr
}
if err := s.RecheckRecoveryReview(ctx, review, operator, resolve); err != nil {
return finish(RecoveryResultRejected, ErrRecoveryActionRejected)
}
if review.summary.Action == RecoveryActionRecord {
return finish(RecoveryResultObserved, nil)
}
result, actionErr := s.executeRecoveryAction(ctx, review, resolve)
return finish(result, actionErr)
}
func (s *Service) executeRecoveryAction(ctx context.Context, review RecoveryReview, resolve func(context.Context, string) (Server, error)) (result string, actionErr error) {
result, actionErr = RecoveryResultRejected, ErrRecoveryActionRejected
record, recordDigest, serverDigest, err := s.reviewRecord(ctx, review.summary.TransactionID, resolve)
if err != nil || recordDigest != review.recordDigest || serverDigest != review.serverDigest {
return
}
server, err := resolve(ctx, record.ServerID)
if err != nil {
return
}
server = server.Normalize()
currentServerDigest, err := reviewServerDigest(server)
if err != nil || currentServerDigest != review.serverDigest || validateRecoveryAction(record, review.summary.Action, s.VerifyRecoveryTarget != nil) != nil {
return
}
client, err := s.factory.Connect(ctx, server)
if err != nil {
return
}
mutated := false
defer func() {
// 在结果审计和释放共同操作门之前关闭连接,关闭失败不能伪装成成功。
if closeErr := client.Close(); closeErr != nil {
if mutated {
result, actionErr = RecoveryResultUncertain, ErrRecoveryActionUncertain
} else {
result, actionErr = RecoveryResultRejected, ErrRecoveryActionRejected
}
}
}()
if err := ctx.Err(); err != nil || !s.now().Before(review.summary.ExpiresAt) {
return
}
switch review.summary.Action {
case RecoveryActionCommit:
err = s.executeRecoveryCommit(ctx, &record, server, client, &mutated)
result = RecoveryResultCommitted
case RecoveryActionRollback:
err = s.executeRecoveryRollback(ctx, &record, server, client, &mutated)
result = RecoveryResultRolledBack
default:
return
}
if err == nil {
return result, nil
}
if mutated {
return RecoveryResultUncertain, ErrRecoveryActionUncertain
}
return RecoveryResultRejected, ErrRecoveryActionRejected
}
func validateRecoveryAction(record TransactionRecord, action RecoveryAction, verifier bool) error {
if record.Evidence == nil || !verifier || len(record.Backups) != len(record.Paths) {
return ErrRecoveryActionRejected
}
switch action {
case RecoveryActionCommit:
if record.Evidence.Baseline.ExpectedFingerprint == "" {
return ErrRecoveryActionRejected
}
switch record.Phase {
case TransactionPhaseReloadAttempted, TransactionPhaseCommitting, TransactionPhaseCommitVerified:
return nil
}
case RecoveryActionRollback:
if record.Evidence.Baseline.PreviousFingerprint == "" {
return ErrRecoveryActionRejected
}
switch record.Phase {
case TransactionPhaseLocked, TransactionPhaseUploaded, TransactionPhaseReplacing, TransactionPhaseInstalled,
TransactionPhaseReloadAttempted, TransactionPhaseCommitting, TransactionPhaseRollbackVerified:
return nil
}
}
return ErrRecoveryActionRejected
}

View File

@ -0,0 +1,428 @@
package deployment
import (
"context"
"errors"
"maps"
"reflect"
"testing"
"time"
)
type executorJournal struct {
memoryTransactionJournal
item RecoveryDisposition
claimed bool
beginCalls, finishCalls int
beginErr, finishErr, updateErr error
finishContextErr error
existing bool
beforeFinish func()
}
func (j *executorJournal) BeginRecoveryDisposition(_ context.Context, intent RecoveryDispositionIntent) (RecoveryDisposition, bool, error) {
j.beginCalls++
if j.beginErr != nil {
return RecoveryDisposition{}, false, j.beginErr
}
if j.existing {
return j.item, false, nil
}
if j.claimed {
return RecoveryDisposition{}, false, ErrRecoveryDispositionBusy
}
if !intent.MatchesRecord(j.pending[0]) {
return RecoveryDisposition{}, false, ErrRecoveryReviewChanged
}
j.item = RecoveryDisposition{Intent: intent, Status: "running"}
j.claimed = true
return j.item, true, nil
}
func (j *executorJournal) FinishRecoveryDisposition(ctx context.Context, id, result string) (RecoveryDisposition, error) {
j.finishCalls++
j.finishContextErr = ctx.Err()
if j.beforeFinish != nil {
j.beforeFinish()
}
if j.finishErr != nil {
return RecoveryDisposition{}, j.finishErr
}
if id != j.item.Intent.ID || !j.item.Intent.AllowsResult(result) || !j.claimed {
return RecoveryDisposition{}, ErrRecoveryDispositionConflict
}
j.item.Status, j.item.ResultCode = "succeeded", result
if result == RecoveryResultRejected || result == RecoveryResultUncertain {
j.item.Status = result
}
j.claimed = result == RecoveryResultUncertain
return j.item, nil
}
func (j *executorJournal) GetRecoveryDisposition(context.Context, string) (RecoveryDisposition, error) {
return j.item, nil
}
func (j *executorJournal) HasRecoveryClaim(context.Context, string) (bool, error) {
return j.claimed, nil
}
func (j *executorJournal) UpdateDeploymentTransaction(ctx context.Context, record TransactionRecord) error {
if j.updateErr != nil {
return j.updateErr
}
return j.memoryTransactionJournal.UpdateDeploymentTransaction(ctx, record)
}
type executorClient struct {
*memoryClient
closeErr error
closed int
beforeClose func()
}
func (c *executorClient) Close() error {
c.closed++
if c.beforeClose != nil {
c.beforeClose()
}
return c.closeErr
}
func executorFixture(t *testing.T) (*Service, *executorJournal, *executorClient, Server, TransactionRecord) {
t.Helper()
server, memory, record := recoveryPreflightFixture()
journal := &executorJournal{memoryTransactionJournal: memoryTransactionJournal{pending: []TransactionRecord{record}}}
client := &executorClient{memoryClient: memory}
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { return nil }
return service, journal, client, server, record
}
func TestRecoveryExecutorThreeExplicitActions(t *testing.T) {
for _, action := range []RecoveryAction{RecoveryActionRecord, RecoveryActionCommit, RecoveryActionRollback} {
t.Run(string(action), func(t *testing.T) {
service, journal, client, server, record := executorFixture(t)
before := maps.Clone(client.files)
resolve := func(context.Context, string) (Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
if err != nil {
t.Fatal(err)
}
var verified []bool
service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, commit bool) error {
verified = append(verified, commit)
return nil
}
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if err != nil || item.Status != "succeeded" || journal.claimed || journal.beginCalls != 1 || journal.finishCalls != 1 || len(journal.finished) != 0 {
t.Fatalf("execution failed: %+v, %v", item, err)
}
if action == RecoveryActionRecord {
if item.ResultCode != RecoveryResultObserved || len(verified) != 0 || len(journal.updated) != 0 || !reflect.DeepEqual(before, client.files) {
t.Fatal("record-only action changed original state")
}
} else {
expected, phase, result := "new", TransactionPhaseCommitVerified, RecoveryResultCommitted
if action == RecoveryActionRollback {
expected, phase, result = "old", TransactionPhaseRollbackVerified, RecoveryResultRolledBack
}
if !reflect.DeepEqual(verified, []bool{action == RecoveryActionCommit}) || item.ResultCode != result || len(journal.updated) != 1 || journal.updated[0].Phase != phase {
t.Fatal("wrong action, verification or phase")
}
for _, path := range record.Paths {
if string(client.files[path].data) != expected {
t.Fatal("wrong certificate files")
}
}
if len(client.files) != 3 {
t.Fatal("transaction leftovers remain")
}
}
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, ErrRecoveryConfirmation) || journal.beginCalls != 1 {
t.Fatal("execution replayed")
}
})
}
}
func TestRecoveryExecutorFailureBoundaries(t *testing.T) {
for _, scenario := range []string{"changed", "foreign-lock", "legacy", "phase", "missing-baseline", "missing-backup-entry", "begin", "finish", "persist", "cleanup", "reload", "commit-https", "rollback-https", "cancel", "close", "existing"} {
t.Run(scenario, func(t *testing.T) {
service, journal, client, server, record := executorFixture(t)
action := RecoveryActionCommit
if scenario == "reload" || scenario == "rollback-https" || scenario == "missing-backup-entry" {
action = RecoveryActionRollback
}
if scenario == "legacy" {
record.Evidence = nil
action = RecoveryActionRecord
}
if scenario == "phase" {
record.Phase = TransactionPhaseUploaded
}
if scenario == "missing-baseline" {
record.Evidence.Baseline.ExpectedFingerprint = ""
}
if scenario == "missing-backup-entry" {
record.Backups = record.Backups[:2]
}
journal.pending[0] = record
resolve := func(context.Context, string) (Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
verifyCalls := 0
service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, commit bool) error {
verifyCalls++
if commit != (action == RecoveryActionCommit) {
t.Fatal("failed action silently switched direction")
}
if scenario == "commit-https" || scenario == "rollback-https" {
return errors.New("private upstream details")
}
if scenario == "cancel" {
cancel()
}
if scenario == "close" {
client.closeErr = errors.New("private close details")
}
return nil
}
switch scenario {
case "changed":
client.files[record.Paths[0]] = memoryFile{data: []byte("foreign")}
case "foreign-lock":
client.files[record.LockPaths[0]+"/owner"] = memoryFile{data: []byte("foreign")}
case "begin":
journal.beginErr = errors.New("private database error")
case "finish":
journal.finishErr = errors.New("private audit error")
case "persist":
journal.updateErr = errors.New("private update error")
case "cleanup":
client.removeError = ".scert-backup-"
case "reload":
client.runErrors = []error{errors.New("private command error")}
case "existing":
journal.existing = true
journal.item = RecoveryDisposition{Status: "uncertain", ResultCode: RecoveryResultUncertain}
}
before := maps.Clone(client.files)
closedBefore := client.closed
item, err := service.ExecuteRecoveryDisposition(ctx, confirmation, "unix:0", token, resolve)
if scenario == "legacy" {
if err != nil || item.ResultCode != RecoveryResultObserved || len(journal.updated) != 0 || !reflect.DeepEqual(before, client.files) {
t.Fatal("legacy inspection mutated history")
}
return
}
if scenario == "existing" {
if err != nil || item.Status != "uncertain" || verifyCalls != 0 || journal.finishCalls != 0 || client.closed != closedBefore || !reflect.DeepEqual(before, client.files) {
t.Fatal("existing request executed again")
}
return
}
if err == nil || item.Status == "succeeded" || journal.finishContextErr != nil {
t.Fatalf("failure reported success or skipped independent audit: %+v %v", item, err)
}
uncertain := scenario == "finish" || scenario == "persist" || scenario == "cleanup" || scenario == "reload" || scenario == "rollback-https" || scenario == "close"
if journal.claimed != uncertain {
t.Fatalf("wrong claim retention: %v", journal.claimed)
}
if !uncertain && !reflect.DeepEqual(before, client.files) {
t.Fatal("preflight rejection changed remote files")
}
if len(journal.finished) != 0 {
t.Fatal("executor bypassed atomic disposition completion")
}
})
}
}
func TestRecoveryExecutorWaitCancellationConsumesConfirmation(t *testing.T) {
service, journal, _, server, record := executorFixture(t)
resolve := func(context.Context, string) (Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", RecoveryActionRecord, resolve)
if err != nil {
t.Fatal(err)
}
release, err := service.acquireOperation(t.Context())
if err != nil {
t.Fatal(err)
}
defer release()
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
defer cancel()
if _, err := service.ExecuteRecoveryDisposition(ctx, confirmation, "unix:0", token, resolve); !errors.Is(err, context.DeadlineExceeded) || journal.beginCalls != 0 {
t.Fatal("cancelled waiter acquired persistent claim")
}
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, ErrRecoveryConfirmation) {
t.Fatal("cancelled confirmation replayed")
}
}
func TestRecoveryExecutorHoldsOperationThroughVerificationCloseAndAudit(t *testing.T) {
service, journal, client, server, record := executorFixture(t)
resolve := func(context.Context, string) (Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", RecoveryActionCommit, resolve)
if err != nil {
t.Fatal(err)
}
checked := map[string]int{}
checkHeld := func(stage string) {
checked[stage]++
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Millisecond)
defer cancel()
release, err := service.acquireOperation(ctx)
if err == nil {
release()
t.Fatalf("operation gate released during %s", stage)
}
if !errors.Is(err, context.DeadlineExceeded) || !journal.claimed {
t.Fatalf("missing durable claim or invalid wait during %s", stage)
}
}
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error {
checkHeld("verify")
return nil
}
client.beforeClose = func() { checkHeld("close") }
journal.beforeFinish = func() { checkHeld("audit") }
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); err != nil {
t.Fatal(err)
}
if checked["verify"] != 1 || checked["close"] != 2 || checked["audit"] != 1 {
t.Fatalf("missing gate coverage: %v", checked)
}
release, err := service.acquireOperation(t.Context())
if err != nil {
t.Fatal(err)
}
release()
}
func TestRecoveryExecutorVerifiedHistoryStillRechecksHTTPS(t *testing.T) {
for _, action := range []RecoveryAction{RecoveryActionCommit, RecoveryActionRollback} {
for _, failVerification := range []bool{false, true} {
t.Run(string(action)+map[bool]string{false: "/success", true: "/failure"}[failVerification], func(t *testing.T) {
service, journal, client, server, record := executorFixture(t)
record.Phase = TransactionPhaseCommitVerified
if action == RecoveryActionRollback {
record.Phase = TransactionPhaseRollbackVerified
for _, path := range record.Paths {
client.files[path] = memoryFile{data: []byte("old")}
}
}
for _, path := range record.TemporaryPaths {
delete(client.files, path)
}
for _, backup := range record.Backups {
delete(client.files, backup.Saved)
}
delete(client.files, record.LockPaths[0])
delete(client.files, record.LockPaths[0]+"/owner")
journal.pending[0] = record
resolve := func(context.Context, string) (Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
if err != nil {
t.Fatal(err)
}
verified := 0
service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, commit bool) error {
verified++
if commit != (action == RecoveryActionCommit) {
t.Fatal("wrong HTTPS verification direction")
}
if failVerification {
return errors.New("HTTPS mismatch")
}
return nil
}
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if verified != 1 || (err != nil) != failVerification || (item.Status == "succeeded") == failVerification {
t.Fatalf("historical phase bypassed new verification: %+v %v", item, err)
}
if !failVerification && len(client.files) != 3 {
t.Fatal("partial unlock history failed to finish safely")
}
if failVerification {
if _, exists := client.files[record.LockPaths[1]]; !exists {
t.Fatal("failed verification removed retained lock")
}
}
})
}
}
}
func TestRecoveryExecutorRechecksAfterHTTPSAndAuditsCancellation(t *testing.T) {
for _, scenario := range []string{"target", "backup", "lock", "cancel-after-write"} {
t.Run(scenario, func(t *testing.T) {
service, journal, client, server, record := executorFixture(t)
action := RecoveryActionCommit
if scenario == "cancel-after-write" {
action = RecoveryActionRollback
}
resolve := func(context.Context, string) (Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error {
path := record.Paths[0]
switch scenario {
case "backup":
path = record.Backups[0].Saved
case "lock":
path = record.LockPaths[0] + "/owner"
case "cancel-after-write":
cancel()
return ctx.Err()
}
client.files[path] = memoryFile{data: []byte("foreign-change")}
return nil
}
item, err := service.ExecuteRecoveryDisposition(ctx, confirmation, "unix:0", token, resolve)
if err == nil || item.Status == "succeeded" || journal.finishCalls != 1 || journal.finishContextErr != nil {
t.Fatal("changed or cancelled execution skipped audit or reported success")
}
if journal.claimed != (scenario == "cancel-after-write") {
t.Fatal("wrong claim retention after cancelled write")
}
if _, exists := client.files[record.LockPaths[0]]; !exists {
t.Fatal("changed state unlocked")
}
})
}
}
func TestRecoveryExecutorRejectsUnlistedTemporaryFiles(t *testing.T) {
for _, action := range []RecoveryAction{RecoveryActionCommit, RecoveryActionRollback} {
t.Run(string(action), func(t *testing.T) {
service, journal, client, server, record := executorFixture(t)
delete(record.Evidence.TemporaryFiles, record.TemporaryPaths[0])
record.TemporaryPaths = record.TemporaryPaths[1:]
journal.pending[0] = record
resolve := func(context.Context, string) (Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
if err != nil {
t.Fatal(err)
}
before := maps.Clone(client.files)
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error {
t.Fatal("unknown temporary file reached HTTPS verification")
return nil
}
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if !errors.Is(err, ErrRecoveryActionRejected) || item.ResultCode != RecoveryResultRejected || journal.claimed || !reflect.DeepEqual(before, client.files) {
t.Fatal("unlisted temporary file ignored or modified")
}
})
}
}

View File

@ -0,0 +1,79 @@
package deployment
import (
"context"
"strings"
"time"
)
// RecoveryDispositionSummary 是本机查询的明确输出边界,不包含请求键、绑定摘要或操作者凭据。
type RecoveryDispositionSummary struct {
ID string `json:"id"`
TransactionID string `json:"transaction_id"`
ServerID string `json:"server_id"`
Action RecoveryAction `json:"action"`
Status string `json:"status"`
ResultCode string `json:"result_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
func (r RecoveryDisposition) Summary() RecoveryDispositionSummary {
return RecoveryDispositionSummary{
ID: r.Intent.ID, TransactionID: r.Intent.TransactionID, ServerID: r.Intent.ServerID,
Action: r.Intent.Action, Status: r.Status, ResultCode: r.ResultCode,
CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt,
}
}
func validDispositionID(id string) bool {
return validDigest(id) && id == strings.ToLower(id) && id != strings.Repeat("0", 64)
}
func (r RecoveryDispositionSummary) Validate() error {
if !validDispositionID(r.ID) || !validInspectionID(r.TransactionID) || !validRecoverySummary(r.ServerID) ||
!validRecoveryAction(r.Action) || r.CreatedAt.IsZero() || r.UpdatedAt.IsZero() || r.UpdatedAt.Before(r.CreatedAt) {
return ErrRecoveryDispositionInvalid
}
intent := RecoveryDispositionIntent{Action: r.Action}
switch r.Status {
case "running":
if r.ResultCode == "" {
return nil
}
case "succeeded":
if r.ResultCode != RecoveryResultRejected && r.ResultCode != RecoveryResultUncertain && intent.AllowsResult(r.ResultCode) {
return nil
}
case RecoveryResultRejected, RecoveryResultUncertain:
if r.ResultCode == r.Status {
return nil
}
}
return ErrRecoveryDispositionInvalid
}
// GetRecoveryDisposition 不取得写操作门,执行中或重启后均可查询;查询本身不会重放动作或释放占用。
func (s *Service) GetRecoveryDisposition(ctx context.Context, id string) (RecoveryDispositionSummary, error) {
if !validDispositionID(id) {
return RecoveryDispositionSummary{}, ErrRecoveryDispositionInvalid
}
if s == nil {
return RecoveryDispositionSummary{}, ErrRecoveryDispositionUnavailable
}
journal, ok := s.journal.(RecoveryDispositionJournal)
if !ok {
return RecoveryDispositionSummary{}, ErrRecoveryDispositionUnavailable
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
item, err := journal.GetRecoveryDisposition(ctx, id)
if err != nil || ctx.Err() != nil || item.Intent.ID != id || item.Intent.Validate() != nil {
return RecoveryDispositionSummary{}, ErrRecoveryDispositionUnavailable
}
summary := item.Summary()
if summary.Validate() != nil {
return RecoveryDispositionSummary{}, ErrRecoveryDispositionUnavailable
}
return summary, nil
}

View File

@ -0,0 +1,67 @@
package deployment
import (
"encoding/json"
"strings"
"testing"
"time"
)
func TestRecoveryDispositionSummaryValidation(t *testing.T) {
now := time.Now().UTC()
for _, action := range []RecoveryAction{RecoveryActionRecord, RecoveryActionCommit, RecoveryActionRollback} {
for _, result := range []string{"", RecoveryResultObserved, RecoveryResultCommitted, RecoveryResultRolledBack, RecoveryResultRejected, RecoveryResultUncertain} {
status := "succeeded"
if result == "" {
status = "running"
}
if result == RecoveryResultRejected || result == RecoveryResultUncertain {
status = result
}
summary := RecoveryDispositionSummary{ID: strings.Repeat("a", 64), TransactionID: "transaction", ServerID: "server", Action: action, Status: status, ResultCode: result, CreatedAt: now, UpdatedAt: now}
want := result == "" || (RecoveryDispositionIntent{Action: action}).AllowsResult(result)
if (summary.Validate() == nil) != want {
t.Fatalf("wrong result validation: %+v", summary)
}
}
}
}
func TestRecoveryDispositionQueryIsReadOnlyAndPrivate(t *testing.T) {
service, journal, _, _, _ := executorFixture(t)
now := time.Now().UTC()
journal.item = RecoveryDisposition{
Intent: RecoveryDispositionIntent{
ID: strings.Repeat("a", 64), RequestKey: strings.Repeat("b", 64), RecordDigest: strings.Repeat("c", 64),
ServerDigest: strings.Repeat("d", 64), ObservationDigest: strings.Repeat("e", 64),
TransactionID: "transaction", ServerID: "server", Operator: "unix:private", Action: RecoveryActionCommit,
}, Status: "running", CreatedAt: now, UpdatedAt: now,
}
journal.claimed = true
release, err := service.acquireOperation(t.Context())
if err != nil {
t.Fatal(err)
}
defer release()
result, err := service.GetRecoveryDisposition(t.Context(), journal.item.Intent.ID)
if err != nil || result.Status != "running" || !journal.claimed || journal.beginCalls != 0 || journal.finishCalls != 0 {
t.Fatal("query changed state or blocked behind writer", err)
}
encoded, err := json.Marshal(result)
if err != nil {
t.Fatal(err)
}
for _, private := range []string{journal.item.Intent.RequestKey, journal.item.Intent.RecordDigest, journal.item.Intent.ServerDigest, journal.item.Intent.ObservationDigest, journal.item.Intent.Operator} {
if strings.Contains(string(encoded), private) {
t.Fatal("private binding leaked in result")
}
}
if _, err := service.GetRecoveryDisposition(t.Context(), strings.Repeat("f", 64)); err == nil {
t.Fatal("wrong result identifier accepted")
}
journal.item.Status = "succeeded"
journal.item.ResultCode = RecoveryResultRolledBack
if _, err := service.GetRecoveryDisposition(t.Context(), journal.item.Intent.ID); err == nil {
t.Fatal("inconsistent action result accepted")
}
}

View File

@ -0,0 +1,236 @@
package repository
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"errors"
"maps"
"reflect"
"strings"
"testing"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
)
// executorIntegrationClient 仅模拟远端文件;处置占用、审计和终态使用真实 SQLite。
// 未列出的远端方法故意保持不可用,避免测试意外放行额外写操作。
type executorIntegrationClient struct {
deployment.Client
files map[string]string
}
var _ deployment.EvidenceClient = (*executorIntegrationClient)(nil)
func (c *executorIntegrationClient) CreateOwnedLock(context.Context, string, string) error {
return errors.New("人工处置不能创建新的部署锁")
}
func (c *executorIntegrationClient) Connect(context.Context, deployment.Server) (deployment.Client, error) {
return c, nil
}
func (c *executorIntegrationClient) Close() error { return nil }
func (c *executorIntegrationClient) Exists(_ context.Context, path string) (bool, error) {
_, exists := c.files[path]
return exists, nil
}
func (c *executorIntegrationClient) FileDigest(_ context.Context, path string) (string, error) {
value, exists := c.files[path]
if !exists {
return "", errors.New("测试文件不存在")
}
return executorDigest(value), nil
}
func (c *executorIntegrationClient) ReadLockOwner(_ context.Context, path string) (string, error) {
return c.files[path+"/owner"], nil
}
func (c *executorIntegrationClient) Remove(_ context.Context, path string) error {
delete(c.files, path)
return nil
}
func (c *executorIntegrationClient) Rename(_ context.Context, from, to string) error {
value, exists := c.files[from]
if !exists {
return errors.New("测试源文件不存在")
}
c.files[to] = value
delete(c.files, from)
return nil
}
func (c *executorIntegrationClient) Unlock(_ context.Context, path string) error {
delete(c.files, path)
delete(c.files, path+"/owner")
return nil
}
func (c *executorIntegrationClient) Run(context.Context, []string) (deployment.CommandResult, error) {
return deployment.CommandResult{}, nil
}
func executorDigest(value string) string {
digest := sha256.Sum256([]byte(value))
return hex.EncodeToString(digest[:])
}
func executorIntegrationFixture(t *testing.T) (*SQLDeploymentTransactionRepository, *sql.DB, *deployment.Service, *executorIntegrationClient, deployment.Server, deployment.TransactionRecord) {
t.Helper()
_, db := newTaskRepositoryTest(t)
t.Cleanup(func() { _ = db.Close() })
repo := NewSQLDeploymentTransactionRepository(db)
server := deployment.Server{
ID: "executor-server", Name: "executor-server", Address: "192.0.2.10", Port: 22, Username: "deploy",
AuthMethod: deployment.SSHAuthPassword, PasswordRef: "secret://ssh/password/test", HostKeyFingerprint: "SHA256:test",
CertificatePath: "/cert.pem", CertificateChainPath: "/chain.pem", PrivateKeyPath: "/key.pem", WebServer: deployment.WebServerNginx,
}.Normalize()
record := deployment.TransactionRecord{
ID: "executor-transaction", TaskID: "task", AttemptID: "attempt", AttemptNumber: 1, ServerID: server.ID, ServerName: server.Name,
Paths: []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath},
Status: deployment.TransactionStatusRecoveryRequired, Phase: deployment.TransactionPhaseReloadAttempted, ReloadAttempted: true,
Evidence: &deployment.RecoveryEvidence{
Version: 1, LockToken: executorDigest("owner"), Address: server.Address, Port: server.Port, HostKeyFingerprint: server.HostKeyFingerprint,
ValidationCommand: server.ValidationCommand, ReloadCommand: server.ReloadCommand, Files: map[string]string{}, TemporaryFiles: map[string]string{},
Baseline: deployment.RecoveryBaseline{Endpoint: "https://example.com", Domains: []string{"example.com"}, PreviousFingerprint: "old", ExpectedFingerprint: "new"},
},
}
client := &executorIntegrationClient{files: map[string]string{}}
for _, path := range record.Paths {
lock, backup, temporary := path+".scert-lock", path+".scert-backup-"+record.ID, path+".scert-tmp-"+record.ID
record.LockPaths = append(record.LockPaths, lock)
record.TemporaryPaths = append(record.TemporaryPaths, temporary)
record.InstalledPaths = append(record.InstalledPaths, path)
record.Backups = append(record.Backups, deployment.TransactionBackup{Original: path, Saved: backup, OriginalExists: true, OriginalDigest: executorDigest("old")})
record.Evidence.Files[path], record.Evidence.TemporaryFiles[temporary] = executorDigest("new"), executorDigest("temporary")
client.files[path], client.files[backup], client.files[temporary] = "new", "old", "temporary"
client.files[lock], client.files[lock+"/owner"] = "", record.Evidence.LockToken
}
if err := repo.CreateDeploymentTransaction(t.Context(), record); err != nil {
t.Fatal(err)
}
service := deployment.NewServiceWithJournal(client, repo)
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error { return nil }
return repo, db, service, client, server, record
}
func TestRecoveryExecutorSQLiteAtomicLifecycle(t *testing.T) {
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionRecord, deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
t.Run(string(action), func(t *testing.T) {
repo, db, service, client, server, record := executorIntegrationFixture(t)
before, err := repo.ListPendingDeploymentTransactions(t.Context())
if err != nil {
t.Fatal(err)
}
filesBefore := maps.Clone(client.files)
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
if err != nil {
t.Fatal(err)
}
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if err != nil || item.Status != "succeeded" {
t.Fatalf("execution failed: %+v %v", item, err)
}
recoveryCount(t, db, "recovery_dispositions", 1)
recoveryCount(t, db, "recovery_events", 2)
recoveryCount(t, db, "recovery_claims", 0)
stored, err := repo.GetRecoveryDisposition(t.Context(), confirmation.DispositionID())
if err != nil || !reflect.DeepEqual(stored, item) {
t.Fatal("result query lost completed result", err)
}
view, err := service.GetRecoveryDisposition(t.Context(), confirmation.DispositionID())
if err != nil || view.Validate() != nil || !reflect.DeepEqual(view, item.Summary()) {
t.Fatal("safe query omitted or changed persisted result", err)
}
var status string
if err := db.QueryRowContext(t.Context(), "SELECT status FROM deployment_transactions WHERE id = ?", record.ID).Scan(&status); err != nil {
t.Fatal(err)
}
switch action {
case deployment.RecoveryActionRecord:
after, err := repo.ListPendingDeploymentTransactions(t.Context())
if err != nil || !reflect.DeepEqual(before, after) || !reflect.DeepEqual(filesBefore, client.files) {
t.Fatal("record-only changed original state", err)
}
case deployment.RecoveryActionCommit:
if status != deployment.TransactionStatusCommitted || client.files[record.Paths[0]] != "new" {
t.Fatal("commit not atomic with audit")
}
case deployment.RecoveryActionRollback:
if status != deployment.TransactionStatusRolledBack || client.files[record.Paths[0]] != "old" {
t.Fatal("rollback not atomic with audit")
}
}
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, deployment.ErrRecoveryConfirmation) {
t.Fatal("completed confirmation replayed")
}
})
}
}
func TestRecoveryExecutorSQLiteAuditFailuresAndRestart(t *testing.T) {
for _, stage := range []string{"started", "finished"} {
t.Run(stage, func(t *testing.T) {
repo, db, service, client, server, record := executorIntegrationFixture(t)
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", deployment.RecoveryActionCommit, resolve)
if err != nil {
t.Fatal(err)
}
query := "CREATE TRIGGER executor_audit_failure BEFORE INSERT ON recovery_events WHEN NEW.event_type = '" + stage + "' BEGIN SELECT RAISE(ABORT, '测试审计失败'); END"
if _, err := db.ExecContext(t.Context(), query); err != nil {
t.Fatal(err)
}
filesBefore := maps.Clone(client.files)
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if !errors.Is(err, deployment.ErrRecoveryDispositionUnavailable) || item.Status == "succeeded" {
t.Fatal("audit failure reported success")
}
if stage == "started" {
recoveryCount(t, db, "recovery_dispositions", 0)
recoveryCount(t, db, "recovery_claims", 0)
recoveryCount(t, db, "recovery_events", 0)
if !reflect.DeepEqual(filesBefore, client.files) {
t.Fatal("begin audit failure modified remote")
}
return
}
recoveryCount(t, db, "recovery_claims", 1)
recoveryCount(t, db, "recovery_events", 1)
var sequence int
var name, path string
if err := db.QueryRowContext(t.Context(), "PRAGMA database_list").Scan(&sequence, &name, &path); err != nil {
t.Fatal(err)
}
if err := db.Close(); err != nil {
t.Fatal(err)
}
reopened, err := sql.Open("sqlite", path)
if err != nil {
t.Fatal(err)
}
defer reopened.Close()
repo = NewSQLDeploymentTransactionRepository(reopened)
stored, err := repo.GetRecoveryDisposition(t.Context(), confirmation.DispositionID())
if err != nil || stored.Status != "running" {
t.Fatal("uncertain result lost after reopening", err)
}
pending, err := repo.ListPendingDeploymentTransactions(t.Context())
if err != nil || len(pending) != 1 || pending[0].Status != deployment.TransactionStatusRecoveryRequired || pending[0].Phase != deployment.TransactionPhaseCommitVerified {
t.Fatal("failed finish altered original terminal state", err)
}
if strings.Contains(stored.Intent.RecordDigest, token) {
t.Fatal("confirmation token persisted")
}
restarted := deployment.NewServiceWithJournal(nil, repo)
view, err := restarted.GetRecoveryDisposition(t.Context(), confirmation.DispositionID())
if err != nil || view.Status != "running" || view.ID != confirmation.DispositionID() {
t.Fatal("query after process restart failed", err)
}
if err := restarted.RecoverPending(t.Context(), func(context.Context, string) (deployment.Server, error) {
t.Error("restart re-executed occupied transaction")
return deployment.Server{}, errors.New("unexpected resolver")
}); !errors.Is(err, deployment.ErrRecoveryDispositionBusy) {
t.Fatal("restart did not honor durable claim", err)
}
})
}
}

View File

@ -1,5 +1,17 @@
# Linux 生产流程测试记录 # Linux 生产流程测试记录
## 2026年10月3日20:52 一次性确认与执行器回归
范围为第一台隔离源码目录 `/var/tmp/scert-local-recovery-20261003-1828`,不替代正式安装包;远端文件和 HTTPS 回调在本批 Go 测试中使用测试实现,SQLite 为真实临时数据库。线上业务数据库未清理、正式服务未升级。
- [✓] `scert-executor-20261003-2035.service`:部署全包竞态通过,约 3.254 秒,`EXECUTOR_RACE_EXIT=0`。
- [✓] `scert-executor-extra-20261003-2041.service` 的部署竞态通过,约 3.510 秒;但同组 SQLite 五场景因测试客户端缺少证据接口方法而失败,整组失败记录保留于 `executor-extra.log`,不能记整组成功。
- [x] 测试夹具补齐拒绝建锁的方法及编译期证据接口断言,未放宽生产核查边界。
- [✓] `scert-executor-sqlite-20261003-2045.service`:SQLite 五场景竞态复测通过,仓储约 5.303 秒,`EXECUTOR_SQLITE_FIX_EXIT=0`;完整静态检查和后端全量通过,`EXECUTOR_VET_EXIT=0`、`EXECUTOR_FULL_EXIT=0`。服务已退出,`MainPID=0`、`Result=success`。
- [~] 最新追加未知临时文件漏项拦截与脱敏查询,八份源码归档 SHA-256:`101eb370619bac98d4b069ea3c242206b3035fca7cd097006b83fb8f4494a83f`,本地与第一台一致。重新执行任务 `scert-executor-final-20261003-2052.service`,日志 `executor-final.log`,当前未记通过。
- [x] 本轮第二台根分区复核可用 26 GB、使用率 10%,未继续删除文件;第一台 Scert、Nginx 运行,可用约 9.7 GB。
- [ ] 本机协议/终端、PostgreSQL 等必要实库和正式双机生产闭环仍待完成;生产验收未通过。
## 2026年10月3日20:20 后端全量最终结果 ## 2026年10月3日20:20 后端全量最终结果
- [✓] 第一台 `scert-operation-full-20261003-2012.service` 已终止,`MainPID=0`、`Result=success`、`ActiveState=inactive`;整组约四分五十八秒,`OPERATION_FULL_RUN_EXIT=0`。 - [✓] 第一台 `scert-operation-full-20261003-2012.service` 已终止,`MainPID=0`、`Result=success`、`ActiveState=inactive`;整组约四分五十八秒,`OPERATION_FULL_RUN_EXIT=0`。

View File

@ -1,10 +1,13 @@
# Scert 开发任务清单 # Scert 开发任务清单
## 2026年10月3日20:20 回归收口 ## 2026年10月3日20:52 当前开发进度
- [✓] 完整部署互斥与占用防护批次的原生 Linux 后端全量、部署竞态、补充竞态及完整静态检查通过;代码 `b3f27e8` 已推送。 - [x] 一次性确认、记录/提交/回滚执行器、只读脱敏结果查询已编码;执行器共用部署互斥,绑定持久化意图与原子审计,不在失败后切换动作。
- [ ] 下一开发项:一次性本机确认、三动作执行器、受保护协议及终端接入。PostgreSQL 实库及正式生产验收未完成。 - [x] 补上未知临时文件完整集合检查、核验后再检查、部分解锁历史的重新 HTTPS 核验,以及请求取消后独立有界结束审计。
- [~] 本节覆盖下方“全量运行中”状态;自动回归不等于生产验收。 - [✓] 基础两轮部署竞态和 SQLite 五场景通过,静态检查、后端全量通过一轮;SQLite 夹具首次失败已修复,保留原记录。
- [~] 最新八份源码正在重新执行完整回归,未把运行中的结果提前记为通过。
- [ ] 下一开发项:受保护本机协议的准备、确认、查询和管理员终端;身份与会话失效、混合载荷、重放、断连测试。
- [ ] PostgreSQL 等必要实库和正式生产部署验收仍未完成;本节覆盖下方历史状态。
## 2026年10月3日20:17 当前开发进度 ## 2026年10月3日20:17 当前开发进度

View File

@ -1,10 +1,12 @@
# Scert 整改任务执行清单 # Scert 整改任务执行清单
## 2026年10月3日20:20 最新执行结论 ## 2026年10月3日20:52 最新执行结论
- [✓] 本批完整互斥及人工占用防护通过原生 Linux 后端全量,整组退出码零;竞态及静态检查通过,代码 `b3f27e8` 已推送。 - [x] R3 确认和三动作执行器已编码,并接入 R4 共同操作门、持久占用、原子开始/结束审计;查询结果不触发重放。
- [ ] R3 一次性本机确认、动作执行器和终端仍待接入;R4 PostgreSQL 实库与执行器边界、R5 正式包生产门禁仍未完成。 - [✓] 基础部署竞态、SQLite 原子三动作、开始/结束审计失败和数据库重开拦截通过;完整静态检查及后端全量通过一轮。
- [~] 本节覆盖下方全量待结果描述;未据局部开发回归宣布可上线。 - [~] 最新追加的未知临时文件完整集合检查与安全查询正在重新回归;测试任务 `scert-executor-final-20261003-2052.service`,源码归档哈希已比对。
- [ ] 本机协议和终端尚未接入,因此 R3 整体仍未完成;R4 PostgreSQL 等必要实库与 R5 正式包生产门禁仍未完成。
- [~] 文档同步;本节覆盖下面历史状态,不将自动回归等同于生产验收。
## 2026年10月3日20:17 当前执行状态 ## 2026年10月3日20:17 当前执行状态

View File

@ -1,14 +1,15 @@
# Scert 整改、合并与优化清单 # Scert 整改、合并与优化清单
## 2026年10月3日20:20 最新回归结论 ## 2026年10月3日20:52 最新整改进度
本节覆盖下方“全量运行中”状态。 本节覆盖下面历史状态;仅 Linux,生产验收仍未通过。
- [x] 代码检查点 `b3f27e8` 已推送;部署完整互斥和持久化人工占用防护已接入。 - [x] R3 一次性确认已编码:短时令牌、操作者与准备摘要绑定、复制共享消费状态、错误确认及取消后不可重放。
- [✓] 本批原生 Linux 后端全量回归通过:`OPERATION_FULL_EXIT=0`,整组退出码零,约四分五十八秒;此前部署竞态、补充竞态及完整静态检查亦通过。默认全量未开启外部数据库矩阵,不扩大为三库实机验收。 - [x] 记录、提交收尾、回滚三个明确执行器接入共同操作门和持久化意图/结束审计;失败不切换动作,可能已写入的失败保留占用,历史已核验阶段也须重新核验站点。
- [x] MySQL 审计故障注入已修复,六场景实库复测通过;迁移十二场景通过,原失败记录保留。 - [x] 新增脱敏结果查询,执行中及重启后可读取持久结果,不重放动作、不释放占用;补上历史临时路径列表漏项时的未知文件拦截。
- [ ] 下一项仍为 R3:一次性本机确认、记录/提交/回滚执行器及本机协议和终端接入;随后补充 PostgreSQL 实库、执行器并发重启边界和正式包生产门禁。 - [✓] 执行器基础两轮部署竞态、SQLite 三动作和两类审计故障/数据库重开测试通过,静态检查和后端全量通过一轮。SQLite 首次测试夹具缺少证据接口,修复后五场景通过,原失败日志保留。
- [~] 六份相关文档同步;第一台 Scert、Nginx 运行,未更新线上正式包、未清理业务数据库,整体生产验收仍未通过。 - [~] 最后追加的未知临时文件拦截与安全查询随最新八份源码重新进行 Linux 竞态、SQLite、静态检查和全量回归,当前任务为 `scert-executor-final-20261003-2052.service`。
- [ ] R3 本机协议和管理员终端接入;R4 PostgreSQL 等必要实库及故障矩阵;R5 正式 Linux 包和完整生产门禁。尚未开放人工写入口、未更新线上正式包、未清理业务数据库。
## 2026年10月3日20:17 最新执行进度 ## 2026年10月3日20:17 最新执行进度

View File

@ -1,10 +1,14 @@
# 部署事务人工处置设计 # 部署事务人工处置设计
## 2026年10月3日20:20 互斥批次复核结果 ## 2026年10月3日20:52 确认和执行器实现边界
- [✓] 已接入的完整部署互斥、启动恢复占用识别和失败关闭通过最新原生 Linux 后端全量、竞态与静态检查,代码 `b3f27e8` 已推送。 - [x] 确认对象持有不可公开的准备绑定、令牌摘要和持久请求标识,副本共享原子消费状态;随机令牌不入库。有效期沿用一分钟,失败确认亦消费,不能重放。
- [ ] 一次性本机确认和人工执行器仍须接入此共同操作门、持久化意图与原子结束审计;不把现有历史核验阶段当成本次人工动作已执行的证据。 - [x] 确认消费后等待共同操作门;持久化开始和占用成功后才重新核查。已有幂等记录只返回结果,不再次连接;准备变化只结束为拒绝,可能发生写入的失败结束为不确定并保留占用。
- [ ] 本机协议和终端、PostgreSQL 实库、动作失败及重启矩阵和正式生产门禁仍待完成;本节覆盖此前全量待结果描述。 - [x] 提交必须重新验证新证书,回滚必须具有旧站点基线并重载、验证旧证书;历史已核验阶段也不能跳过本次验证。不使用按阶段自动选择方向的恢复入口,提交与回滚不互相回退。
- [x] 连接关闭和独立有界结束审计均在共同操作门内;审计失败不能返回成功。结果查询输出明确的处置标识、事务、服务器、动作、状态和时间,不输出请求键、绑定摘要或确认令牌。
- [x] 对全部目标推导临时路径,拒绝历史列表缺项掩盖的未知文件;回滚后再次核对基线和残留,避免解锁未知现场。
- [✓] 基础部署竞态、SQLite 三动作原子结束和两类审计失败/数据库重开通过;静态检查、后端全量通过一轮。
- [~] 最新安全查询和未知临时文件补充正在回归。本机协议/终端、PostgreSQL 等实库及正式生产门禁仍待完成,不开放半成品写入口。
## 2026年10月3日20:17 互斥接入状态 ## 2026年10月3日20:17 互斥接入状态

View File

@ -1,5 +1,16 @@
# 部署事务人工核查任务清单 # 部署事务人工核查任务清单
## 2026年10月3日20:52 本轮执行任务
沿用已确认的 Linux 整改范围,不增加公网接口,不处理 Windows。本节优先于下面历史进度。
- [x] R3-B/R3-C:短时一次性确认、操作者/动作/准备摘要绑定,副本共享消费状态,失败确认不可重放。
- [x] R3-D/R3-E:记录、提交、回滚三个明确动作接入共同互斥及持久化开始/结束审计;增加不重放、不释放占用的脱敏查询。
- [✓] R4-C 部分回归:并发消费、取消等待、核验/连接收尾/审计期间互斥、失败不切换方向、历史核验阶段重新检查,以及 SQLite 审计失败后真实关闭重开数据库保留占用通过。
- [~] 最新未知临时文件漏项拦截与查询补充正在回归;尚未通过本机终端执行真实服务器动作,不将 R3/R4 整项标完成。
- [ ] R3 本机协议和管理员终端接入;完成之前不对用户开放写入口。
- [ ] R4 PostgreSQL 等必要实库与 R5 正式包生产门禁,仍未完成。
## 2026年10月3日20:20 最新复核结论 ## 2026年10月3日20:20 最新复核结论
- [✓] 部署和自动恢复共同互斥、持久化占用检查及数据库重开拦截专项通过;本批后端全量、竞态和静态检查通过。 - [✓] 部署和自动恢复共同互斥、持久化占用检查及数据库重开拦截专项通过;本批后端全量、竞态和静态检查通过。