实现人工处置一次性确认和三动作执行及脱敏查询

This commit is contained in:
xuans committed 2026-10-03 20:59:22 +08:00
1 parent e36dd3ddc7
commit bf4caebaae
14 files changed
+1319 -19

No files matched your search

@@ -0,0 +1,128 @@
package deployment
import "context"
// validateManualTemporaryFiles 检查完整目标集合,不能因为历史临时路径列表缺项就忽略未知文件。
func validateManualTemporaryFiles(ctx context.Context, client Client, record TransactionRecord) error {
for _, target := range record.Paths {
path := target + ".scert-tmp-" + record.ID
if exists, err := client.Exists(ctx, path); err != nil {
return err
} else if exists {
if err := checkFileDigest(ctx, client, path, record.Evidence.TemporaryFiles[path]); err != nil {
return err
}
}
}
return ctx.Err()
}
func validateRecoveryCommit(ctx context.Context, client Client, record TransactionRecord, allowMissingLocks bool) error {
if err := validateTransactionLocks(ctx, client, record, allowMissingLocks); err != nil {
return err
}
for _, path := range record.Paths {
if err := checkFileDigest(ctx, client, path, record.Evidence.Files[path]); err != nil {
return err
}
}
if err := validateManualTemporaryFiles(ctx, client, record); err != nil {
return err
}
for _, backup := range record.Backups {
if _, err := validateRecoveryBackup(ctx, client, backup); err != nil {
return err
}
}
return ctx.Err()
}
func (s *Service) executeRecoveryCommit(ctx context.Context, record *TransactionRecord, server Server, client Client, mutated *bool) error {
allowMissingLocks := record.Phase == TransactionPhaseCommitVerified
if err := validateRecoveryCommit(ctx, client, *record, allowMissingLocks); err != nil {
return err
}
// 即使历史阶段已核验,也必须再次核验真实新证书,不能仅凭历史状态清理。
if err := s.VerifyRecoveryTarget(ctx, server, record.Evidence.Baseline, true); err != nil {
return err
}
if err := validateRecoveryCommit(ctx, client, *record, allowMissingLocks); err != nil {
return err
}
*mutated = true
for _, temporary := range record.TemporaryPaths {
if err := client.Remove(ctx, temporary); err != nil {
return err
}
}
for _, backup := range record.Backups {
if err := client.Remove(ctx, backup.Saved); err != nil {
return err
}
}
if err := s.persistRecoveryPhase(ctx, record, TransactionPhaseCommitVerified); err != nil {
return err
}
return releaseTransactionLocks(ctx, client, *record, true, allowMissingLocks)
}
func validateRecoveredBaseline(ctx context.Context, client Client, record TransactionRecord) error {
for _, backup := range record.Backups {
if backup.OriginalExists {
if err := checkFileDigest(ctx, client, backup.Original, backup.OriginalDigest); err != nil {
return err
}
} else if exists, err := client.Exists(ctx, backup.Original); err != nil || exists {
return ErrRecoveryActionRejected
}
// 还原完成后不允许留下新出现或未消费的临时文件、备份。
for _, path := range []string{backup.Original + ".scert-tmp-" + record.ID, backup.Saved} {
if exists, err := client.Exists(ctx, path); err != nil || exists {
return ErrRecoveryActionRejected
}
}
}
return ctx.Err()
}
func (s *Service) executeRecoveryRollback(ctx context.Context, record *TransactionRecord, server Server, client Client, mutated *bool) error {
allowMissingLocks := record.Phase == TransactionPhaseRollbackVerified
if err := validateManualTemporaryFiles(ctx, client, *record); err != nil {
return err
}
if allowMissingLocks {
if err := validateTransactionLocks(ctx, client, *record, true); err != nil {
return err
}
if err := validateRecoveredBaseline(ctx, client, *record); err != nil {
return err
}
} else if err := validateRollbackFiles(ctx, client, *record); err != nil {
return err
}
if err := ctx.Err(); err != nil {
return err
}
*mutated = true
if !allowMissingLocks {
if err := restoreTransactionFiles(ctx, client, *record); err != nil {
return err
}
}
for _, command := range [][]string{server.ValidationCommand, server.ReloadCommand} {
response, err := client.Run(ctx, command)
if err != nil || response.ExitCode != 0 {
return ErrRecoveryActionUncertain
}
}
if err := s.VerifyRecoveryTarget(ctx, server, record.Evidence.Baseline, false); err != nil {
return err
}
if err := validateRecoveredBaseline(ctx, client, *record); err != nil {
return err
}
if err := s.persistRecoveryPhase(ctx, record, TransactionPhaseRollbackVerified); err != nil {
return err
}
return releaseTransactionLocks(ctx, client, *record, true, allowMissingLocks)
}
@@ -0,0 +1,93 @@
package deployment
import (
"context"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"encoding/hex"
"errors"
"sync/atomic"
)
var ErrRecoveryConfirmation = errors.New("人工处置确认无效、已使用或已过期,请重新核查")
// RecoveryConfirmation 的副本共享消费状态,不能通过复制对象重复授权。
// 本机会话持有对象,终端仅收到短时令牌和脱敏摘要;令牌不写入数据库。
type RecoveryConfirmation struct {
state *recoveryConfirmationState
}
type recoveryConfirmationState struct {
review RecoveryReview
token [sha256.Size]byte
intent RecoveryDispositionIntent
used atomic.Bool
}
func (c RecoveryConfirmation) Summary() RecoveryReviewSummary {
if c.state == nil {
return RecoveryReviewSummary{}
}
return c.state.review.Summary()
}
func (c RecoveryConfirmation) DispositionID() string {
if c.state == nil {
return ""
}
return c.state.intent.ID
}
func (s *Service) PrepareRecoveryConfirmation(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryConfirmation, string, error) {
review, err := s.PrepareRecoveryReview(ctx, id, operator, action, resolve)
if err != nil {
return RecoveryConfirmation{}, "", err
}
return s.newRecoveryConfirmation(ctx, review)
}
func (s *Service) newRecoveryConfirmation(ctx context.Context, review RecoveryReview) (RecoveryConfirmation, string, error) {
if s == nil || review.service != s {
return RecoveryConfirmation{}, "", ErrRecoveryConfirmation
}
var entropy [96]byte
if _, err := rand.Read(entropy[:]); err != nil {
return RecoveryConfirmation{}, "", ErrRecoveryConfirmation
}
token := hex.EncodeToString(entropy[:32])
intent, err := review.DispositionIntent(hex.EncodeToString(entropy[32:64]), hex.EncodeToString(entropy[64:]))
if err != nil {
return RecoveryConfirmation{}, "", err
}
if err := ctx.Err(); err != nil {
return RecoveryConfirmation{}, "", err
}
return RecoveryConfirmation{state: &recoveryConfirmationState{
review: review, intent: intent, token: sha256.Sum256([]byte(token)),
}}, token, nil
}
// consumeRecoveryConfirmation 每个对象只允许一次确认尝试,错误身份、错误令牌和取消均不可重放。
// 成功消费仍不是远端写授权,执行器必须取得持久化占用并重新检查现场。
func (s *Service) consumeRecoveryConfirmation(ctx context.Context, c RecoveryConfirmation, operator, token string) (RecoveryReview, RecoveryDispositionIntent, error) {
fail := func(err error) (RecoveryReview, RecoveryDispositionIntent, error) {
return RecoveryReview{}, RecoveryDispositionIntent{}, err
}
state := c.state
if state == nil || !state.used.CompareAndSwap(false, true) {
return fail(ErrRecoveryConfirmation)
}
if s == nil || state.review.service != s || s.now == nil || !validReviewOperator(operator) || state.review.operator != operator ||
!s.now().Before(state.review.summary.ExpiresAt) || len(token) != 64 {
return fail(ErrRecoveryConfirmation)
}
digest := sha256.Sum256([]byte(token))
if subtle.ConstantTimeCompare(digest[:], state.token[:]) != 1 {
return fail(ErrRecoveryConfirmation)
}
if err := ctx.Err(); err != nil {
return fail(err)
}
return state.review, state.intent, nil
}
@@ -0,0 +1,97 @@
package deployment
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
func confirmationFixture(t *testing.T) (*Service, RecoveryConfirmation, string) {
t.Helper()
server, client, record := recoveryPreflightFixture()
service := NewServiceWithJournal(fixedClientFactory{client}, &memoryTransactionJournal{pending: []TransactionRecord{record}})
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", RecoveryActionRecord, func(context.Context, string) (Server, error) { return server, nil })
if err != nil {
t.Fatal(err)
}
return service, confirmation, token
}
func TestRecoveryConfirmationSingleUseAndPrivateBindings(t *testing.T) {
service, confirmation, token := confirmationFixture(t)
if len(token) != 64 || len(confirmation.DispositionID()) != 64 || token == confirmation.DispositionID() {
t.Fatal("invalid confirmation entropy")
}
encoded, err := json.Marshal(confirmation)
if err != nil || string(encoded) != "{}" {
t.Fatal("private confirmation serialized")
}
copy := confirmation
review, intent, err := service.consumeRecoveryConfirmation(t.Context(), confirmation, "unix:0", token)
if err != nil || review.Summary() != confirmation.Summary() || intent.Validate() != nil || intent.ID != confirmation.DispositionID() {
t.Fatalf("confirmation failed: %v", err)
}
encoded, _ = json.Marshal(intent)
if strings.Contains(string(encoded), token) {
t.Fatal("raw token persisted")
}
if _, _, err := service.consumeRecoveryConfirmation(t.Context(), copy, "unix:0", token); !errors.Is(err, ErrRecoveryConfirmation) {
t.Fatal("copied confirmation replayed")
}
}
func TestRecoveryConfirmationRejectsAndConsumesInvalidAttempt(t *testing.T) {
for _, scenario := range []string{"identity", "token", "oversized", "expired", "service", "cancelled", "empty"} {
t.Run(scenario, func(t *testing.T) {
service, confirmation, token := confirmationFixture(t)
originalService, originalToken := service, token
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
operator := "unix:0"
switch scenario {
case "identity":
operator = "unix:1"
case "token":
token = strings.Repeat("0", 64)
case "oversized":
token = strings.Repeat("a", 8192)
case "expired":
service.now = func() time.Time { return confirmation.Summary().ExpiresAt }
case "service":
service = NewService(nil)
case "cancelled":
cancel()
case "empty":
confirmation = RecoveryConfirmation{}
}
if _, _, err := service.consumeRecoveryConfirmation(ctx, confirmation, operator, token); err == nil {
t.Fatal("invalid confirmation accepted")
}
if _, _, err := originalService.consumeRecoveryConfirmation(t.Context(), confirmation, "unix:0", originalToken); err == nil {
t.Fatal("failed attempt could be replayed")
}
})
}
}
func TestRecoveryConfirmationConcurrentCopiesConsumeOnlyOnce(t *testing.T) {
service, confirmation, token := confirmationFixture(t)
var accepted atomic.Int32
var group sync.WaitGroup
for range 32 {
group.Go(func() {
if _, _, err := service.consumeRecoveryConfirmation(t.Context(), confirmation, "unix:0", token); err == nil {
accepted.Add(1)
}
})
}
group.Wait()
if accepted.Load() != 1 {
t.Fatalf("accepted %d confirmations", accepted.Load())
}
}
@@ -0,0 +1,139 @@
package deployment
import (
"context"
"errors"
"time"
)
var (
ErrRecoveryActionRejected = errors.New("人工处置前置核验未通过,未执行远端变更")
ErrRecoveryActionUncertain = errors.New("人工处置结果不确定,已保留占用,请查询记录并核查现场")
)
// ExecuteRecoveryDisposition 仅供受保护本机会话使用,操作者必须来自本机对端身份。
// 一次性确认、共同操作门、持久化占用和远端锁各自独立,缺一不可。
func (s *Service) ExecuteRecoveryDisposition(ctx context.Context, confirmation RecoveryConfirmation, operator, token string, resolve func(context.Context, string) (Server, error)) (RecoveryDisposition, error) {
review, intent, err := s.consumeRecoveryConfirmation(ctx, confirmation, operator, token)
if err != nil {
return RecoveryDisposition{}, err
}
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
release, err := s.acquireOperation(ctx)
if err != nil {
return RecoveryDisposition{}, err
}
defer release()
journal, ok := s.journal.(RecoveryDispositionJournal)
if !ok || resolve == nil {
return RecoveryDisposition{}, ErrRecoveryDispositionUnavailable
}
if !s.now().Before(review.summary.ExpiresAt) {
return RecoveryDisposition{}, ErrRecoveryConfirmation
}
item, created, err := journal.BeginRecoveryDisposition(ctx, intent)
if err != nil {
return RecoveryDisposition{}, ErrRecoveryDispositionUnavailable
}
if !created {
// 响应丢失后的既有请求只返回持久结果,绝不重新连接或执行。
return item, nil
}
finish := func(result string, actionErr error) (RecoveryDisposition, error) {
// 请求取消不能跳过结束审计;保存失败保留原占用且绝不返回成功。
finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer finishCancel()
finished, finishErr := journal.FinishRecoveryDisposition(finishCtx, intent.ID, result)
if finishErr != nil {
return item, ErrRecoveryDispositionUnavailable
}
return finished, actionErr
}
if err := s.RecheckRecoveryReview(ctx, review, operator, resolve); err != nil {
return finish(RecoveryResultRejected, ErrRecoveryActionRejected)
}
if review.summary.Action == RecoveryActionRecord {
return finish(RecoveryResultObserved, nil)
}
result, actionErr := s.executeRecoveryAction(ctx, review, resolve)
return finish(result, actionErr)
}
func (s *Service) executeRecoveryAction(ctx context.Context, review RecoveryReview, resolve func(context.Context, string) (Server, error)) (result string, actionErr error) {
result, actionErr = RecoveryResultRejected, ErrRecoveryActionRejected
record, recordDigest, serverDigest, err := s.reviewRecord(ctx, review.summary.TransactionID, resolve)
if err != nil || recordDigest != review.recordDigest || serverDigest != review.serverDigest {
return
}
server, err := resolve(ctx, record.ServerID)
if err != nil {
return
}
server = server.Normalize()
currentServerDigest, err := reviewServerDigest(server)
if err != nil || currentServerDigest != review.serverDigest || validateRecoveryAction(record, review.summary.Action, s.VerifyRecoveryTarget != nil) != nil {
return
}
client, err := s.factory.Connect(ctx, server)
if err != nil {
return
}
mutated := false
defer func() {
// 在结果审计和释放共同操作门之前关闭连接,关闭失败不能伪装成成功。
if closeErr := client.Close(); closeErr != nil {
if mutated {
result, actionErr = RecoveryResultUncertain, ErrRecoveryActionUncertain
} else {
result, actionErr = RecoveryResultRejected, ErrRecoveryActionRejected
}
}
}()
if err := ctx.Err(); err != nil || !s.now().Before(review.summary.ExpiresAt) {
return
}
switch review.summary.Action {
case RecoveryActionCommit:
err = s.executeRecoveryCommit(ctx, &record, server, client, &mutated)
result = RecoveryResultCommitted
case RecoveryActionRollback:
err = s.executeRecoveryRollback(ctx, &record, server, client, &mutated)
result = RecoveryResultRolledBack
default:
return
}
if err == nil {
return result, nil
}
if mutated {
return RecoveryResultUncertain, ErrRecoveryActionUncertain
}
return RecoveryResultRejected, ErrRecoveryActionRejected
}
func validateRecoveryAction(record TransactionRecord, action RecoveryAction, verifier bool) error {
if record.Evidence == nil || !verifier || len(record.Backups) != len(record.Paths) {
return ErrRecoveryActionRejected
}
switch action {
case RecoveryActionCommit:
if record.Evidence.Baseline.ExpectedFingerprint == "" {
return ErrRecoveryActionRejected
}
switch record.Phase {
case TransactionPhaseReloadAttempted, TransactionPhaseCommitting, TransactionPhaseCommitVerified:
return nil
}
case RecoveryActionRollback:
if record.Evidence.Baseline.PreviousFingerprint == "" {
return ErrRecoveryActionRejected
}
switch record.Phase {
case TransactionPhaseLocked, TransactionPhaseUploaded, TransactionPhaseReplacing, TransactionPhaseInstalled,
TransactionPhaseReloadAttempted, TransactionPhaseCommitting, TransactionPhaseRollbackVerified:
return nil
}
}
return ErrRecoveryActionRejected
}
@@ -0,0 +1,428 @@
package deployment
import (
"context"
"errors"
"maps"
"reflect"
"testing"
"time"
)
type executorJournal struct {
memoryTransactionJournal
item RecoveryDisposition
claimed bool
beginCalls, finishCalls int
beginErr, finishErr, updateErr error
finishContextErr error
existing bool
beforeFinish func()
}
func (j *executorJournal) BeginRecoveryDisposition(_ context.Context, intent RecoveryDispositionIntent) (RecoveryDisposition, bool, error) {
j.beginCalls++
if j.beginErr != nil {
return RecoveryDisposition{}, false, j.beginErr
}
if j.existing {
return j.item, false, nil
}
if j.claimed {
return RecoveryDisposition{}, false, ErrRecoveryDispositionBusy
}
if !intent.MatchesRecord(j.pending[0]) {
return RecoveryDisposition{}, false, ErrRecoveryReviewChanged
}
j.item = RecoveryDisposition{Intent: intent, Status: "running"}
j.claimed = true
return j.item, true, nil
}
func (j *executorJournal) FinishRecoveryDisposition(ctx context.Context, id, result string) (RecoveryDisposition, error) {
j.finishCalls++
j.finishContextErr = ctx.Err()
if j.beforeFinish != nil {
j.beforeFinish()
}
if j.finishErr != nil {
return RecoveryDisposition{}, j.finishErr
}
if id != j.item.Intent.ID || !j.item.Intent.AllowsResult(result) || !j.claimed {
return RecoveryDisposition{}, ErrRecoveryDispositionConflict
}
j.item.Status, j.item.ResultCode = "succeeded", result
if result == RecoveryResultRejected || result == RecoveryResultUncertain {
j.item.Status = result
}
j.claimed = result == RecoveryResultUncertain
return j.item, nil
}
func (j *executorJournal) GetRecoveryDisposition(context.Context, string) (RecoveryDisposition, error) {
return j.item, nil
}
func (j *executorJournal) HasRecoveryClaim(context.Context, string) (bool, error) {
return j.claimed, nil
}
func (j *executorJournal) UpdateDeploymentTransaction(ctx context.Context, record TransactionRecord) error {
if j.updateErr != nil {
return j.updateErr
}
return j.memoryTransactionJournal.UpdateDeploymentTransaction(ctx, record)
}
type executorClient struct {
*memoryClient
closeErr error
closed int
beforeClose func()
}
func (c *executorClient) Close() error {
c.closed++
if c.beforeClose != nil {
c.beforeClose()
}
return c.closeErr
}
func executorFixture(t *testing.T) (*Service, *executorJournal, *executorClient, Server, TransactionRecord) {
t.Helper()
server, memory, record := recoveryPreflightFixture()
journal := &executorJournal{memoryTransactionJournal: memoryTransactionJournal{pending: []TransactionRecord{record}}}
client := &executorClient{memoryClient: memory}
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { return nil }
return service, journal, client, server, record
}
func TestRecoveryExecutorThreeExplicitActions(t *testing.T) {
for _, action := range []RecoveryAction{RecoveryActionRecord, RecoveryActionCommit, RecoveryActionRollback} {
t.Run(string(action), func(t *testing.T) {
service, journal, client, server, record := executorFixture(t)
before := maps.Clone(client.files)
resolve := func(context.Context, string) (Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
if err != nil {
t.Fatal(err)
}
var verified []bool
service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, commit bool) error {
verified = append(verified, commit)
return nil
}
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if err != nil || item.Status != "succeeded" || journal.claimed || journal.beginCalls != 1 || journal.finishCalls != 1 || len(journal.finished) != 0 {
t.Fatalf("execution failed: %+v, %v", item, err)
}
if action == RecoveryActionRecord {
if item.ResultCode != RecoveryResultObserved || len(verified) != 0 || len(journal.updated) != 0 || !reflect.DeepEqual(before, client.files) {
t.Fatal("record-only action changed original state")
}
} else {
expected, phase, result := "new", TransactionPhaseCommitVerified, RecoveryResultCommitted
if action == RecoveryActionRollback {
expected, phase, result = "old", TransactionPhaseRollbackVerified, RecoveryResultRolledBack
}
if !reflect.DeepEqual(verified, []bool{action == RecoveryActionCommit}) || item.ResultCode != result || len(journal.updated) != 1 || journal.updated[0].Phase != phase {
t.Fatal("wrong action, verification or phase")
}
for _, path := range record.Paths {
if string(client.files[path].data) != expected {
t.Fatal("wrong certificate files")
}
}
if len(client.files) != 3 {
t.Fatal("transaction leftovers remain")
}
}
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, ErrRecoveryConfirmation) || journal.beginCalls != 1 {
t.Fatal("execution replayed")
}
})
}
}
func TestRecoveryExecutorFailureBoundaries(t *testing.T) {
for _, scenario := range []string{"changed", "foreign-lock", "legacy", "phase", "missing-baseline", "missing-backup-entry", "begin", "finish", "persist", "cleanup", "reload", "commit-https", "rollback-https", "cancel", "close", "existing"} {
t.Run(scenario, func(t *testing.T) {
service, journal, client, server, record := executorFixture(t)
action := RecoveryActionCommit
if scenario == "reload" || scenario == "rollback-https" || scenario == "missing-backup-entry" {
action = RecoveryActionRollback
}
if scenario == "legacy" {
record.Evidence = nil
action = RecoveryActionRecord
}
if scenario == "phase" {
record.Phase = TransactionPhaseUploaded
}
if scenario == "missing-baseline" {
record.Evidence.Baseline.ExpectedFingerprint = ""
}
if scenario == "missing-backup-entry" {
record.Backups = record.Backups[:2]
}
journal.pending[0] = record
resolve := func(context.Context, string) (Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
verifyCalls := 0
service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, commit bool) error {
verifyCalls++
if commit != (action == RecoveryActionCommit) {
t.Fatal("failed action silently switched direction")
}
if scenario == "commit-https" || scenario == "rollback-https" {
return errors.New("private upstream details")
}
if scenario == "cancel" {
cancel()
}
if scenario == "close" {
client.closeErr = errors.New("private close details")
}
return nil
}
switch scenario {
case "changed":
client.files[record.Paths[0]] = memoryFile{data: []byte("foreign")}
case "foreign-lock":
client.files[record.LockPaths[0]+"/owner"] = memoryFile{data: []byte("foreign")}
case "begin":
journal.beginErr = errors.New("private database error")
case "finish":
journal.finishErr = errors.New("private audit error")
case "persist":
journal.updateErr = errors.New("private update error")
case "cleanup":
client.removeError = ".scert-backup-"
case "reload":
client.runErrors = []error{errors.New("private command error")}
case "existing":
journal.existing = true
journal.item = RecoveryDisposition{Status: "uncertain", ResultCode: RecoveryResultUncertain}
}
before := maps.Clone(client.files)
closedBefore := client.closed
item, err := service.ExecuteRecoveryDisposition(ctx, confirmation, "unix:0", token, resolve)
if scenario == "legacy" {
if err != nil || item.ResultCode != RecoveryResultObserved || len(journal.updated) != 0 || !reflect.DeepEqual(before, client.files) {
t.Fatal("legacy inspection mutated history")
}
return
}
if scenario == "existing" {
if err != nil || item.Status != "uncertain" || verifyCalls != 0 || journal.finishCalls != 0 || client.closed != closedBefore || !reflect.DeepEqual(before, client.files) {
t.Fatal("existing request executed again")
}
return
}
if err == nil || item.Status == "succeeded" || journal.finishContextErr != nil {
t.Fatalf("failure reported success or skipped independent audit: %+v %v", item, err)
}
uncertain := scenario == "finish" || scenario == "persist" || scenario == "cleanup" || scenario == "reload" || scenario == "rollback-https" || scenario == "close"
if journal.claimed != uncertain {
t.Fatalf("wrong claim retention: %v", journal.claimed)
}
if !uncertain && !reflect.DeepEqual(before, client.files) {
t.Fatal("preflight rejection changed remote files")
}
if len(journal.finished) != 0 {
t.Fatal("executor bypassed atomic disposition completion")
}
})
}
}
func TestRecoveryExecutorWaitCancellationConsumesConfirmation(t *testing.T) {
service, journal, _, server, record := executorFixture(t)
resolve := func(context.Context, string) (Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", RecoveryActionRecord, resolve)
if err != nil {
t.Fatal(err)
}
release, err := service.acquireOperation(t.Context())
if err != nil {
t.Fatal(err)
}
defer release()
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
defer cancel()
if _, err := service.ExecuteRecoveryDisposition(ctx, confirmation, "unix:0", token, resolve); !errors.Is(err, context.DeadlineExceeded) || journal.beginCalls != 0 {
t.Fatal("cancelled waiter acquired persistent claim")
}
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, ErrRecoveryConfirmation) {
t.Fatal("cancelled confirmation replayed")
}
}
func TestRecoveryExecutorHoldsOperationThroughVerificationCloseAndAudit(t *testing.T) {
service, journal, client, server, record := executorFixture(t)
resolve := func(context.Context, string) (Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", RecoveryActionCommit, resolve)
if err != nil {
t.Fatal(err)
}
checked := map[string]int{}
checkHeld := func(stage string) {
checked[stage]++
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Millisecond)
defer cancel()
release, err := service.acquireOperation(ctx)
if err == nil {
release()
t.Fatalf("operation gate released during %s", stage)
}
if !errors.Is(err, context.DeadlineExceeded) || !journal.claimed {
t.Fatalf("missing durable claim or invalid wait during %s", stage)
}
}
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error {
checkHeld("verify")
return nil
}
client.beforeClose = func() { checkHeld("close") }
journal.beforeFinish = func() { checkHeld("audit") }
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); err != nil {
t.Fatal(err)
}
if checked["verify"] != 1 || checked["close"] != 2 || checked["audit"] != 1 {
t.Fatalf("missing gate coverage: %v", checked)
}
release, err := service.acquireOperation(t.Context())
if err != nil {
t.Fatal(err)
}
release()
}
func TestRecoveryExecutorVerifiedHistoryStillRechecksHTTPS(t *testing.T) {
for _, action := range []RecoveryAction{RecoveryActionCommit, RecoveryActionRollback} {
for _, failVerification := range []bool{false, true} {
t.Run(string(action)+map[bool]string{false: "/success", true: "/failure"}[failVerification], func(t *testing.T) {
service, journal, client, server, record := executorFixture(t)
record.Phase = TransactionPhaseCommitVerified
if action == RecoveryActionRollback {
record.Phase = TransactionPhaseRollbackVerified
for _, path := range record.Paths {
client.files[path] = memoryFile{data: []byte("old")}
}
}
for _, path := range record.TemporaryPaths {
delete(client.files, path)
}
for _, backup := range record.Backups {
delete(client.files, backup.Saved)
}
delete(client.files, record.LockPaths[0])
delete(client.files, record.LockPaths[0]+"/owner")
journal.pending[0] = record
resolve := func(context.Context, string) (Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
if err != nil {
t.Fatal(err)
}
verified := 0
service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, commit bool) error {
verified++
if commit != (action == RecoveryActionCommit) {
t.Fatal("wrong HTTPS verification direction")
}
if failVerification {
return errors.New("HTTPS mismatch")
}
return nil
}
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if verified != 1 || (err != nil) != failVerification || (item.Status == "succeeded") == failVerification {
t.Fatalf("historical phase bypassed new verification: %+v %v", item, err)
}
if !failVerification && len(client.files) != 3 {
t.Fatal("partial unlock history failed to finish safely")
}
if failVerification {
if _, exists := client.files[record.LockPaths[1]]; !exists {
t.Fatal("failed verification removed retained lock")
}
}
})
}
}
}
func TestRecoveryExecutorRechecksAfterHTTPSAndAuditsCancellation(t *testing.T) {
for _, scenario := range []string{"target", "backup", "lock", "cancel-after-write"} {
t.Run(scenario, func(t *testing.T) {
service, journal, client, server, record := executorFixture(t)
action := RecoveryActionCommit
if scenario == "cancel-after-write" {
action = RecoveryActionRollback
}
resolve := func(context.Context, string) (Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error {
path := record.Paths[0]
switch scenario {
case "backup":
path = record.Backups[0].Saved
case "lock":
path = record.LockPaths[0] + "/owner"
case "cancel-after-write":
cancel()
return ctx.Err()
}
client.files[path] = memoryFile{data: []byte("foreign-change")}
return nil
}
item, err := service.ExecuteRecoveryDisposition(ctx, confirmation, "unix:0", token, resolve)
if err == nil || item.Status == "succeeded" || journal.finishCalls != 1 || journal.finishContextErr != nil {
t.Fatal("changed or cancelled execution skipped audit or reported success")
}
if journal.claimed != (scenario == "cancel-after-write") {
t.Fatal("wrong claim retention after cancelled write")
}
if _, exists := client.files[record.LockPaths[0]]; !exists {
t.Fatal("changed state unlocked")
}
})
}
}
func TestRecoveryExecutorRejectsUnlistedTemporaryFiles(t *testing.T) {
for _, action := range []RecoveryAction{RecoveryActionCommit, RecoveryActionRollback} {
t.Run(string(action), func(t *testing.T) {
service, journal, client, server, record := executorFixture(t)
delete(record.Evidence.TemporaryFiles, record.TemporaryPaths[0])
record.TemporaryPaths = record.TemporaryPaths[1:]
journal.pending[0] = record
resolve := func(context.Context, string) (Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
if err != nil {
t.Fatal(err)
}
before := maps.Clone(client.files)
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error {
t.Fatal("unknown temporary file reached HTTPS verification")
return nil
}
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if !errors.Is(err, ErrRecoveryActionRejected) || item.ResultCode != RecoveryResultRejected || journal.claimed || !reflect.DeepEqual(before, client.files) {
t.Fatal("unlisted temporary file ignored or modified")
}
})
}
}
@@ -0,0 +1,79 @@
package deployment
import (
"context"
"strings"
"time"
)
// RecoveryDispositionSummary 是本机查询的明确输出边界,不包含请求键、绑定摘要或操作者凭据。
type RecoveryDispositionSummary struct {
ID string `json:"id"`
TransactionID string `json:"transaction_id"`
ServerID string `json:"server_id"`
Action RecoveryAction `json:"action"`
Status string `json:"status"`
ResultCode string `json:"result_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
func (r RecoveryDisposition) Summary() RecoveryDispositionSummary {
return RecoveryDispositionSummary{
ID: r.Intent.ID, TransactionID: r.Intent.TransactionID, ServerID: r.Intent.ServerID,
Action: r.Intent.Action, Status: r.Status, ResultCode: r.ResultCode,
CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt,
}
}
func validDispositionID(id string) bool {
return validDigest(id) && id == strings.ToLower(id) && id != strings.Repeat("0", 64)
}
func (r RecoveryDispositionSummary) Validate() error {
if !validDispositionID(r.ID) || !validInspectionID(r.TransactionID) || !validRecoverySummary(r.ServerID) ||
!validRecoveryAction(r.Action) || r.CreatedAt.IsZero() || r.UpdatedAt.IsZero() || r.UpdatedAt.Before(r.CreatedAt) {
return ErrRecoveryDispositionInvalid
}
intent := RecoveryDispositionIntent{Action: r.Action}
switch r.Status {
case "running":
if r.ResultCode == "" {
return nil
}
case "succeeded":
if r.ResultCode != RecoveryResultRejected && r.ResultCode != RecoveryResultUncertain && intent.AllowsResult(r.ResultCode) {
return nil
}
case RecoveryResultRejected, RecoveryResultUncertain:
if r.ResultCode == r.Status {
return nil
}
}
return ErrRecoveryDispositionInvalid
}
// GetRecoveryDisposition 不取得写操作门,执行中或重启后均可查询;查询本身不会重放动作或释放占用。
func (s *Service) GetRecoveryDisposition(ctx context.Context, id string) (RecoveryDispositionSummary, error) {
if !validDispositionID(id) {
return RecoveryDispositionSummary{}, ErrRecoveryDispositionInvalid
}
if s == nil {
return RecoveryDispositionSummary{}, ErrRecoveryDispositionUnavailable
}
journal, ok := s.journal.(RecoveryDispositionJournal)
if !ok {
return RecoveryDispositionSummary{}, ErrRecoveryDispositionUnavailable
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
item, err := journal.GetRecoveryDisposition(ctx, id)
if err != nil || ctx.Err() != nil || item.Intent.ID != id || item.Intent.Validate() != nil {
return RecoveryDispositionSummary{}, ErrRecoveryDispositionUnavailable
}
summary := item.Summary()
if summary.Validate() != nil {
return RecoveryDispositionSummary{}, ErrRecoveryDispositionUnavailable
}
return summary, nil
}
@@ -0,0 +1,67 @@
package deployment
import (
"encoding/json"
"strings"
"testing"
"time"
)
func TestRecoveryDispositionSummaryValidation(t *testing.T) {
now := time.Now().UTC()
for _, action := range []RecoveryAction{RecoveryActionRecord, RecoveryActionCommit, RecoveryActionRollback} {
for _, result := range []string{"", RecoveryResultObserved, RecoveryResultCommitted, RecoveryResultRolledBack, RecoveryResultRejected, RecoveryResultUncertain} {
status := "succeeded"
if result == "" {
status = "running"
}
if result == RecoveryResultRejected || result == RecoveryResultUncertain {
status = result
}
summary := RecoveryDispositionSummary{ID: strings.Repeat("a", 64), TransactionID: "transaction", ServerID: "server", Action: action, Status: status, ResultCode: result, CreatedAt: now, UpdatedAt: now}
want := result == "" || (RecoveryDispositionIntent{Action: action}).AllowsResult(result)
if (summary.Validate() == nil) != want {
t.Fatalf("wrong result validation: %+v", summary)
}
}
}
}
func TestRecoveryDispositionQueryIsReadOnlyAndPrivate(t *testing.T) {
service, journal, _, _, _ := executorFixture(t)
now := time.Now().UTC()
journal.item = RecoveryDisposition{
Intent: RecoveryDispositionIntent{
ID: strings.Repeat("a", 64), RequestKey: strings.Repeat("b", 64), RecordDigest: strings.Repeat("c", 64),
ServerDigest: strings.Repeat("d", 64), ObservationDigest: strings.Repeat("e", 64),
TransactionID: "transaction", ServerID: "server", Operator: "unix:private", Action: RecoveryActionCommit,
}, Status: "running", CreatedAt: now, UpdatedAt: now,
}
journal.claimed = true
release, err := service.acquireOperation(t.Context())
if err != nil {
t.Fatal(err)
}
defer release()
result, err := service.GetRecoveryDisposition(t.Context(), journal.item.Intent.ID)
if err != nil || result.Status != "running" || !journal.claimed || journal.beginCalls != 0 || journal.finishCalls != 0 {
t.Fatal("query changed state or blocked behind writer", err)
}
encoded, err := json.Marshal(result)
if err != nil {
t.Fatal(err)
}
for _, private := range []string{journal.item.Intent.RequestKey, journal.item.Intent.RecordDigest, journal.item.Intent.ServerDigest, journal.item.Intent.ObservationDigest, journal.item.Intent.Operator} {
if strings.Contains(string(encoded), private) {
t.Fatal("private binding leaked in result")
}
}
if _, err := service.GetRecoveryDisposition(t.Context(), strings.Repeat("f", 64)); err == nil {
t.Fatal("wrong result identifier accepted")
}
journal.item.Status = "succeeded"
journal.item.ResultCode = RecoveryResultRolledBack
if _, err := service.GetRecoveryDisposition(t.Context(), journal.item.Intent.ID); err == nil {
t.Fatal("inconsistent action result accepted")
}
}