实现人工处置一次性确认和三动作执行及脱敏查询
This commit is contained in:
1 parent
e36dd3ddc7
commit
bf4caebaae
14 files changed
+1319
-19
No files matched your search
@@ -0,0 +1,128 @@
|
||||
package deployment
|
||||
|
||||
import "context"
|
||||
|
||||
// validateManualTemporaryFiles 检查完整目标集合,不能因为历史临时路径列表缺项就忽略未知文件。
|
||||
func validateManualTemporaryFiles(ctx context.Context, client Client, record TransactionRecord) error {
|
||||
for _, target := range record.Paths {
|
||||
path := target + ".scert-tmp-" + record.ID
|
||||
if exists, err := client.Exists(ctx, path); err != nil {
|
||||
return err
|
||||
} else if exists {
|
||||
if err := checkFileDigest(ctx, client, path, record.Evidence.TemporaryFiles[path]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func validateRecoveryCommit(ctx context.Context, client Client, record TransactionRecord, allowMissingLocks bool) error {
|
||||
if err := validateTransactionLocks(ctx, client, record, allowMissingLocks); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, path := range record.Paths {
|
||||
if err := checkFileDigest(ctx, client, path, record.Evidence.Files[path]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := validateManualTemporaryFiles(ctx, client, record); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, backup := range record.Backups {
|
||||
if _, err := validateRecoveryBackup(ctx, client, backup); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func (s *Service) executeRecoveryCommit(ctx context.Context, record *TransactionRecord, server Server, client Client, mutated *bool) error {
|
||||
allowMissingLocks := record.Phase == TransactionPhaseCommitVerified
|
||||
if err := validateRecoveryCommit(ctx, client, *record, allowMissingLocks); err != nil {
|
||||
return err
|
||||
}
|
||||
// 即使历史阶段已核验,也必须再次核验真实新证书,不能仅凭历史状态清理。
|
||||
if err := s.VerifyRecoveryTarget(ctx, server, record.Evidence.Baseline, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateRecoveryCommit(ctx, client, *record, allowMissingLocks); err != nil {
|
||||
return err
|
||||
}
|
||||
*mutated = true
|
||||
for _, temporary := range record.TemporaryPaths {
|
||||
if err := client.Remove(ctx, temporary); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, backup := range record.Backups {
|
||||
if err := client.Remove(ctx, backup.Saved); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := s.persistRecoveryPhase(ctx, record, TransactionPhaseCommitVerified); err != nil {
|
||||
return err
|
||||
}
|
||||
return releaseTransactionLocks(ctx, client, *record, true, allowMissingLocks)
|
||||
}
|
||||
|
||||
func validateRecoveredBaseline(ctx context.Context, client Client, record TransactionRecord) error {
|
||||
for _, backup := range record.Backups {
|
||||
if backup.OriginalExists {
|
||||
if err := checkFileDigest(ctx, client, backup.Original, backup.OriginalDigest); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if exists, err := client.Exists(ctx, backup.Original); err != nil || exists {
|
||||
return ErrRecoveryActionRejected
|
||||
}
|
||||
// 还原完成后不允许留下新出现或未消费的临时文件、备份。
|
||||
for _, path := range []string{backup.Original + ".scert-tmp-" + record.ID, backup.Saved} {
|
||||
if exists, err := client.Exists(ctx, path); err != nil || exists {
|
||||
return ErrRecoveryActionRejected
|
||||
}
|
||||
}
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func (s *Service) executeRecoveryRollback(ctx context.Context, record *TransactionRecord, server Server, client Client, mutated *bool) error {
|
||||
allowMissingLocks := record.Phase == TransactionPhaseRollbackVerified
|
||||
if err := validateManualTemporaryFiles(ctx, client, *record); err != nil {
|
||||
return err
|
||||
}
|
||||
if allowMissingLocks {
|
||||
if err := validateTransactionLocks(ctx, client, *record, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateRecoveredBaseline(ctx, client, *record); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if err := validateRollbackFiles(ctx, client, *record); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
*mutated = true
|
||||
if !allowMissingLocks {
|
||||
if err := restoreTransactionFiles(ctx, client, *record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, command := range [][]string{server.ValidationCommand, server.ReloadCommand} {
|
||||
response, err := client.Run(ctx, command)
|
||||
if err != nil || response.ExitCode != 0 {
|
||||
return ErrRecoveryActionUncertain
|
||||
}
|
||||
}
|
||||
if err := s.VerifyRecoveryTarget(ctx, server, record.Evidence.Baseline, false); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateRecoveredBaseline(ctx, client, *record); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.persistRecoveryPhase(ctx, record, TransactionPhaseRollbackVerified); err != nil {
|
||||
return err
|
||||
}
|
||||
return releaseTransactionLocks(ctx, client, *record, true, allowMissingLocks)
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
var ErrRecoveryConfirmation = errors.New("人工处置确认无效、已使用或已过期,请重新核查")
|
||||
|
||||
// RecoveryConfirmation 的副本共享消费状态,不能通过复制对象重复授权。
|
||||
// 本机会话持有对象,终端仅收到短时令牌和脱敏摘要;令牌不写入数据库。
|
||||
type RecoveryConfirmation struct {
|
||||
state *recoveryConfirmationState
|
||||
}
|
||||
|
||||
type recoveryConfirmationState struct {
|
||||
review RecoveryReview
|
||||
token [sha256.Size]byte
|
||||
intent RecoveryDispositionIntent
|
||||
used atomic.Bool
|
||||
}
|
||||
|
||||
func (c RecoveryConfirmation) Summary() RecoveryReviewSummary {
|
||||
if c.state == nil {
|
||||
return RecoveryReviewSummary{}
|
||||
}
|
||||
return c.state.review.Summary()
|
||||
}
|
||||
|
||||
func (c RecoveryConfirmation) DispositionID() string {
|
||||
if c.state == nil {
|
||||
return ""
|
||||
}
|
||||
return c.state.intent.ID
|
||||
}
|
||||
|
||||
func (s *Service) PrepareRecoveryConfirmation(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryConfirmation, string, error) {
|
||||
review, err := s.PrepareRecoveryReview(ctx, id, operator, action, resolve)
|
||||
if err != nil {
|
||||
return RecoveryConfirmation{}, "", err
|
||||
}
|
||||
return s.newRecoveryConfirmation(ctx, review)
|
||||
}
|
||||
|
||||
func (s *Service) newRecoveryConfirmation(ctx context.Context, review RecoveryReview) (RecoveryConfirmation, string, error) {
|
||||
if s == nil || review.service != s {
|
||||
return RecoveryConfirmation{}, "", ErrRecoveryConfirmation
|
||||
}
|
||||
var entropy [96]byte
|
||||
if _, err := rand.Read(entropy[:]); err != nil {
|
||||
return RecoveryConfirmation{}, "", ErrRecoveryConfirmation
|
||||
}
|
||||
token := hex.EncodeToString(entropy[:32])
|
||||
intent, err := review.DispositionIntent(hex.EncodeToString(entropy[32:64]), hex.EncodeToString(entropy[64:]))
|
||||
if err != nil {
|
||||
return RecoveryConfirmation{}, "", err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return RecoveryConfirmation{}, "", err
|
||||
}
|
||||
return RecoveryConfirmation{state: &recoveryConfirmationState{
|
||||
review: review, intent: intent, token: sha256.Sum256([]byte(token)),
|
||||
}}, token, nil
|
||||
}
|
||||
|
||||
// consumeRecoveryConfirmation 每个对象只允许一次确认尝试,错误身份、错误令牌和取消均不可重放。
|
||||
// 成功消费仍不是远端写授权,执行器必须取得持久化占用并重新检查现场。
|
||||
func (s *Service) consumeRecoveryConfirmation(ctx context.Context, c RecoveryConfirmation, operator, token string) (RecoveryReview, RecoveryDispositionIntent, error) {
|
||||
fail := func(err error) (RecoveryReview, RecoveryDispositionIntent, error) {
|
||||
return RecoveryReview{}, RecoveryDispositionIntent{}, err
|
||||
}
|
||||
state := c.state
|
||||
if state == nil || !state.used.CompareAndSwap(false, true) {
|
||||
return fail(ErrRecoveryConfirmation)
|
||||
}
|
||||
if s == nil || state.review.service != s || s.now == nil || !validReviewOperator(operator) || state.review.operator != operator ||
|
||||
!s.now().Before(state.review.summary.ExpiresAt) || len(token) != 64 {
|
||||
return fail(ErrRecoveryConfirmation)
|
||||
}
|
||||
digest := sha256.Sum256([]byte(token))
|
||||
if subtle.ConstantTimeCompare(digest[:], state.token[:]) != 1 {
|
||||
return fail(ErrRecoveryConfirmation)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
return state.review, state.intent, nil
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func confirmationFixture(t *testing.T) (*Service, RecoveryConfirmation, string) {
|
||||
t.Helper()
|
||||
server, client, record := recoveryPreflightFixture()
|
||||
service := NewServiceWithJournal(fixedClientFactory{client}, &memoryTransactionJournal{pending: []TransactionRecord{record}})
|
||||
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", RecoveryActionRecord, func(context.Context, string) (Server, error) { return server, nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return service, confirmation, token
|
||||
}
|
||||
|
||||
func TestRecoveryConfirmationSingleUseAndPrivateBindings(t *testing.T) {
|
||||
service, confirmation, token := confirmationFixture(t)
|
||||
if len(token) != 64 || len(confirmation.DispositionID()) != 64 || token == confirmation.DispositionID() {
|
||||
t.Fatal("invalid confirmation entropy")
|
||||
}
|
||||
encoded, err := json.Marshal(confirmation)
|
||||
if err != nil || string(encoded) != "{}" {
|
||||
t.Fatal("private confirmation serialized")
|
||||
}
|
||||
copy := confirmation
|
||||
review, intent, err := service.consumeRecoveryConfirmation(t.Context(), confirmation, "unix:0", token)
|
||||
if err != nil || review.Summary() != confirmation.Summary() || intent.Validate() != nil || intent.ID != confirmation.DispositionID() {
|
||||
t.Fatalf("confirmation failed: %v", err)
|
||||
}
|
||||
encoded, _ = json.Marshal(intent)
|
||||
if strings.Contains(string(encoded), token) {
|
||||
t.Fatal("raw token persisted")
|
||||
}
|
||||
if _, _, err := service.consumeRecoveryConfirmation(t.Context(), copy, "unix:0", token); !errors.Is(err, ErrRecoveryConfirmation) {
|
||||
t.Fatal("copied confirmation replayed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryConfirmationRejectsAndConsumesInvalidAttempt(t *testing.T) {
|
||||
for _, scenario := range []string{"identity", "token", "oversized", "expired", "service", "cancelled", "empty"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
service, confirmation, token := confirmationFixture(t)
|
||||
originalService, originalToken := service, token
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
operator := "unix:0"
|
||||
switch scenario {
|
||||
case "identity":
|
||||
operator = "unix:1"
|
||||
case "token":
|
||||
token = strings.Repeat("0", 64)
|
||||
case "oversized":
|
||||
token = strings.Repeat("a", 8192)
|
||||
case "expired":
|
||||
service.now = func() time.Time { return confirmation.Summary().ExpiresAt }
|
||||
case "service":
|
||||
service = NewService(nil)
|
||||
case "cancelled":
|
||||
cancel()
|
||||
case "empty":
|
||||
confirmation = RecoveryConfirmation{}
|
||||
}
|
||||
if _, _, err := service.consumeRecoveryConfirmation(ctx, confirmation, operator, token); err == nil {
|
||||
t.Fatal("invalid confirmation accepted")
|
||||
}
|
||||
if _, _, err := originalService.consumeRecoveryConfirmation(t.Context(), confirmation, "unix:0", originalToken); err == nil {
|
||||
t.Fatal("failed attempt could be replayed")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryConfirmationConcurrentCopiesConsumeOnlyOnce(t *testing.T) {
|
||||
service, confirmation, token := confirmationFixture(t)
|
||||
var accepted atomic.Int32
|
||||
var group sync.WaitGroup
|
||||
for range 32 {
|
||||
group.Go(func() {
|
||||
if _, _, err := service.consumeRecoveryConfirmation(t.Context(), confirmation, "unix:0", token); err == nil {
|
||||
accepted.Add(1)
|
||||
}
|
||||
})
|
||||
}
|
||||
group.Wait()
|
||||
if accepted.Load() != 1 {
|
||||
t.Fatalf("accepted %d confirmations", accepted.Load())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrRecoveryActionRejected = errors.New("人工处置前置核验未通过,未执行远端变更")
|
||||
ErrRecoveryActionUncertain = errors.New("人工处置结果不确定,已保留占用,请查询记录并核查现场")
|
||||
)
|
||||
|
||||
// ExecuteRecoveryDisposition 仅供受保护本机会话使用,操作者必须来自本机对端身份。
|
||||
// 一次性确认、共同操作门、持久化占用和远端锁各自独立,缺一不可。
|
||||
func (s *Service) ExecuteRecoveryDisposition(ctx context.Context, confirmation RecoveryConfirmation, operator, token string, resolve func(context.Context, string) (Server, error)) (RecoveryDisposition, error) {
|
||||
review, intent, err := s.consumeRecoveryConfirmation(ctx, confirmation, operator, token)
|
||||
if err != nil {
|
||||
return RecoveryDisposition{}, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
release, err := s.acquireOperation(ctx)
|
||||
if err != nil {
|
||||
return RecoveryDisposition{}, err
|
||||
}
|
||||
defer release()
|
||||
journal, ok := s.journal.(RecoveryDispositionJournal)
|
||||
if !ok || resolve == nil {
|
||||
return RecoveryDisposition{}, ErrRecoveryDispositionUnavailable
|
||||
}
|
||||
if !s.now().Before(review.summary.ExpiresAt) {
|
||||
return RecoveryDisposition{}, ErrRecoveryConfirmation
|
||||
}
|
||||
item, created, err := journal.BeginRecoveryDisposition(ctx, intent)
|
||||
if err != nil {
|
||||
return RecoveryDisposition{}, ErrRecoveryDispositionUnavailable
|
||||
}
|
||||
if !created {
|
||||
// 响应丢失后的既有请求只返回持久结果,绝不重新连接或执行。
|
||||
return item, nil
|
||||
}
|
||||
finish := func(result string, actionErr error) (RecoveryDisposition, error) {
|
||||
// 请求取消不能跳过结束审计;保存失败保留原占用且绝不返回成功。
|
||||
finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer finishCancel()
|
||||
finished, finishErr := journal.FinishRecoveryDisposition(finishCtx, intent.ID, result)
|
||||
if finishErr != nil {
|
||||
return item, ErrRecoveryDispositionUnavailable
|
||||
}
|
||||
return finished, actionErr
|
||||
}
|
||||
if err := s.RecheckRecoveryReview(ctx, review, operator, resolve); err != nil {
|
||||
return finish(RecoveryResultRejected, ErrRecoveryActionRejected)
|
||||
}
|
||||
if review.summary.Action == RecoveryActionRecord {
|
||||
return finish(RecoveryResultObserved, nil)
|
||||
}
|
||||
result, actionErr := s.executeRecoveryAction(ctx, review, resolve)
|
||||
return finish(result, actionErr)
|
||||
}
|
||||
|
||||
func (s *Service) executeRecoveryAction(ctx context.Context, review RecoveryReview, resolve func(context.Context, string) (Server, error)) (result string, actionErr error) {
|
||||
result, actionErr = RecoveryResultRejected, ErrRecoveryActionRejected
|
||||
record, recordDigest, serverDigest, err := s.reviewRecord(ctx, review.summary.TransactionID, resolve)
|
||||
if err != nil || recordDigest != review.recordDigest || serverDigest != review.serverDigest {
|
||||
return
|
||||
}
|
||||
server, err := resolve(ctx, record.ServerID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
server = server.Normalize()
|
||||
currentServerDigest, err := reviewServerDigest(server)
|
||||
if err != nil || currentServerDigest != review.serverDigest || validateRecoveryAction(record, review.summary.Action, s.VerifyRecoveryTarget != nil) != nil {
|
||||
return
|
||||
}
|
||||
client, err := s.factory.Connect(ctx, server)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
mutated := false
|
||||
defer func() {
|
||||
// 在结果审计和释放共同操作门之前关闭连接,关闭失败不能伪装成成功。
|
||||
if closeErr := client.Close(); closeErr != nil {
|
||||
if mutated {
|
||||
result, actionErr = RecoveryResultUncertain, ErrRecoveryActionUncertain
|
||||
} else {
|
||||
result, actionErr = RecoveryResultRejected, ErrRecoveryActionRejected
|
||||
}
|
||||
}
|
||||
}()
|
||||
if err := ctx.Err(); err != nil || !s.now().Before(review.summary.ExpiresAt) {
|
||||
return
|
||||
}
|
||||
switch review.summary.Action {
|
||||
case RecoveryActionCommit:
|
||||
err = s.executeRecoveryCommit(ctx, &record, server, client, &mutated)
|
||||
result = RecoveryResultCommitted
|
||||
case RecoveryActionRollback:
|
||||
err = s.executeRecoveryRollback(ctx, &record, server, client, &mutated)
|
||||
result = RecoveryResultRolledBack
|
||||
default:
|
||||
return
|
||||
}
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
if mutated {
|
||||
return RecoveryResultUncertain, ErrRecoveryActionUncertain
|
||||
}
|
||||
return RecoveryResultRejected, ErrRecoveryActionRejected
|
||||
}
|
||||
|
||||
func validateRecoveryAction(record TransactionRecord, action RecoveryAction, verifier bool) error {
|
||||
if record.Evidence == nil || !verifier || len(record.Backups) != len(record.Paths) {
|
||||
return ErrRecoveryActionRejected
|
||||
}
|
||||
switch action {
|
||||
case RecoveryActionCommit:
|
||||
if record.Evidence.Baseline.ExpectedFingerprint == "" {
|
||||
return ErrRecoveryActionRejected
|
||||
}
|
||||
switch record.Phase {
|
||||
case TransactionPhaseReloadAttempted, TransactionPhaseCommitting, TransactionPhaseCommitVerified:
|
||||
return nil
|
||||
}
|
||||
case RecoveryActionRollback:
|
||||
if record.Evidence.Baseline.PreviousFingerprint == "" {
|
||||
return ErrRecoveryActionRejected
|
||||
}
|
||||
switch record.Phase {
|
||||
case TransactionPhaseLocked, TransactionPhaseUploaded, TransactionPhaseReplacing, TransactionPhaseInstalled,
|
||||
TransactionPhaseReloadAttempted, TransactionPhaseCommitting, TransactionPhaseRollbackVerified:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return ErrRecoveryActionRejected
|
||||
}
|
||||
@@ -0,0 +1,428 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"maps"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type executorJournal struct {
|
||||
memoryTransactionJournal
|
||||
item RecoveryDisposition
|
||||
claimed bool
|
||||
beginCalls, finishCalls int
|
||||
beginErr, finishErr, updateErr error
|
||||
finishContextErr error
|
||||
existing bool
|
||||
beforeFinish func()
|
||||
}
|
||||
|
||||
func (j *executorJournal) BeginRecoveryDisposition(_ context.Context, intent RecoveryDispositionIntent) (RecoveryDisposition, bool, error) {
|
||||
j.beginCalls++
|
||||
if j.beginErr != nil {
|
||||
return RecoveryDisposition{}, false, j.beginErr
|
||||
}
|
||||
if j.existing {
|
||||
return j.item, false, nil
|
||||
}
|
||||
if j.claimed {
|
||||
return RecoveryDisposition{}, false, ErrRecoveryDispositionBusy
|
||||
}
|
||||
if !intent.MatchesRecord(j.pending[0]) {
|
||||
return RecoveryDisposition{}, false, ErrRecoveryReviewChanged
|
||||
}
|
||||
j.item = RecoveryDisposition{Intent: intent, Status: "running"}
|
||||
j.claimed = true
|
||||
return j.item, true, nil
|
||||
}
|
||||
|
||||
func (j *executorJournal) FinishRecoveryDisposition(ctx context.Context, id, result string) (RecoveryDisposition, error) {
|
||||
j.finishCalls++
|
||||
j.finishContextErr = ctx.Err()
|
||||
if j.beforeFinish != nil {
|
||||
j.beforeFinish()
|
||||
}
|
||||
if j.finishErr != nil {
|
||||
return RecoveryDisposition{}, j.finishErr
|
||||
}
|
||||
if id != j.item.Intent.ID || !j.item.Intent.AllowsResult(result) || !j.claimed {
|
||||
return RecoveryDisposition{}, ErrRecoveryDispositionConflict
|
||||
}
|
||||
j.item.Status, j.item.ResultCode = "succeeded", result
|
||||
if result == RecoveryResultRejected || result == RecoveryResultUncertain {
|
||||
j.item.Status = result
|
||||
}
|
||||
j.claimed = result == RecoveryResultUncertain
|
||||
return j.item, nil
|
||||
}
|
||||
|
||||
func (j *executorJournal) GetRecoveryDisposition(context.Context, string) (RecoveryDisposition, error) {
|
||||
return j.item, nil
|
||||
}
|
||||
|
||||
func (j *executorJournal) HasRecoveryClaim(context.Context, string) (bool, error) {
|
||||
return j.claimed, nil
|
||||
}
|
||||
|
||||
func (j *executorJournal) UpdateDeploymentTransaction(ctx context.Context, record TransactionRecord) error {
|
||||
if j.updateErr != nil {
|
||||
return j.updateErr
|
||||
}
|
||||
return j.memoryTransactionJournal.UpdateDeploymentTransaction(ctx, record)
|
||||
}
|
||||
|
||||
type executorClient struct {
|
||||
*memoryClient
|
||||
closeErr error
|
||||
closed int
|
||||
beforeClose func()
|
||||
}
|
||||
|
||||
func (c *executorClient) Close() error {
|
||||
c.closed++
|
||||
if c.beforeClose != nil {
|
||||
c.beforeClose()
|
||||
}
|
||||
return c.closeErr
|
||||
}
|
||||
|
||||
func executorFixture(t *testing.T) (*Service, *executorJournal, *executorClient, Server, TransactionRecord) {
|
||||
t.Helper()
|
||||
server, memory, record := recoveryPreflightFixture()
|
||||
journal := &executorJournal{memoryTransactionJournal: memoryTransactionJournal{pending: []TransactionRecord{record}}}
|
||||
client := &executorClient{memoryClient: memory}
|
||||
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
|
||||
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { return nil }
|
||||
return service, journal, client, server, record
|
||||
}
|
||||
|
||||
func TestRecoveryExecutorThreeExplicitActions(t *testing.T) {
|
||||
for _, action := range []RecoveryAction{RecoveryActionRecord, RecoveryActionCommit, RecoveryActionRollback} {
|
||||
t.Run(string(action), func(t *testing.T) {
|
||||
service, journal, client, server, record := executorFixture(t)
|
||||
before := maps.Clone(client.files)
|
||||
resolve := func(context.Context, string) (Server, error) { return server, nil }
|
||||
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var verified []bool
|
||||
service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, commit bool) error {
|
||||
verified = append(verified, commit)
|
||||
return nil
|
||||
}
|
||||
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
|
||||
if err != nil || item.Status != "succeeded" || journal.claimed || journal.beginCalls != 1 || journal.finishCalls != 1 || len(journal.finished) != 0 {
|
||||
t.Fatalf("execution failed: %+v, %v", item, err)
|
||||
}
|
||||
if action == RecoveryActionRecord {
|
||||
if item.ResultCode != RecoveryResultObserved || len(verified) != 0 || len(journal.updated) != 0 || !reflect.DeepEqual(before, client.files) {
|
||||
t.Fatal("record-only action changed original state")
|
||||
}
|
||||
} else {
|
||||
expected, phase, result := "new", TransactionPhaseCommitVerified, RecoveryResultCommitted
|
||||
if action == RecoveryActionRollback {
|
||||
expected, phase, result = "old", TransactionPhaseRollbackVerified, RecoveryResultRolledBack
|
||||
}
|
||||
if !reflect.DeepEqual(verified, []bool{action == RecoveryActionCommit}) || item.ResultCode != result || len(journal.updated) != 1 || journal.updated[0].Phase != phase {
|
||||
t.Fatal("wrong action, verification or phase")
|
||||
}
|
||||
for _, path := range record.Paths {
|
||||
if string(client.files[path].data) != expected {
|
||||
t.Fatal("wrong certificate files")
|
||||
}
|
||||
}
|
||||
if len(client.files) != 3 {
|
||||
t.Fatal("transaction leftovers remain")
|
||||
}
|
||||
}
|
||||
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, ErrRecoveryConfirmation) || journal.beginCalls != 1 {
|
||||
t.Fatal("execution replayed")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryExecutorFailureBoundaries(t *testing.T) {
|
||||
for _, scenario := range []string{"changed", "foreign-lock", "legacy", "phase", "missing-baseline", "missing-backup-entry", "begin", "finish", "persist", "cleanup", "reload", "commit-https", "rollback-https", "cancel", "close", "existing"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
service, journal, client, server, record := executorFixture(t)
|
||||
action := RecoveryActionCommit
|
||||
if scenario == "reload" || scenario == "rollback-https" || scenario == "missing-backup-entry" {
|
||||
action = RecoveryActionRollback
|
||||
}
|
||||
if scenario == "legacy" {
|
||||
record.Evidence = nil
|
||||
action = RecoveryActionRecord
|
||||
}
|
||||
if scenario == "phase" {
|
||||
record.Phase = TransactionPhaseUploaded
|
||||
}
|
||||
if scenario == "missing-baseline" {
|
||||
record.Evidence.Baseline.ExpectedFingerprint = ""
|
||||
}
|
||||
if scenario == "missing-backup-entry" {
|
||||
record.Backups = record.Backups[:2]
|
||||
}
|
||||
journal.pending[0] = record
|
||||
resolve := func(context.Context, string) (Server, error) { return server, nil }
|
||||
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
verifyCalls := 0
|
||||
service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, commit bool) error {
|
||||
verifyCalls++
|
||||
if commit != (action == RecoveryActionCommit) {
|
||||
t.Fatal("failed action silently switched direction")
|
||||
}
|
||||
if scenario == "commit-https" || scenario == "rollback-https" {
|
||||
return errors.New("private upstream details")
|
||||
}
|
||||
if scenario == "cancel" {
|
||||
cancel()
|
||||
}
|
||||
if scenario == "close" {
|
||||
client.closeErr = errors.New("private close details")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
switch scenario {
|
||||
case "changed":
|
||||
client.files[record.Paths[0]] = memoryFile{data: []byte("foreign")}
|
||||
case "foreign-lock":
|
||||
client.files[record.LockPaths[0]+"/owner"] = memoryFile{data: []byte("foreign")}
|
||||
case "begin":
|
||||
journal.beginErr = errors.New("private database error")
|
||||
case "finish":
|
||||
journal.finishErr = errors.New("private audit error")
|
||||
case "persist":
|
||||
journal.updateErr = errors.New("private update error")
|
||||
case "cleanup":
|
||||
client.removeError = ".scert-backup-"
|
||||
case "reload":
|
||||
client.runErrors = []error{errors.New("private command error")}
|
||||
case "existing":
|
||||
journal.existing = true
|
||||
journal.item = RecoveryDisposition{Status: "uncertain", ResultCode: RecoveryResultUncertain}
|
||||
}
|
||||
before := maps.Clone(client.files)
|
||||
closedBefore := client.closed
|
||||
item, err := service.ExecuteRecoveryDisposition(ctx, confirmation, "unix:0", token, resolve)
|
||||
if scenario == "legacy" {
|
||||
if err != nil || item.ResultCode != RecoveryResultObserved || len(journal.updated) != 0 || !reflect.DeepEqual(before, client.files) {
|
||||
t.Fatal("legacy inspection mutated history")
|
||||
}
|
||||
return
|
||||
}
|
||||
if scenario == "existing" {
|
||||
if err != nil || item.Status != "uncertain" || verifyCalls != 0 || journal.finishCalls != 0 || client.closed != closedBefore || !reflect.DeepEqual(before, client.files) {
|
||||
t.Fatal("existing request executed again")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || item.Status == "succeeded" || journal.finishContextErr != nil {
|
||||
t.Fatalf("failure reported success or skipped independent audit: %+v %v", item, err)
|
||||
}
|
||||
uncertain := scenario == "finish" || scenario == "persist" || scenario == "cleanup" || scenario == "reload" || scenario == "rollback-https" || scenario == "close"
|
||||
if journal.claimed != uncertain {
|
||||
t.Fatalf("wrong claim retention: %v", journal.claimed)
|
||||
}
|
||||
if !uncertain && !reflect.DeepEqual(before, client.files) {
|
||||
t.Fatal("preflight rejection changed remote files")
|
||||
}
|
||||
if len(journal.finished) != 0 {
|
||||
t.Fatal("executor bypassed atomic disposition completion")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryExecutorWaitCancellationConsumesConfirmation(t *testing.T) {
|
||||
service, journal, _, server, record := executorFixture(t)
|
||||
resolve := func(context.Context, string) (Server, error) { return server, nil }
|
||||
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", RecoveryActionRecord, resolve)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release, err := service.acquireOperation(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer release()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
if _, err := service.ExecuteRecoveryDisposition(ctx, confirmation, "unix:0", token, resolve); !errors.Is(err, context.DeadlineExceeded) || journal.beginCalls != 0 {
|
||||
t.Fatal("cancelled waiter acquired persistent claim")
|
||||
}
|
||||
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, ErrRecoveryConfirmation) {
|
||||
t.Fatal("cancelled confirmation replayed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryExecutorHoldsOperationThroughVerificationCloseAndAudit(t *testing.T) {
|
||||
service, journal, client, server, record := executorFixture(t)
|
||||
resolve := func(context.Context, string) (Server, error) { return server, nil }
|
||||
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", RecoveryActionCommit, resolve)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checked := map[string]int{}
|
||||
checkHeld := func(stage string) {
|
||||
checked[stage]++
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Millisecond)
|
||||
defer cancel()
|
||||
release, err := service.acquireOperation(ctx)
|
||||
if err == nil {
|
||||
release()
|
||||
t.Fatalf("operation gate released during %s", stage)
|
||||
}
|
||||
if !errors.Is(err, context.DeadlineExceeded) || !journal.claimed {
|
||||
t.Fatalf("missing durable claim or invalid wait during %s", stage)
|
||||
}
|
||||
}
|
||||
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error {
|
||||
checkHeld("verify")
|
||||
return nil
|
||||
}
|
||||
client.beforeClose = func() { checkHeld("close") }
|
||||
journal.beforeFinish = func() { checkHeld("audit") }
|
||||
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if checked["verify"] != 1 || checked["close"] != 2 || checked["audit"] != 1 {
|
||||
t.Fatalf("missing gate coverage: %v", checked)
|
||||
}
|
||||
release, err := service.acquireOperation(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release()
|
||||
}
|
||||
|
||||
func TestRecoveryExecutorVerifiedHistoryStillRechecksHTTPS(t *testing.T) {
|
||||
for _, action := range []RecoveryAction{RecoveryActionCommit, RecoveryActionRollback} {
|
||||
for _, failVerification := range []bool{false, true} {
|
||||
t.Run(string(action)+map[bool]string{false: "/success", true: "/failure"}[failVerification], func(t *testing.T) {
|
||||
service, journal, client, server, record := executorFixture(t)
|
||||
record.Phase = TransactionPhaseCommitVerified
|
||||
if action == RecoveryActionRollback {
|
||||
record.Phase = TransactionPhaseRollbackVerified
|
||||
for _, path := range record.Paths {
|
||||
client.files[path] = memoryFile{data: []byte("old")}
|
||||
}
|
||||
}
|
||||
for _, path := range record.TemporaryPaths {
|
||||
delete(client.files, path)
|
||||
}
|
||||
for _, backup := range record.Backups {
|
||||
delete(client.files, backup.Saved)
|
||||
}
|
||||
delete(client.files, record.LockPaths[0])
|
||||
delete(client.files, record.LockPaths[0]+"/owner")
|
||||
journal.pending[0] = record
|
||||
resolve := func(context.Context, string) (Server, error) { return server, nil }
|
||||
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
verified := 0
|
||||
service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, commit bool) error {
|
||||
verified++
|
||||
if commit != (action == RecoveryActionCommit) {
|
||||
t.Fatal("wrong HTTPS verification direction")
|
||||
}
|
||||
if failVerification {
|
||||
return errors.New("HTTPS mismatch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
|
||||
if verified != 1 || (err != nil) != failVerification || (item.Status == "succeeded") == failVerification {
|
||||
t.Fatalf("historical phase bypassed new verification: %+v %v", item, err)
|
||||
}
|
||||
if !failVerification && len(client.files) != 3 {
|
||||
t.Fatal("partial unlock history failed to finish safely")
|
||||
}
|
||||
if failVerification {
|
||||
if _, exists := client.files[record.LockPaths[1]]; !exists {
|
||||
t.Fatal("failed verification removed retained lock")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryExecutorRechecksAfterHTTPSAndAuditsCancellation(t *testing.T) {
|
||||
for _, scenario := range []string{"target", "backup", "lock", "cancel-after-write"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
service, journal, client, server, record := executorFixture(t)
|
||||
action := RecoveryActionCommit
|
||||
if scenario == "cancel-after-write" {
|
||||
action = RecoveryActionRollback
|
||||
}
|
||||
resolve := func(context.Context, string) (Server, error) { return server, nil }
|
||||
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error {
|
||||
path := record.Paths[0]
|
||||
switch scenario {
|
||||
case "backup":
|
||||
path = record.Backups[0].Saved
|
||||
case "lock":
|
||||
path = record.LockPaths[0] + "/owner"
|
||||
case "cancel-after-write":
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
}
|
||||
client.files[path] = memoryFile{data: []byte("foreign-change")}
|
||||
return nil
|
||||
}
|
||||
item, err := service.ExecuteRecoveryDisposition(ctx, confirmation, "unix:0", token, resolve)
|
||||
if err == nil || item.Status == "succeeded" || journal.finishCalls != 1 || journal.finishContextErr != nil {
|
||||
t.Fatal("changed or cancelled execution skipped audit or reported success")
|
||||
}
|
||||
if journal.claimed != (scenario == "cancel-after-write") {
|
||||
t.Fatal("wrong claim retention after cancelled write")
|
||||
}
|
||||
if _, exists := client.files[record.LockPaths[0]]; !exists {
|
||||
t.Fatal("changed state unlocked")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryExecutorRejectsUnlistedTemporaryFiles(t *testing.T) {
|
||||
for _, action := range []RecoveryAction{RecoveryActionCommit, RecoveryActionRollback} {
|
||||
t.Run(string(action), func(t *testing.T) {
|
||||
service, journal, client, server, record := executorFixture(t)
|
||||
delete(record.Evidence.TemporaryFiles, record.TemporaryPaths[0])
|
||||
record.TemporaryPaths = record.TemporaryPaths[1:]
|
||||
journal.pending[0] = record
|
||||
resolve := func(context.Context, string) (Server, error) { return server, nil }
|
||||
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := maps.Clone(client.files)
|
||||
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error {
|
||||
t.Fatal("unknown temporary file reached HTTPS verification")
|
||||
return nil
|
||||
}
|
||||
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
|
||||
if !errors.Is(err, ErrRecoveryActionRejected) || item.ResultCode != RecoveryResultRejected || journal.claimed || !reflect.DeepEqual(before, client.files) {
|
||||
t.Fatal("unlisted temporary file ignored or modified")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RecoveryDispositionSummary 是本机查询的明确输出边界,不包含请求键、绑定摘要或操作者凭据。
|
||||
type RecoveryDispositionSummary struct {
|
||||
ID string `json:"id"`
|
||||
TransactionID string `json:"transaction_id"`
|
||||
ServerID string `json:"server_id"`
|
||||
Action RecoveryAction `json:"action"`
|
||||
Status string `json:"status"`
|
||||
ResultCode string `json:"result_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
func (r RecoveryDisposition) Summary() RecoveryDispositionSummary {
|
||||
return RecoveryDispositionSummary{
|
||||
ID: r.Intent.ID, TransactionID: r.Intent.TransactionID, ServerID: r.Intent.ServerID,
|
||||
Action: r.Intent.Action, Status: r.Status, ResultCode: r.ResultCode,
|
||||
CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func validDispositionID(id string) bool {
|
||||
return validDigest(id) && id == strings.ToLower(id) && id != strings.Repeat("0", 64)
|
||||
}
|
||||
|
||||
func (r RecoveryDispositionSummary) Validate() error {
|
||||
if !validDispositionID(r.ID) || !validInspectionID(r.TransactionID) || !validRecoverySummary(r.ServerID) ||
|
||||
!validRecoveryAction(r.Action) || r.CreatedAt.IsZero() || r.UpdatedAt.IsZero() || r.UpdatedAt.Before(r.CreatedAt) {
|
||||
return ErrRecoveryDispositionInvalid
|
||||
}
|
||||
intent := RecoveryDispositionIntent{Action: r.Action}
|
||||
switch r.Status {
|
||||
case "running":
|
||||
if r.ResultCode == "" {
|
||||
return nil
|
||||
}
|
||||
case "succeeded":
|
||||
if r.ResultCode != RecoveryResultRejected && r.ResultCode != RecoveryResultUncertain && intent.AllowsResult(r.ResultCode) {
|
||||
return nil
|
||||
}
|
||||
case RecoveryResultRejected, RecoveryResultUncertain:
|
||||
if r.ResultCode == r.Status {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return ErrRecoveryDispositionInvalid
|
||||
}
|
||||
|
||||
// GetRecoveryDisposition 不取得写操作门,执行中或重启后均可查询;查询本身不会重放动作或释放占用。
|
||||
func (s *Service) GetRecoveryDisposition(ctx context.Context, id string) (RecoveryDispositionSummary, error) {
|
||||
if !validDispositionID(id) {
|
||||
return RecoveryDispositionSummary{}, ErrRecoveryDispositionInvalid
|
||||
}
|
||||
if s == nil {
|
||||
return RecoveryDispositionSummary{}, ErrRecoveryDispositionUnavailable
|
||||
}
|
||||
journal, ok := s.journal.(RecoveryDispositionJournal)
|
||||
if !ok {
|
||||
return RecoveryDispositionSummary{}, ErrRecoveryDispositionUnavailable
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
item, err := journal.GetRecoveryDisposition(ctx, id)
|
||||
if err != nil || ctx.Err() != nil || item.Intent.ID != id || item.Intent.Validate() != nil {
|
||||
return RecoveryDispositionSummary{}, ErrRecoveryDispositionUnavailable
|
||||
}
|
||||
summary := item.Summary()
|
||||
if summary.Validate() != nil {
|
||||
return RecoveryDispositionSummary{}, ErrRecoveryDispositionUnavailable
|
||||
}
|
||||
return summary, nil
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRecoveryDispositionSummaryValidation(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
for _, action := range []RecoveryAction{RecoveryActionRecord, RecoveryActionCommit, RecoveryActionRollback} {
|
||||
for _, result := range []string{"", RecoveryResultObserved, RecoveryResultCommitted, RecoveryResultRolledBack, RecoveryResultRejected, RecoveryResultUncertain} {
|
||||
status := "succeeded"
|
||||
if result == "" {
|
||||
status = "running"
|
||||
}
|
||||
if result == RecoveryResultRejected || result == RecoveryResultUncertain {
|
||||
status = result
|
||||
}
|
||||
summary := RecoveryDispositionSummary{ID: strings.Repeat("a", 64), TransactionID: "transaction", ServerID: "server", Action: action, Status: status, ResultCode: result, CreatedAt: now, UpdatedAt: now}
|
||||
want := result == "" || (RecoveryDispositionIntent{Action: action}).AllowsResult(result)
|
||||
if (summary.Validate() == nil) != want {
|
||||
t.Fatalf("wrong result validation: %+v", summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryDispositionQueryIsReadOnlyAndPrivate(t *testing.T) {
|
||||
service, journal, _, _, _ := executorFixture(t)
|
||||
now := time.Now().UTC()
|
||||
journal.item = RecoveryDisposition{
|
||||
Intent: RecoveryDispositionIntent{
|
||||
ID: strings.Repeat("a", 64), RequestKey: strings.Repeat("b", 64), RecordDigest: strings.Repeat("c", 64),
|
||||
ServerDigest: strings.Repeat("d", 64), ObservationDigest: strings.Repeat("e", 64),
|
||||
TransactionID: "transaction", ServerID: "server", Operator: "unix:private", Action: RecoveryActionCommit,
|
||||
}, Status: "running", CreatedAt: now, UpdatedAt: now,
|
||||
}
|
||||
journal.claimed = true
|
||||
release, err := service.acquireOperation(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer release()
|
||||
result, err := service.GetRecoveryDisposition(t.Context(), journal.item.Intent.ID)
|
||||
if err != nil || result.Status != "running" || !journal.claimed || journal.beginCalls != 0 || journal.finishCalls != 0 {
|
||||
t.Fatal("query changed state or blocked behind writer", err)
|
||||
}
|
||||
encoded, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, private := range []string{journal.item.Intent.RequestKey, journal.item.Intent.RecordDigest, journal.item.Intent.ServerDigest, journal.item.Intent.ObservationDigest, journal.item.Intent.Operator} {
|
||||
if strings.Contains(string(encoded), private) {
|
||||
t.Fatal("private binding leaked in result")
|
||||
}
|
||||
}
|
||||
if _, err := service.GetRecoveryDisposition(t.Context(), strings.Repeat("f", 64)); err == nil {
|
||||
t.Fatal("wrong result identifier accepted")
|
||||
}
|
||||
journal.item.Status = "succeeded"
|
||||
journal.item.ResultCode = RecoveryResultRolledBack
|
||||
if _, err := service.GetRecoveryDisposition(t.Context(), journal.item.Intent.ID); err == nil {
|
||||
t.Fatal("inconsistent action result accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"maps"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||
)
|
||||
|
||||
// executorIntegrationClient 仅模拟远端文件;处置占用、审计和终态使用真实 SQLite。
|
||||
// 未列出的远端方法故意保持不可用,避免测试意外放行额外写操作。
|
||||
type executorIntegrationClient struct {
|
||||
deployment.Client
|
||||
files map[string]string
|
||||
}
|
||||
|
||||
var _ deployment.EvidenceClient = (*executorIntegrationClient)(nil)
|
||||
|
||||
func (c *executorIntegrationClient) CreateOwnedLock(context.Context, string, string) error {
|
||||
return errors.New("人工处置不能创建新的部署锁")
|
||||
}
|
||||
|
||||
func (c *executorIntegrationClient) Connect(context.Context, deployment.Server) (deployment.Client, error) {
|
||||
return c, nil
|
||||
}
|
||||
func (c *executorIntegrationClient) Close() error { return nil }
|
||||
func (c *executorIntegrationClient) Exists(_ context.Context, path string) (bool, error) {
|
||||
_, exists := c.files[path]
|
||||
return exists, nil
|
||||
}
|
||||
func (c *executorIntegrationClient) FileDigest(_ context.Context, path string) (string, error) {
|
||||
value, exists := c.files[path]
|
||||
if !exists {
|
||||
return "", errors.New("测试文件不存在")
|
||||
}
|
||||
return executorDigest(value), nil
|
||||
}
|
||||
func (c *executorIntegrationClient) ReadLockOwner(_ context.Context, path string) (string, error) {
|
||||
return c.files[path+"/owner"], nil
|
||||
}
|
||||
func (c *executorIntegrationClient) Remove(_ context.Context, path string) error {
|
||||
delete(c.files, path)
|
||||
return nil
|
||||
}
|
||||
func (c *executorIntegrationClient) Rename(_ context.Context, from, to string) error {
|
||||
value, exists := c.files[from]
|
||||
if !exists {
|
||||
return errors.New("测试源文件不存在")
|
||||
}
|
||||
c.files[to] = value
|
||||
delete(c.files, from)
|
||||
return nil
|
||||
}
|
||||
func (c *executorIntegrationClient) Unlock(_ context.Context, path string) error {
|
||||
delete(c.files, path)
|
||||
delete(c.files, path+"/owner")
|
||||
return nil
|
||||
}
|
||||
func (c *executorIntegrationClient) Run(context.Context, []string) (deployment.CommandResult, error) {
|
||||
return deployment.CommandResult{}, nil
|
||||
}
|
||||
|
||||
func executorDigest(value string) string {
|
||||
digest := sha256.Sum256([]byte(value))
|
||||
return hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
func executorIntegrationFixture(t *testing.T) (*SQLDeploymentTransactionRepository, *sql.DB, *deployment.Service, *executorIntegrationClient, deployment.Server, deployment.TransactionRecord) {
|
||||
t.Helper()
|
||||
_, db := newTaskRepositoryTest(t)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
repo := NewSQLDeploymentTransactionRepository(db)
|
||||
server := deployment.Server{
|
||||
ID: "executor-server", Name: "executor-server", Address: "192.0.2.10", Port: 22, Username: "deploy",
|
||||
AuthMethod: deployment.SSHAuthPassword, PasswordRef: "secret://ssh/password/test", HostKeyFingerprint: "SHA256:test",
|
||||
CertificatePath: "/cert.pem", CertificateChainPath: "/chain.pem", PrivateKeyPath: "/key.pem", WebServer: deployment.WebServerNginx,
|
||||
}.Normalize()
|
||||
record := deployment.TransactionRecord{
|
||||
ID: "executor-transaction", TaskID: "task", AttemptID: "attempt", AttemptNumber: 1, ServerID: server.ID, ServerName: server.Name,
|
||||
Paths: []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath},
|
||||
Status: deployment.TransactionStatusRecoveryRequired, Phase: deployment.TransactionPhaseReloadAttempted, ReloadAttempted: true,
|
||||
Evidence: &deployment.RecoveryEvidence{
|
||||
Version: 1, LockToken: executorDigest("owner"), Address: server.Address, Port: server.Port, HostKeyFingerprint: server.HostKeyFingerprint,
|
||||
ValidationCommand: server.ValidationCommand, ReloadCommand: server.ReloadCommand, Files: map[string]string{}, TemporaryFiles: map[string]string{},
|
||||
Baseline: deployment.RecoveryBaseline{Endpoint: "https://example.com", Domains: []string{"example.com"}, PreviousFingerprint: "old", ExpectedFingerprint: "new"},
|
||||
},
|
||||
}
|
||||
client := &executorIntegrationClient{files: map[string]string{}}
|
||||
for _, path := range record.Paths {
|
||||
lock, backup, temporary := path+".scert-lock", path+".scert-backup-"+record.ID, path+".scert-tmp-"+record.ID
|
||||
record.LockPaths = append(record.LockPaths, lock)
|
||||
record.TemporaryPaths = append(record.TemporaryPaths, temporary)
|
||||
record.InstalledPaths = append(record.InstalledPaths, path)
|
||||
record.Backups = append(record.Backups, deployment.TransactionBackup{Original: path, Saved: backup, OriginalExists: true, OriginalDigest: executorDigest("old")})
|
||||
record.Evidence.Files[path], record.Evidence.TemporaryFiles[temporary] = executorDigest("new"), executorDigest("temporary")
|
||||
client.files[path], client.files[backup], client.files[temporary] = "new", "old", "temporary"
|
||||
client.files[lock], client.files[lock+"/owner"] = "", record.Evidence.LockToken
|
||||
}
|
||||
if err := repo.CreateDeploymentTransaction(t.Context(), record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service := deployment.NewServiceWithJournal(client, repo)
|
||||
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error { return nil }
|
||||
return repo, db, service, client, server, record
|
||||
}
|
||||
|
||||
func TestRecoveryExecutorSQLiteAtomicLifecycle(t *testing.T) {
|
||||
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionRecord, deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
|
||||
t.Run(string(action), func(t *testing.T) {
|
||||
repo, db, service, client, server, record := executorIntegrationFixture(t)
|
||||
before, err := repo.ListPendingDeploymentTransactions(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
filesBefore := maps.Clone(client.files)
|
||||
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
|
||||
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
|
||||
if err != nil || item.Status != "succeeded" {
|
||||
t.Fatalf("execution failed: %+v %v", item, err)
|
||||
}
|
||||
recoveryCount(t, db, "recovery_dispositions", 1)
|
||||
recoveryCount(t, db, "recovery_events", 2)
|
||||
recoveryCount(t, db, "recovery_claims", 0)
|
||||
stored, err := repo.GetRecoveryDisposition(t.Context(), confirmation.DispositionID())
|
||||
if err != nil || !reflect.DeepEqual(stored, item) {
|
||||
t.Fatal("result query lost completed result", err)
|
||||
}
|
||||
view, err := service.GetRecoveryDisposition(t.Context(), confirmation.DispositionID())
|
||||
if err != nil || view.Validate() != nil || !reflect.DeepEqual(view, item.Summary()) {
|
||||
t.Fatal("safe query omitted or changed persisted result", err)
|
||||
}
|
||||
var status string
|
||||
if err := db.QueryRowContext(t.Context(), "SELECT status FROM deployment_transactions WHERE id = ?", record.ID).Scan(&status); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
switch action {
|
||||
case deployment.RecoveryActionRecord:
|
||||
after, err := repo.ListPendingDeploymentTransactions(t.Context())
|
||||
if err != nil || !reflect.DeepEqual(before, after) || !reflect.DeepEqual(filesBefore, client.files) {
|
||||
t.Fatal("record-only changed original state", err)
|
||||
}
|
||||
case deployment.RecoveryActionCommit:
|
||||
if status != deployment.TransactionStatusCommitted || client.files[record.Paths[0]] != "new" {
|
||||
t.Fatal("commit not atomic with audit")
|
||||
}
|
||||
case deployment.RecoveryActionRollback:
|
||||
if status != deployment.TransactionStatusRolledBack || client.files[record.Paths[0]] != "old" {
|
||||
t.Fatal("rollback not atomic with audit")
|
||||
}
|
||||
}
|
||||
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, deployment.ErrRecoveryConfirmation) {
|
||||
t.Fatal("completed confirmation replayed")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryExecutorSQLiteAuditFailuresAndRestart(t *testing.T) {
|
||||
for _, stage := range []string{"started", "finished"} {
|
||||
t.Run(stage, func(t *testing.T) {
|
||||
repo, db, service, client, server, record := executorIntegrationFixture(t)
|
||||
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
|
||||
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", deployment.RecoveryActionCommit, resolve)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
query := "CREATE TRIGGER executor_audit_failure BEFORE INSERT ON recovery_events WHEN NEW.event_type = '" + stage + "' BEGIN SELECT RAISE(ABORT, '测试审计失败'); END"
|
||||
if _, err := db.ExecContext(t.Context(), query); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
filesBefore := maps.Clone(client.files)
|
||||
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
|
||||
if !errors.Is(err, deployment.ErrRecoveryDispositionUnavailable) || item.Status == "succeeded" {
|
||||
t.Fatal("audit failure reported success")
|
||||
}
|
||||
if stage == "started" {
|
||||
recoveryCount(t, db, "recovery_dispositions", 0)
|
||||
recoveryCount(t, db, "recovery_claims", 0)
|
||||
recoveryCount(t, db, "recovery_events", 0)
|
||||
if !reflect.DeepEqual(filesBefore, client.files) {
|
||||
t.Fatal("begin audit failure modified remote")
|
||||
}
|
||||
return
|
||||
}
|
||||
recoveryCount(t, db, "recovery_claims", 1)
|
||||
recoveryCount(t, db, "recovery_events", 1)
|
||||
var sequence int
|
||||
var name, path string
|
||||
if err := db.QueryRowContext(t.Context(), "PRAGMA database_list").Scan(&sequence, &name, &path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reopened.Close()
|
||||
repo = NewSQLDeploymentTransactionRepository(reopened)
|
||||
stored, err := repo.GetRecoveryDisposition(t.Context(), confirmation.DispositionID())
|
||||
if err != nil || stored.Status != "running" {
|
||||
t.Fatal("uncertain result lost after reopening", err)
|
||||
}
|
||||
pending, err := repo.ListPendingDeploymentTransactions(t.Context())
|
||||
if err != nil || len(pending) != 1 || pending[0].Status != deployment.TransactionStatusRecoveryRequired || pending[0].Phase != deployment.TransactionPhaseCommitVerified {
|
||||
t.Fatal("failed finish altered original terminal state", err)
|
||||
}
|
||||
if strings.Contains(stored.Intent.RecordDigest, token) {
|
||||
t.Fatal("confirmation token persisted")
|
||||
}
|
||||
restarted := deployment.NewServiceWithJournal(nil, repo)
|
||||
view, err := restarted.GetRecoveryDisposition(t.Context(), confirmation.DispositionID())
|
||||
if err != nil || view.Status != "running" || view.ID != confirmation.DispositionID() {
|
||||
t.Fatal("query after process restart failed", err)
|
||||
}
|
||||
if err := restarted.RecoverPending(t.Context(), func(context.Context, string) (deployment.Server, error) {
|
||||
t.Error("restart re-executed occupied transaction")
|
||||
return deployment.Server{}, errors.New("unexpected resolver")
|
||||
}); !errors.Is(err, deployment.ErrRecoveryDispositionBusy) {
|
||||
t.Fatal("restart did not honor durable claim", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user