429 lines
16 KiB
Go
429 lines
16 KiB
Go
package deployment
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"maps"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
type executorJournal struct {
|
|
memoryTransactionJournal
|
|
item RecoveryDisposition
|
|
claimed bool
|
|
beginCalls, finishCalls int
|
|
beginErr, finishErr, updateErr error
|
|
finishContextErr error
|
|
existing bool
|
|
beforeFinish func()
|
|
}
|
|
|
|
func (j *executorJournal) BeginRecoveryDisposition(_ context.Context, intent RecoveryDispositionIntent) (RecoveryDisposition, bool, error) {
|
|
j.beginCalls++
|
|
if j.beginErr != nil {
|
|
return RecoveryDisposition{}, false, j.beginErr
|
|
}
|
|
if j.existing {
|
|
return j.item, false, nil
|
|
}
|
|
if j.claimed {
|
|
return RecoveryDisposition{}, false, ErrRecoveryDispositionBusy
|
|
}
|
|
if !intent.MatchesRecord(j.pending[0]) {
|
|
return RecoveryDisposition{}, false, ErrRecoveryReviewChanged
|
|
}
|
|
j.item = RecoveryDisposition{Intent: intent, Status: "running"}
|
|
j.claimed = true
|
|
return j.item, true, nil
|
|
}
|
|
|
|
func (j *executorJournal) FinishRecoveryDisposition(ctx context.Context, id, result string) (RecoveryDisposition, error) {
|
|
j.finishCalls++
|
|
j.finishContextErr = ctx.Err()
|
|
if j.beforeFinish != nil {
|
|
j.beforeFinish()
|
|
}
|
|
if j.finishErr != nil {
|
|
return RecoveryDisposition{}, j.finishErr
|
|
}
|
|
if id != j.item.Intent.ID || !j.item.Intent.AllowsResult(result) || !j.claimed {
|
|
return RecoveryDisposition{}, ErrRecoveryDispositionConflict
|
|
}
|
|
j.item.Status, j.item.ResultCode = "succeeded", result
|
|
if result == RecoveryResultRejected || result == RecoveryResultUncertain {
|
|
j.item.Status = result
|
|
}
|
|
j.claimed = result == RecoveryResultUncertain
|
|
return j.item, nil
|
|
}
|
|
|
|
func (j *executorJournal) GetRecoveryDisposition(context.Context, string) (RecoveryDisposition, error) {
|
|
return j.item, nil
|
|
}
|
|
|
|
func (j *executorJournal) HasRecoveryClaim(context.Context, string) (bool, error) {
|
|
return j.claimed, nil
|
|
}
|
|
|
|
func (j *executorJournal) UpdateDeploymentTransaction(ctx context.Context, record TransactionRecord) error {
|
|
if j.updateErr != nil {
|
|
return j.updateErr
|
|
}
|
|
return j.memoryTransactionJournal.UpdateDeploymentTransaction(ctx, record)
|
|
}
|
|
|
|
type executorClient struct {
|
|
*memoryClient
|
|
closeErr error
|
|
closed int
|
|
beforeClose func()
|
|
}
|
|
|
|
func (c *executorClient) Close() error {
|
|
c.closed++
|
|
if c.beforeClose != nil {
|
|
c.beforeClose()
|
|
}
|
|
return c.closeErr
|
|
}
|
|
|
|
func executorFixture(t *testing.T) (*Service, *executorJournal, *executorClient, Server, TransactionRecord) {
|
|
t.Helper()
|
|
server, memory, record := recoveryPreflightFixture()
|
|
journal := &executorJournal{memoryTransactionJournal: memoryTransactionJournal{pending: []TransactionRecord{record}}}
|
|
client := &executorClient{memoryClient: memory}
|
|
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
|
|
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { return nil }
|
|
return service, journal, client, server, record
|
|
}
|
|
|
|
func TestRecoveryExecutorThreeExplicitActions(t *testing.T) {
|
|
for _, action := range []RecoveryAction{RecoveryActionRecord, RecoveryActionCommit, RecoveryActionRollback} {
|
|
t.Run(string(action), func(t *testing.T) {
|
|
service, journal, client, server, record := executorFixture(t)
|
|
before := maps.Clone(client.files)
|
|
resolve := func(context.Context, string) (Server, error) { return server, nil }
|
|
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var verified []bool
|
|
service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, commit bool) error {
|
|
verified = append(verified, commit)
|
|
return nil
|
|
}
|
|
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
|
|
if err != nil || item.Status != "succeeded" || journal.claimed || journal.beginCalls != 1 || journal.finishCalls != 1 || len(journal.finished) != 0 {
|
|
t.Fatalf("execution failed: %+v, %v", item, err)
|
|
}
|
|
if action == RecoveryActionRecord {
|
|
if item.ResultCode != RecoveryResultObserved || len(verified) != 0 || len(journal.updated) != 0 || !reflect.DeepEqual(before, client.files) {
|
|
t.Fatal("record-only action changed original state")
|
|
}
|
|
} else {
|
|
expected, phase, result := "new", TransactionPhaseCommitVerified, RecoveryResultCommitted
|
|
if action == RecoveryActionRollback {
|
|
expected, phase, result = "old", TransactionPhaseRollbackVerified, RecoveryResultRolledBack
|
|
}
|
|
if !reflect.DeepEqual(verified, []bool{action == RecoveryActionCommit}) || item.ResultCode != result || len(journal.updated) != 1 || journal.updated[0].Phase != phase {
|
|
t.Fatal("wrong action, verification or phase")
|
|
}
|
|
for _, path := range record.Paths {
|
|
if string(client.files[path].data) != expected {
|
|
t.Fatal("wrong certificate files")
|
|
}
|
|
}
|
|
if len(client.files) != 3 {
|
|
t.Fatal("transaction leftovers remain")
|
|
}
|
|
}
|
|
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, ErrRecoveryConfirmation) || journal.beginCalls != 1 {
|
|
t.Fatal("execution replayed")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRecoveryExecutorFailureBoundaries(t *testing.T) {
|
|
for _, scenario := range []string{"changed", "foreign-lock", "legacy", "phase", "missing-baseline", "missing-backup-entry", "begin", "finish", "persist", "cleanup", "reload", "commit-https", "rollback-https", "cancel", "close", "existing"} {
|
|
t.Run(scenario, func(t *testing.T) {
|
|
service, journal, client, server, record := executorFixture(t)
|
|
action := RecoveryActionCommit
|
|
if scenario == "reload" || scenario == "rollback-https" || scenario == "missing-backup-entry" {
|
|
action = RecoveryActionRollback
|
|
}
|
|
if scenario == "legacy" {
|
|
record.Evidence = nil
|
|
action = RecoveryActionRecord
|
|
}
|
|
if scenario == "phase" {
|
|
record.Phase = TransactionPhaseUploaded
|
|
}
|
|
if scenario == "missing-baseline" {
|
|
record.Evidence.Baseline.ExpectedFingerprint = ""
|
|
}
|
|
if scenario == "missing-backup-entry" {
|
|
record.Backups = record.Backups[:2]
|
|
}
|
|
journal.pending[0] = record
|
|
resolve := func(context.Context, string) (Server, error) { return server, nil }
|
|
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
defer cancel()
|
|
verifyCalls := 0
|
|
service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, commit bool) error {
|
|
verifyCalls++
|
|
if commit != (action == RecoveryActionCommit) {
|
|
t.Fatal("failed action silently switched direction")
|
|
}
|
|
if scenario == "commit-https" || scenario == "rollback-https" {
|
|
return errors.New("private upstream details")
|
|
}
|
|
if scenario == "cancel" {
|
|
cancel()
|
|
}
|
|
if scenario == "close" {
|
|
client.closeErr = errors.New("private close details")
|
|
}
|
|
return nil
|
|
}
|
|
switch scenario {
|
|
case "changed":
|
|
client.files[record.Paths[0]] = memoryFile{data: []byte("foreign")}
|
|
case "foreign-lock":
|
|
client.files[record.LockPaths[0]+"/owner"] = memoryFile{data: []byte("foreign")}
|
|
case "begin":
|
|
journal.beginErr = errors.New("private database error")
|
|
case "finish":
|
|
journal.finishErr = errors.New("private audit error")
|
|
case "persist":
|
|
journal.updateErr = errors.New("private update error")
|
|
case "cleanup":
|
|
client.removeError = ".scert-backup-"
|
|
case "reload":
|
|
client.runErrors = []error{errors.New("private command error")}
|
|
case "existing":
|
|
journal.existing = true
|
|
journal.item = RecoveryDisposition{Status: "uncertain", ResultCode: RecoveryResultUncertain}
|
|
}
|
|
before := maps.Clone(client.files)
|
|
closedBefore := client.closed
|
|
item, err := service.ExecuteRecoveryDisposition(ctx, confirmation, "unix:0", token, resolve)
|
|
if scenario == "legacy" {
|
|
if err != nil || item.ResultCode != RecoveryResultObserved || len(journal.updated) != 0 || !reflect.DeepEqual(before, client.files) {
|
|
t.Fatal("legacy inspection mutated history")
|
|
}
|
|
return
|
|
}
|
|
if scenario == "existing" {
|
|
if err != nil || item.Status != "uncertain" || verifyCalls != 0 || journal.finishCalls != 0 || client.closed != closedBefore || !reflect.DeepEqual(before, client.files) {
|
|
t.Fatal("existing request executed again")
|
|
}
|
|
return
|
|
}
|
|
if err == nil || item.Status == "succeeded" || journal.finishContextErr != nil {
|
|
t.Fatalf("failure reported success or skipped independent audit: %+v %v", item, err)
|
|
}
|
|
uncertain := scenario == "finish" || scenario == "persist" || scenario == "cleanup" || scenario == "reload" || scenario == "rollback-https" || scenario == "close"
|
|
if journal.claimed != uncertain {
|
|
t.Fatalf("wrong claim retention: %v", journal.claimed)
|
|
}
|
|
if !uncertain && !reflect.DeepEqual(before, client.files) {
|
|
t.Fatal("preflight rejection changed remote files")
|
|
}
|
|
if len(journal.finished) != 0 {
|
|
t.Fatal("executor bypassed atomic disposition completion")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRecoveryExecutorWaitCancellationConsumesConfirmation(t *testing.T) {
|
|
service, journal, _, server, record := executorFixture(t)
|
|
resolve := func(context.Context, string) (Server, error) { return server, nil }
|
|
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", RecoveryActionRecord, resolve)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
release, err := service.acquireOperation(t.Context())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer release()
|
|
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
|
|
defer cancel()
|
|
if _, err := service.ExecuteRecoveryDisposition(ctx, confirmation, "unix:0", token, resolve); !errors.Is(err, context.DeadlineExceeded) || journal.beginCalls != 0 {
|
|
t.Fatal("cancelled waiter acquired persistent claim")
|
|
}
|
|
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, ErrRecoveryConfirmation) {
|
|
t.Fatal("cancelled confirmation replayed")
|
|
}
|
|
}
|
|
|
|
func TestRecoveryExecutorHoldsOperationThroughVerificationCloseAndAudit(t *testing.T) {
|
|
service, journal, client, server, record := executorFixture(t)
|
|
resolve := func(context.Context, string) (Server, error) { return server, nil }
|
|
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", RecoveryActionCommit, resolve)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
checked := map[string]int{}
|
|
checkHeld := func(stage string) {
|
|
checked[stage]++
|
|
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Millisecond)
|
|
defer cancel()
|
|
release, err := service.acquireOperation(ctx)
|
|
if err == nil {
|
|
release()
|
|
t.Fatalf("operation gate released during %s", stage)
|
|
}
|
|
if !errors.Is(err, context.DeadlineExceeded) || !journal.claimed {
|
|
t.Fatalf("missing durable claim or invalid wait during %s", stage)
|
|
}
|
|
}
|
|
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error {
|
|
checkHeld("verify")
|
|
return nil
|
|
}
|
|
client.beforeClose = func() { checkHeld("close") }
|
|
journal.beforeFinish = func() { checkHeld("audit") }
|
|
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if checked["verify"] != 1 || checked["close"] != 2 || checked["audit"] != 1 {
|
|
t.Fatalf("missing gate coverage: %v", checked)
|
|
}
|
|
release, err := service.acquireOperation(t.Context())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
release()
|
|
}
|
|
|
|
func TestRecoveryExecutorVerifiedHistoryStillRechecksHTTPS(t *testing.T) {
|
|
for _, action := range []RecoveryAction{RecoveryActionCommit, RecoveryActionRollback} {
|
|
for _, failVerification := range []bool{false, true} {
|
|
t.Run(string(action)+map[bool]string{false: "/success", true: "/failure"}[failVerification], func(t *testing.T) {
|
|
service, journal, client, server, record := executorFixture(t)
|
|
record.Phase = TransactionPhaseCommitVerified
|
|
if action == RecoveryActionRollback {
|
|
record.Phase = TransactionPhaseRollbackVerified
|
|
for _, path := range record.Paths {
|
|
client.files[path] = memoryFile{data: []byte("old")}
|
|
}
|
|
}
|
|
for _, path := range record.TemporaryPaths {
|
|
delete(client.files, path)
|
|
}
|
|
for _, backup := range record.Backups {
|
|
delete(client.files, backup.Saved)
|
|
}
|
|
delete(client.files, record.LockPaths[0])
|
|
delete(client.files, record.LockPaths[0]+"/owner")
|
|
journal.pending[0] = record
|
|
resolve := func(context.Context, string) (Server, error) { return server, nil }
|
|
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
verified := 0
|
|
service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, commit bool) error {
|
|
verified++
|
|
if commit != (action == RecoveryActionCommit) {
|
|
t.Fatal("wrong HTTPS verification direction")
|
|
}
|
|
if failVerification {
|
|
return errors.New("HTTPS mismatch")
|
|
}
|
|
return nil
|
|
}
|
|
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
|
|
if verified != 1 || (err != nil) != failVerification || (item.Status == "succeeded") == failVerification {
|
|
t.Fatalf("historical phase bypassed new verification: %+v %v", item, err)
|
|
}
|
|
if !failVerification && len(client.files) != 3 {
|
|
t.Fatal("partial unlock history failed to finish safely")
|
|
}
|
|
if failVerification {
|
|
if _, exists := client.files[record.LockPaths[1]]; !exists {
|
|
t.Fatal("failed verification removed retained lock")
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRecoveryExecutorRechecksAfterHTTPSAndAuditsCancellation(t *testing.T) {
|
|
for _, scenario := range []string{"target", "backup", "lock", "cancel-after-write"} {
|
|
t.Run(scenario, func(t *testing.T) {
|
|
service, journal, client, server, record := executorFixture(t)
|
|
action := RecoveryActionCommit
|
|
if scenario == "cancel-after-write" {
|
|
action = RecoveryActionRollback
|
|
}
|
|
resolve := func(context.Context, string) (Server, error) { return server, nil }
|
|
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
defer cancel()
|
|
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error {
|
|
path := record.Paths[0]
|
|
switch scenario {
|
|
case "backup":
|
|
path = record.Backups[0].Saved
|
|
case "lock":
|
|
path = record.LockPaths[0] + "/owner"
|
|
case "cancel-after-write":
|
|
cancel()
|
|
return ctx.Err()
|
|
}
|
|
client.files[path] = memoryFile{data: []byte("foreign-change")}
|
|
return nil
|
|
}
|
|
item, err := service.ExecuteRecoveryDisposition(ctx, confirmation, "unix:0", token, resolve)
|
|
if err == nil || item.Status == "succeeded" || journal.finishCalls != 1 || journal.finishContextErr != nil {
|
|
t.Fatal("changed or cancelled execution skipped audit or reported success")
|
|
}
|
|
if journal.claimed != (scenario == "cancel-after-write") {
|
|
t.Fatal("wrong claim retention after cancelled write")
|
|
}
|
|
if _, exists := client.files[record.LockPaths[0]]; !exists {
|
|
t.Fatal("changed state unlocked")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRecoveryExecutorRejectsUnlistedTemporaryFiles(t *testing.T) {
|
|
for _, action := range []RecoveryAction{RecoveryActionCommit, RecoveryActionRollback} {
|
|
t.Run(string(action), func(t *testing.T) {
|
|
service, journal, client, server, record := executorFixture(t)
|
|
delete(record.Evidence.TemporaryFiles, record.TemporaryPaths[0])
|
|
record.TemporaryPaths = record.TemporaryPaths[1:]
|
|
journal.pending[0] = record
|
|
resolve := func(context.Context, string) (Server, error) { return server, nil }
|
|
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := maps.Clone(client.files)
|
|
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error {
|
|
t.Fatal("unknown temporary file reached HTTPS verification")
|
|
return nil
|
|
}
|
|
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
|
|
if !errors.Is(err, ErrRecoveryActionRejected) || item.ResultCode != RecoveryResultRejected || journal.claimed || !reflect.DeepEqual(before, client.files) {
|
|
t.Fatal("unlisted temporary file ignored or modified")
|
|
}
|
|
})
|
|
}
|
|
}
|