Files
Scert/backend/internal/deployment/transaction_recovery_test.go
T

216 lines
10 KiB
Go

package deployment
import (
"context"
"testing"
)
type memoryTransactionJournal struct {
pending []TransactionRecord
finished []struct {
id string
status string
message string
}
updated []TransactionRecord
}
func (j *memoryTransactionJournal) CreateDeploymentTransaction(_ context.Context, item TransactionRecord) error {
j.pending = append(j.pending, item)
return nil
}
func (j *memoryTransactionJournal) UpdateDeploymentTransaction(_ context.Context, item TransactionRecord) error {
j.updated = append(j.updated, item)
return nil
}
func (j *memoryTransactionJournal) FinishDeploymentTransaction(_ context.Context, id, status, message string) error {
j.finished = append(j.finished, struct {
id string
status string
message string
}{id: id, status: status, message: message})
return nil
}
func (j *memoryTransactionJournal) ListPendingDeploymentTransactions(context.Context) ([]TransactionRecord, error) {
return append([]TransactionRecord(nil), j.pending...), nil
}
func TestRecoverPendingRollsBackOwnedDeployment(t *testing.T) {
server := testServer("recovery")
server = server.Normalize()
client := newMemoryClient()
id := "recovery-transaction"
paths := []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath}
locks := make([]string, 0, len(paths))
backups := make([]TransactionBackup, 0, len(paths))
temporary := make([]string, 0, len(paths))
files := make(map[string]string, len(paths))
for _, path := range paths {
lockPath := path + ".scert-lock"
backupPath := path + ".scert-backup-" + id
temporaryPath := path + ".scert-tmp-" + id
locks = append(locks, lockPath)
backups = append(backups, TransactionBackup{Original: path, Saved: backupPath, OriginalExists: true, OriginalDigest: contentDigest([]byte("old"))})
temporary = append(temporary, temporaryPath)
client.files[lockPath] = memoryFile{}
client.files[lockPath+"/owner"] = memoryFile{data: []byte(contentDigest([]byte("scert-lock:" + id))), mode: 0o600}
client.files[backupPath] = memoryFile{data: []byte("old"), mode: 0o600}
client.files[path] = memoryFile{data: []byte("new"), mode: 0o600}
client.files[temporaryPath] = memoryFile{data: []byte("partial"), mode: 0o600}
files[path] = contentDigest([]byte("new"))
}
journal := &memoryTransactionJournal{pending: []TransactionRecord{{
ID: id, TaskID: "task", AttemptID: "attempt", AttemptNumber: 1, ResultIndex: 0,
ServerID: server.ID, ServerName: server.Name, Paths: paths, TemporaryPaths: temporary,
InstalledPaths: paths, Backups: backups, LockPaths: locks, ReloadAttempted: true,
Phase: TransactionPhaseReloadAttempted, Status: TransactionStatusPending,
Evidence: &RecoveryEvidence{Version: 1, LockToken: contentDigest([]byte("scert-lock:" + id)), Address: server.Address, Port: server.Port, HostKeyFingerprint: server.HostKeyFingerprint, ValidationCommand: append([]string(nil), server.ValidationCommand...), ReloadCommand: append([]string(nil), server.ReloadCommand...), Files: files, Baseline: RecoveryBaseline{PreviousFingerprint: "old", Endpoint: "https://example.com", Domains: []string{"example.com"}}},
}}}
journal.pending[0].Evidence.TemporaryFiles = make(map[string]string)
for _, path := range temporary {
journal.pending[0].Evidence.TemporaryFiles[path] = contentDigest([]byte("partial"))
}
service := NewServiceWithJournal(memoryFactory{clients: map[string]*memoryClient{server.ID: client}}, journal)
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { return nil }
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err != nil {
t.Fatalf("事务恢复失败:%v", err)
}
for _, path := range paths {
file, ok := client.files[path]
if !ok || string(file.data) != "old" {
t.Fatalf("目标文件未恢复:%s,文件:%+v", path, client.files)
}
if _, ok := client.files[path+".scert-lock"]; ok {
t.Fatalf("恢复成功后不应残留锁:%s", path)
}
if _, ok := client.files[path+".scert-tmp-"+id]; ok {
t.Fatalf("恢复成功后不应残留临时文件:%s", path)
}
}
if len(client.commands) != 2 || len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRolledBack {
t.Fatalf("恢复收尾记录不正确:命令=%d,完成记录=%+v", len(client.commands), journal.finished)
}
}
func TestRecoverPendingKeepsUnprovableState(t *testing.T) {
server := testServer("unprovable")
client := newMemoryClient()
journal := &memoryTransactionJournal{pending: []TransactionRecord{{
ID: "unprovable-transaction", TaskID: "task", AttemptID: "attempt", AttemptNumber: 1,
ResultIndex: 0, ServerID: server.ID, ServerName: server.Name,
Paths: []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath},
LockPaths: []string{server.PrivateKeyPath + ".scert-lock"}, Status: TransactionStatusPending,
Phase: TransactionPhaseInstalled,
}}}
service := NewServiceWithJournal(memoryFactory{clients: map[string]*memoryClient{server.ID: client}}, journal)
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err == nil {
t.Fatal("无法证明锁归属时必须返回人工核查错误")
}
if len(journal.updated) != 1 || journal.updated[0].Status != TransactionStatusRecoveryRequired {
t.Fatalf("未记录人工核查状态:%+v", journal.updated)
}
}
func TestDeployLockConflictFinalizesTransaction(t *testing.T) {
server := testServer("lock-conflict").Normalize()
client := newMemoryClient()
blockedLock := server.CertificatePath + ".scert-lock"
client.files[blockedLock] = memoryFile{}
journal := &memoryTransactionJournal{}
service := NewServiceWithJournal(memoryFactory{clients: map[string]*memoryClient{server.ID: client}}, journal)
ctx := WithTransactionContext(t.Context(), TransactionContext{TaskID: "task", AttemptID: "attempt", AttemptNumber: 1})
report := service.Deploy(ctx, []Target{{Server: server}}, testArtifact(t))
if report.Status != "failed" || len(report.Results) != 1 || report.Results[0].FailureCode != FailureLock {
t.Fatalf("锁冲突应失败:%+v", report)
}
if len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRolledBack {
t.Fatalf("锁冲突后必须完成回滚事务:%+v", journal.finished)
}
if _, ok := client.files[blockedLock]; !ok {
t.Fatal("已有锁不能被清理")
}
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
lockPath := path + ".scert-lock"
if lockPath != blockedLock {
if _, ok := client.files[lockPath]; ok {
t.Fatalf("锁冲突失败后不应残留本次锁:%s", lockPath)
}
}
}
}
func TestDeployPartialLockConflictReleasesOwnedLocks(t *testing.T) {
server := testServer("partial-lock-conflict").Normalize()
client := newMemoryClient()
blockedLock := server.PrivateKeyPath + ".scert-lock"
client.files[blockedLock] = memoryFile{}
journal := &memoryTransactionJournal{}
service := NewServiceWithJournal(memoryFactory{clients: map[string]*memoryClient{server.ID: client}}, journal)
ctx := WithTransactionContext(t.Context(), TransactionContext{TaskID: "task", AttemptID: "attempt", AttemptNumber: 1})
report := service.Deploy(ctx, []Target{{Server: server}}, testArtifact(t))
if report.Status != "failed" || len(report.Results) != 1 || report.Results[0].FailureCode != FailureLock {
t.Fatalf("部分锁冲突应失败:%+v", report)
}
if len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRolledBack {
t.Fatalf("部分锁冲突后必须完成回滚事务:%+v", journal.finished)
}
if _, ok := client.files[blockedLock]; !ok {
t.Fatal("已有锁不能被清理")
}
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
lockPath := path + ".scert-lock"
if lockPath == blockedLock {
continue
}
if _, ok := client.files[lockPath]; ok {
t.Fatalf("部分锁冲突失败后不应残留本次锁:%s", lockPath)
}
if _, ok := client.files[lockPath+"/owner"]; ok {
t.Fatalf("部分锁冲突失败后不应残留锁归属:%s", lockPath)
}
}
}
func TestRecoverPendingAcquiringPreservesLocksWithoutOwner(t *testing.T) {
server := testServer("acquiring-missing-owner").Normalize()
client := newMemoryClient()
id := "acquiring-missing-owner-transaction"
paths := []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath}
locks := make([]string, 0, len(paths))
files := make(map[string]string, len(paths))
for _, path := range paths {
lockPath := path + ".scert-lock"
locks = append(locks, lockPath)
client.files[lockPath] = memoryFile{}
files[path] = contentDigest([]byte("old"))
}
journal := &memoryTransactionJournal{pending: []TransactionRecord{{
ID: id, TaskID: "task", AttemptID: "attempt", AttemptNumber: 1, ResultIndex: 0,
ServerID: server.ID, ServerName: server.Name, Paths: paths, LockPaths: locks,
Phase: TransactionPhaseAcquiring, Status: TransactionStatusRecoveryRequired,
Evidence: &RecoveryEvidence{
Version: 1, LockToken: contentDigest([]byte("scert-lock:" + id)),
Address: server.Address, Port: server.Port, HostKeyFingerprint: server.HostKeyFingerprint,
ValidationCommand: append([]string(nil), server.ValidationCommand...),
ReloadCommand: append([]string(nil), server.ReloadCommand...), Files: files,
},
}}}
service := NewServiceWithJournal(memoryFactory{clients: map[string]*memoryClient{server.ID: client}}, journal)
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err == nil {
t.Fatal("无归属锁不能证明属于本次中断事务,必须拒绝自动清理")
}
for _, lockPath := range locks {
if _, ok := client.files[lockPath]; !ok {
t.Fatalf("不得删除无归属锁目录:%s", lockPath)
}
}
if len(journal.finished) != 0 || len(journal.updated) != 1 || journal.updated[0].Status != TransactionStatusRecoveryRequired {
t.Fatal("无归属锁必须保留事务等待人工核查")
}
}