281 lines
13 KiB
Go
281 lines
13 KiB
Go
package deployment
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"testing"
|
||
)
|
||
|
||
type memoryTransactionJournal struct {
|
||
pending []TransactionRecord
|
||
finished []struct {
|
||
id string
|
||
status string
|
||
message string
|
||
}
|
||
updated []TransactionRecord
|
||
}
|
||
|
||
func (j *memoryTransactionJournal) CreateDeploymentTransaction(_ context.Context, item TransactionRecord) error {
|
||
j.pending = append(j.pending, item)
|
||
return nil
|
||
}
|
||
|
||
func (j *memoryTransactionJournal) UpdateDeploymentTransaction(_ context.Context, item TransactionRecord) error {
|
||
j.updated = append(j.updated, item)
|
||
return nil
|
||
}
|
||
|
||
func (j *memoryTransactionJournal) FinishDeploymentTransaction(_ context.Context, id, status, message string) error {
|
||
j.finished = append(j.finished, struct {
|
||
id string
|
||
status string
|
||
message string
|
||
}{id: id, status: status, message: message})
|
||
return nil
|
||
}
|
||
|
||
func (j *memoryTransactionJournal) ListPendingDeploymentTransactions(context.Context) ([]TransactionRecord, error) {
|
||
return append([]TransactionRecord(nil), j.pending...), nil
|
||
}
|
||
|
||
func TestRecoverPendingRollsBackOwnedDeployment(t *testing.T) {
|
||
server := testServer("recovery")
|
||
server = server.Normalize()
|
||
client := newMemoryClient()
|
||
id := "recovery-transaction"
|
||
paths := []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath}
|
||
locks := make([]string, 0, len(paths))
|
||
backups := make([]TransactionBackup, 0, len(paths))
|
||
temporary := make([]string, 0, len(paths))
|
||
files := make(map[string]string, len(paths))
|
||
for _, path := range paths {
|
||
lockPath := path + ".scert-lock"
|
||
backupPath := path + ".scert-backup-" + id
|
||
temporaryPath := path + ".scert-tmp-" + id
|
||
locks = append(locks, lockPath)
|
||
backups = append(backups, TransactionBackup{Original: path, Saved: backupPath, OriginalExists: true, OriginalDigest: contentDigest([]byte("old"))})
|
||
temporary = append(temporary, temporaryPath)
|
||
client.files[lockPath] = memoryFile{}
|
||
client.files[lockPath+"/owner"] = memoryFile{data: []byte(contentDigest([]byte("scert-lock:" + id))), mode: 0o600}
|
||
client.files[backupPath] = memoryFile{data: []byte("old"), mode: 0o600}
|
||
client.files[path] = memoryFile{data: []byte("new"), mode: 0o600}
|
||
client.files[temporaryPath] = memoryFile{data: []byte("partial"), mode: 0o600}
|
||
files[path] = contentDigest([]byte("new"))
|
||
}
|
||
journal := &memoryTransactionJournal{pending: []TransactionRecord{{
|
||
ID: id, TaskID: "task", AttemptID: "attempt", AttemptNumber: 1, ResultIndex: 0,
|
||
ServerID: server.ID, ServerName: server.Name, Paths: paths, TemporaryPaths: temporary,
|
||
InstalledPaths: paths, Backups: backups, LockPaths: locks, ReloadAttempted: true,
|
||
Phase: TransactionPhaseReloadAttempted, Status: TransactionStatusPending,
|
||
Evidence: &RecoveryEvidence{Version: 1, LockToken: contentDigest([]byte("scert-lock:" + id)), Address: server.Address, Port: server.Port, HostKeyFingerprint: server.HostKeyFingerprint, ValidationCommand: append([]string(nil), server.ValidationCommand...), ReloadCommand: append([]string(nil), server.ReloadCommand...), Files: files, Baseline: RecoveryBaseline{PreviousFingerprint: "old", Endpoint: "https://example.com", Domains: []string{"example.com"}}},
|
||
}}}
|
||
journal.pending[0].Evidence.TemporaryFiles = make(map[string]string)
|
||
for _, path := range temporary {
|
||
journal.pending[0].Evidence.TemporaryFiles[path] = contentDigest([]byte("partial"))
|
||
}
|
||
service := NewServiceWithJournal(memoryFactory{clients: map[string]*memoryClient{server.ID: client}}, journal)
|
||
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { return nil }
|
||
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err != nil {
|
||
t.Fatalf("事务恢复失败：%v", err)
|
||
}
|
||
for _, path := range paths {
|
||
file, ok := client.files[path]
|
||
if !ok || string(file.data) != "old" {
|
||
t.Fatalf("目标文件未恢复：%s，文件：%+v", path, client.files)
|
||
}
|
||
if _, ok := client.files[path+".scert-lock"]; ok {
|
||
t.Fatalf("恢复成功后不应残留锁：%s", path)
|
||
}
|
||
if _, ok := client.files[path+".scert-tmp-"+id]; ok {
|
||
t.Fatalf("恢复成功后不应残留临时文件：%s", path)
|
||
}
|
||
}
|
||
if len(client.commands) != 2 || len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRolledBack {
|
||
t.Fatalf("恢复收尾记录不正确：命令=%d，完成记录=%+v", len(client.commands), journal.finished)
|
||
}
|
||
}
|
||
|
||
func TestRecoverPendingKeepsUnprovableState(t *testing.T) {
|
||
server := testServer("unprovable")
|
||
client := newMemoryClient()
|
||
journal := &memoryTransactionJournal{pending: []TransactionRecord{{
|
||
ID: "unprovable-transaction", TaskID: "task", AttemptID: "attempt", AttemptNumber: 1,
|
||
ResultIndex: 0, ServerID: server.ID, ServerName: server.Name,
|
||
Paths: []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath},
|
||
LockPaths: []string{server.PrivateKeyPath + ".scert-lock"}, Status: TransactionStatusPending,
|
||
Phase: TransactionPhaseInstalled,
|
||
}}}
|
||
service := NewServiceWithJournal(memoryFactory{clients: map[string]*memoryClient{server.ID: client}}, journal)
|
||
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err == nil {
|
||
t.Fatal("无法证明锁归属时必须返回人工核查错误")
|
||
}
|
||
if len(journal.updated) != 1 || journal.updated[0].Status != TransactionStatusRecoveryRequired {
|
||
t.Fatalf("未记录人工核查状态：%+v", journal.updated)
|
||
}
|
||
}
|
||
|
||
func TestDeployLockConflictFinalizesTransaction(t *testing.T) {
|
||
server := testServer("lock-conflict").Normalize()
|
||
client := newMemoryClient()
|
||
blockedLock := server.CertificatePath + ".scert-lock"
|
||
client.files[blockedLock] = memoryFile{}
|
||
journal := &memoryTransactionJournal{}
|
||
service := NewServiceWithJournal(memoryFactory{clients: map[string]*memoryClient{server.ID: client}}, journal)
|
||
ctx := WithTransactionContext(t.Context(), TransactionContext{TaskID: "task", AttemptID: "attempt", AttemptNumber: 1})
|
||
|
||
report := service.Deploy(ctx, []Target{{Server: server}}, testArtifact(t))
|
||
if report.Status != "failed" || len(report.Results) != 1 || report.Results[0].FailureCode != FailureLock {
|
||
t.Fatalf("锁冲突应失败：%+v", report)
|
||
}
|
||
if len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRolledBack {
|
||
t.Fatalf("锁冲突后必须完成回滚事务：%+v", journal.finished)
|
||
}
|
||
if _, ok := client.files[blockedLock]; !ok {
|
||
t.Fatal("已有锁不能被清理")
|
||
}
|
||
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
|
||
lockPath := path + ".scert-lock"
|
||
if lockPath != blockedLock {
|
||
if _, ok := client.files[lockPath]; ok {
|
||
t.Fatalf("锁冲突失败后不应残留本次锁：%s", lockPath)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestDeployPartialLockConflictReleasesOwnedLocks(t *testing.T) {
|
||
server := testServer("partial-lock-conflict").Normalize()
|
||
client := newMemoryClient()
|
||
blockedLock := server.PrivateKeyPath + ".scert-lock"
|
||
client.files[blockedLock] = memoryFile{}
|
||
journal := &memoryTransactionJournal{}
|
||
service := NewServiceWithJournal(memoryFactory{clients: map[string]*memoryClient{server.ID: client}}, journal)
|
||
ctx := WithTransactionContext(t.Context(), TransactionContext{TaskID: "task", AttemptID: "attempt", AttemptNumber: 1})
|
||
|
||
report := service.Deploy(ctx, []Target{{Server: server}}, testArtifact(t))
|
||
if report.Status != "failed" || len(report.Results) != 1 || report.Results[0].FailureCode != FailureLock {
|
||
t.Fatalf("部分锁冲突应失败：%+v", report)
|
||
}
|
||
if len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRolledBack {
|
||
t.Fatalf("部分锁冲突后必须完成回滚事务：%+v", journal.finished)
|
||
}
|
||
if _, ok := client.files[blockedLock]; !ok {
|
||
t.Fatal("已有锁不能被清理")
|
||
}
|
||
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
|
||
lockPath := path + ".scert-lock"
|
||
if lockPath == blockedLock {
|
||
continue
|
||
}
|
||
if _, ok := client.files[lockPath]; ok {
|
||
t.Fatalf("部分锁冲突失败后不应残留本次锁：%s", lockPath)
|
||
}
|
||
if _, ok := client.files[lockPath+"/owner"]; ok {
|
||
t.Fatalf("部分锁冲突失败后不应残留锁归属：%s", lockPath)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestRecoverPendingAcquiringPreservesLocksWithoutOwner(t *testing.T) {
|
||
server := testServer("acquiring-missing-owner").Normalize()
|
||
client := newMemoryClient()
|
||
id := "acquiring-missing-owner-transaction"
|
||
paths := []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath}
|
||
locks := make([]string, 0, len(paths))
|
||
files := make(map[string]string, len(paths))
|
||
for _, path := range paths {
|
||
lockPath := path + ".scert-lock"
|
||
locks = append(locks, lockPath)
|
||
client.files[lockPath] = memoryFile{}
|
||
files[path] = contentDigest([]byte("old"))
|
||
}
|
||
journal := &memoryTransactionJournal{pending: []TransactionRecord{{
|
||
ID: id, TaskID: "task", AttemptID: "attempt", AttemptNumber: 1, ResultIndex: 0,
|
||
ServerID: server.ID, ServerName: server.Name, Paths: paths, LockPaths: locks,
|
||
Phase: TransactionPhaseAcquiring, Status: TransactionStatusRecoveryRequired,
|
||
Evidence: &RecoveryEvidence{
|
||
Version: 1, LockToken: contentDigest([]byte("scert-lock:" + id)),
|
||
Address: server.Address, Port: server.Port, HostKeyFingerprint: server.HostKeyFingerprint,
|
||
ValidationCommand: append([]string(nil), server.ValidationCommand...),
|
||
ReloadCommand: append([]string(nil), server.ReloadCommand...), Files: files,
|
||
},
|
||
}}}
|
||
service := NewServiceWithJournal(memoryFactory{clients: map[string]*memoryClient{server.ID: client}}, journal)
|
||
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err == nil {
|
||
t.Fatal("无归属锁不能证明属于本次中断事务，必须拒绝自动清理")
|
||
}
|
||
for _, lockPath := range locks {
|
||
if _, ok := client.files[lockPath]; !ok {
|
||
t.Fatalf("不得删除无归属锁目录：%s", lockPath)
|
||
}
|
||
}
|
||
if len(journal.finished) != 0 || len(journal.updated) != 1 || journal.updated[0].Status != TransactionStatusRecoveryRequired {
|
||
t.Fatal("无归属锁必须保留事务等待人工核查")
|
||
}
|
||
}
|
||
func TestRecoverPendingCommitsVerifiedFirstDeployment(t *testing.T) {
|
||
server, client, record := recoveryPreflightFixture()
|
||
for index := range record.Backups {
|
||
record.Backups[index].OriginalExists = false
|
||
record.Backups[index].OriginalDigest = ""
|
||
delete(client.files, record.Backups[index].Saved)
|
||
}
|
||
journal := &memoryTransactionJournal{pending: []TransactionRecord{record}}
|
||
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
|
||
service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, committed bool) error {
|
||
if !committed {
|
||
t.Fatal("首次部署现场应按新证书提交核验")
|
||
}
|
||
return nil
|
||
}
|
||
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err != nil {
|
||
t.Fatalf("首次部署提交恢复失败：%v", err)
|
||
}
|
||
if len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusCommitted {
|
||
t.Fatalf("首次部署提交状态不正确：%+v", journal.finished)
|
||
}
|
||
for index, target := range record.Paths {
|
||
if _, ok := client.files[target+".scert-lock"]; ok {
|
||
t.Fatalf("提交完成后不应残留部署锁：%s", target)
|
||
}
|
||
if _, ok := client.files[record.TemporaryPaths[index]]; ok {
|
||
t.Fatalf("提交完成后不应残留临时文件：%s", record.TemporaryPaths[index])
|
||
}
|
||
if string(client.files[target].data) != "new" {
|
||
t.Fatalf("提交完成后目标文件被错误修改：%s", target)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestRecoverPendingKeepsFirstDeploymentForManualReviewWhenVerificationFails(t *testing.T) {
|
||
server, client, record := recoveryPreflightFixture()
|
||
for index := range record.Backups {
|
||
record.Backups[index].OriginalExists = false
|
||
record.Backups[index].OriginalDigest = ""
|
||
delete(client.files, record.Backups[index].Saved)
|
||
}
|
||
journal := &memoryTransactionJournal{pending: []TransactionRecord{record}}
|
||
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
|
||
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error {
|
||
return errors.New("站点尚未确认新证书")
|
||
}
|
||
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err == nil {
|
||
t.Fatal("首次部署站点未确认时必须保留人工核查状态")
|
||
}
|
||
if len(journal.finished) != 0 || len(journal.updated) != 1 || journal.updated[0].Status != TransactionStatusRecoveryRequired {
|
||
t.Fatalf("失败核验状态不正确：%+v", journal)
|
||
}
|
||
for index, target := range record.Paths {
|
||
if _, ok := client.files[target+".scert-lock"]; !ok {
|
||
t.Fatalf("失败核验时不得释放部署锁：%s", target)
|
||
}
|
||
if string(client.files[target].data) != "new" {
|
||
t.Fatalf("失败核验时不得删除新目标文件：%s", target)
|
||
}
|
||
if _, ok := client.files[record.TemporaryPaths[index]]; !ok {
|
||
t.Fatalf("失败核验时应保留临时现场：%s", record.TemporaryPaths[index])
|
||
}
|
||
}
|
||
}
|