修复部署恢复前证据核验并补齐原生负向回归

This commit is contained in:
xuans committed 2026-10-02 13:28:27 +08:00
1 parent 9a3ef1711f
commit 46a9fa0006
14 files changed
+538 -15

No files matched your search

@@ -0,0 +1,83 @@
package deployment
import (
"context"
"errors"
"fmt"
)
// validateTransactionLocks 在任何文件变更或解锁之前核验完整锁集合。
// 已完成核验的收尾阶段允许锁已不存在,但存在的无归属锁绝不自动接管。
func validateTransactionLocks(ctx context.Context, client Client, record TransactionRecord, allowMissing bool) error {
if record.Evidence == nil || !validDigest(record.Evidence.LockToken) {
return errors.New("缺少可信部署锁证据")
}
for _, path := range record.LockPaths {
exists, err := client.Exists(ctx, path)
if err != nil {
return fmt.Errorf("检查部署锁失败:%w", err)
}
if !exists && allowMissing {
continue
}
if err := checkOwnedLock(ctx, client, path, record.Evidence.LockToken, false); err != nil {
return fmt.Errorf("部署锁归属预检失败 %s:%w", path, err)
}
}
return nil
}
// validateRollbackFiles 先核验所有旧文件和目标文件,再允许逐个还原,避免部分覆盖现场。
func validateRollbackFiles(ctx context.Context, client Client, record TransactionRecord) error {
if err := validateTransactionLocks(ctx, client, record, false); err != nil {
return err
}
for _, backup := range record.Backups {
backupExists, err := validateRecoveryBackup(ctx, client, backup)
if err != nil {
return err
}
targetExists, err := client.Exists(ctx, backup.Original)
if err != nil {
return fmt.Errorf("检查待恢复目标失败:%w", err)
}
if backup.OriginalExists && !backupExists {
if !targetExists {
return fmt.Errorf("备份文件和目标文件均不存在:%s", backup.Original)
}
if err := checkFileDigest(ctx, client, backup.Original, backup.OriginalDigest); err != nil {
return fmt.Errorf("无法证明旧文件已恢复:%w", err)
}
continue
}
if !targetExists {
continue
}
if err := checkFileDigest(ctx, client, backup.Original, record.Evidence.Files[backup.Original]); err != nil {
if !backup.OriginalExists {
return fmt.Errorf("新增目标文件身份不一致:%w", err)
}
if oldErr := checkFileDigest(ctx, client, backup.Original, backup.OriginalDigest); oldErr != nil {
return fmt.Errorf("目标既不是本次部署文件也不是原始文件:%w", oldErr)
}
}
}
return nil
}
func validateRecoveryBackup(ctx context.Context, client Client, backup TransactionBackup) (bool, error) {
exists, err := client.Exists(ctx, backup.Saved)
if err != nil {
return false, fmt.Errorf("检查恢复备份失败:%w", err)
}
if !exists {
return false, nil
}
if !backup.OriginalExists {
return true, fmt.Errorf("发现不属于本次事务的备份文件:%s", backup.Saved)
}
if err := checkFileDigest(ctx, client, backup.Saved, backup.OriginalDigest); err != nil {
return true, fmt.Errorf("恢复备份身份不一致:%w", err)
}
return true, nil
}
@@ -0,0 +1,126 @@
package deployment
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"golang.org/x/crypto/ssh"
)
// TestRecoveryPreflightOverPasswordSFTP 使用回环 SSH 和真实临时文件核验协议行为,不连接业务服务器。
func TestRecoveryPreflightOverPasswordSFTP(t *testing.T) {
hostSigner := newTestECDSAHostSigner(t)
endpoint := startTestPasswordSSHServerWithHandler(t, func(password string) error {
if password != "test-password" {
return errors.New("隔离测试密码不匹配")
}
return nil
}, serveTestSFTP, hostSigner)
for _, scenario := range []string{"foreign_lock", "missing_owner", "changed_backup", "changed_target", "valid_restore"} {
t.Run(scenario, func(t *testing.T) {
root := t.TempDir()
server, memory, r := recoveryPreflightFixture(root)
model := testSSHServerModel(endpoint, "secret://ssh/preflight", ssh.FingerprintSHA256(hostSigner.PublicKey()))
server.Address, server.Port, server.Username = model.Address, model.Port, model.Username
server.AuthMethod, server.PasswordRef, server.HostKeyFingerprint = model.AuthMethod, model.PasswordRef, model.HostKeyFingerprint
r.Evidence.Address, r.Evidence.Port, r.Evidence.HostKeyFingerprint = server.Address, server.Port, server.HostKeyFingerprint
r.ReloadAttempted, r.Phase = false, TransactionPhaseInstalled
switch scenario {
case "foreign_lock":
memory.files[r.LockPaths[0]+"/owner"] = memoryFile{data: []byte(contentDigest([]byte("another-owner")))}
case "missing_owner":
delete(memory.files, r.LockPaths[0]+"/owner")
case "changed_backup":
memory.files[r.Backups[0].Saved] = memoryFile{data: []byte("unexpected-backup")}
case "changed_target":
memory.files[r.Paths[0]] = memoryFile{data: []byte("unexpected-target")}
}
for path, file := range memory.files {
if strings.HasSuffix(path, ".scert-lock") {
if err := os.MkdirAll(path, 0o700); err != nil {
t.Fatal(err)
}
continue
}
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, file.data, 0o600); err != nil {
t.Fatal(err)
}
}
before := snapshotRecoveryDirectory(t, root)
factory := SSHFactory{
Secrets: staticSecretReader{server.PasswordRef: {"password": "test-password"}},
Timeout: 5 * time.Second,
}
journal := &memoryTransactionJournal{pending: []TransactionRecord{r}}
service := NewServiceWithJournal(factory, journal)
err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil })
if scenario == "valid_restore" {
if err != nil {
t.Fatalf("真实 SFTP 可信恢复失败:%v", err)
}
if len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRolledBack {
t.Fatal("真实 SFTP 恢复未完成")
}
for _, target := range r.Paths {
data, err := os.ReadFile(target)
if err != nil || string(data) != "old" {
t.Fatalf("真实文件恢复错误:%s %v", target, err)
}
}
if len(snapshotRecoveryDirectory(t, root)) != 4 {
t.Fatal("恢复后应仅保留目录和三个原始文件")
}
return
}
if err == nil {
t.Fatal("真实 SFTP 必须拒绝不可信恢复")
}
if !reflect.DeepEqual(before, snapshotRecoveryDirectory(t, root)) {
t.Fatal("真实 SFTP 拒绝恢复时修改了磁盘现场")
}
if len(journal.finished) != 0 || len(journal.updated) != 1 || journal.updated[0].Status != TransactionStatusRecoveryRequired {
t.Fatal("真实 SFTP 未保留待核查事务")
}
})
}
}
func snapshotRecoveryDirectory(t *testing.T, root string) map[string]string {
t.Helper()
result := map[string]string{}
err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
info, err := entry.Info()
if err != nil {
return err
}
if entry.IsDir() {
result[path] = info.Mode().String()
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
result[path] = fmt.Sprintf("%s:%x", info.Mode(), sha256.Sum256(data))
return nil
})
if err != nil {
t.Fatal(err)
}
return result
}
@@ -0,0 +1,251 @@
package deployment
import (
"context"
"errors"
"maps"
"path/filepath"
"reflect"
"testing"
)
func recoveryPreflightFixture(roots ...string) (Server, *memoryClient, TransactionRecord) {
server := testServer("preflight").Normalize()
if len(roots) == 1 {
server.CertificatePath = filepath.ToSlash(filepath.Join(roots[0], "cert.pem"))
server.CertificateChainPath = filepath.ToSlash(filepath.Join(roots[0], "fullchain.pem"))
server.PrivateKeyPath = filepath.ToSlash(filepath.Join(roots[0], "key.pem"))
}
client := newMemoryClient()
r := TransactionRecord{
ID: "preflight-transaction", TaskID: "task", AttemptID: "attempt", AttemptNumber: 1,
ServerID: server.ID, ServerName: server.Name,
Paths: []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath},
Phase: TransactionPhaseReloadAttempted, Status: TransactionStatusPending, ReloadAttempted: true,
Evidence: &RecoveryEvidence{
Version: 1, LockToken: contentDigest([]byte("preflight-owner")),
Address: server.Address, Port: server.Port, HostKeyFingerprint: server.HostKeyFingerprint,
ValidationCommand: server.ValidationCommand, ReloadCommand: server.ReloadCommand,
Files: map[string]string{}, Baseline: RecoveryBaseline{
Endpoint: "https://example.com", Domains: []string{"example.com"},
PreviousFingerprint: "old", ExpectedFingerprint: "new",
},
},
}
for _, path := range r.Paths {
lock, backup, temporary := path+".scert-lock", path+".scert-backup-"+r.ID, path+".scert-tmp-"+r.ID
r.LockPaths = append(r.LockPaths, lock)
r.TemporaryPaths = append(r.TemporaryPaths, temporary)
r.InstalledPaths = append(r.InstalledPaths, path)
r.Backups = append(r.Backups, TransactionBackup{
Original: path, Saved: backup, OriginalExists: true, OriginalDigest: contentDigest([]byte("old")),
})
r.Evidence.Files[path] = contentDigest([]byte("new"))
client.files[lock] = memoryFile{}
client.files[lock+"/owner"] = memoryFile{data: []byte(r.Evidence.LockToken), mode: 0o600}
client.files[backup] = memoryFile{data: []byte("old"), mode: 0o600}
client.files[path] = memoryFile{data: []byte("new"), mode: 0o600}
client.files[temporary] = memoryFile{data: []byte("partial"), mode: 0o600}
}
return server, client, r
}
func TestRecoveryPreflightRejectsUntrustedStateWithoutMutation(t *testing.T) {
for _, scenario := range []string{
"foreign_lock", "missing_owner", "changed_backup", "changed_target", "missing_backup_and_target",
"commit_changed_backup", "acquiring_without_owner", "verified_foreign_lock",
} {
t.Run(scenario, func(t *testing.T) {
server, client, r := recoveryPreflightFixture()
switch scenario {
case "foreign_lock", "verified_foreign_lock":
client.files[r.LockPaths[0]+"/owner"] = memoryFile{data: []byte(contentDigest([]byte("another-owner")))}
if scenario == "verified_foreign_lock" {
r.Phase = TransactionPhaseRollbackVerified
}
case "missing_owner", "acquiring_without_owner":
delete(client.files, r.LockPaths[0]+"/owner")
if scenario == "acquiring_without_owner" {
r.Phase = TransactionPhaseAcquiring
}
case "changed_backup":
client.files[r.Backups[0].Saved] = memoryFile{data: []byte("unexpected-backup")}
case "changed_target":
client.files[r.Paths[0]] = memoryFile{data: []byte("unexpected-target")}
case "missing_backup_and_target":
delete(client.files, r.Backups[0].Saved)
delete(client.files, r.Paths[0])
case "commit_changed_backup":
r.Phase = TransactionPhaseCommitting
client.files[r.Backups[2].Saved] = memoryFile{data: []byte("unexpected-backup")}
}
before := maps.Clone(client.files)
journal := &memoryTransactionJournal{pending: []TransactionRecord{r}}
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { return nil }
err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil })
if err == nil {
t.Error("证据不可信时必须拒绝恢复")
}
if !reflect.DeepEqual(before, client.files) || len(client.commands) != 0 {
t.Error("证据核验失败前不得修改文件、释放锁或执行命令")
}
if len(journal.finished) != 0 || len(journal.updated) != 1 || journal.updated[0].Status != TransactionStatusRecoveryRequired {
t.Fatalf("必须保留事务等待人工核查:%+v", journal)
}
})
}
}
func TestRecoveryVerifiedCleanupIsRestartable(t *testing.T) {
for _, phase := range []string{TransactionPhaseRollbackVerified, TransactionPhaseCommitVerified} {
t.Run(phase, func(t *testing.T) {
server, client, r := recoveryPreflightFixture()
r.Phase = phase
delete(client.files, r.LockPaths[2])
delete(client.files, r.LockPaths[2]+"/owner")
journal := &memoryTransactionJournal{pending: []TransactionRecord{r}}
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err != nil {
t.Fatalf("已核验事务的部分解锁必须可重入:%v", err)
}
want := TransactionStatusRolledBack
if phase == TransactionPhaseCommitVerified {
want = TransactionStatusCommitted
}
if len(journal.finished) != 1 || journal.finished[0].status != want {
t.Fatalf("收尾状态不正确:%+v", journal.finished)
}
})
}
}
func TestRecoveryPreflightAllowsInterruptedFileStates(t *testing.T) {
for _, scenario := range []string{"before_backup_rename", "after_backup_rename", "partly_restored", "new_target", "commit_partly_cleaned", "acquiring_partly_locked"} {
t.Run(scenario, func(t *testing.T) {
server, client, r := recoveryPreflightFixture()
switch scenario {
case "before_backup_rename", "partly_restored":
delete(client.files, r.Backups[0].Saved)
client.files[r.Paths[0]] = memoryFile{data: []byte("old"), mode: 0o600}
case "after_backup_rename":
delete(client.files, r.Paths[0])
case "new_target":
r.Backups[0].OriginalExists = false
r.Backups[0].OriginalDigest = ""
delete(client.files, r.Backups[0].Saved)
case "commit_partly_cleaned":
r.Phase = TransactionPhaseCommitting
delete(client.files, r.Backups[0].Saved)
case "acquiring_partly_locked":
r.Phase = TransactionPhaseAcquiring
delete(client.files, r.LockPaths[2])
delete(client.files, r.LockPaths[2]+"/owner")
}
journal := &memoryTransactionJournal{pending: []TransactionRecord{r}}
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, committed bool) error {
if committed != (scenario == "commit_partly_cleaned") {
t.Fatal("恢复核验方向错误")
}
return nil
}
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err != nil {
t.Fatalf("可信中断状态必须可以恢复:%v", err)
}
wantStatus := TransactionStatusRolledBack
if scenario == "commit_partly_cleaned" {
wantStatus = TransactionStatusCommitted
}
if len(journal.finished) != 1 || journal.finished[0].status != wantStatus {
t.Fatalf("恢复状态不正确:%+v", journal.finished)
}
for index, target := range r.Paths {
if _, ok := client.files[r.LockPaths[index]]; ok {
t.Fatal("可信恢复完成后不应残留锁")
}
if scenario == "new_target" && index == 0 {
if _, ok := client.files[target]; ok {
t.Fatal("新增目标未回滚")
}
continue
}
want := "old"
if scenario == "commit_partly_cleaned" || scenario == "acquiring_partly_locked" {
want = "new"
}
if string(client.files[target].data) != want {
t.Fatal("目标文件恢复内容不正确")
}
}
})
}
}
func TestRecoveryPreflightHTTPSFailureKeepsLocksUntilReverified(t *testing.T) {
server, client, r := recoveryPreflightFixture()
journal := &memoryTransactionJournal{pending: []TransactionRecord{r}}
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error {
return errors.New("模拟旧证书未实际生效")
}
resolve := func(context.Context, string) (Server, error) { return server, nil }
if err := service.RecoverPending(t.Context(), resolve); err == nil {
t.Fatal("HTTPS 核验失败不得宣称恢复成功")
}
if len(journal.finished) != 0 || len(journal.updated) != 1 || journal.updated[0].Status != TransactionStatusRecoveryRequired {
t.Fatal("HTTPS 核验失败必须保留待核查事务")
}
for index, target := range r.Paths {
if string(client.files[target].data) != "old" {
t.Fatal("文件应已恢复为旧版本")
}
if _, ok := client.files[r.LockPaths[index]]; !ok {
t.Fatal("站点未核验时不得释放部署锁")
}
}
journal.pending = append([]TransactionRecord(nil), journal.updated...)
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { return nil }
if err := service.RecoverPending(t.Context(), resolve); err != nil {
t.Fatalf("再次核验可信旧文件应可恢复:%v", err)
}
if len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRolledBack {
t.Fatal("再次核验后没有完成回滚")
}
}
type recoveryConnectCounter struct{ calls int }
func (f *recoveryConnectCounter) Connect(context.Context, Server) (Client, error) {
f.calls++
return nil, errors.New("不应尝试连接身份已变更的服务器")
}
func TestRecoveryPreflightChecksIdentityBeforeAuthentication(t *testing.T) {
for _, scenario := range []string{"address", "port", "host_key", "command", "missing_evidence"} {
t.Run(scenario, func(t *testing.T) {
server, _, r := recoveryPreflightFixture()
switch scenario {
case "address":
server.Address = "192.0.2.55"
case "port":
server.Port++
case "host_key":
server.HostKeyFingerprint = "SHA256:changed"
case "command":
server.ReloadCommand = []string{"true"}
case "missing_evidence":
r.Evidence = nil
}
factory := &recoveryConnectCounter{}
journal := &memoryTransactionJournal{pending: []TransactionRecord{r}}
service := NewServiceWithJournal(factory, journal)
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err == nil {
t.Fatal("恢复身份不一致时必须拒绝恢复")
}
if factory.calls != 0 {
t.Fatal("验证恢复身份之前不得向服务器发送认证请求")
}
})
}
}
+17 -4
View File
@@ -646,6 +646,9 @@ func (s *Service) recoverTransaction(ctx context.Context, record *TransactionRec
if err := server.Validate(); err != nil {
return fmt.Errorf("服务器配置无效:%w", err)
}
if err := validateRecoveryEvidence(*record, server); err != nil {
return err
}
recoveryContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
defer cancel()
client, err := s.factory.Connect(recoveryContext, server)
@@ -653,11 +656,8 @@ func (s *Service) recoverTransaction(ctx context.Context, record *TransactionRec
return fmt.Errorf("连接服务器失败:%w", err)
}
defer client.Close()
if err := validateRecoveryEvidence(*record, server); err != nil {
return err
}
if record.Phase == TransactionPhaseAcquiring || record.Phase == TransactionPhaseRollbackVerified || record.Phase == TransactionPhaseCommitVerified {
return releaseTransactionLocks(recoveryContext, client, *record, true, record.Phase == TransactionPhaseAcquiring)
return releaseTransactionLocks(recoveryContext, client, *record, true, true)
}
if record.Phase == TransactionPhaseCommitting {
return s.recoverCommittedTransaction(recoveryContext, record, server, client)
@@ -699,6 +699,11 @@ func (s *Service) recoverCommittedTransaction(ctx context.Context, record *Trans
return err
}
}
for _, backup := range record.Backups {
if _, err := validateRecoveryBackup(ctx, client, backup); err != nil {
return err
}
}
if s.VerifyRecoveryTarget == nil || record.Evidence.Baseline.ExpectedFingerprint == "" {
return errors.New("缺少可信提交站点基线,已保留现场等待人工核查")
}
@@ -729,6 +734,9 @@ func (s *Service) recoverCommittedTransaction(ctx context.Context, record *Trans
}
func restoreTransactionFiles(ctx context.Context, client Client, record TransactionRecord) error {
if err := validateRollbackFiles(ctx, client, record); err != nil {
return err
}
for _, temporaryPath := range record.TemporaryPaths {
if exists, err := client.Exists(ctx, temporaryPath); err != nil {
return fmt.Errorf("检查临时文件失败:%w", err)
@@ -778,6 +786,11 @@ func restoreTransactionFiles(ctx context.Context, client Client, record Transact
}
func releaseTransactionLocks(ctx context.Context, client Client, record TransactionRecord, verifyOwner, allowMissingOwner bool) error {
if verifyOwner {
if err := validateTransactionLocks(ctx, client, record, allowMissingOwner); err != nil {
return err
}
}
for index := len(record.LockPaths) - 1; index >= 0; index-- {
lockPath := record.LockPaths[index]
if verifyOwner {
@@ -60,7 +60,7 @@ func TestRecoverPendingRollsBackOwnedDeployment(t *testing.T) {
client.files[backupPath] = memoryFile{data: []byte("old"), mode: 0o600}
client.files[path] = memoryFile{data: []byte("new"), mode: 0o600}
client.files[temporaryPath] = memoryFile{data: []byte("partial"), mode: 0o600}
files[path] = contentDigest([]byte("old"))
files[path] = contentDigest([]byte("new"))
}
journal := &memoryTransactionJournal{pending: []TransactionRecord{{
ID: id, TaskID: "task", AttemptID: "attempt", AttemptNumber: 1, ResultIndex: 0,
@@ -172,7 +172,7 @@ func TestDeployPartialLockConflictReleasesOwnedLocks(t *testing.T) {
}
}
func TestRecoverPendingAcquiringCleansLocksWithoutOwner(t *testing.T) {
func TestRecoverPendingAcquiringPreservesLocksWithoutOwner(t *testing.T) {
server := testServer("acquiring-missing-owner").Normalize()
client := newMemoryClient()
id := "acquiring-missing-owner-transaction"
@@ -197,15 +197,15 @@ func TestRecoverPendingAcquiringCleansLocksWithoutOwner(t *testing.T) {
},
}}}
service := NewServiceWithJournal(memoryFactory{clients: map[string]*memoryClient{server.ID: client}}, journal)
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err != nil {
t.Fatalf("无归属的中断获取阶段恢复失败:%v", err)
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err == nil {
t.Fatal("无归属锁不能证明属于本次中断事务,必须拒绝自动清理")
}
for _, lockPath := range locks {
if _, ok := client.files[lockPath]; ok {
t.Fatalf("无归属锁目录未清理:%s", lockPath)
if _, ok := client.files[lockPath]; !ok {
t.Fatalf("不得删除无归属锁目录:%s", lockPath)
}
}
if len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRolledBack {
t.Fatalf("无归属锁恢复未完成回滚:%+v", journal.finished)
if len(journal.finished) != 0 || len(journal.updated) != 1 || journal.updated[0].Status != TransactionStatusRecoveryRequired {
t.Fatal("无归属锁必须保留事务等待人工核查")
}
}