修复部署恢复前证据核验并补齐原生负向回归
This commit is contained in:
parent
9a3ef1711f
commit
46a9fa0006
83
backend/internal/deployment/recovery_preflight.go
Normal file
83
backend/internal/deployment/recovery_preflight.go
Normal file
@ -0,0 +1,83 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// validateTransactionLocks 在任何文件变更或解锁之前核验完整锁集合。
|
||||
// 已完成核验的收尾阶段允许锁已不存在,但存在的无归属锁绝不自动接管。
|
||||
func validateTransactionLocks(ctx context.Context, client Client, record TransactionRecord, allowMissing bool) error {
|
||||
if record.Evidence == nil || !validDigest(record.Evidence.LockToken) {
|
||||
return errors.New("缺少可信部署锁证据")
|
||||
}
|
||||
for _, path := range record.LockPaths {
|
||||
exists, err := client.Exists(ctx, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("检查部署锁失败:%w", err)
|
||||
}
|
||||
if !exists && allowMissing {
|
||||
continue
|
||||
}
|
||||
if err := checkOwnedLock(ctx, client, path, record.Evidence.LockToken, false); err != nil {
|
||||
return fmt.Errorf("部署锁归属预检失败 %s:%w", path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateRollbackFiles 先核验所有旧文件和目标文件,再允许逐个还原,避免部分覆盖现场。
|
||||
func validateRollbackFiles(ctx context.Context, client Client, record TransactionRecord) error {
|
||||
if err := validateTransactionLocks(ctx, client, record, false); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, backup := range record.Backups {
|
||||
backupExists, err := validateRecoveryBackup(ctx, client, backup)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
targetExists, err := client.Exists(ctx, backup.Original)
|
||||
if err != nil {
|
||||
return fmt.Errorf("检查待恢复目标失败:%w", err)
|
||||
}
|
||||
if backup.OriginalExists && !backupExists {
|
||||
if !targetExists {
|
||||
return fmt.Errorf("备份文件和目标文件均不存在:%s", backup.Original)
|
||||
}
|
||||
if err := checkFileDigest(ctx, client, backup.Original, backup.OriginalDigest); err != nil {
|
||||
return fmt.Errorf("无法证明旧文件已恢复:%w", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !targetExists {
|
||||
continue
|
||||
}
|
||||
if err := checkFileDigest(ctx, client, backup.Original, record.Evidence.Files[backup.Original]); err != nil {
|
||||
if !backup.OriginalExists {
|
||||
return fmt.Errorf("新增目标文件身份不一致:%w", err)
|
||||
}
|
||||
if oldErr := checkFileDigest(ctx, client, backup.Original, backup.OriginalDigest); oldErr != nil {
|
||||
return fmt.Errorf("目标既不是本次部署文件也不是原始文件:%w", oldErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRecoveryBackup(ctx context.Context, client Client, backup TransactionBackup) (bool, error) {
|
||||
exists, err := client.Exists(ctx, backup.Saved)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("检查恢复备份失败:%w", err)
|
||||
}
|
||||
if !exists {
|
||||
return false, nil
|
||||
}
|
||||
if !backup.OriginalExists {
|
||||
return true, fmt.Errorf("发现不属于本次事务的备份文件:%s", backup.Saved)
|
||||
}
|
||||
if err := checkFileDigest(ctx, client, backup.Saved, backup.OriginalDigest); err != nil {
|
||||
return true, fmt.Errorf("恢复备份身份不一致:%w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
126
backend/internal/deployment/recovery_preflight_sftp_test.go
Normal file
126
backend/internal/deployment/recovery_preflight_sftp_test.go
Normal file
@ -0,0 +1,126 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// TestRecoveryPreflightOverPasswordSFTP 使用回环 SSH 和真实临时文件核验协议行为,不连接业务服务器。
|
||||
func TestRecoveryPreflightOverPasswordSFTP(t *testing.T) {
|
||||
hostSigner := newTestECDSAHostSigner(t)
|
||||
endpoint := startTestPasswordSSHServerWithHandler(t, func(password string) error {
|
||||
if password != "test-password" {
|
||||
return errors.New("隔离测试密码不匹配")
|
||||
}
|
||||
return nil
|
||||
}, serveTestSFTP, hostSigner)
|
||||
for _, scenario := range []string{"foreign_lock", "missing_owner", "changed_backup", "changed_target", "valid_restore"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
server, memory, r := recoveryPreflightFixture(root)
|
||||
model := testSSHServerModel(endpoint, "secret://ssh/preflight", ssh.FingerprintSHA256(hostSigner.PublicKey()))
|
||||
server.Address, server.Port, server.Username = model.Address, model.Port, model.Username
|
||||
server.AuthMethod, server.PasswordRef, server.HostKeyFingerprint = model.AuthMethod, model.PasswordRef, model.HostKeyFingerprint
|
||||
r.Evidence.Address, r.Evidence.Port, r.Evidence.HostKeyFingerprint = server.Address, server.Port, server.HostKeyFingerprint
|
||||
r.ReloadAttempted, r.Phase = false, TransactionPhaseInstalled
|
||||
switch scenario {
|
||||
case "foreign_lock":
|
||||
memory.files[r.LockPaths[0]+"/owner"] = memoryFile{data: []byte(contentDigest([]byte("another-owner")))}
|
||||
case "missing_owner":
|
||||
delete(memory.files, r.LockPaths[0]+"/owner")
|
||||
case "changed_backup":
|
||||
memory.files[r.Backups[0].Saved] = memoryFile{data: []byte("unexpected-backup")}
|
||||
case "changed_target":
|
||||
memory.files[r.Paths[0]] = memoryFile{data: []byte("unexpected-target")}
|
||||
}
|
||||
for path, file := range memory.files {
|
||||
if strings.HasSuffix(path, ".scert-lock") {
|
||||
if err := os.MkdirAll(path, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, file.data, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
before := snapshotRecoveryDirectory(t, root)
|
||||
factory := SSHFactory{
|
||||
Secrets: staticSecretReader{server.PasswordRef: {"password": "test-password"}},
|
||||
Timeout: 5 * time.Second,
|
||||
}
|
||||
journal := &memoryTransactionJournal{pending: []TransactionRecord{r}}
|
||||
service := NewServiceWithJournal(factory, journal)
|
||||
err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil })
|
||||
if scenario == "valid_restore" {
|
||||
if err != nil {
|
||||
t.Fatalf("真实 SFTP 可信恢复失败:%v", err)
|
||||
}
|
||||
if len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRolledBack {
|
||||
t.Fatal("真实 SFTP 恢复未完成")
|
||||
}
|
||||
for _, target := range r.Paths {
|
||||
data, err := os.ReadFile(target)
|
||||
if err != nil || string(data) != "old" {
|
||||
t.Fatalf("真实文件恢复错误:%s %v", target, err)
|
||||
}
|
||||
}
|
||||
if len(snapshotRecoveryDirectory(t, root)) != 4 {
|
||||
t.Fatal("恢复后应仅保留目录和三个原始文件")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("真实 SFTP 必须拒绝不可信恢复")
|
||||
}
|
||||
if !reflect.DeepEqual(before, snapshotRecoveryDirectory(t, root)) {
|
||||
t.Fatal("真实 SFTP 拒绝恢复时修改了磁盘现场")
|
||||
}
|
||||
if len(journal.finished) != 0 || len(journal.updated) != 1 || journal.updated[0].Status != TransactionStatusRecoveryRequired {
|
||||
t.Fatal("真实 SFTP 未保留待核查事务")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func snapshotRecoveryDirectory(t *testing.T, root string) map[string]string {
|
||||
t.Helper()
|
||||
result := map[string]string{}
|
||||
err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
result[path] = info.Mode().String()
|
||||
return nil
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result[path] = fmt.Sprintf("%s:%x", info.Mode(), sha256.Sum256(data))
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
251
backend/internal/deployment/recovery_preflight_test.go
Normal file
251
backend/internal/deployment/recovery_preflight_test.go
Normal file
@ -0,0 +1,251 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"maps"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func recoveryPreflightFixture(roots ...string) (Server, *memoryClient, TransactionRecord) {
|
||||
server := testServer("preflight").Normalize()
|
||||
if len(roots) == 1 {
|
||||
server.CertificatePath = filepath.ToSlash(filepath.Join(roots[0], "cert.pem"))
|
||||
server.CertificateChainPath = filepath.ToSlash(filepath.Join(roots[0], "fullchain.pem"))
|
||||
server.PrivateKeyPath = filepath.ToSlash(filepath.Join(roots[0], "key.pem"))
|
||||
}
|
||||
client := newMemoryClient()
|
||||
r := TransactionRecord{
|
||||
ID: "preflight-transaction", TaskID: "task", AttemptID: "attempt", AttemptNumber: 1,
|
||||
ServerID: server.ID, ServerName: server.Name,
|
||||
Paths: []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath},
|
||||
Phase: TransactionPhaseReloadAttempted, Status: TransactionStatusPending, ReloadAttempted: true,
|
||||
Evidence: &RecoveryEvidence{
|
||||
Version: 1, LockToken: contentDigest([]byte("preflight-owner")),
|
||||
Address: server.Address, Port: server.Port, HostKeyFingerprint: server.HostKeyFingerprint,
|
||||
ValidationCommand: server.ValidationCommand, ReloadCommand: server.ReloadCommand,
|
||||
Files: map[string]string{}, Baseline: RecoveryBaseline{
|
||||
Endpoint: "https://example.com", Domains: []string{"example.com"},
|
||||
PreviousFingerprint: "old", ExpectedFingerprint: "new",
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, path := range r.Paths {
|
||||
lock, backup, temporary := path+".scert-lock", path+".scert-backup-"+r.ID, path+".scert-tmp-"+r.ID
|
||||
r.LockPaths = append(r.LockPaths, lock)
|
||||
r.TemporaryPaths = append(r.TemporaryPaths, temporary)
|
||||
r.InstalledPaths = append(r.InstalledPaths, path)
|
||||
r.Backups = append(r.Backups, TransactionBackup{
|
||||
Original: path, Saved: backup, OriginalExists: true, OriginalDigest: contentDigest([]byte("old")),
|
||||
})
|
||||
r.Evidence.Files[path] = contentDigest([]byte("new"))
|
||||
client.files[lock] = memoryFile{}
|
||||
client.files[lock+"/owner"] = memoryFile{data: []byte(r.Evidence.LockToken), mode: 0o600}
|
||||
client.files[backup] = memoryFile{data: []byte("old"), mode: 0o600}
|
||||
client.files[path] = memoryFile{data: []byte("new"), mode: 0o600}
|
||||
client.files[temporary] = memoryFile{data: []byte("partial"), mode: 0o600}
|
||||
}
|
||||
return server, client, r
|
||||
}
|
||||
|
||||
func TestRecoveryPreflightRejectsUntrustedStateWithoutMutation(t *testing.T) {
|
||||
for _, scenario := range []string{
|
||||
"foreign_lock", "missing_owner", "changed_backup", "changed_target", "missing_backup_and_target",
|
||||
"commit_changed_backup", "acquiring_without_owner", "verified_foreign_lock",
|
||||
} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
server, client, r := recoveryPreflightFixture()
|
||||
switch scenario {
|
||||
case "foreign_lock", "verified_foreign_lock":
|
||||
client.files[r.LockPaths[0]+"/owner"] = memoryFile{data: []byte(contentDigest([]byte("another-owner")))}
|
||||
if scenario == "verified_foreign_lock" {
|
||||
r.Phase = TransactionPhaseRollbackVerified
|
||||
}
|
||||
case "missing_owner", "acquiring_without_owner":
|
||||
delete(client.files, r.LockPaths[0]+"/owner")
|
||||
if scenario == "acquiring_without_owner" {
|
||||
r.Phase = TransactionPhaseAcquiring
|
||||
}
|
||||
case "changed_backup":
|
||||
client.files[r.Backups[0].Saved] = memoryFile{data: []byte("unexpected-backup")}
|
||||
case "changed_target":
|
||||
client.files[r.Paths[0]] = memoryFile{data: []byte("unexpected-target")}
|
||||
case "missing_backup_and_target":
|
||||
delete(client.files, r.Backups[0].Saved)
|
||||
delete(client.files, r.Paths[0])
|
||||
case "commit_changed_backup":
|
||||
r.Phase = TransactionPhaseCommitting
|
||||
client.files[r.Backups[2].Saved] = memoryFile{data: []byte("unexpected-backup")}
|
||||
}
|
||||
before := maps.Clone(client.files)
|
||||
journal := &memoryTransactionJournal{pending: []TransactionRecord{r}}
|
||||
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
|
||||
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { return nil }
|
||||
err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil })
|
||||
if err == nil {
|
||||
t.Error("证据不可信时必须拒绝恢复")
|
||||
}
|
||||
if !reflect.DeepEqual(before, client.files) || len(client.commands) != 0 {
|
||||
t.Error("证据核验失败前不得修改文件、释放锁或执行命令")
|
||||
}
|
||||
if len(journal.finished) != 0 || len(journal.updated) != 1 || journal.updated[0].Status != TransactionStatusRecoveryRequired {
|
||||
t.Fatalf("必须保留事务等待人工核查:%+v", journal)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryVerifiedCleanupIsRestartable(t *testing.T) {
|
||||
for _, phase := range []string{TransactionPhaseRollbackVerified, TransactionPhaseCommitVerified} {
|
||||
t.Run(phase, func(t *testing.T) {
|
||||
server, client, r := recoveryPreflightFixture()
|
||||
r.Phase = phase
|
||||
delete(client.files, r.LockPaths[2])
|
||||
delete(client.files, r.LockPaths[2]+"/owner")
|
||||
journal := &memoryTransactionJournal{pending: []TransactionRecord{r}}
|
||||
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
|
||||
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err != nil {
|
||||
t.Fatalf("已核验事务的部分解锁必须可重入:%v", err)
|
||||
}
|
||||
want := TransactionStatusRolledBack
|
||||
if phase == TransactionPhaseCommitVerified {
|
||||
want = TransactionStatusCommitted
|
||||
}
|
||||
if len(journal.finished) != 1 || journal.finished[0].status != want {
|
||||
t.Fatalf("收尾状态不正确:%+v", journal.finished)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryPreflightAllowsInterruptedFileStates(t *testing.T) {
|
||||
for _, scenario := range []string{"before_backup_rename", "after_backup_rename", "partly_restored", "new_target", "commit_partly_cleaned", "acquiring_partly_locked"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
server, client, r := recoveryPreflightFixture()
|
||||
switch scenario {
|
||||
case "before_backup_rename", "partly_restored":
|
||||
delete(client.files, r.Backups[0].Saved)
|
||||
client.files[r.Paths[0]] = memoryFile{data: []byte("old"), mode: 0o600}
|
||||
case "after_backup_rename":
|
||||
delete(client.files, r.Paths[0])
|
||||
case "new_target":
|
||||
r.Backups[0].OriginalExists = false
|
||||
r.Backups[0].OriginalDigest = ""
|
||||
delete(client.files, r.Backups[0].Saved)
|
||||
case "commit_partly_cleaned":
|
||||
r.Phase = TransactionPhaseCommitting
|
||||
delete(client.files, r.Backups[0].Saved)
|
||||
case "acquiring_partly_locked":
|
||||
r.Phase = TransactionPhaseAcquiring
|
||||
delete(client.files, r.LockPaths[2])
|
||||
delete(client.files, r.LockPaths[2]+"/owner")
|
||||
}
|
||||
journal := &memoryTransactionJournal{pending: []TransactionRecord{r}}
|
||||
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
|
||||
service.VerifyRecoveryTarget = func(_ context.Context, _ Server, _ RecoveryBaseline, committed bool) error {
|
||||
if committed != (scenario == "commit_partly_cleaned") {
|
||||
t.Fatal("恢复核验方向错误")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err != nil {
|
||||
t.Fatalf("可信中断状态必须可以恢复:%v", err)
|
||||
}
|
||||
wantStatus := TransactionStatusRolledBack
|
||||
if scenario == "commit_partly_cleaned" {
|
||||
wantStatus = TransactionStatusCommitted
|
||||
}
|
||||
if len(journal.finished) != 1 || journal.finished[0].status != wantStatus {
|
||||
t.Fatalf("恢复状态不正确:%+v", journal.finished)
|
||||
}
|
||||
for index, target := range r.Paths {
|
||||
if _, ok := client.files[r.LockPaths[index]]; ok {
|
||||
t.Fatal("可信恢复完成后不应残留锁")
|
||||
}
|
||||
if scenario == "new_target" && index == 0 {
|
||||
if _, ok := client.files[target]; ok {
|
||||
t.Fatal("新增目标未回滚")
|
||||
}
|
||||
continue
|
||||
}
|
||||
want := "old"
|
||||
if scenario == "commit_partly_cleaned" || scenario == "acquiring_partly_locked" {
|
||||
want = "new"
|
||||
}
|
||||
if string(client.files[target].data) != want {
|
||||
t.Fatal("目标文件恢复内容不正确")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryPreflightHTTPSFailureKeepsLocksUntilReverified(t *testing.T) {
|
||||
server, client, r := recoveryPreflightFixture()
|
||||
journal := &memoryTransactionJournal{pending: []TransactionRecord{r}}
|
||||
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
|
||||
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error {
|
||||
return errors.New("模拟旧证书未实际生效")
|
||||
}
|
||||
resolve := func(context.Context, string) (Server, error) { return server, nil }
|
||||
if err := service.RecoverPending(t.Context(), resolve); err == nil {
|
||||
t.Fatal("HTTPS 核验失败不得宣称恢复成功")
|
||||
}
|
||||
if len(journal.finished) != 0 || len(journal.updated) != 1 || journal.updated[0].Status != TransactionStatusRecoveryRequired {
|
||||
t.Fatal("HTTPS 核验失败必须保留待核查事务")
|
||||
}
|
||||
for index, target := range r.Paths {
|
||||
if string(client.files[target].data) != "old" {
|
||||
t.Fatal("文件应已恢复为旧版本")
|
||||
}
|
||||
if _, ok := client.files[r.LockPaths[index]]; !ok {
|
||||
t.Fatal("站点未核验时不得释放部署锁")
|
||||
}
|
||||
}
|
||||
journal.pending = append([]TransactionRecord(nil), journal.updated...)
|
||||
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { return nil }
|
||||
if err := service.RecoverPending(t.Context(), resolve); err != nil {
|
||||
t.Fatalf("再次核验可信旧文件应可恢复:%v", err)
|
||||
}
|
||||
if len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRolledBack {
|
||||
t.Fatal("再次核验后没有完成回滚")
|
||||
}
|
||||
}
|
||||
|
||||
type recoveryConnectCounter struct{ calls int }
|
||||
|
||||
func (f *recoveryConnectCounter) Connect(context.Context, Server) (Client, error) {
|
||||
f.calls++
|
||||
return nil, errors.New("不应尝试连接身份已变更的服务器")
|
||||
}
|
||||
|
||||
func TestRecoveryPreflightChecksIdentityBeforeAuthentication(t *testing.T) {
|
||||
for _, scenario := range []string{"address", "port", "host_key", "command", "missing_evidence"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
server, _, r := recoveryPreflightFixture()
|
||||
switch scenario {
|
||||
case "address":
|
||||
server.Address = "192.0.2.55"
|
||||
case "port":
|
||||
server.Port++
|
||||
case "host_key":
|
||||
server.HostKeyFingerprint = "SHA256:changed"
|
||||
case "command":
|
||||
server.ReloadCommand = []string{"true"}
|
||||
case "missing_evidence":
|
||||
r.Evidence = nil
|
||||
}
|
||||
factory := &recoveryConnectCounter{}
|
||||
journal := &memoryTransactionJournal{pending: []TransactionRecord{r}}
|
||||
service := NewServiceWithJournal(factory, journal)
|
||||
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err == nil {
|
||||
t.Fatal("恢复身份不一致时必须拒绝恢复")
|
||||
}
|
||||
if factory.calls != 0 {
|
||||
t.Fatal("验证恢复身份之前不得向服务器发送认证请求")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -646,6 +646,9 @@ func (s *Service) recoverTransaction(ctx context.Context, record *TransactionRec
|
||||
if err := server.Validate(); err != nil {
|
||||
return fmt.Errorf("服务器配置无效:%w", err)
|
||||
}
|
||||
if err := validateRecoveryEvidence(*record, server); err != nil {
|
||||
return err
|
||||
}
|
||||
recoveryContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
|
||||
defer cancel()
|
||||
client, err := s.factory.Connect(recoveryContext, server)
|
||||
@ -653,11 +656,8 @@ func (s *Service) recoverTransaction(ctx context.Context, record *TransactionRec
|
||||
return fmt.Errorf("连接服务器失败:%w", err)
|
||||
}
|
||||
defer client.Close()
|
||||
if err := validateRecoveryEvidence(*record, server); err != nil {
|
||||
return err
|
||||
}
|
||||
if record.Phase == TransactionPhaseAcquiring || record.Phase == TransactionPhaseRollbackVerified || record.Phase == TransactionPhaseCommitVerified {
|
||||
return releaseTransactionLocks(recoveryContext, client, *record, true, record.Phase == TransactionPhaseAcquiring)
|
||||
return releaseTransactionLocks(recoveryContext, client, *record, true, true)
|
||||
}
|
||||
if record.Phase == TransactionPhaseCommitting {
|
||||
return s.recoverCommittedTransaction(recoveryContext, record, server, client)
|
||||
@ -699,6 +699,11 @@ func (s *Service) recoverCommittedTransaction(ctx context.Context, record *Trans
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, backup := range record.Backups {
|
||||
if _, err := validateRecoveryBackup(ctx, client, backup); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if s.VerifyRecoveryTarget == nil || record.Evidence.Baseline.ExpectedFingerprint == "" {
|
||||
return errors.New("缺少可信提交站点基线,已保留现场等待人工核查")
|
||||
}
|
||||
@ -729,6 +734,9 @@ func (s *Service) recoverCommittedTransaction(ctx context.Context, record *Trans
|
||||
}
|
||||
|
||||
func restoreTransactionFiles(ctx context.Context, client Client, record TransactionRecord) error {
|
||||
if err := validateRollbackFiles(ctx, client, record); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, temporaryPath := range record.TemporaryPaths {
|
||||
if exists, err := client.Exists(ctx, temporaryPath); err != nil {
|
||||
return fmt.Errorf("检查临时文件失败:%w", err)
|
||||
@ -778,6 +786,11 @@ func restoreTransactionFiles(ctx context.Context, client Client, record Transact
|
||||
}
|
||||
|
||||
func releaseTransactionLocks(ctx context.Context, client Client, record TransactionRecord, verifyOwner, allowMissingOwner bool) error {
|
||||
if verifyOwner {
|
||||
if err := validateTransactionLocks(ctx, client, record, allowMissingOwner); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for index := len(record.LockPaths) - 1; index >= 0; index-- {
|
||||
lockPath := record.LockPaths[index]
|
||||
if verifyOwner {
|
||||
|
||||
@ -60,7 +60,7 @@ func TestRecoverPendingRollsBackOwnedDeployment(t *testing.T) {
|
||||
client.files[backupPath] = memoryFile{data: []byte("old"), mode: 0o600}
|
||||
client.files[path] = memoryFile{data: []byte("new"), mode: 0o600}
|
||||
client.files[temporaryPath] = memoryFile{data: []byte("partial"), mode: 0o600}
|
||||
files[path] = contentDigest([]byte("old"))
|
||||
files[path] = contentDigest([]byte("new"))
|
||||
}
|
||||
journal := &memoryTransactionJournal{pending: []TransactionRecord{{
|
||||
ID: id, TaskID: "task", AttemptID: "attempt", AttemptNumber: 1, ResultIndex: 0,
|
||||
@ -172,7 +172,7 @@ func TestDeployPartialLockConflictReleasesOwnedLocks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverPendingAcquiringCleansLocksWithoutOwner(t *testing.T) {
|
||||
func TestRecoverPendingAcquiringPreservesLocksWithoutOwner(t *testing.T) {
|
||||
server := testServer("acquiring-missing-owner").Normalize()
|
||||
client := newMemoryClient()
|
||||
id := "acquiring-missing-owner-transaction"
|
||||
@ -197,15 +197,15 @@ func TestRecoverPendingAcquiringCleansLocksWithoutOwner(t *testing.T) {
|
||||
},
|
||||
}}}
|
||||
service := NewServiceWithJournal(memoryFactory{clients: map[string]*memoryClient{server.ID: client}}, journal)
|
||||
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err != nil {
|
||||
t.Fatalf("无归属的中断获取阶段恢复失败:%v", err)
|
||||
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err == nil {
|
||||
t.Fatal("无归属锁不能证明属于本次中断事务,必须拒绝自动清理")
|
||||
}
|
||||
for _, lockPath := range locks {
|
||||
if _, ok := client.files[lockPath]; ok {
|
||||
t.Fatalf("无归属锁目录未清理:%s", lockPath)
|
||||
if _, ok := client.files[lockPath]; !ok {
|
||||
t.Fatalf("不得删除无归属锁目录:%s", lockPath)
|
||||
}
|
||||
}
|
||||
if len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRolledBack {
|
||||
t.Fatalf("无归属锁恢复未完成回滚:%+v", journal.finished)
|
||||
if len(journal.finished) != 0 || len(journal.updated) != 1 || journal.updated[0].Status != TransactionStatusRecoveryRequired {
|
||||
t.Fatal("无归属锁必须保留事务等待人工核查")
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -11,6 +12,7 @@ import (
|
||||
|
||||
func TestProjectAuditReaderScopesLogsAndHonorsRevocation(t *testing.T) {
|
||||
cfg := config.Default(t.TempDir()).Database
|
||||
cfg.DSN = filepath.Join(t.TempDir(), "project-audit.db")
|
||||
db, err := database.Open(t.Context(), cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@ -3,6 +3,7 @@ package repository
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/config"
|
||||
@ -12,7 +13,9 @@ import (
|
||||
)
|
||||
|
||||
func TestProjectDirectorySQLite(t *testing.T) {
|
||||
checkProjectDirectory(t, config.Default(t.TempDir()).Database)
|
||||
cfg := config.Default(t.TempDir()).Database
|
||||
cfg.DSN = filepath.Join(t.TempDir(), "project-directory.db")
|
||||
checkProjectDirectory(t, cfg)
|
||||
}
|
||||
|
||||
func TestProjectDirectoryMySQLReal(t *testing.T) {
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/config"
|
||||
@ -14,6 +15,7 @@ import (
|
||||
|
||||
func TestProjectDomainRepositorySQLite(t *testing.T) {
|
||||
cfg := config.Default(t.TempDir()).Database
|
||||
cfg.DSN = filepath.Join(t.TempDir(), "project-domains.db")
|
||||
db, err := database.Open(t.Context(), cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@ -264,6 +264,7 @@ func TestProjectLifecycleSQLiteAuditRollback(t *testing.T) {
|
||||
for _, mode := range []string{"ABORT, 'audit-failure'", "IGNORE"} {
|
||||
t.Run(action+mode, func(t *testing.T) {
|
||||
cfg := config.Default(t.TempDir()).Database
|
||||
cfg.DSN = filepath.Join(t.TempDir(), "project-audit-rollback.db")
|
||||
db, err := database.Open(t.Context(), cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/config"
|
||||
@ -13,7 +14,9 @@ import (
|
||||
)
|
||||
|
||||
func TestProjectMemberListingSQLite(t *testing.T) {
|
||||
checkProjectMemberListing(t, config.Default(t.TempDir()).Database)
|
||||
cfg := config.Default(t.TempDir()).Database
|
||||
cfg.DSN = filepath.Join(t.TempDir(), "project-members.db")
|
||||
checkProjectMemberListing(t, cfg)
|
||||
}
|
||||
|
||||
func TestProjectMemberListingMySQLReal(t *testing.T) {
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/config"
|
||||
@ -14,7 +15,9 @@ import (
|
||||
)
|
||||
|
||||
func TestProjectResourcesSQLite(t *testing.T) {
|
||||
checkProjectResources(t, config.Default(t.TempDir()).Database, nil)
|
||||
cfg := config.Default(t.TempDir()).Database
|
||||
cfg.DSN = filepath.Join(t.TempDir(), "project-resources.db")
|
||||
checkProjectResources(t, cfg, nil)
|
||||
}
|
||||
|
||||
func TestProjectResourcesMySQLReal(t *testing.T) {
|
||||
|
||||
@ -1,5 +1,19 @@
|
||||
# Linux 生产流程测试记录
|
||||
|
||||
## 2026年10月2日13:30 恢复预检原生 Linux 回归
|
||||
|
||||
本轮为第二台隔离源码测试,不是清库首次生产验收。使用 nobody、Go 1.26.6 和临时目录;未修改业务数据库、服务配置、Nginx 或公网端口。线上仍为此前 1150c39 对应发行包。
|
||||
|
||||
- [x] 隔离证据目录:`/var/tmp/scert-recovery-preflight-20261002-1308/`。`baseline.log` 记录旧代码八个不可信状态、两个部分解锁重入场景失败,退出码为 1。
|
||||
- [x] 修复后部署包完整测试退出零;`regression-final.log` 中 deployment、repository、app 三包均通过,退出码为 0;`vet.log` 对应三包 go vet 退出零。
|
||||
- [x] 新增八个不可信状态、两个已验证阶段重入、六个合法中断状态、HTTPS 失败后再恢复、五个认证前身份检查及五个真实回环密码 SSH/SFTP 子场景。SFTP 测试检查磁盘内容和权限,拒绝不可信现场时不能有文件变更。
|
||||
- [x] `repeat-final.log`:`go test ./internal/deployment -run 'TestRecoveryPreflight|TestRecoveryVerifiedCleanup' -count=20` 实际退出零。
|
||||
- [x] 首次扩大回归中仓储测试错误尝试写入 `/scert/data`,被 nobody 拒绝;六个相关测试改用临时 SQLite DSN 后完整重跑。未提权、未放宽正式路径权限;原失败保留在 `regression.log`。
|
||||
- [!] `race-repeat.log` 首次因 CGO 未启用退出 2;`race-repeat-cgo.log` 显式启用后因找不到 gcc 退出 1。均未完成竞争检测,不记为通过。
|
||||
- [x] 北京时间 13:25 第二台 Scert 为 active,Scert 8443、MySQL 3306/33060 均仅监听 127.0.0.1;这不代表修复版已经部署。
|
||||
- [~] 上传碰撞与事务日志组合仍需核查,本批正式构建部署、业务实机负向恢复和历史事务证据核查待完成。可选真实数据库用例未显式开启,不宣称三数据库验收通过。
|
||||
|
||||
|
||||
## 2026年10月2日12:54 双机正式候选升级记录
|
||||
|
||||
本节属于升级与安全复核,不是清库首次生产验收。未开放额外端口,未修改 Nginx,未关闭 TLS 或 SSH 主机指纹校验,未创建辅助验收脚本。只使用正式安装器自带发布事务,没有额外远端备份。
|
||||
|
||||
@ -1,5 +1,17 @@
|
||||
# Scert 整改任务执行清单
|
||||
|
||||
## 2026年10月2日13:30 当前执行入口
|
||||
|
||||
需求确认:仅 Linux,沿用 P0-10 恢复安全和生产门禁;未知归属不得自动接管,公网只走 Nginx 80/443,不放宽校验,不使用辅助脚本作为验收依据。
|
||||
|
||||
- [x] T2-A:恢复前核验完整锁、备份及目标身份;SSH 认证前核验服务器证据,已验证阶段支持部分解锁重入。
|
||||
- [x] T2-B:原实现负向复现;修复后部署、仓储、应用三包原生 Linux 测试及 go vet 通过,恢复新用例重复二十次通过。
|
||||
- [x] T2-C:真实回环密码 SSH/SFTP 的五个场景通过;测试数据库使用临时目录,不提权写入正式安装路径。
|
||||
- [!] T2-D:竞争检测因缺少 gcc 未执行成功;普通重复测试不是其替代验收。
|
||||
- [~] T2-E:带事务日志上传碰撞的路径所有权核查。
|
||||
- [ ] T2-F:本批修复的干净提交正式构建、部署及业务实机恢复复审。
|
||||
- [~] P0-10 整项仍进行中;历史缺证据事务、升级 502、系统通知投递、三数据库清库首次 Web 流程、备份恢复和统一复审继续保留。
|
||||
|
||||
## 2026年10月2日12:54 双机安全升级当前入口
|
||||
|
||||
本节覆盖下方本轮执行态记录;历史失败和未通过门禁继续保留。
|
||||
|
||||
@ -1,5 +1,13 @@
|
||||
# Scert 整改、合并与优化清单
|
||||
|
||||
## 2026年10月2日13:30 恢复安全整改进度
|
||||
|
||||
- [x] P0-10 子项:恢复前完整核验锁集合、备份摘要及当前目标身份;存在但无归属的锁拒绝自动处理,已验证阶段允许部分解锁后重入。
|
||||
- [x] 恢复服务器身份在 SSH 认证前核验;提交恢复先核验全部备份再清理,避免检查失败时已经修改部分现场。
|
||||
- [x] 原代码的八个负向和两个重入场景均复现失败;修复后部署、仓储、应用三包原生 Linux 回归及 go vet 通过,新增恢复用例重复二十次通过。真实回环密码 SSH/SFTP 覆盖文件内容及权限保护,不扩大为业务服务器恢复验收。
|
||||
- [!] 竞争检测未通过执行门槛:启用 CGO 后缺少 gcc;没有安装工具链或将普通重复测试冒充竞争检测。
|
||||
- [~] 待核查带事务日志的上传碰撞路径归属;本批尚未正式构建部署,P0-10 保持进行中,总体生产验收未通过。
|
||||
|
||||
## 2026年10月2日12:54 最新进度入口
|
||||
|
||||
- [x] Linux 非回环监听风险已完成代码收口、六包原生回归、负向发行程序验证和双机配套部署;两台当前均仅监听 127.0.0.1:8443。
|
||||
@ -1119,6 +1127,8 @@ updated_by
|
||||
|
||||
**状态:** `[~] 进行中`
|
||||
|
||||
**恢复安全更新(2026-10-02 13:30):** 已复现并修复先改文件后验锁、未知归属锁清理及已验证阶段部分解锁无法重入的问题;补齐认证前身份核验和整组备份预检。三包原生 Linux 回归、go vet 和新用例重复二十次通过;竞争检测因缺少 gcc 未运行成功,修复尚未正式部署。上传碰撞与事务日志的组合仍需核查,历史缺证据事务继续保留,不能标记整项通过。
|
||||
|
||||
**中断恢复实测(2026-09-30 22:35):** 真实部署校验中强制终止主进程,systemd 自动重启,任务、尝试及审计正确标记中断失败;远端锁和事务旧文件保留,直接重试被拒绝。人工核查并隔离现场后可通过产品重新部署。已开发固定阶段诊断和先核查后重试提示,专项通过但未部署;远端崩溃事务自动恢复仍未完成,不能标记整项通过。
|
||||
|
||||
**最新整改(2026-09-30):** 已复现并修复后置 HTTPS 校验接受旧有效证书、校验失败仍返回成功部署报告的问题;身份来自实际待部署文件。报告以 verification_status 单独记录站点生效校验,整体失败不改写单台真实文件推送事实;任务、尝试和接口保持失败,页面不再汇总为完成。本批本地测试不等于实机恢复通过;该校验仍位于文件部署事务之后,不具备据此宣称自动回滚的依据。
|
||||
|
||||
Loading…
Reference in New Issue
Block a user