修复事务上传碰撞误删并记录临时文件归属证据

This commit is contained in:
xuans committed 2026-10-02 13:40:34 +08:00
1 parent 46a9fa0006
commit 3b254496b0
11 files changed
+258 -7

No files matched your search

@@ -27,6 +27,7 @@ type RecoveryEvidence struct {
ValidationCommand []string `json:"validation_command"`
ReloadCommand []string `json:"reload_command"`
Files map[string]string `json:"files"`
TemporaryFiles map[string]string `json:"temporary_files,omitempty"`
Baseline RecoveryBaseline `json:"baseline"`
}
@@ -112,14 +113,21 @@ func validateRecoveryEvidence(record TransactionRecord, server Server) error {
}
locks[lock] = true
}
temporaryPaths := make(map[string]bool)
for _, temporary := range record.TemporaryPaths {
matched := false
for path := range paths {
matched = matched || temporary == path+".scert-tmp-"+record.ID
}
if !matched {
if !matched || temporaryPaths[temporary] {
return errors.New("部署事务临时路径无效")
}
temporaryPaths[temporary] = true
}
for path, digest := range e.TemporaryFiles {
if !temporaryPaths[path] || !validDigest(digest) {
return errors.New("部署事务临时文件归属证据无效")
}
}
seen := make(map[string]bool)
for _, backup := range record.Backups {
@@ -32,6 +32,9 @@ func validateRollbackFiles(ctx context.Context, client Client, record Transactio
if err := validateTransactionLocks(ctx, client, record, false); err != nil {
return err
}
if err := validateTemporaryFiles(ctx, client, record); err != nil {
return err
}
for _, backup := range record.Backups {
backupExists, err := validateRecoveryBackup(ctx, client, backup)
if err != nil {
@@ -25,7 +25,7 @@ func TestRecoveryPreflightOverPasswordSFTP(t *testing.T) {
}
return nil
}, serveTestSFTP, hostSigner)
for _, scenario := range []string{"foreign_lock", "missing_owner", "changed_backup", "changed_target", "valid_restore"} {
for _, scenario := range []string{"foreign_lock", "missing_owner", "changed_backup", "changed_target", "changed_temporary", "unrecorded_temporary", "valid_restore"} {
t.Run(scenario, func(t *testing.T) {
root := t.TempDir()
server, memory, r := recoveryPreflightFixture(root)
@@ -43,6 +43,10 @@ func TestRecoveryPreflightOverPasswordSFTP(t *testing.T) {
memory.files[r.Backups[0].Saved] = memoryFile{data: []byte("unexpected-backup")}
case "changed_target":
memory.files[r.Paths[0]] = memoryFile{data: []byte("unexpected-target")}
case "changed_temporary":
memory.files[r.TemporaryPaths[2]] = memoryFile{data: []byte("unexpected-temporary")}
case "unrecorded_temporary":
delete(r.Evidence.TemporaryFiles, r.TemporaryPaths[2])
}
for path, file := range memory.files {
if strings.HasSuffix(path, ".scert-lock") {
@@ -26,7 +26,7 @@ func recoveryPreflightFixture(roots ...string) (Server, *memoryClient, Transacti
Version: 1, LockToken: contentDigest([]byte("preflight-owner")),
Address: server.Address, Port: server.Port, HostKeyFingerprint: server.HostKeyFingerprint,
ValidationCommand: server.ValidationCommand, ReloadCommand: server.ReloadCommand,
Files: map[string]string{}, Baseline: RecoveryBaseline{
Files: map[string]string{}, TemporaryFiles: map[string]string{}, Baseline: RecoveryBaseline{
Endpoint: "https://example.com", Domains: []string{"example.com"},
PreviousFingerprint: "old", ExpectedFingerprint: "new",
},
@@ -41,6 +41,7 @@ func recoveryPreflightFixture(roots ...string) (Server, *memoryClient, Transacti
Original: path, Saved: backup, OriginalExists: true, OriginalDigest: contentDigest([]byte("old")),
})
r.Evidence.Files[path] = contentDigest([]byte("new"))
r.Evidence.TemporaryFiles[temporary] = contentDigest([]byte("partial"))
client.files[lock] = memoryFile{}
client.files[lock+"/owner"] = memoryFile{data: []byte(r.Evidence.LockToken), mode: 0o600}
client.files[backup] = memoryFile{data: []byte("old"), mode: 0o600}
+15 -4
View File
@@ -368,13 +368,21 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
temporaryPath := file.path + ".scert-tmp-" + backupID
// 上传失败也可能留下半份私钥,必须在开始上传前登记清理路径。
temporary = append(temporary, temporaryPath)
if err := client.Upload(ctx, temporaryPath, file.data, file.mode); err != nil {
uploadErr := client.Upload(ctx, temporaryPath, file.data, file.mode)
if uploadErr != nil {
var notCreated *uploadNotCreatedError
if errors.As(err, &notCreated) {
if errors.As(uploadErr, &notCreated) {
temporary = temporary[:len(temporary)-1]
return failed(fmt.Sprintf("上传失败且未确认临时路径所有权,已保留 %s 供人工核查:%v", temporaryPath, err))
return failed(fmt.Sprintf("上传失败且未确认临时路径所有权,已保留 %s 供人工核查:%v", temporaryPath, uploadErr))
}
return failed(fmt.Sprintf("上传远程证书失败:%v", err))
}
if transactionEnabled {
if err := s.captureUploadEvidence(ctx, server, &client, journalRecord, temporaryPath, file.data, uploadErr); err != nil {
return failed(fmt.Sprintf("保存临时文件归属证据失败:%v", err))
}
}
if uploadErr != nil {
return failed(fmt.Sprintf("上传远程证书失败:%v", uploadErr))
}
}
if err := persistTransaction(ctx, TransactionPhaseUploaded); err != nil {
@@ -699,6 +707,9 @@ func (s *Service) recoverCommittedTransaction(ctx context.Context, record *Trans
return err
}
}
if err := validateTemporaryFiles(ctx, client, *record); err != nil {
return err
}
for _, backup := range record.Backups {
if _, err := validateRecoveryBackup(ctx, client, backup); err != nil {
return err
@@ -69,6 +69,10 @@ func TestRecoverPendingRollsBackOwnedDeployment(t *testing.T) {
Phase: TransactionPhaseReloadAttempted, Status: TransactionStatusPending,
Evidence: &RecoveryEvidence{Version: 1, LockToken: contentDigest([]byte("scert-lock:" + id)), Address: server.Address, Port: server.Port, HostKeyFingerprint: server.HostKeyFingerprint, ValidationCommand: append([]string(nil), server.ValidationCommand...), ReloadCommand: append([]string(nil), server.ReloadCommand...), Files: files, Baseline: RecoveryBaseline{PreviousFingerprint: "old", Endpoint: "https://example.com", Domains: []string{"example.com"}}},
}}}
journal.pending[0].Evidence.TemporaryFiles = make(map[string]string)
for _, path := range temporary {
journal.pending[0].Evidence.TemporaryFiles[path] = contentDigest([]byte("partial"))
}
service := NewServiceWithJournal(memoryFactory{clients: map[string]*memoryClient{server.ID: client}}, journal)
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { return nil }
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err != nil {
@@ -0,0 +1,58 @@
package deployment
import (
"context"
"fmt"
)
// captureUploadEvidence 只记录已确认独占创建的文件,计划路径不构成所有权证据。
// 创建响应丢失、摘要读取失败或证据未落库时,重启恢复必须保留未知文件。
func (s *Service) captureUploadEvidence(ctx context.Context, server Server, client *Client, record *TransactionRecord, path string, data []byte, uploadErr error) error {
evidenceContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
defer cancel()
digest := contentDigest(data)
if uploadErr != nil {
if err := reconnectDeploymentClient(evidenceContext, s.factory, server, client); err != nil {
return fmt.Errorf("读取部分上传证据前重连失败:%w", err)
}
reader, ok := (*client).(EvidenceClient)
if !ok {
return fmt.Errorf("客户端无法核验部分上传文件")
}
var err error
digest, err = reader.FileDigest(evidenceContext, path)
if err != nil {
return fmt.Errorf("读取部分上传摘要失败:%w", err)
}
if !validDigest(digest) {
return fmt.Errorf("部分上传摘要无效")
}
}
if record.Evidence.TemporaryFiles == nil {
record.Evidence.TemporaryFiles = make(map[string]string)
}
record.Evidence.TemporaryFiles[path] = digest
record.UpdatedAt = s.now().UTC()
return s.journal.UpdateDeploymentTransaction(evidenceContext, *record)
}
// validateTemporaryFiles 在删除任何临时文件前完整检查实际归属及内容身份。
// 旧日志只有计划路径而没有创建证据时,不自动删除仍存在的文件。
func validateTemporaryFiles(ctx context.Context, client Client, record TransactionRecord) error {
for _, path := range record.TemporaryPaths {
exists, err := client.Exists(ctx, path)
if err != nil {
return fmt.Errorf("检查临时文件失败:%w", err)
}
if !exists {
continue
}
if record.Evidence == nil || !validDigest(record.Evidence.TemporaryFiles[path]) {
return fmt.Errorf("无法证明临时文件归属,已保留现场:%s", path)
}
if err := checkFileDigest(ctx, client, path, record.Evidence.TemporaryFiles[path]); err != nil {
return fmt.Errorf("临时文件身份已变更,已保留现场:%w", err)
}
}
return nil
}
@@ -0,0 +1,137 @@
package deployment
import (
"context"
"errors"
"maps"
"reflect"
"strconv"
"testing"
)
type journalCollisionClient struct {
*memoryClient
collisionAt int
calls int
collided string
}
func (c *journalCollisionClient) Upload(ctx context.Context, path string, data []byte, mode uint32) error {
c.calls++
if c.calls == c.collisionAt {
c.collided = path
c.files[path] = memoryFile{data: []byte("not-owned"), mode: 0o600}
return &uploadNotCreatedError{err: errors.New("模拟独占创建冲突")}
}
return c.memoryClient.Upload(ctx, path, data, mode)
}
func TestJournalUploadCollisionPreservesUnownedFile(t *testing.T) {
for _, position := range []int{1, 3} {
t.Run(strconv.Itoa(position), func(t *testing.T) {
client := &journalCollisionClient{memoryClient: newMemoryClient(), collisionAt: position}
server := testServer("journal-collision")
client.files[server.CertificatePath] = memoryFile{data: []byte("current"), mode: 0o600}
journal := &memoryTransactionJournal{}
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
ctx := WithTransactionContext(t.Context(), TransactionContext{TaskID: "task", AttemptID: "attempt", AttemptNumber: 1})
report := service.Deploy(ctx, []Target{{Server: server}}, testArtifact(t))
if report.Status != "failed" || client.collided == "" {
t.Fatalf("必须触发上传碰撞:%+v", report)
}
if string(client.files[client.collided].data) != "not-owned" {
t.Error("事务日志不得把计划路径当作已取得所有权的临时文件删除")
}
if string(client.files[server.CertificatePath].data) != "current" || len(client.commands) != 0 {
t.Fatal("上传碰撞不得改变当前证书或执行命令")
}
if len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRecoveryRequired {
t.Fatal("碰撞路径归属未知,必须保留锁及事务等待核查")
}
for _, target := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
if _, exists := client.files[target+".scert-lock"]; !exists {
t.Fatal("未知文件仍存在时不得自动释放部署锁")
}
}
})
}
}
func TestRecoveryRejectsUnprovenTemporaryFileBeforeMutation(t *testing.T) {
for _, scenario := range []string{"changed", "unrecorded", "commit_changed", "legacy", "legacy_commit"} {
t.Run(scenario, func(t *testing.T) {
server, client, record := recoveryPreflightFixture()
switch scenario {
case "legacy", "legacy_commit":
record.Evidence.TemporaryFiles = nil
case "unrecorded":
delete(record.Evidence.TemporaryFiles, record.TemporaryPaths[2])
}
if scenario == "commit_changed" || scenario == "legacy_commit" {
record.Phase = TransactionPhaseCommitting
}
client.files[record.TemporaryPaths[2]] = memoryFile{data: []byte("foreign-file"), mode: 0o600}
journal := &memoryTransactionJournal{pending: []TransactionRecord{record}}
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { return nil }
before := maps.Clone(client.files)
err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil })
if err == nil || !reflect.DeepEqual(client.files, before) || len(client.commands) != 0 {
t.Fatal("无法证明临时文件身份时必须保留完整现场,不得先删除其他文件")
}
})
}
}
func TestJournalPartialUploadCleansOnlyProvenFiles(t *testing.T) {
client := newMemoryClient()
client.uploadError = "key.pem"
server := testServer("journal-partial")
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
client.files[path] = memoryFile{data: []byte("original"), mode: 0o600}
}
before := maps.Clone(client.files)
journal := &memoryTransactionJournal{}
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
ctx := WithTransactionContext(t.Context(), TransactionContext{TaskID: "task", AttemptID: "attempt", AttemptNumber: 1})
report := service.Deploy(ctx, []Target{{Server: server}}, testArtifact(t))
if report.Status != "failed" || !reflect.DeepEqual(client.files, before) || len(client.commands) != 0 {
t.Fatalf("已取得归属的半份私钥应可安全清理,且不改变旧证书:%+v", report)
}
if len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRolledBack {
t.Fatal("已证明归属的上传失败应完成回滚")
}
}
func TestRecoveryLegacyWithoutTemporaryFilesStillRestores(t *testing.T) {
server, client, record := recoveryPreflightFixture()
record.Evidence.TemporaryFiles = nil
for _, path := range record.TemporaryPaths {
delete(client.files, path)
}
journal := &memoryTransactionJournal{pending: []TransactionRecord{record}}
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { return nil }
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err != nil {
t.Fatalf("没有残留临时文件且其他证据完整的旧事务仍应恢复:%v", err)
}
}
func TestRecoveryRejectsMalformedTemporaryOwnershipBeforeAuthentication(t *testing.T) {
for _, scenario := range []string{"foreign_path", "invalid_digest", "duplicate_path"} {
t.Run(scenario, func(t *testing.T) {
server, _, record := recoveryPreflightFixture()
switch scenario {
case "foreign_path":
record.Evidence.TemporaryFiles["/foreign/file"] = contentDigest([]byte("foreign"))
case "invalid_digest":
record.Evidence.TemporaryFiles[record.TemporaryPaths[0]] = "invalid"
case "duplicate_path":
record.TemporaryPaths = append(record.TemporaryPaths, record.TemporaryPaths[0])
}
if err := validateRecoveryEvidence(record, server); err == nil {
t.Fatal("认证前必须拒绝无效临时文件证据")
}
})
}
}