修复配置事务准备残留阻塞启动并完成崩溃恢复实机回归

This commit is contained in:
xuans committed 2026-09-22 00:09:13 +08:00
1 parent a55e2b68eb
commit ccc1d5af03
5 files changed
+297 -11

No files matched your search

+2 -2
View File
@@ -132,7 +132,7 @@ func TestUnavailableSelectedDatabaseDoesNotCreateSQLite(t *testing.T) {
t.Fatal(err)
}
for _, entry := range entries {
if entry.Name() != ".config-state.lock" && entry.Name() != "transaction" {
if entry.Name() != ".config-state.lock" && entry.Name() != "transaction" && entry.Name() != "transaction-staging" {
t.Fatalf("远程数据库失败后创建了额外文件:%s", entry.Name())
}
}
@@ -263,7 +263,7 @@ func TestSelectedDatabaseLiveIntegration(t *testing.T) {
t.Fatal(err)
}
for _, entry := range entries {
if entry.Name() == ".config-state.lock" || entry.Name() == "transaction" || kind == DatabaseTypeSQLite && entry.Name() == "smail.db" {
if entry.Name() == ".config-state.lock" || entry.Name() == "transaction" || entry.Name() == "transaction-staging" || kind == DatabaseTypeSQLite && entry.Name() == "smail.db" {
continue
}
t.Fatalf("真实数据库模式产生额外数据库:%s", entry.Name())
+57 -1
View File
@@ -41,6 +41,9 @@ type TransactionManifest struct {
type TransactionHooks struct {
BeforePhase func(TransactionPhase) error
// 准备钩子仅供内部测试使用,不通过运行配置或网络请求开放。
BeforePrepare func() error
BeforePublish func() error
}
type TransactionStore struct {
@@ -119,11 +122,16 @@ func (s *TransactionStore) Commit(current, candidate RuntimeConfig) error {
if err != nil {
return err
}
staging, err := secureTransactionStagingDirectory(transactions)
if err != nil {
return err
}
id, err := newTransactionID()
if err != nil {
return err
}
transactionDirectory := filepath.Join(transactions, id)
// 清单及归档完整前,不得把目录暴露给启动恢复扫描。
transactionDirectory := filepath.Join(staging, id)
candidateDirectory := filepath.Join(transactionDirectory, "candidate")
originalDirectory := filepath.Join(transactionDirectory, "original")
manifestWritten := false
@@ -141,6 +149,11 @@ func (s *TransactionStore) Commit(current, candidate RuntimeConfig) error {
if err := validateTransactionDirectory(transactionDirectory); err != nil {
return err
}
if s.hooks.BeforePrepare != nil {
if err := s.hooks.BeforePrepare(); err != nil {
return err
}
}
for name, data := range currentFiles {
if err := atomicfile.Write(filepath.Join(originalDirectory, name), data, 0600); err != nil {
return err
@@ -164,7 +177,36 @@ func (s *TransactionStore) Commit(current, candidate RuntimeConfig) error {
if err := writeTransactionManifest(transactionDirectory, manifest); err != nil {
return err
}
if s.hooks.BeforePublish != nil {
if err := s.hooks.BeforePublish(); err != nil {
return err
}
}
if err := atomicfile.SyncDirectory(transactionDirectory); err != nil {
return err
}
if err := atomicfile.SyncDirectory(staging); err != nil {
return err
}
activeDirectory := filepath.Join(transactions, id)
if _, err := os.Lstat(activeDirectory); err == nil {
return ErrConfigRecoveryRequired
} else if !os.IsNotExist(err) {
return err
}
if err := os.Rename(transactionDirectory, activeDirectory); err != nil {
return err
}
transactionDirectory = activeDirectory
candidateDirectory = filepath.Join(transactionDirectory, "candidate")
// 发布后即使目录同步失败也必须保留清单,由启动恢复决定回滚或保留。
manifestWritten = true
if err := atomicfile.SyncDirectory(transactions); err != nil {
return err
}
if err := atomicfile.SyncDirectory(staging); err != nil {
return err
}
if err := s.beforePhase(manifest.Phase); err != nil {
return err
}
@@ -563,6 +605,20 @@ func secureTransactionDirectory(configDirectory string) (string, error) {
return directory, nil
}
func secureTransactionStagingDirectory(transactions string) (string, error) {
directory := filepath.Join(filepath.Dir(transactions), "transaction-staging")
if err := os.MkdirAll(directory, 0700); err != nil {
return "", err
}
if err := ensureDirectory(directory); err != nil {
return "", err
}
if err := atomicfile.SyncDirectory(filepath.Dir(transactions)); err != nil {
return "", err
}
return directory, nil
}
func validateTransactionDirectory(directory string) error {
if err := ensureDirectory(directory); err != nil {
return err
+97 -8
View File
@@ -8,6 +8,7 @@ import (
"os/exec"
"path/filepath"
"reflect"
"runtime"
"testing"
"time"
)
@@ -58,7 +59,28 @@ func TestTransactionCrashChild(t *testing.T) {
if point == "root_before_manifest" {
phase = TransactionSectionsCommitted
}
pause := func() error {
if err := os.Mkdir(filepath.Join(root, "crash-ready"), 0700); err != nil {
return err
}
// 必须等待父进程强制终止,不能返回错误而执行提交函数的 defer。
for {
time.Sleep(time.Hour)
}
}
store := NewTransactionStore(root, crashOptions().Validation).WithHooks(TransactionHooks{
BeforePrepare: func() error {
if point == "before_prepare" {
return pause()
}
return nil
},
BeforePublish: func() error {
if point == "before_publish" {
return pause()
}
return nil
},
BeforePhase: func(reached TransactionPhase) error {
if reached != phase {
return nil
@@ -74,13 +96,7 @@ func TestTransactionCrashChild(t *testing.T) {
return err
}
}
if err := os.Mkdir(filepath.Join(root, "crash-ready"), 0700); err != nil {
return err
}
// 必须等待父进程强制终止,不能返回错误而执行提交函数的 defer。
for {
time.Sleep(time.Hour)
}
return pause()
},
})
if err := store.Commit(current, crashCandidate(current)); err != nil {
@@ -251,7 +267,7 @@ func TestTransactionProcessCrashRecovery(t *testing.T) {
t.Fatal(err)
}
for _, entry := range entries {
if entry.Name() != ".config-state.lock" && entry.Name() != "transaction" {
if entry.Name() != ".config-state.lock" && entry.Name() != "transaction" && entry.Name() != "transaction-staging" {
t.Fatalf("纯配置恢复意外创建数据文件:%s", entry.Name())
}
}
@@ -306,3 +322,76 @@ func TestTransactionUnpublishedDirectoryFailsClosed(t *testing.T) {
}
}
}
func TestTransactionPreparationProcessCrashRecovery(t *testing.T) {
for _, kind := range []DatabaseType{DatabaseTypeSQLite, DatabaseTypeMySQL, DatabaseTypePostgreSQL} {
t.Run(string(kind), func(t *testing.T) {
for _, point := range []string{"before_prepare", "before_publish"} {
t.Run(point, func(t *testing.T) {
root, current := crashFixture(t, kind)
files := readCrashConfigFiles(t, filepath.Join(root, "config"))
killTransactionChild(t, root, point)
active, err := os.ReadDir(filepath.Join(root, "data", "transaction"))
if err != nil || len(active) != 0 {
t.Fatalf("准备中的事务被提前发布:%v", err)
}
base := filepath.Join(root, "data", "transaction-staging")
staged, err := os.ReadDir(base)
if err != nil || len(staged) != 1 {
t.Fatalf("没有保留强制终止时的暂存现场:%v", err)
}
stagedDirectory := filepath.Join(base, staged[0].Name())
if point == "before_publish" {
manifest, err := readTransactionManifest(stagedDirectory)
if err != nil || manifest.Phase != TransactionPrepared {
t.Fatalf("发布前清单不完整:%v", err)
}
} else if _, err := os.Stat(filepath.Join(stagedDirectory, "manifest.json")); !os.IsNotExist(err) {
t.Fatalf("准备开始前不应有事务清单:%v", err)
}
for attempt := 0; attempt < 2; attempt++ {
manager, err := NewConfigManager(root, crashOptions())
if err != nil || manager.Snapshot().Generation != current.Generation {
t.Fatalf("准备阶段崩溃后无法启动原配置:%v", err)
}
assertCrashConfigFiles(t, root, files)
}
// 旧暂存仅保留为证据,不得阻塞后续正式配置提交。
store := NewTransactionStore(root, crashOptions().Validation)
if err := store.Commit(current, crashCandidate(current)); err != nil {
t.Fatalf("准备残留阻塞后续提交:%v", err)
}
manager, err := NewConfigManager(root, crashOptions())
if err != nil || manager.Snapshot().Generation != current.Generation+1 {
t.Fatalf("后续提交未能正常启动:%v", err)
}
staged, err = os.ReadDir(base)
if err != nil || len(staged) != 1 || filepath.Join(base, staged[0].Name()) != stagedDirectory {
t.Fatal("后续提交删除旧证据或遗留了新的暂存目录")
}
})
}
})
}
}
func TestTransactionStagingRejectsSymlink(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("Windows 符号链接需要额外权限;在 Linux 云端验证")
}
root, current := crashFixture(t, DatabaseTypeSQLite)
files := readCrashConfigFiles(t, filepath.Join(root, "config"))
outside := t.TempDir()
if err := os.Symlink(outside, filepath.Join(root, "data", "transaction-staging")); err != nil {
t.Fatal(err)
}
store := NewTransactionStore(root, crashOptions().Validation)
if err := store.Commit(current, crashCandidate(current)); !errors.Is(err, ErrTransactionInvalid) {
t.Fatalf("未拒绝暂存目录符号链接:%v", err)
}
entries, err := os.ReadDir(outside)
if err != nil || len(entries) != 0 {
t.Fatal("事务写入逃逸到外部目录")
}
assertCrashConfigFiles(t, root, files)
}