修复配置事务准备残留阻塞启动并完成崩溃恢复实机回归
This commit is contained in:
1 parent
a55e2b68eb
commit
ccc1d5af03
5 files changed
+297
-11
No files matched your search
@@ -132,7 +132,7 @@ func TestUnavailableSelectedDatabaseDoesNotCreateSQLite(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.Name() != ".config-state.lock" && entry.Name() != "transaction" {
|
||||
if entry.Name() != ".config-state.lock" && entry.Name() != "transaction" && entry.Name() != "transaction-staging" {
|
||||
t.Fatalf("远程数据库失败后创建了额外文件:%s", entry.Name())
|
||||
}
|
||||
}
|
||||
@@ -263,7 +263,7 @@ func TestSelectedDatabaseLiveIntegration(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.Name() == ".config-state.lock" || entry.Name() == "transaction" || kind == DatabaseTypeSQLite && entry.Name() == "smail.db" {
|
||||
if entry.Name() == ".config-state.lock" || entry.Name() == "transaction" || entry.Name() == "transaction-staging" || kind == DatabaseTypeSQLite && entry.Name() == "smail.db" {
|
||||
continue
|
||||
}
|
||||
t.Fatalf("真实数据库模式产生额外数据库:%s", entry.Name())
|
||||
|
||||
@@ -41,6 +41,9 @@ type TransactionManifest struct {
|
||||
|
||||
type TransactionHooks struct {
|
||||
BeforePhase func(TransactionPhase) error
|
||||
// 准备钩子仅供内部测试使用,不通过运行配置或网络请求开放。
|
||||
BeforePrepare func() error
|
||||
BeforePublish func() error
|
||||
}
|
||||
|
||||
type TransactionStore struct {
|
||||
@@ -119,11 +122,16 @@ func (s *TransactionStore) Commit(current, candidate RuntimeConfig) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
staging, err := secureTransactionStagingDirectory(transactions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
id, err := newTransactionID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
transactionDirectory := filepath.Join(transactions, id)
|
||||
// 清单及归档完整前,不得把目录暴露给启动恢复扫描。
|
||||
transactionDirectory := filepath.Join(staging, id)
|
||||
candidateDirectory := filepath.Join(transactionDirectory, "candidate")
|
||||
originalDirectory := filepath.Join(transactionDirectory, "original")
|
||||
manifestWritten := false
|
||||
@@ -141,6 +149,11 @@ func (s *TransactionStore) Commit(current, candidate RuntimeConfig) error {
|
||||
if err := validateTransactionDirectory(transactionDirectory); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.hooks.BeforePrepare != nil {
|
||||
if err := s.hooks.BeforePrepare(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for name, data := range currentFiles {
|
||||
if err := atomicfile.Write(filepath.Join(originalDirectory, name), data, 0600); err != nil {
|
||||
return err
|
||||
@@ -164,7 +177,36 @@ func (s *TransactionStore) Commit(current, candidate RuntimeConfig) error {
|
||||
if err := writeTransactionManifest(transactionDirectory, manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.hooks.BeforePublish != nil {
|
||||
if err := s.hooks.BeforePublish(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := atomicfile.SyncDirectory(transactionDirectory); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := atomicfile.SyncDirectory(staging); err != nil {
|
||||
return err
|
||||
}
|
||||
activeDirectory := filepath.Join(transactions, id)
|
||||
if _, err := os.Lstat(activeDirectory); err == nil {
|
||||
return ErrConfigRecoveryRequired
|
||||
} else if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(transactionDirectory, activeDirectory); err != nil {
|
||||
return err
|
||||
}
|
||||
transactionDirectory = activeDirectory
|
||||
candidateDirectory = filepath.Join(transactionDirectory, "candidate")
|
||||
// 发布后即使目录同步失败也必须保留清单,由启动恢复决定回滚或保留。
|
||||
manifestWritten = true
|
||||
if err := atomicfile.SyncDirectory(transactions); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := atomicfile.SyncDirectory(staging); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.beforePhase(manifest.Phase); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -563,6 +605,20 @@ func secureTransactionDirectory(configDirectory string) (string, error) {
|
||||
return directory, nil
|
||||
}
|
||||
|
||||
func secureTransactionStagingDirectory(transactions string) (string, error) {
|
||||
directory := filepath.Join(filepath.Dir(transactions), "transaction-staging")
|
||||
if err := os.MkdirAll(directory, 0700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := ensureDirectory(directory); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := atomicfile.SyncDirectory(filepath.Dir(transactions)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return directory, nil
|
||||
}
|
||||
|
||||
func validateTransactionDirectory(directory string) error {
|
||||
if err := ensureDirectory(directory); err != nil {
|
||||
return err
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -58,7 +59,28 @@ func TestTransactionCrashChild(t *testing.T) {
|
||||
if point == "root_before_manifest" {
|
||||
phase = TransactionSectionsCommitted
|
||||
}
|
||||
pause := func() error {
|
||||
if err := os.Mkdir(filepath.Join(root, "crash-ready"), 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
// 必须等待父进程强制终止,不能返回错误而执行提交函数的 defer。
|
||||
for {
|
||||
time.Sleep(time.Hour)
|
||||
}
|
||||
}
|
||||
store := NewTransactionStore(root, crashOptions().Validation).WithHooks(TransactionHooks{
|
||||
BeforePrepare: func() error {
|
||||
if point == "before_prepare" {
|
||||
return pause()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
BeforePublish: func() error {
|
||||
if point == "before_publish" {
|
||||
return pause()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
BeforePhase: func(reached TransactionPhase) error {
|
||||
if reached != phase {
|
||||
return nil
|
||||
@@ -74,13 +96,7 @@ func TestTransactionCrashChild(t *testing.T) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := os.Mkdir(filepath.Join(root, "crash-ready"), 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
// 必须等待父进程强制终止,不能返回错误而执行提交函数的 defer。
|
||||
for {
|
||||
time.Sleep(time.Hour)
|
||||
}
|
||||
return pause()
|
||||
},
|
||||
})
|
||||
if err := store.Commit(current, crashCandidate(current)); err != nil {
|
||||
@@ -251,7 +267,7 @@ func TestTransactionProcessCrashRecovery(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.Name() != ".config-state.lock" && entry.Name() != "transaction" {
|
||||
if entry.Name() != ".config-state.lock" && entry.Name() != "transaction" && entry.Name() != "transaction-staging" {
|
||||
t.Fatalf("纯配置恢复意外创建数据文件:%s", entry.Name())
|
||||
}
|
||||
}
|
||||
@@ -306,3 +322,76 @@ func TestTransactionUnpublishedDirectoryFailsClosed(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransactionPreparationProcessCrashRecovery(t *testing.T) {
|
||||
for _, kind := range []DatabaseType{DatabaseTypeSQLite, DatabaseTypeMySQL, DatabaseTypePostgreSQL} {
|
||||
t.Run(string(kind), func(t *testing.T) {
|
||||
for _, point := range []string{"before_prepare", "before_publish"} {
|
||||
t.Run(point, func(t *testing.T) {
|
||||
root, current := crashFixture(t, kind)
|
||||
files := readCrashConfigFiles(t, filepath.Join(root, "config"))
|
||||
killTransactionChild(t, root, point)
|
||||
active, err := os.ReadDir(filepath.Join(root, "data", "transaction"))
|
||||
if err != nil || len(active) != 0 {
|
||||
t.Fatalf("准备中的事务被提前发布:%v", err)
|
||||
}
|
||||
base := filepath.Join(root, "data", "transaction-staging")
|
||||
staged, err := os.ReadDir(base)
|
||||
if err != nil || len(staged) != 1 {
|
||||
t.Fatalf("没有保留强制终止时的暂存现场:%v", err)
|
||||
}
|
||||
stagedDirectory := filepath.Join(base, staged[0].Name())
|
||||
if point == "before_publish" {
|
||||
manifest, err := readTransactionManifest(stagedDirectory)
|
||||
if err != nil || manifest.Phase != TransactionPrepared {
|
||||
t.Fatalf("发布前清单不完整:%v", err)
|
||||
}
|
||||
} else if _, err := os.Stat(filepath.Join(stagedDirectory, "manifest.json")); !os.IsNotExist(err) {
|
||||
t.Fatalf("准备开始前不应有事务清单:%v", err)
|
||||
}
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
manager, err := NewConfigManager(root, crashOptions())
|
||||
if err != nil || manager.Snapshot().Generation != current.Generation {
|
||||
t.Fatalf("准备阶段崩溃后无法启动原配置:%v", err)
|
||||
}
|
||||
assertCrashConfigFiles(t, root, files)
|
||||
}
|
||||
// 旧暂存仅保留为证据,不得阻塞后续正式配置提交。
|
||||
store := NewTransactionStore(root, crashOptions().Validation)
|
||||
if err := store.Commit(current, crashCandidate(current)); err != nil {
|
||||
t.Fatalf("准备残留阻塞后续提交:%v", err)
|
||||
}
|
||||
manager, err := NewConfigManager(root, crashOptions())
|
||||
if err != nil || manager.Snapshot().Generation != current.Generation+1 {
|
||||
t.Fatalf("后续提交未能正常启动:%v", err)
|
||||
}
|
||||
staged, err = os.ReadDir(base)
|
||||
if err != nil || len(staged) != 1 || filepath.Join(base, staged[0].Name()) != stagedDirectory {
|
||||
t.Fatal("后续提交删除旧证据或遗留了新的暂存目录")
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransactionStagingRejectsSymlink(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Windows 符号链接需要额外权限;在 Linux 云端验证")
|
||||
}
|
||||
root, current := crashFixture(t, DatabaseTypeSQLite)
|
||||
files := readCrashConfigFiles(t, filepath.Join(root, "config"))
|
||||
outside := t.TempDir()
|
||||
if err := os.Symlink(outside, filepath.Join(root, "data", "transaction-staging")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := NewTransactionStore(root, crashOptions().Validation)
|
||||
if err := store.Commit(current, crashCandidate(current)); !errors.Is(err, ErrTransactionInvalid) {
|
||||
t.Fatalf("未拒绝暂存目录符号链接:%v", err)
|
||||
}
|
||||
entries, err := os.ReadDir(outside)
|
||||
if err != nil || len(entries) != 0 {
|
||||
t.Fatal("事务写入逃逸到外部目录")
|
||||
}
|
||||
assertCrashConfigFiles(t, root, files)
|
||||
}
|
||||
Reference in new issue
Block a user