From b8dbd3bb2b36a08d7017eed1652fcfeec0c5df23 Mon Sep 17 00:00:00 2001 From: smail Date: Mon, 21 Sep 2026 23:50:27 +0800 Subject: [PATCH] =?UTF-8?q?=E8=A1=A5=E9=BD=90=E9=85=8D=E7=BD=AE=E5=88=87?= =?UTF-8?q?=E6=8D=A2=E7=9C=9F=E5=AE=9E=E8=BF=9B=E7=A8=8B=E7=BB=88=E6=AD=A2?= =?UTF-8?q?=E4=B8=8E=E6=8D=9F=E5=9D=8F=E6=8B=92=E7=BB=9D=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/config/transaction_crash_test.go | 308 +++++++++++++++++++++++ 1 file changed, 308 insertions(+) create mode 100644 backend/config/transaction_crash_test.go diff --git a/backend/config/transaction_crash_test.go b/backend/config/transaction_crash_test.go new file mode 100644 index 0000000..d5455a1 --- /dev/null +++ b/backend/config/transaction_crash_test.go @@ -0,0 +1,308 @@ +package config + +import ( + "bytes" + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "reflect" + "testing" + "time" +) + +const transactionCrashRoot = "SMAIL_TEST_TRANSACTION_CRASH_ROOT" +const transactionCrashPoint = "SMAIL_TEST_TRANSACTION_CRASH_POINT" + +func crashOptions() ConfigLoaderOptions { + return ConfigLoaderOptions{Validation: RuntimeValidationOptions{HasSuperAdmin: true, Production: true}} +} + +func crashCandidate(current RuntimeConfig) RuntimeConfig { + candidate := cloneRuntimeConfig(current) + setCandidateGeneration(&candidate, current.Generation+1) + candidate.Email.Policy.MaxMessageBytes = 9 << 20 + if candidate.Root.Database.Type == DatabaseTypeSQLite { + candidate.Root.Database.LocalRef = "next.db" + } else { + candidate.Root.Database.Name = "smail_next" + } + return candidate +} + +func crashTransactionDirectory(t *testing.T, root string) string { + t.Helper() + base := filepath.Join(root, "data", "transaction") + entries, err := os.ReadDir(base) + if err != nil || len(entries) != 1 { + t.Fatalf("事务目录数量异常：%d，%v", len(entries), err) + } + return filepath.Join(base, entries[0].Name()) +} + +// TestTransactionCrashChild 只在测试二进制中运行，不向正式程序添加故障注入入口。 +func TestTransactionCrashChild(t *testing.T) { + root, point := os.Getenv(transactionCrashRoot), os.Getenv(transactionCrashPoint) + if root == "" { + return + } + current, err := NewConfigLoader(root, crashOptions()).Load() + if err != nil { + t.Fatal(err) + } + phase := TransactionPhase(point) + if point == "partial_sections" { + phase = TransactionPrepared + } + if point == "root_before_manifest" { + phase = TransactionSectionsCommitted + } + store := NewTransactionStore(root, crashOptions().Validation).WithHooks(TransactionHooks{ + BeforePhase: func(reached TransactionPhase) error { + if reached != phase { + return nil + } + // 复用正式原子替换函数构造阶段内部的落盘状态；不声称命中了实际指令间隙。 + if point == "partial_sections" || point == "root_before_manifest" { + name := "config.json" + if point == "partial_sections" { + name, _ = FixedSectionFile(requiredPartitionSections[0]) + } + source := filepath.Join(crashTransactionDirectory(t, root), "candidate", name) + if err := replaceLiveConfigFile(filepath.Join(root, "config"), source, name); err != nil { + return err + } + } + if err := os.Mkdir(filepath.Join(root, "crash-ready"), 0700); err != nil { + return err + } + // 必须等待父进程强制终止，不能返回错误而执行提交函数的 defer。 + for { + time.Sleep(time.Hour) + } + }, + }) + if err := store.Commit(current, crashCandidate(current)); err != nil { + t.Fatal(err) + } + t.Fatal("子进程没有停在要求的提交阶段") +} + +func killTransactionChild(t *testing.T, root, point string) { + t.Helper() + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + command := exec.CommandContext(ctx, executable, "-test.run=^TestTransactionCrashChild$", "-test.timeout=45s") + command.Env = append(os.Environ(), transactionCrashRoot+"="+root, transactionCrashPoint+"="+point) + var output bytes.Buffer + command.Stdout, command.Stderr = &output, &output + if err := command.Start(); err != nil { + t.Fatal(err) + } + wait := make(chan error, 1) + go func() { wait <- command.Wait() }() + finished := false + defer func() { + if !finished { + _ = command.Process.Kill() + <-wait + } + }() + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + select { + case err := <-wait: + finished = true + t.Fatalf("到达故障点前子进程退出：%v\n%s", err, output.String()) + case <-ctx.Done(): + t.Fatal("等待事务故障点超时") + case <-ticker.C: + ready, err := os.Stat(filepath.Join(root, "crash-ready")) + if os.IsNotExist(err) { + continue + } + if err != nil || !ready.IsDir() { + t.Fatalf("故障点确认失败：%v", err) + } + if err := command.Process.Kill(); err != nil { + t.Fatal(err) + } + err = <-wait + finished = true + var exitError *exec.ExitError + if !errors.As(err, &exitError) || command.ProcessState.Success() { + t.Fatalf("子进程不是强制终止：%v", err) + } + t.Logf("已确认阶段 %s，子进程 %d，退出状态 %s", point, command.Process.Pid, command.ProcessState) + return + } + } +} + +func crashFixture(t *testing.T, kind DatabaseType) (string, RuntimeConfig) { + t.Helper() + root := t.TempDir() + if err := InitializeDeployment(root); err != nil { + t.Fatal(err) + } + current := validRuntimeConfig() + if kind != DatabaseTypeSQLite { + current = validMySQLRuntimeConfig(t, root) + current.Root.Database.Type = kind + // 本专项只验证配置落盘恢复，不访问共享数据库或创建替代 SQLite。 + current.Root.Database.Port = 1 + } + writeRuntimeFixture(t, root, current) + loaded, err := NewConfigLoader(root, crashOptions()).Load() + if err != nil { + t.Fatal(err) + } + current = loaded + return root, current +} + +func readCrashConfigFiles(t *testing.T, directory string) map[string][]byte { + t.Helper() + files := make(map[string][]byte) + for _, name := range transactionFileNames() { + data, err := os.ReadFile(filepath.Join(directory, name)) + if err != nil { + t.Fatal(err) + } + files[name] = data + } + return files +} + +func assertCrashConfigFiles(t *testing.T, root string, expected map[string][]byte) { + t.Helper() + for name, want := range expected { + got, err := os.ReadFile(filepath.Join(root, "config", name)) + if err != nil || !bytes.Equal(got, want) { + t.Fatalf("恢复后的配置文件不匹配：%s，%v", name, err) + } + } +} + +func TestTransactionProcessCrashRecovery(t *testing.T) { + cases := []struct { + point string + phase TransactionPhase + new bool + }{ + {"prepared", TransactionPrepared, false}, + {"partial_sections", TransactionPrepared, false}, + {"sections_committed", TransactionSectionsCommitted, false}, + {"root_before_manifest", TransactionSectionsCommitted, true}, + {"root_committed", TransactionRootCommitted, true}, + {"verified", TransactionVerified, true}, + } + for _, kind := range []DatabaseType{DatabaseTypeSQLite, DatabaseTypeMySQL, DatabaseTypePostgreSQL} { + t.Run(string(kind), func(t *testing.T) { + for _, test := range cases { + t.Run(test.point, func(t *testing.T) { + root, current := crashFixture(t, kind) + killTransactionChild(t, root, test.point) + manifest, err := readTransactionManifest(crashTransactionDirectory(t, root)) + if err != nil || manifest.Phase != test.phase { + t.Fatalf("强制退出后事务清单状态异常：%v", err) + } + before, loadErr := NewConfigLoader(root, crashOptions()).Load() + partial := test.point == "partial_sections" || test.point == "sections_committed" + if partial && loadErr == nil { + t.Fatal("尚未恢复的不完整配置被错误接受") + } + want := current + if test.new { + want = crashCandidate(current) + } + part := "original" + if test.new { + part = "candidate" + } + files := readCrashConfigFiles(t, filepath.Join(crashTransactionDirectory(t, root), part)) + if !partial && (loadErr != nil || before.Generation != want.Generation) { + t.Fatalf("恢复前完整配置代次异常：%v", loadErr) + } + // 两次走正式启动恢复路径，同时验证锁释放和恢复幂等。 + for attempt := 0; attempt < 2; attempt++ { + manager, err := NewConfigManager(root, crashOptions()) + if err != nil { + t.Fatalf("第 %d 次启动恢复失败：%v", attempt+1, err) + } + got := manager.Snapshot() + if got.Generation != want.Generation || !reflect.DeepEqual(got.Root.Database, want.Root.Database) { + t.Fatal("启动后选中的数据库或配置代次不匹配") + } + assertCrashConfigFiles(t, root, files) + } + entries, err := os.ReadDir(filepath.Join(root, "data", "transaction")) + if err != nil || len(entries) != 0 { + t.Fatalf("恢复后仍有活动事务：%v", err) + } + entries, err = os.ReadDir(filepath.Join(root, "data")) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.Name() != ".config-state.lock" && entry.Name() != "transaction" { + t.Fatalf("纯配置恢复意外创建数据文件：%s", entry.Name()) + } + } + }) + } + }) + } +} + +func TestTransactionProcessCrashRejectsTamperedRecovery(t *testing.T) { + for _, part := range []string{"original", "candidate"} { + t.Run(part, func(t *testing.T) { + root, _ := crashFixture(t, DatabaseTypeSQLite) + files := readCrashConfigFiles(t, filepath.Join(root, "config")) + killTransactionChild(t, root, "prepared") + directory := crashTransactionDirectory(t, root) + path := filepath.Join(directory, part, "config.json") + if err := os.WriteFile(path, []byte("{}"), 0600); err != nil { + t.Fatal(err) + } + for attempt := 0; attempt < 2; attempt++ { + if _, err := NewConfigManager(root, crashOptions()); !errors.Is(err, ErrConfigDigestMismatch) { + t.Fatalf("损坏事务没有失败关闭：%v", err) + } + assertCrashConfigFiles(t, root, files) + if _, err := os.Stat(path); err != nil { + t.Fatal("失败关闭不应删除损坏证据") + } + } + }) + } +} + +// TestTransactionUnpublishedDirectoryFailsClosed 复现首份清单发布前的磁盘状态。 +// 无法证明来源的残留必须保留并阻止启动，不能把安全拒绝报告成自动恢复成功。 +func TestTransactionUnpublishedDirectoryFailsClosed(t *testing.T) { + root, _ := crashFixture(t, DatabaseTypeSQLite) + files := readCrashConfigFiles(t, filepath.Join(root, "config")) + directory := filepath.Join(root, "data", "transaction", "0123456789abcdef0123456789abcdef") + for _, part := range []string{"original", "candidate"} { + if err := os.MkdirAll(filepath.Join(directory, part), 0700); err != nil { + t.Fatal(err) + } + } + for attempt := 0; attempt < 2; attempt++ { + if _, err := NewConfigManager(root, crashOptions()); !errors.Is(err, ErrTransactionInvalid) { + t.Fatalf("无清单残留没有阻止启动：%v", err) + } + assertCrashConfigFiles(t, root, files) + if _, err := os.Stat(directory); err != nil { + t.Fatal("无清单残留被错误删除") + } + } +}