补齐人工处置PostgreSQL和MySQL实库执行器验证

This commit is contained in:
xuans committed 2026-10-03 21:49:06 +08:00
1 parent 0f2e6bfcce
commit e0d16a6ab1
10 files changed
+539 -1

No files matched your search

@@ -0,0 +1,67 @@
package database
import (
"fmt"
"testing"
"git.pcux.cn/xuans/Scert/backend/internal/testsupport/postgrestest"
)
func TestPostgresRealRecoveryDispositionMigration(t *testing.T) {
server := postgrestest.New(t)
for version := 22; version <= 24; version++ {
t.Run(fmt.Sprint(version), func(t *testing.T) {
cfg := server.Database(t)
db, err := Open(t.Context(), cfg)
if err != nil {
t.Fatal("隔离迁移连接失败")
}
t.Cleanup(func() { _ = db.Close() })
if _, err := db.DB.ExecContext(t.Context(), migrationTableSQL(cfg.Driver)); err != nil {
t.Fatal(err)
}
for previous := 1; previous < version; previous++ {
if err := applyMigration(t.Context(), db.DB, cfg.Driver, previous); err != nil {
t.Fatal(err)
}
}
content, err := migrationFiles.ReadFile(fmt.Sprintf("migrations/postgres/%04d_%s.sql", version, migrationName(version)))
if err != nil {
t.Fatal(err)
}
// 已执行 DDL 但版本尚未落库时中止事务,不能留下半迁移结构。
tx, err := db.DB.BeginTx(t.Context(), nil)
if err != nil {
t.Fatal(err)
}
for _, query := range migrationStatements(content) {
if _, err := tx.ExecContext(t.Context(), query); err != nil {
_ = tx.Rollback()
t.Fatal(err)
}
}
if err := tx.Rollback(); err != nil {
t.Fatal(err)
}
current, err := currentMigrationVersion(t.Context(), db.DB)
if err != nil || current != version-1 {
t.Fatal("回滚后版本记录改变", err)
}
for range 2 {
if err := Migrate(t.Context(), db.DB, cfg.Driver); err != nil {
t.Fatal("中断后的迁移重入失败", err)
}
}
current, err = currentMigrationVersion(t.Context(), db.DB)
if err != nil || current != latestMigrationVersion {
t.Fatal("迁移未达到当前版本", err)
}
for _, table := range []string{"recovery_dispositions", "recovery_claims", "recovery_events"} {
var count int
if err := db.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM "+table).Scan(&count); err != nil || count != 0 {
t.Fatal("处置表未正确初始化", err)
}
}
})
}
}
@@ -0,0 +1,212 @@
package repository
import (
"context"
"database/sql"
"errors"
"maps"
"reflect"
"strings"
"sync"
"sync/atomic"
"testing"
"github.com/go-sql-driver/mysql"
"git.pcux.cn/xuans/Scert/backend/internal/config"
"git.pcux.cn/xuans/Scert/backend/internal/database"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
"git.pcux.cn/xuans/Scert/backend/internal/testsupport/mysqltest"
"git.pcux.cn/xuans/Scert/backend/internal/testsupport/postgrestest"
)
type recoveryExternalFixture struct {
database func(*testing.T) config.DatabaseConfig
inject func(*testing.T, *sql.DB, config.DatabaseConfig, string)
}
func TestRecoveryExecutorPostgresReal(t *testing.T) {
server := postgrestest.New(t)
runExternalRecovery(t, recoveryExternalFixture{database: server.Database, inject: func(t *testing.T, db *sql.DB, _ config.DatabaseConfig, stage string) {
for _, query := range []string{
`CREATE FUNCTION reject_recovery_event() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.event_type = '` + stage + `' THEN RAISE EXCEPTION 'injected'; END IF; RETURN NEW; END $$`,
`CREATE TRIGGER reject_recovery_event BEFORE INSERT ON recovery_events FOR EACH ROW EXECUTE FUNCTION reject_recovery_event()`,
} {
if _, err := db.ExecContext(t.Context(), query); err != nil {
t.Fatal("安装隔离审计故障失败", err)
}
}
}})
}
func TestRecoveryExecutorMySQLReal(t *testing.T) {
server := mysqltest.New(t)
runExternalRecovery(t, recoveryExternalFixture{database: server.Database, inject: func(t *testing.T, _ *sql.DB, cfg config.DatabaseConfig, stage string) {
parsed, err := mysql.ParseDSN(cfg.DSN)
if err != nil || !strings.HasPrefix(parsed.DBName, "scert_test_") {
t.Fatal("拒绝对非隔离库注入故障")
}
schema := "`" + strings.ReplaceAll(parsed.DBName, "`", "``") + "`"
query := `CREATE TRIGGER ` + schema + `.reject_recovery_event BEFORE INSERT ON ` + schema + `.recovery_events FOR EACH ROW BEGIN IF NEW.event_type = '` + stage + `' THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'injected'; END IF; END`
// MySQL 开启二进制日志时,仅隔离管理连接创建触发器;业务账户不提权。
if _, err := server.Admin().ExecContext(t.Context(), query); err != nil {
t.Fatal("安装隔离审计故障失败", err)
}
}})
}
func openRecoveryExternal(t *testing.T, cfg config.DatabaseConfig) *sql.DB {
t.Helper()
db, err := database.Open(t.Context(), cfg)
if err != nil {
t.Fatal("连接隔离实库失败")
}
t.Cleanup(func() { _ = db.Close() })
return db.DB
}
// 只替换远端文件实现,准备、确认、执行、迁移、占用和审计均走实际产品代码。
func runExternalRecovery(t *testing.T, fixture recoveryExternalFixture) {
t.Helper()
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionRecord, deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
for _, stage := range []string{"none", "started", "finished"} {
t.Run(string(action)+"/"+stage, func(t *testing.T) {
cfg := fixture.database(t)
db := openRecoveryExternal(t, cfg)
if err := database.Migrate(t.Context(), db, cfg.Driver); err != nil {
t.Fatal(err)
}
repo, _, service, client, server, record := seedRecoveryExecutor(t, NewSQLDeploymentTransactionRepository(db, cfg.Driver), db)
before, err := repo.ListPendingDeploymentTransactions(t.Context())
if err != nil {
t.Fatal(err)
}
filesBefore := maps.Clone(client.files)
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
if err != nil {
t.Fatal("实库准备失败", err)
}
if stage != "none" {
fixture.inject(t, db, cfg, stage)
}
item, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if stage == "none" {
if err != nil || item.Status != "succeeded" {
t.Fatal("实库执行未成功", err)
}
recoveryCount(t, db, "recovery_dispositions", 1)
recoveryCount(t, db, "recovery_claims", 0)
recoveryCount(t, db, "recovery_events", 2)
} else if !errors.Is(err, deployment.ErrRecoveryDispositionUnavailable) || item.Status == "succeeded" {
t.Fatal("审计故障未拒绝成功结果", err)
}
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, deployment.ErrRecoveryConfirmation) {
t.Fatal("实库执行确认被重放", err)
}
// 实际关闭连接池后重新打开,不能靠旧仓储对象的内存证明持久性。
if err := db.Close(); err != nil {
t.Fatal(err)
}
db = openRecoveryExternal(t, cfg)
if err := database.Migrate(t.Context(), db, cfg.Driver); err != nil {
t.Fatal("实库重开迁移失败", err)
}
repo = NewSQLDeploymentTransactionRepository(db, cfg.Driver)
pending, err := repo.ListPendingDeploymentTransactions(t.Context())
if err != nil {
t.Fatal(err)
}
if stage == "started" {
for _, table := range []string{"recovery_dispositions", "recovery_claims", "recovery_events"} {
recoveryCount(t, db, table, 0)
}
if !reflect.DeepEqual(before, pending) || !reflect.DeepEqual(filesBefore, client.files) {
t.Fatal("开始审计失败修改了原事务或远端")
}
return
}
stored, err := repo.GetRecoveryDisposition(t.Context(), confirmation.DispositionID())
if err != nil {
t.Fatal("重开后处置记录丢失", err)
}
restarted := deployment.NewServiceWithJournal(nil, repo)
view, err := restarted.GetRecoveryDisposition(t.Context(), confirmation.DispositionID())
if err != nil || view.Validate() != nil || !reflect.DeepEqual(view, stored.Summary()) {
t.Fatal("重开后脱敏查询与持久结果不一致", err)
}
if stage == "finished" {
recoveryCount(t, db, "recovery_claims", 1)
recoveryCount(t, db, "recovery_events", 1)
if stored.Status != "running" || len(pending) != 1 || pending[0].Status != deployment.TransactionStatusRecoveryRequired {
t.Fatal("结束审计失败丢失占用或发布了事务终态")
}
if err := restarted.RecoverPending(t.Context(), func(context.Context, string) (deployment.Server, error) {
t.Error("重开后自动恢复重复执行人工占用事务")
return deployment.Server{}, errors.New("禁止解析远端")
}); !errors.Is(err, deployment.ErrRecoveryDispositionBusy) {
t.Fatal("重开后未遵守持久占用", err)
}
} else if stored.Status != "succeeded" {
t.Fatal("成功结果重开后丢失")
}
if action == deployment.RecoveryActionRecord {
if !reflect.DeepEqual(before, pending) || !reflect.DeepEqual(filesBefore, client.files) {
t.Fatal("仅记录动作改变原事务、证据或远端文件")
}
} else {
expected, result := "new", deployment.RecoveryResultCommitted
if action == deployment.RecoveryActionRollback {
expected, result = "old", deployment.RecoveryResultRolledBack
}
for _, path := range record.Paths {
if client.files[path] != expected {
t.Fatal("远端动作与明确选择不一致")
}
}
if stage == "none" && (len(pending) != 0 || stored.ResultCode != result) {
t.Fatal("成功结果与原事务终态不一致")
}
}
})
}
}
t.Run("concurrent_idempotency", func(t *testing.T) {
cfg := fixture.database(t)
db := openRecoveryExternal(t, cfg)
if err := database.Migrate(t.Context(), db, cfg.Driver); err != nil {
t.Fatal(err)
}
repo, _, intent := seedRecoveryDisposition(t, NewSQLDeploymentTransactionRepository(db, cfg.Driver), db, deployment.RecoveryActionRecord)
var created atomic.Int32
var group sync.WaitGroup
for range 8 {
group.Go(func() {
_, fresh, err := repo.BeginRecoveryDisposition(t.Context(), intent)
if err != nil {
t.Errorf("并发幂等请求失败:%v", err)
}
if fresh {
created.Add(1)
}
})
}
group.Wait()
if created.Load() != 1 {
t.Fatal("并发请求没有精确创建一次处置")
}
conflict := intent
conflict.Operator = "unix:1"
if _, _, err := repo.BeginRecoveryDisposition(t.Context(), conflict); !errors.Is(err, deployment.ErrRecoveryDispositionConflict) {
t.Fatal("不同操作者复用了幂等结果", err)
}
conflict = intent
conflict.ID, conflict.RequestKey = strings.Repeat("e", 64), strings.Repeat("f", 64)
if _, _, err := repo.BeginRecoveryDisposition(t.Context(), conflict); !errors.Is(err, deployment.ErrRecoveryDispositionBusy) {
t.Fatal("其他请求覆盖已有占用", err)
}
for _, table := range []string{"recovery_dispositions", "recovery_claims", "recovery_events"} {
recoveryCount(t, db, table, 1)
}
})
}
@@ -77,6 +77,11 @@ func executorIntegrationFixture(t *testing.T) (*SQLDeploymentTransactionReposito
_, db := newTaskRepositoryTest(t)
t.Cleanup(func() { _ = db.Close() })
repo := NewSQLDeploymentTransactionRepository(db)
return seedRecoveryExecutor(t, repo, db)
}
func seedRecoveryExecutor(t *testing.T, repo *SQLDeploymentTransactionRepository, db *sql.DB) (*SQLDeploymentTransactionRepository, *sql.DB, *deployment.Service, *executorIntegrationClient, deployment.Server, deployment.TransactionRecord) {
t.Helper()
server := deployment.Server{
ID: "executor-server", Name: "executor-server", Address: "192.0.2.10", Port: 22, Username: "deploy",
AuthMethod: deployment.SSHAuthPassword, PasswordRef: "secret://ssh/password/test", HostKeyFingerprint: "SHA256:test",
@@ -0,0 +1,197 @@
package postgrestest
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"database/sql"
"encoding/hex"
"encoding/pem"
"fmt"
"math/big"
"net"
"net/url"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
_ "github.com/jackc/pgx/v5/stdlib"
"git.pcux.cn/xuans/Scert/backend/internal/config"
)
// Server 只管理本次测试创建的隔离集群,不接受已有实例的连接串。
type Server struct {
root string
bin string
port string
password string
admin *sql.DB
sequence int
}
func New(t *testing.T) *Server {
t.Helper()
bin, root := os.Getenv("SCERT_POSTGRES_TEST_BIN"), os.Getenv("SCERT_POSTGRES_TEST_ROOT")
if bin == "" {
t.Skip("未启用 PostgreSQL 隔离实库测试")
}
if !filepath.IsAbs(bin) || !filepath.IsAbs(root) {
t.Fatal("PostgreSQL 工具与隔离产物目录必须使用绝对路径")
}
if err := os.MkdirAll(root, 0o700); err != nil {
t.Fatal(err)
}
root, err := os.MkdirTemp(root, "scert-postgres-integration-")
if err != nil {
t.Fatal(err)
}
var secret [32]byte
if _, err := rand.Read(secret[:]); err != nil {
t.Fatal(err)
}
f := &Server{root: root, bin: bin, password: hex.EncodeToString(secret[:])}
t.Logf("PostgreSQL 隔离产物保留于:%s", root)
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, filepath.Join(bin, "postgres"), "--version")
cmd.Env = f.environment()
version, err := cmd.Output()
if err != nil || !strings.HasPrefix(string(version), "postgres (PostgreSQL) ") {
t.Fatal("无法确认隔离 PostgreSQL 版本")
}
t.Logf("隔离服务端:%s", strings.TrimSpace(string(version)))
passwordFile := filepath.Join(root, "bootstrap-password.txt")
write(t, passwordFile, []byte(f.password+"\n"))
data := filepath.Join(root, "cluster")
f.control(t, "initdb", "-D", data, "-U", "postgres", "--pwfile="+passwordFile, "--auth=scram-sha-256", "--encoding=UTF8", "--no-locale")
f.createTLS(t)
listener, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
f.port = strconv.Itoa(listener.Addr().(*net.TCPAddr).Port)
if err := listener.Close(); err != nil {
t.Fatal(err)
}
settings := "shared_buffers = '16MB'\nwork_mem = '1MB'\nmaintenance_work_mem = '16MB'\nmax_connections = 24\nmax_parallel_workers = 0\nmax_worker_processes = 0\n"
settings += "listen_addresses = '127.0.0.1'\nport = " + f.port + "\nssl = on\nssl_cert_file = 'server.crt'\nssl_key_file = 'server.key'\npassword_encryption = 'scram-sha-256'\nunix_socket_directories = ''\n"
write(t, filepath.Join(data, "postgresql.conf"), []byte(settings))
write(t, filepath.Join(data, "pg_hba.conf"), []byte("hostssl all all 127.0.0.1/32 scram-sha-256\nhost all all 0.0.0.0/0 reject\nhost all all ::0/0 reject\n"))
// 即使启动中途失败,也只停止本测试目录中的集群;不删除产物。
t.Cleanup(func() {
if f.admin != nil {
_ = f.admin.Close()
}
if _, err := os.Stat(filepath.Join(data, "postmaster.pid")); os.IsNotExist(err) {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, filepath.Join(bin, "pg_ctl"), "-D", data, "-m", "fast", "-w", "-t", "20", "stop")
cmd.Env = f.environment()
cmd.WaitDelay = 2 * time.Second
if err := cmd.Run(); err != nil {
t.Errorf("停止隔离 PostgreSQL 失败:%v", err)
}
})
f.control(t, "pg_ctl", "-D", data, "-l", filepath.Join(root, "postgres.log"), "-w", "-t", "30", "start")
f.admin, err = sql.Open("pgx", f.dsn("postgres", "postgres"))
if err != nil || f.admin.PingContext(t.Context()) != nil {
t.Fatal("隔离 PostgreSQL 严格 TLS 管理连接失败")
}
return f
}
func (f *Server) Database(t *testing.T) config.DatabaseConfig {
t.Helper()
f.sequence++
name, owner := fmt.Sprintf("scert_test_%d", f.sequence), fmt.Sprintf("scert_owner_%d", f.sequence)
// 每个用例都有独立数据库和无超级权限的属主,不能访问业务数据库。
for _, query := range []string{
"CREATE ROLE " + owner + " LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE PASSWORD '" + f.password + "'",
"CREATE DATABASE " + name + " OWNER " + owner,
"REVOKE ALL ON DATABASE " + name + " FROM PUBLIC",
} {
if _, err := f.admin.ExecContext(t.Context(), query); err != nil {
t.Fatal("创建隔离 PostgreSQL 数据库及受限属主失败")
}
}
return config.DatabaseConfig{Driver: "postgres", DSN: f.dsn(name, owner), MaxOpenConns: 8, MaxIdleConns: 2}
}
func (f *Server) dsn(name, user string) string {
u := &url.URL{Scheme: "postgres", Host: net.JoinHostPort("127.0.0.1", f.port), Path: "/" + name, User: url.UserPassword(user, f.password)}
u.RawQuery = url.Values{"sslmode": {"verify-full"}, "sslrootcert": {filepath.Join(f.root, "cluster", "server.crt")}, "connect_timeout": {"5"}}.Encode()
return u.String()
}
func (f *Server) environment() []string {
env := []string{}
for _, entry := range os.Environ() {
key, _, _ := strings.Cut(entry, "=")
upper := strings.ToUpper(key)
if strings.HasPrefix(upper, "PG") || upper == "PATH" || upper == "LC_ALL" || upper == "LANG" {
continue
}
env = append(env, entry)
}
return append(env, "PATH="+f.bin+string(os.PathListSeparator)+os.Getenv("PATH"), "LC_ALL=C", "LANG=C")
}
func (f *Server) control(t *testing.T, tool string, args ...string) {
t.Helper()
ctx, cancel := context.WithTimeout(t.Context(), time.Minute)
defer cancel()
cmd := exec.CommandContext(ctx, filepath.Join(f.bin, tool), args...)
cmd.Env = f.environment()
// 后台进程不能继承 Go 输出复制管道,否则父进程会一直等待。
output, err := os.OpenFile(filepath.Join(f.root, tool+".log"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
if err != nil {
t.Fatal(err)
}
defer output.Close()
cmd.Stdout, cmd.Stderr = output, output
cmd.WaitDelay = 2 * time.Second
if err := cmd.Run(); err != nil {
t.Fatalf("隔离工具 %s 失败,请检查保留日志:%v", tool, err)
}
}
func write(t *testing.T, path string, value []byte) {
t.Helper()
if err := os.WriteFile(path, value, 0o600); err != nil {
t.Fatal(err)
}
}
func (f *Server) createTLS(t *testing.T) {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
certificate := &x509.Certificate{
SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "Scert isolated PostgreSQL"},
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * time.Hour),
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, IsCA: true, BasicConstraintsValid: true,
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
}
der, err := x509.CreateCertificate(rand.Reader, certificate, certificate, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
private, err := x509.MarshalPKCS8PrivateKey(key)
if err != nil {
t.Fatal(err)
}
write(t, filepath.Join(f.root, "cluster", "server.crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
write(t, filepath.Join(f.root, "cluster", "server.key"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: private}))
}