保存不确定部署处置重新核查实现及测试检查点

This commit is contained in:
xuans committed 2026-10-03 23:49:31 +08:00
1 parent 7fa5e9b3d2
commit f9cd19f631
29 files changed
+907 -49

No files matched your search

@@ -43,6 +43,11 @@ func (r *SQLDeploymentTransactionRepository) FinishRecoveryDisposition(ctx conte
if err := tx.QueryRowContext(ctx, r.query(`SELECT event_type FROM recovery_events WHERE disposition_id = ? AND event_type = 'started'`), id).Scan(&start); err != nil {
return deployment.RecoveryDisposition{}, dispositionError(ctx)
}
if item.Intent.SourceDispositionID != "" {
if err := r.validateReconciliationSource(ctx, tx, item.Intent); err != nil {
return deployment.RecoveryDisposition{}, err
}
}
now := time.Now().UTC()
if err := r.finishRecoveryOriginal(ctx, tx, item.Intent, resultCode, now); err != nil {
return deployment.RecoveryDisposition{}, err
@@ -58,7 +63,13 @@ func (r *SQLDeploymentTransactionRepository) FinishRecoveryDisposition(ctx conte
if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_events (disposition_id, event_type, result_code, created_at) VALUES (?, 'finished', ?, ?)`), id, resultCode, now); err != nil {
return deployment.RecoveryDisposition{}, dispositionError(ctx)
}
if resultCode != deployment.RecoveryResultUncertain {
if item.Intent.SourceDispositionID != "" && (resultCode == deployment.RecoveryResultRejected || resultCode == deployment.RecoveryResultUncertain) {
// 只读核查未通过时归还原占用,保留两次审计;绝不解锁原事务。
result, err := tx.ExecContext(ctx, r.query(`UPDATE recovery_claims SET disposition_id = ?, created_at = ? WHERE transaction_id = ? AND disposition_id = ?`), item.Intent.SourceDispositionID, now, item.Intent.TransactionID, id)
if err != nil || !oneRecoveryRow(result) {
return deployment.RecoveryDisposition{}, dispositionError(ctx)
}
} else if resultCode != deployment.RecoveryResultUncertain {
result, err := tx.ExecContext(ctx, r.query(`DELETE FROM recovery_claims WHERE transaction_id = ? AND disposition_id = ?`), item.Intent.TransactionID, id)
if err != nil || !oneRecoveryRow(result) {
return deployment.RecoveryDisposition{}, dispositionError(ctx)
@@ -96,6 +107,9 @@ func (r *SQLDeploymentTransactionRepository) finishRecoveryOriginal(ctx context.
(record.Status != deployment.TransactionStatusPending && record.Status != deployment.TransactionStatusRecoveryRequired) {
return deployment.ErrRecoveryDispositionConflict
}
if intent.SourceDispositionID != "" && !intent.MatchesRecord(record) {
return deployment.ErrRecoveryReviewChanged
}
result, err := tx.ExecContext(ctx, r.query(`UPDATE deployment_transactions SET status = ?, phase = ?, message = ?, updated_at = ?
WHERE id = ? AND phase = ? AND status IN ('pending', 'recovery_required')`), status, status, message, now, intent.TransactionID, phase)
if err != nil || !oneRecoveryRow(result) {
@@ -12,13 +12,13 @@ import (
var _ deployment.RecoveryDispositionJournal = (*SQLDeploymentTransactionRepository)(nil)
const recoveryDispositionColumns = `id, transaction_id, server_id, operator_id, action, request_key,
record_digest, server_digest, observation_digest, status, result_code, created_at, updated_at`
record_digest, server_digest, observation_digest, status, result_code, created_at, updated_at, source_disposition_id`
func scanRecoveryDisposition(row deploymentTransactionScanner) (deployment.RecoveryDisposition, error) {
var item deployment.RecoveryDisposition
i := &item.Intent
err := row.Scan(&i.ID, &i.TransactionID, &i.ServerID, &i.Operator, &i.Action, &i.RequestKey,
&i.RecordDigest, &i.ServerDigest, &i.ObservationDigest, &item.Status, &item.ResultCode, &item.CreatedAt, &item.UpdatedAt)
&i.RecordDigest, &i.ServerDigest, &i.ObservationDigest, &item.Status, &item.ResultCode, &item.CreatedAt, &item.UpdatedAt, &i.SourceDispositionID)
return item, err
}
@@ -75,9 +75,9 @@ func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx contex
defer tx.Rollback()
now := time.Now().UTC()
item := deployment.RecoveryDisposition{Intent: intent, Status: "running", CreatedAt: now, UpdatedAt: now}
_, err = tx.ExecContext(ctx, r.query(`INSERT INTO recovery_dispositions (`+recoveryDispositionColumns+`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`),
_, err = tx.ExecContext(ctx, r.query(`INSERT INTO recovery_dispositions (`+recoveryDispositionColumns+`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`),
intent.ID, intent.TransactionID, intent.ServerID, intent.Operator, intent.Action, intent.RequestKey,
intent.RecordDigest, intent.ServerDigest, intent.ObservationDigest, item.Status, "", now, now)
intent.RecordDigest, intent.ServerDigest, intent.ObservationDigest, item.Status, "", now, now, intent.SourceDispositionID)
if err != nil {
_ = tx.Rollback()
if existing, readErr := r.findRecoveryRequest(ctx, intent.RequestKey); readErr == nil {
@@ -85,7 +85,7 @@ func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx contex
}
return deployment.RecoveryDisposition{}, false, dispositionError(ctx)
}
if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_claims (transaction_id, disposition_id, created_at) VALUES (?, ?, ?)`), intent.TransactionID, intent.ID, now); err != nil {
if err := r.claimRecoveryDisposition(ctx, tx, intent, now); err != nil {
_ = tx.Rollback()
if claimed, readErr := r.HasRecoveryClaim(ctx, intent.TransactionID); readErr == nil && claimed {
return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionBusy
@@ -109,6 +109,39 @@ func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx contex
return item, true, nil
}
func (r *SQLDeploymentTransactionRepository) claimRecoveryDisposition(ctx context.Context, tx *sql.Tx, intent deployment.RecoveryDispositionIntent, now time.Time) error {
if intent.SourceDispositionID == "" {
_, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_claims (transaction_id, disposition_id, created_at) VALUES (?, ?, ?)`), intent.TransactionID, intent.ID, now)
return err
}
if err := r.validateReconciliationSource(ctx, tx, intent); err != nil {
return err
}
// 只从指定的已结束处置接管占用;运行中、其他处置或丢失的占用均不接管。
result, err := tx.ExecContext(ctx, r.query(`UPDATE recovery_claims SET disposition_id = ?, created_at = ? WHERE transaction_id = ? AND disposition_id = ?`), intent.ID, now, intent.TransactionID, intent.SourceDispositionID)
if err != nil || !oneRecoveryRow(result) {
return deployment.ErrRecoveryDispositionBusy
}
return nil
}
func (r *SQLDeploymentTransactionRepository) validateReconciliationSource(ctx context.Context, tx *sql.Tx, intent deployment.RecoveryDispositionIntent) error {
query := `SELECT ` + recoveryDispositionColumns + ` FROM recovery_dispositions WHERE id = ?`
if r.driver != "sqlite" {
query += " FOR UPDATE"
}
source, err := scanRecoveryDisposition(tx.QueryRowContext(ctx, r.query(query), intent.SourceDispositionID))
if err != nil || !intent.Reconciles(source) {
return deployment.ErrRecoveryDispositionConflict
}
var count int
if err := tx.QueryRowContext(ctx, r.query(`SELECT COUNT(*) FROM recovery_events WHERE disposition_id = ? AND
((event_type = 'started' AND result_code = '') OR (event_type = 'finished' AND result_code = 'uncertain'))`), intent.SourceDispositionID).Scan(&count); err != nil || count != 2 {
return deployment.ErrRecoveryDispositionConflict
}
return nil
}
func recoveryRetry(existing deployment.RecoveryDisposition, intent deployment.RecoveryDispositionIntent) (deployment.RecoveryDisposition, bool, error) {
if !existing.Intent.SameRequest(intent) {
return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionConflict
@@ -68,6 +68,16 @@ func openRecoveryExternal(t *testing.T, cfg config.DatabaseConfig) *sql.DB {
// 只替换远端文件实现,准备、确认、执行、迁移、占用和审计均走实际产品代码。
func runExternalRecovery(t *testing.T, fixture recoveryExternalFixture) {
t.Helper()
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
t.Run("reconciliation/"+string(action), func(t *testing.T) {
cfg := fixture.database(t)
db := openRecoveryExternal(t, cfg)
if err := database.Migrate(t.Context(), db, cfg.Driver); err != nil {
t.Fatal(err)
}
runReconciliationSuccess(t, NewSQLDeploymentTransactionRepository(db, cfg.Driver), db, action)
})
}
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionRecord, deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
for _, stage := range []string{"none", "started", "finished"} {
t.Run(string(action)+"/"+stage, func(t *testing.T) {
@@ -0,0 +1,250 @@
package repository
import (
"context"
"database/sql"
"errors"
"maps"
"reflect"
"testing"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
)
// 远端文件是测试替身;事务、关联、占用交接与审计均经过真实数据库和产品执行器。
type reconciliationClient struct {
*executorIntegrationClient
failClosed, readOnly bool
connections, writes int
}
func (c *reconciliationClient) Connect(context.Context, deployment.Server) (deployment.Client, error) {
c.connections++
return c, nil
}
func (c *reconciliationClient) Close() error {
if c.failClosed {
if _, exists := c.files["/cert.pem.scert-lock"]; !exists {
return errors.New("注入动作完成后的连接关闭失败")
}
}
return nil
}
func (c *reconciliationClient) writable() error {
c.writes++
if c.readOnly {
return errors.New("重新核查不得执行远端写操作")
}
return nil
}
func (c *reconciliationClient) Remove(ctx context.Context, path string) error {
if err := c.writable(); err != nil {
return err
}
return c.executorIntegrationClient.Remove(ctx, path)
}
func (c *reconciliationClient) Rename(ctx context.Context, from, to string) error {
if err := c.writable(); err != nil {
return err
}
return c.executorIntegrationClient.Rename(ctx, from, to)
}
func (c *reconciliationClient) Unlock(ctx context.Context, path string) error {
if err := c.writable(); err != nil {
return err
}
return c.executorIntegrationClient.Unlock(ctx, path)
}
func (c *reconciliationClient) Run(ctx context.Context, command []string) (deployment.CommandResult, error) {
if err := c.writable(); err != nil {
return deployment.CommandResult{}, err
}
return c.executorIntegrationClient.Run(ctx, command)
}
func seedUncertainReconciliation(t *testing.T, repo *SQLDeploymentTransactionRepository, db *sql.DB, action deployment.RecoveryAction) (*deployment.Service, *reconciliationClient, deployment.Server, deployment.TransactionRecord, deployment.RecoveryDisposition) {
t.Helper()
_, _, _, remote, server, record := seedRecoveryExecutor(t, repo, db)
client := &reconciliationClient{executorIntegrationClient: remote, failClosed: true}
service := deployment.NewServiceWithJournal(client, repo)
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error { return nil }
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
if err != nil {
t.Fatal(err)
}
source, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if !errors.Is(err, deployment.ErrRecoveryActionUncertain) || source.Status != deployment.RecoveryResultUncertain {
t.Fatalf("未形成真实执行器不确定结果:%+v %v", source, err)
}
client.failClosed, client.readOnly = false, true
client.writes = 0
return service, client, server, record, source
}
func assertReconciliationSource(t *testing.T, repo *SQLDeploymentTransactionRepository, source deployment.RecoveryDisposition) {
t.Helper()
stored, err := repo.GetRecoveryDisposition(t.Context(), source.Intent.ID)
if err != nil || !reflect.DeepEqual(stored, source) {
t.Fatalf("重新核查改写了原不确定历史:%+v %v", stored, err)
}
}
func runReconciliationSuccess(t *testing.T, repo *SQLDeploymentTransactionRepository, db *sql.DB, action deployment.RecoveryAction) {
t.Helper()
service, client, server, record, source := seedUncertainReconciliation(t, repo, db, action)
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
files := maps.Clone(client.files)
verified := false
service.VerifyRecoveryTarget = func(_ context.Context, _ deployment.Server, _ deployment.RecoveryBaseline, committed bool) error {
if committed != (action == deployment.RecoveryActionCommit) {
t.Fatal("重新核查擅自切换方向")
}
verified = true
return nil
}
confirmation, token, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve)
if err != nil || confirmation.Summary().Validate() != nil {
t.Fatal("准备重新核查失败", err)
}
other, otherToken, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve)
if err != nil {
t.Fatal(err)
}
result, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if err != nil || result.Status != "succeeded" || result.Intent.SourceDispositionID != source.Intent.ID || !verified {
t.Fatalf("重新核查失败:%+v %v", result, err)
}
if client.writes != 0 || !reflect.DeepEqual(files, client.files) {
t.Fatal("只读重新核查修改了远端")
}
assertReconciliationSource(t, repo, source)
recoveryCount(t, db, "recovery_dispositions", 2)
recoveryCount(t, db, "recovery_events", 4)
recoveryCount(t, db, "recovery_claims", 0)
pending, err := repo.ListPendingDeploymentTransactions(t.Context())
if err != nil || len(pending) != 0 {
t.Fatal("原事务未原子结束", err)
}
view, err := service.GetRecoveryDisposition(t.Context(), result.Intent.ID)
if err != nil || view.SourceDispositionID != source.Intent.ID || view.Validate() != nil {
t.Fatal("查询缺少可信关联", err)
}
connections := client.connections
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, deployment.ErrRecoveryConfirmation) {
t.Fatal("重新核查确认被重放", err)
}
if _, err := service.ExecuteRecoveryDisposition(t.Context(), other, "unix:0", otherToken, resolve); err == nil {
t.Fatal("其他旧确认再次接管已完成的事务")
}
if client.connections != connections {
t.Fatal("完成后仍使用旧确认连接远端")
}
if record.ID != result.Intent.TransactionID {
t.Fatal("结束了其他事务")
}
}
func TestRecoveryReconciliationSQLiteSuccess(t *testing.T) {
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
t.Run(string(action), func(t *testing.T) {
_, db := newTaskRepositoryTest(t)
t.Cleanup(func() { _ = db.Close() })
runReconciliationSuccess(t, NewSQLDeploymentTransactionRepository(db), db, action)
})
}
}
func TestRecoveryReconciliationSQLiteRejectsUnsafeCompletion(t *testing.T) {
for _, failure := range []string{"file", "backup", "temporary", "lock", "https", "during-https", "close", "changed-after-prepare", "started-audit", "finished-audit", "identity", "phase", "running", "direction"} {
t.Run(failure, func(t *testing.T) {
_, db := newTaskRepositoryTest(t)
t.Cleanup(func() { _ = db.Close() })
repo := NewSQLDeploymentTransactionRepository(db)
service, client, server, record, source := seedUncertainReconciliation(t, repo, db, deployment.RecoveryActionCommit)
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
switch failure {
case "file":
client.files[record.Paths[0]] = "tampered"
case "backup":
client.files[record.Backups[0].Saved] = "old"
case "temporary":
client.files[record.Paths[0]+".scert-tmp-"+record.ID] = "unexpected"
case "lock":
client.files[record.LockPaths[0]] = ""
case "https":
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error {
return errors.New("严格 HTTPS 未通过")
}
case "during-https":
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error {
client.files[record.Paths[0]] = "changed"
return nil
}
case "identity":
server.PasswordRef = "secret://ssh/password/changed"
case "phase":
if _, err := db.ExecContext(t.Context(), `UPDATE deployment_transactions SET phase = 'reload_attempted'`); err != nil {
t.Fatal(err)
}
case "running":
if _, err := db.ExecContext(t.Context(), `UPDATE recovery_dispositions SET status = 'running', result_code = ''`); err != nil {
t.Fatal(err)
}
case "direction":
if _, err := db.ExecContext(t.Context(), `UPDATE recovery_dispositions SET action = 'restore_baseline'`); err != nil {
t.Fatal(err)
}
}
confirmation, token, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve)
if failure == "identity" || failure == "phase" || failure == "running" || failure == "direction" {
if err == nil {
t.Fatal("危险来源允许重新核查")
}
recoveryCount(t, db, "recovery_claims", 1)
return
}
if err != nil {
t.Fatal("准备失败", err)
}
if failure == "close" {
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error {
client.failClosed = true
return nil
}
}
if failure == "changed-after-prepare" {
client.files[record.Paths[1]] = "changed"
}
if failure == "started-audit" || failure == "finished-audit" {
stage := "started"
if failure == "finished-audit" {
stage = "finished"
}
if _, err := db.ExecContext(t.Context(), `CREATE TRIGGER reject_recheck BEFORE INSERT ON recovery_events WHEN NEW.event_type = '`+stage+`' BEGIN SELECT RAISE(ABORT, 'injected'); END`); err != nil {
t.Fatal(err)
}
}
result, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if err == nil || result.Status == "succeeded" || client.writes != 0 {
t.Fatalf("危险收尾未拒绝或执行了写操作:%+v %v", result, err)
}
assertReconciliationSource(t, repo, source)
var owner string
if err := db.QueryRowContext(t.Context(), `SELECT disposition_id FROM recovery_claims WHERE transaction_id = ?`, record.ID).Scan(&owner); err != nil {
t.Fatal(err)
}
expected := source.Intent.ID
if failure == "finished-audit" {
expected = confirmation.DispositionID()
}
if owner != expected {
t.Fatal("失败后占用归属不正确", owner, expected)
}
pending, err := repo.ListPendingDeploymentTransactions(t.Context())
if err != nil || len(pending) != 1 || pending[0].Status != deployment.TransactionStatusRecoveryRequired {
t.Fatal("失败改变了原事务", err)
}
})
}
}