保存人工处置持久化与实库回归检查点
This commit is contained in:
1 parent
ed0c97b957
commit
17eb2fc295
25 files changed
+1070
-3
No files matched your search
@@ -0,0 +1,105 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||
)
|
||||
|
||||
// FinishRecoveryDisposition 仅由完成现场核验的内部执行器调用,不接受公网客户端结果。
|
||||
// 不确定结果保留占用;成功结果、必要的原事务终态与结束审计在同一事务内提交。
|
||||
func (r *SQLDeploymentTransactionRepository) FinishRecoveryDisposition(ctx context.Context, id, resultCode string) (deployment.RecoveryDisposition, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||
}
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
query := `SELECT ` + recoveryDispositionColumns + ` FROM recovery_dispositions WHERE id = ?`
|
||||
if r.driver != "sqlite" {
|
||||
query += " FOR UPDATE"
|
||||
}
|
||||
item, err := scanRecoveryDisposition(tx.QueryRowContext(ctx, r.query(query), id))
|
||||
if err != nil {
|
||||
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||
}
|
||||
if !item.Intent.AllowsResult(resultCode) {
|
||||
return deployment.RecoveryDisposition{}, deployment.ErrRecoveryDispositionInvalid
|
||||
}
|
||||
if item.Status != "running" {
|
||||
if item.ResultCode == resultCode {
|
||||
return item, nil
|
||||
}
|
||||
return deployment.RecoveryDisposition{}, deployment.ErrRecoveryDispositionConflict
|
||||
}
|
||||
var claim, start string
|
||||
if err := tx.QueryRowContext(ctx, r.query(`SELECT disposition_id FROM recovery_claims WHERE transaction_id = ?`), item.Intent.TransactionID).Scan(&claim); err != nil || claim != id {
|
||||
return deployment.RecoveryDisposition{}, deployment.ErrRecoveryDispositionConflict
|
||||
}
|
||||
if err := tx.QueryRowContext(ctx, r.query(`SELECT event_type FROM recovery_events WHERE disposition_id = ? AND event_type = 'started'`), id).Scan(&start); err != nil {
|
||||
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if err := r.finishRecoveryOriginal(ctx, tx, item.Intent, resultCode, now); err != nil {
|
||||
return deployment.RecoveryDisposition{}, err
|
||||
}
|
||||
item.Status = "succeeded"
|
||||
if resultCode == deployment.RecoveryResultRejected || resultCode == deployment.RecoveryResultUncertain {
|
||||
item.Status = resultCode
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, r.query(`UPDATE recovery_dispositions SET status = ?, result_code = ?, updated_at = ? WHERE id = ? AND status = 'running'`), item.Status, resultCode, now, id)
|
||||
if err != nil || !oneRecoveryRow(result) {
|
||||
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_events (disposition_id, event_type, result_code, created_at) VALUES (?, 'finished', ?, ?)`), id, resultCode, now); err != nil {
|
||||
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||
}
|
||||
if resultCode != deployment.RecoveryResultUncertain {
|
||||
result, err := tx.ExecContext(ctx, r.query(`DELETE FROM recovery_claims WHERE transaction_id = ? AND disposition_id = ?`), item.Intent.TransactionID, id)
|
||||
if err != nil || !oneRecoveryRow(result) {
|
||||
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||
}
|
||||
item.ResultCode, item.UpdatedAt = resultCode, now
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func oneRecoveryRow(result sql.Result) bool {
|
||||
if result == nil {
|
||||
return false
|
||||
}
|
||||
count, err := result.RowsAffected()
|
||||
return err == nil && count == 1
|
||||
}
|
||||
|
||||
func (r *SQLDeploymentTransactionRepository) finishRecoveryOriginal(ctx context.Context, tx *sql.Tx, intent deployment.RecoveryDispositionIntent, resultCode string, now time.Time) error {
|
||||
if resultCode != deployment.RecoveryResultCommitted && resultCode != deployment.RecoveryResultRolledBack {
|
||||
// 只记录核查、核验拒绝与不确定结果均不改变原事务;不能以已阅读代替恢复成功。
|
||||
return nil
|
||||
}
|
||||
record, err := r.lockRecoveryTransaction(ctx, tx, intent.TransactionID)
|
||||
if err != nil {
|
||||
return dispositionError(ctx)
|
||||
}
|
||||
phase, status, message := deployment.TransactionPhaseCommitVerified, deployment.TransactionStatusCommitted, "人工处置已完成提交核验"
|
||||
if resultCode == deployment.RecoveryResultRolledBack {
|
||||
phase, status, message = deployment.TransactionPhaseRollbackVerified, deployment.TransactionStatusRolledBack, "人工处置已完成回滚核验"
|
||||
}
|
||||
if record.ServerID != intent.ServerID || record.Evidence == nil || record.Phase != phase ||
|
||||
(record.Status != deployment.TransactionStatusPending && record.Status != deployment.TransactionStatusRecoveryRequired) {
|
||||
return deployment.ErrRecoveryDispositionConflict
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, r.query(`UPDATE deployment_transactions SET status = ?, phase = ?, message = ?, updated_at = ?
|
||||
WHERE id = ? AND phase = ? AND status IN ('pending', 'recovery_required')`), status, status, message, now, intent.TransactionID, phase)
|
||||
if err != nil || !oneRecoveryRow(result) {
|
||||
return dispositionError(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||
)
|
||||
|
||||
func TestRecoveryDispositionVerifiedCompletionAtomicity(t *testing.T) {
|
||||
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
|
||||
for _, failure := range []string{"none", "original", "result", "audit", "release"} {
|
||||
t.Run(string(action)+"/"+failure, func(t *testing.T) {
|
||||
repo, db, intent := recoveryDispositionFixture(t, action)
|
||||
if _, _, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
phase, resultCode := deployment.TransactionPhaseCommitVerified, deployment.RecoveryResultCommitted
|
||||
if action == deployment.RecoveryActionRollback {
|
||||
phase, resultCode = deployment.TransactionPhaseRollbackVerified, deployment.RecoveryResultRolledBack
|
||||
}
|
||||
if _, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, resultCode); !errors.Is(err, deployment.ErrRecoveryDispositionConflict) {
|
||||
t.Fatal("unverified original finalized", err)
|
||||
}
|
||||
if _, err := db.ExecContext(t.Context(), `UPDATE deployment_transactions SET phase = ?`, phase); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
triggers := map[string]string{
|
||||
"original": "BEFORE UPDATE ON deployment_transactions",
|
||||
"result": "BEFORE UPDATE ON recovery_dispositions",
|
||||
"audit": "BEFORE INSERT ON recovery_events",
|
||||
"release": "BEFORE DELETE ON recovery_claims",
|
||||
}
|
||||
if trigger, ok := triggers[failure]; ok {
|
||||
if _, err := db.ExecContext(t.Context(), `CREATE TRIGGER reject_finish `+trigger+` BEGIN SELECT RAISE(ABORT, 'injected'); END`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
item, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, resultCode)
|
||||
var status, storedPhase string
|
||||
if readErr := db.QueryRowContext(t.Context(), `SELECT status, phase FROM deployment_transactions WHERE id = ?`, intent.TransactionID).Scan(&status, &storedPhase); readErr != nil {
|
||||
t.Fatal(readErr)
|
||||
}
|
||||
if failure == "none" {
|
||||
if err != nil || item.Status != "succeeded" || status != resultCode || storedPhase != resultCode {
|
||||
t.Fatal("atomic completion failed", err)
|
||||
}
|
||||
recoveryCount(t, db, "recovery_claims", 0)
|
||||
recoveryCount(t, db, "recovery_events", 2)
|
||||
if _, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, resultCode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
if err == nil || status != deployment.TransactionStatusRecoveryRequired || storedPhase != phase {
|
||||
t.Fatal("failed completion changed original")
|
||||
}
|
||||
stored, err := repo.GetRecoveryDisposition(t.Context(), intent.ID)
|
||||
if err != nil || stored.Status != "running" || stored.ResultCode != "" {
|
||||
t.Fatal("partial disposition outcome saved", err)
|
||||
}
|
||||
recoveryCount(t, db, "recovery_claims", 1)
|
||||
recoveryCount(t, db, "recovery_events", 1)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryDispositionRejectsCrossActionCompletion(t *testing.T) {
|
||||
repo, db, intent := recoveryDispositionFixture(t, deployment.RecoveryActionRecord)
|
||||
if _, _, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, result := range []string{deployment.RecoveryResultCommitted, deployment.RecoveryResultRolledBack, "unknown", ""} {
|
||||
if _, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, result); !errors.Is(err, deployment.ErrRecoveryDispositionInvalid) {
|
||||
t.Fatal("invalid result accepted", err)
|
||||
}
|
||||
}
|
||||
if _, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, deployment.RecoveryResultRejected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recoveryCount(t, db, "recovery_claims", 0)
|
||||
pending, err := repo.ListPendingDeploymentTransactions(t.Context())
|
||||
if err != nil || len(pending) != 1 || pending[0].Evidence != nil {
|
||||
t.Fatal("rejected review changed original", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/database"
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/testsupport/mysqltest"
|
||||
)
|
||||
|
||||
func TestRecoveryDispositionMySQLReal(t *testing.T) {
|
||||
server := mysqltest.New(t)
|
||||
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionRecord, deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
|
||||
for _, auditFailure := range []bool{false, true} {
|
||||
name := string(action)
|
||||
if auditFailure {
|
||||
name += "/audit_failure"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cfg := server.Database(t)
|
||||
db, err := database.Open(t.Context(), cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
if err := database.Migrate(t.Context(), db.DB, cfg.Driver); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repo, raw, intent := seedRecoveryDisposition(t, NewSQLDeploymentTransactionRepository(db.DB, cfg.Driver), db.DB, action)
|
||||
if _, created, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil || !created {
|
||||
t.Fatal("start failed", err)
|
||||
}
|
||||
if _, created, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil || created {
|
||||
t.Fatal("retry authorized replay", err)
|
||||
}
|
||||
changed := intent
|
||||
changed.Operator = "unix:1"
|
||||
if _, _, err := repo.BeginRecoveryDisposition(t.Context(), changed); !errors.Is(err, deployment.ErrRecoveryDispositionConflict) {
|
||||
t.Fatal("conflicting replay accepted", err)
|
||||
}
|
||||
changed = intent
|
||||
changed.ID, changed.RequestKey = strings.Repeat("e", 64), strings.Repeat("f", 64)
|
||||
if _, _, err := repo.BeginRecoveryDisposition(t.Context(), changed); !errors.Is(err, deployment.ErrRecoveryDispositionBusy) {
|
||||
t.Fatal("duplicate claim accepted", err)
|
||||
}
|
||||
result, phase := deployment.RecoveryResultObserved, deployment.TransactionPhaseInstalled
|
||||
if action == deployment.RecoveryActionCommit {
|
||||
result, phase = deployment.RecoveryResultCommitted, deployment.TransactionPhaseCommitVerified
|
||||
}
|
||||
if action == deployment.RecoveryActionRollback {
|
||||
result, phase = deployment.RecoveryResultRolledBack, deployment.TransactionPhaseRollbackVerified
|
||||
}
|
||||
if _, err := raw.ExecContext(t.Context(), `UPDATE deployment_transactions SET phase = ? WHERE id = ?`, phase, intent.TransactionID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if auditFailure {
|
||||
if _, err := raw.ExecContext(t.Context(), `CREATE TRIGGER reject_finish BEFORE INSERT ON recovery_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'injected'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
item, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, result)
|
||||
var status, storedPhase string
|
||||
if readErr := raw.QueryRowContext(t.Context(), `SELECT status, phase FROM deployment_transactions WHERE id = ?`, intent.TransactionID).Scan(&status, &storedPhase); readErr != nil {
|
||||
t.Fatal(readErr)
|
||||
}
|
||||
if auditFailure {
|
||||
if err == nil || status != deployment.TransactionStatusRecoveryRequired || storedPhase != phase {
|
||||
t.Fatal("audit failure committed partial outcome")
|
||||
}
|
||||
stored, err := repo.GetRecoveryDisposition(t.Context(), intent.ID)
|
||||
if err != nil || stored.Status != "running" {
|
||||
t.Fatal("failed completion changed disposition", err)
|
||||
}
|
||||
recoveryCount(t, raw, "recovery_events", 1)
|
||||
recoveryCount(t, raw, "recovery_claims", 1)
|
||||
} else {
|
||||
if err != nil || item.Status != "succeeded" {
|
||||
t.Fatal("completion failed", err)
|
||||
}
|
||||
if action != deployment.RecoveryActionRecord && (status != result || storedPhase != result) {
|
||||
t.Fatal("original terminal state missing")
|
||||
}
|
||||
if action == deployment.RecoveryActionRecord && (status != deployment.TransactionStatusRecoveryRequired || storedPhase != phase) {
|
||||
t.Fatal("observation changed original")
|
||||
}
|
||||
if _, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, result); err != nil {
|
||||
t.Fatal("completion retry failed", err)
|
||||
}
|
||||
recoveryCount(t, raw, "recovery_events", 2)
|
||||
recoveryCount(t, raw, "recovery_claims", 0)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||
)
|
||||
|
||||
var _ deployment.RecoveryDispositionJournal = (*SQLDeploymentTransactionRepository)(nil)
|
||||
|
||||
const recoveryDispositionColumns = `id, transaction_id, server_id, operator_id, action, request_key,
|
||||
record_digest, server_digest, observation_digest, status, result_code, created_at, updated_at`
|
||||
|
||||
func scanRecoveryDisposition(row deploymentTransactionScanner) (deployment.RecoveryDisposition, error) {
|
||||
var item deployment.RecoveryDisposition
|
||||
i := &item.Intent
|
||||
err := row.Scan(&i.ID, &i.TransactionID, &i.ServerID, &i.Operator, &i.Action, &i.RequestKey,
|
||||
&i.RecordDigest, &i.ServerDigest, &i.ObservationDigest, &item.Status, &item.ResultCode, &item.CreatedAt, &item.UpdatedAt)
|
||||
return item, err
|
||||
}
|
||||
|
||||
func dispositionError(ctx context.Context) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return deployment.ErrRecoveryDispositionUnavailable
|
||||
}
|
||||
|
||||
func (r *SQLDeploymentTransactionRepository) GetRecoveryDisposition(ctx context.Context, id string) (deployment.RecoveryDisposition, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||
}
|
||||
item, err := scanRecoveryDisposition(r.db.QueryRowContext(ctx, r.query(`SELECT `+recoveryDispositionColumns+` FROM recovery_dispositions WHERE id = ?`), id))
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
err = dispositionError(ctx)
|
||||
}
|
||||
return item, err
|
||||
}
|
||||
|
||||
func (r *SQLDeploymentTransactionRepository) HasRecoveryClaim(ctx context.Context, transactionID string) (bool, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return false, dispositionError(ctx)
|
||||
}
|
||||
var id string
|
||||
err := r.db.QueryRowContext(ctx, r.query(`SELECT disposition_id FROM recovery_claims WHERE transaction_id = ?`), transactionID).Scan(&id)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, dispositionError(ctx)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx context.Context, intent deployment.RecoveryDispositionIntent) (deployment.RecoveryDisposition, bool, error) {
|
||||
if err := intent.Validate(); err != nil {
|
||||
return deployment.RecoveryDisposition{}, false, err
|
||||
}
|
||||
if r == nil || r.db == nil {
|
||||
return deployment.RecoveryDisposition{}, false, dispositionError(ctx)
|
||||
}
|
||||
// 重试先读取既有结果;唯一约束负责并发首次请求,不能靠进程内缓存判定幂等。
|
||||
if existing, err := r.findRecoveryRequest(ctx, intent.RequestKey); err == nil {
|
||||
return recoveryRetry(existing, intent)
|
||||
} else if !errors.Is(err, sql.ErrNoRows) {
|
||||
return deployment.RecoveryDisposition{}, false, dispositionError(ctx)
|
||||
}
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return deployment.RecoveryDisposition{}, false, dispositionError(ctx)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
now := time.Now().UTC()
|
||||
item := deployment.RecoveryDisposition{Intent: intent, Status: "running", CreatedAt: now, UpdatedAt: now}
|
||||
_, err = tx.ExecContext(ctx, r.query(`INSERT INTO recovery_dispositions (`+recoveryDispositionColumns+`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`),
|
||||
intent.ID, intent.TransactionID, intent.ServerID, intent.Operator, intent.Action, intent.RequestKey,
|
||||
intent.RecordDigest, intent.ServerDigest, intent.ObservationDigest, item.Status, "", now, now)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
if existing, readErr := r.findRecoveryRequest(ctx, intent.RequestKey); readErr == nil {
|
||||
return recoveryRetry(existing, intent)
|
||||
}
|
||||
return deployment.RecoveryDisposition{}, false, dispositionError(ctx)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_claims (transaction_id, disposition_id, created_at) VALUES (?, ?, ?)`), intent.TransactionID, intent.ID, now); err != nil {
|
||||
_ = tx.Rollback()
|
||||
if claimed, readErr := r.HasRecoveryClaim(ctx, intent.TransactionID); readErr == nil && claimed {
|
||||
return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionBusy
|
||||
}
|
||||
return deployment.RecoveryDisposition{}, false, dispositionError(ctx)
|
||||
}
|
||||
record, err := r.lockRecoveryTransaction(ctx, tx, intent.TransactionID)
|
||||
if err != nil {
|
||||
return deployment.RecoveryDisposition{}, false, dispositionError(ctx)
|
||||
}
|
||||
if !intent.MatchesRecord(record) {
|
||||
return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryReviewChanged
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_events (disposition_id, event_type, result_code, created_at) VALUES (?, 'started', '', ?)`), intent.ID, now); err != nil {
|
||||
return deployment.RecoveryDisposition{}, false, dispositionError(ctx)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
// 提交结果不确定时不得授权动作;下次凭同一请求键读取持久记录。
|
||||
return deployment.RecoveryDisposition{}, false, dispositionError(ctx)
|
||||
}
|
||||
return item, true, nil
|
||||
}
|
||||
|
||||
func recoveryRetry(existing deployment.RecoveryDisposition, intent deployment.RecoveryDispositionIntent) (deployment.RecoveryDisposition, bool, error) {
|
||||
if !existing.Intent.SameRequest(intent) {
|
||||
return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionConflict
|
||||
}
|
||||
return existing, false, nil
|
||||
}
|
||||
|
||||
func (r *SQLDeploymentTransactionRepository) findRecoveryRequest(ctx context.Context, key string) (deployment.RecoveryDisposition, error) {
|
||||
return scanRecoveryDisposition(r.db.QueryRowContext(ctx, r.query(`SELECT `+recoveryDispositionColumns+` FROM recovery_dispositions WHERE request_key = ?`), key))
|
||||
}
|
||||
|
||||
func (r *SQLDeploymentTransactionRepository) lockRecoveryTransaction(ctx context.Context, tx *sql.Tx, id string) (deployment.TransactionRecord, error) {
|
||||
query := `SELECT id, task_id, attempt_id, attempt_number, result_index,
|
||||
server_id, server_name, certificate_path, certificate_chain_path, private_key_path,
|
||||
temporary_paths_json, installed_paths_json, backups_json, lock_paths_json,
|
||||
reload_attempted, phase, status, message, updated_at, recovery_evidence_json
|
||||
FROM deployment_transactions WHERE id = ?`
|
||||
if r.driver != "sqlite" {
|
||||
query += " FOR UPDATE"
|
||||
}
|
||||
return scanDeploymentTransaction(tx.QueryRowContext(ctx, r.query(query), id))
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||
)
|
||||
|
||||
func recoveryDispositionFixture(t *testing.T, action deployment.RecoveryAction) (*SQLDeploymentTransactionRepository, *sql.DB, deployment.RecoveryDispositionIntent) {
|
||||
t.Helper()
|
||||
_, db := newTaskRepositoryTest(t)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
repo := NewSQLDeploymentTransactionRepository(db)
|
||||
return seedRecoveryDisposition(t, repo, db, action)
|
||||
}
|
||||
|
||||
func seedRecoveryDisposition(t *testing.T, repo *SQLDeploymentTransactionRepository, db *sql.DB, action deployment.RecoveryAction) (*SQLDeploymentTransactionRepository, *sql.DB, deployment.RecoveryDispositionIntent) {
|
||||
t.Helper()
|
||||
record := deployment.TransactionRecord{
|
||||
ID: "recovery-transaction", TaskID: "recovery-task", AttemptID: "recovery-attempt", AttemptNumber: 1,
|
||||
ServerID: "recovery-server", ServerName: "server", Paths: []string{"/cert.pem", "/chain.pem", "/key.pem"},
|
||||
Phase: deployment.TransactionPhaseInstalled, Status: deployment.TransactionStatusRecoveryRequired,
|
||||
}
|
||||
if action != deployment.RecoveryActionRecord {
|
||||
record.Evidence = &deployment.RecoveryEvidence{Version: 1, LockToken: "fixture-secret-not-for-disposition"}
|
||||
}
|
||||
if err := repo.CreateDeploymentTransaction(t.Context(), record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stored, err := repo.ListPendingDeploymentTransactions(t.Context())
|
||||
if err != nil || len(stored) != 1 {
|
||||
t.Fatal("missing fixture transaction", err)
|
||||
}
|
||||
encoded, err := json.Marshal(stored[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256(encoded)
|
||||
intent := deployment.RecoveryDispositionIntent{
|
||||
ID: strings.Repeat("a", 64), TransactionID: record.ID, ServerID: record.ServerID, Operator: "unix:0", Action: action,
|
||||
RequestKey: strings.Repeat("b", 64), RecordDigest: hex.EncodeToString(digest[:]),
|
||||
ServerDigest: strings.Repeat("c", 64), ObservationDigest: strings.Repeat("d", 64),
|
||||
}
|
||||
return repo, db, intent
|
||||
}
|
||||
|
||||
func recoveryCount(t *testing.T, db *sql.DB, table string, want int) {
|
||||
t.Helper()
|
||||
var count int
|
||||
if err := db.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM "+table).Scan(&count); err != nil || count != want {
|
||||
t.Fatalf("%s rows: got %d want %d: %v", table, count, want, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryDispositionRecordLifecycle(t *testing.T) {
|
||||
repo, db, intent := recoveryDispositionFixture(t, deployment.RecoveryActionRecord)
|
||||
before, _ := repo.ListPendingDeploymentTransactions(t.Context())
|
||||
item, created, err := repo.BeginRecoveryDisposition(t.Context(), intent)
|
||||
if err != nil || !created || item.Status != "running" {
|
||||
t.Fatal("begin failed", err)
|
||||
}
|
||||
for _, table := range []string{"recovery_dispositions", "recovery_claims", "recovery_events"} {
|
||||
recoveryCount(t, db, table, 1)
|
||||
}
|
||||
reopened := NewSQLDeploymentTransactionRepository(db)
|
||||
if claimed, err := reopened.HasRecoveryClaim(t.Context(), intent.TransactionID); err != nil || !claimed {
|
||||
t.Fatal("claim lost across repository restart", err)
|
||||
}
|
||||
if item, created, err := reopened.BeginRecoveryDisposition(t.Context(), intent); err != nil || created || item.Intent.ID != intent.ID {
|
||||
t.Fatal("retry may not authorize a second execution", err)
|
||||
}
|
||||
for range 2 {
|
||||
item, err = reopened.FinishRecoveryDisposition(t.Context(), intent.ID, deployment.RecoveryResultObserved)
|
||||
if err != nil || item.Status != "succeeded" || item.ResultCode != deployment.RecoveryResultObserved {
|
||||
t.Fatal("completion/retry failed", err)
|
||||
}
|
||||
}
|
||||
recoveryCount(t, db, "recovery_claims", 0)
|
||||
recoveryCount(t, db, "recovery_events", 2)
|
||||
after, _ := repo.ListPendingDeploymentTransactions(t.Context())
|
||||
beforeJSON, _ := json.Marshal(before)
|
||||
afterJSON, _ := json.Marshal(after)
|
||||
if string(beforeJSON) != string(afterJSON) {
|
||||
t.Fatal("recording observation modified original transaction or manufactured evidence")
|
||||
}
|
||||
if _, created, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil || created {
|
||||
t.Fatal("completed request replayed", err)
|
||||
}
|
||||
if _, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, deployment.RecoveryResultUncertain); !errors.Is(err, deployment.ErrRecoveryDispositionConflict) {
|
||||
t.Fatal("changed terminal outcome accepted", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryDispositionRequestBinding(t *testing.T) {
|
||||
for _, change := range []string{"transaction", "server", "operator", "action", "record", "configuration", "observation"} {
|
||||
t.Run(change, func(t *testing.T) {
|
||||
repo, db, intent := recoveryDispositionFixture(t, deployment.RecoveryActionRecord)
|
||||
if _, _, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
changed := intent
|
||||
switch change {
|
||||
case "transaction":
|
||||
changed.TransactionID = "different"
|
||||
case "server":
|
||||
changed.ServerID = "different"
|
||||
case "operator":
|
||||
changed.Operator = "unix:1"
|
||||
case "action":
|
||||
changed.Action = deployment.RecoveryActionCommit
|
||||
case "record":
|
||||
changed.RecordDigest = strings.Repeat("e", 64)
|
||||
case "configuration":
|
||||
changed.ServerDigest = strings.Repeat("e", 64)
|
||||
case "observation":
|
||||
changed.ObservationDigest = strings.Repeat("e", 64)
|
||||
}
|
||||
if _, created, err := repo.BeginRecoveryDisposition(t.Context(), changed); !errors.Is(err, deployment.ErrRecoveryDispositionConflict) || created {
|
||||
t.Fatal("conflicting idempotency key accepted", err)
|
||||
}
|
||||
changed = intent
|
||||
changed.ID = strings.Repeat("f", 64)
|
||||
if item, created, err := repo.BeginRecoveryDisposition(t.Context(), changed); err != nil || created || item.Intent.ID != intent.ID {
|
||||
t.Fatal("same request must return original ID", err)
|
||||
}
|
||||
recoveryCount(t, db, "recovery_dispositions", 1)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryDispositionBeginFailuresAreAtomic(t *testing.T) {
|
||||
for _, failure := range []string{"recovery_dispositions", "recovery_claims", "recovery_events", "record_changed", "cancelled"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
repo, db, intent := recoveryDispositionFixture(t, deployment.RecoveryActionRecord)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
switch failure {
|
||||
case "record_changed":
|
||||
if _, err := db.ExecContext(t.Context(), `UPDATE deployment_transactions SET message = 'changed'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "cancelled":
|
||||
cancel()
|
||||
default:
|
||||
if _, err := db.ExecContext(t.Context(), `CREATE TRIGGER reject_begin BEFORE INSERT ON `+failure+` BEGIN SELECT RAISE(ABORT, 'injected'); END`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, created, err := repo.BeginRecoveryDisposition(ctx, intent); err == nil || created {
|
||||
t.Fatal("failed start authorized execution")
|
||||
}
|
||||
for _, table := range []string{"recovery_dispositions", "recovery_claims", "recovery_events"} {
|
||||
recoveryCount(t, db, table, 0)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryDispositionUncertainKeepsClaim(t *testing.T) {
|
||||
repo, db, intent := recoveryDispositionFixture(t, deployment.RecoveryActionCommit)
|
||||
if _, _, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, deployment.RecoveryResultUncertain); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.ExecContext(t.Context(), `UPDATE recovery_claims SET created_at = '2000-01-01 00:00:00'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened := NewSQLDeploymentTransactionRepository(db)
|
||||
intent.ID, intent.RequestKey = strings.Repeat("e", 64), strings.Repeat("f", 64)
|
||||
if _, created, err := reopened.BeginRecoveryDisposition(t.Context(), intent); !errors.Is(err, deployment.ErrRecoveryDispositionBusy) || created {
|
||||
t.Fatal("old uncertain claim must not auto-expire", err)
|
||||
}
|
||||
recoveryCount(t, db, "recovery_dispositions", 1)
|
||||
recoveryCount(t, db, "recovery_claims", 1)
|
||||
recoveryCount(t, db, "recovery_events", 2)
|
||||
}
|
||||
|
||||
func TestRecoveryDispositionConcurrentRequests(t *testing.T) {
|
||||
for _, sameKey := range []bool{true, false} {
|
||||
t.Run(fmt.Sprint(sameKey), func(t *testing.T) {
|
||||
repo, db, intent := recoveryDispositionFixture(t, deployment.RecoveryActionRecord)
|
||||
db.SetMaxOpenConns(1)
|
||||
var started atomic.Int32
|
||||
var wg sync.WaitGroup
|
||||
for i := range 8 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
candidate := intent
|
||||
if !sameKey {
|
||||
candidate.ID, candidate.RequestKey = fmt.Sprintf("%064x", i+1), fmt.Sprintf("%064x", i+10)
|
||||
}
|
||||
_, created, err := repo.BeginRecoveryDisposition(t.Context(), candidate)
|
||||
if created {
|
||||
started.Add(1)
|
||||
}
|
||||
if sameKey && err != nil {
|
||||
t.Errorf("same request failed: %v", err)
|
||||
}
|
||||
if !sameKey && err != nil && !errors.Is(err, deployment.ErrRecoveryDispositionBusy) {
|
||||
t.Errorf("unexpected competing request error: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if started.Load() != 1 {
|
||||
t.Fatal("multiple remote executions authorized")
|
||||
}
|
||||
for _, table := range []string{"recovery_dispositions", "recovery_claims", "recovery_events"} {
|
||||
recoveryCount(t, db, table, 1)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryDispositionClaimSurvivesDatabaseReopen(t *testing.T) {
|
||||
repo, db, intent := recoveryDispositionFixture(t, deployment.RecoveryActionCommit)
|
||||
if _, _, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var sequence int
|
||||
var name, path string
|
||||
if err := db.QueryRowContext(t.Context(), "PRAGMA database_list").Scan(&sequence, &name, &path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reopened.Close()
|
||||
repo = NewSQLDeploymentTransactionRepository(reopened)
|
||||
if claimed, err := repo.HasRecoveryClaim(t.Context(), intent.TransactionID); err != nil || !claimed {
|
||||
t.Fatal("incomplete operation lost its persistent claim", err)
|
||||
}
|
||||
if existing, created, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil || created || existing.Status != "running" {
|
||||
t.Fatal("reopened request must not authorize replay", err)
|
||||
}
|
||||
intent.ID, intent.RequestKey = strings.Repeat("e", 64), strings.Repeat("f", 64)
|
||||
if _, _, err := repo.BeginRecoveryDisposition(t.Context(), intent); !errors.Is(err, deployment.ErrRecoveryDispositionBusy) {
|
||||
t.Fatal("restart released unfinished claim", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user