保存不确定部署处置重新核查实现及测试检查点

This commit is contained in:
xuans committed 2026-10-03 23:49:31 +08:00
1 parent 7fa5e9b3d2
commit f9cd19f631
29 files changed
+907 -49

No files matched your search

+5 -2
View File
@@ -60,7 +60,7 @@ func runConnected(ctx context.Context, conn net.Conn, screen terminal) error {
if client.RecoveryAvailable() { if client.RecoveryAvailable() {
prompt := "本机管理:1 重置超级管理员密码;2 只读核查部署事务;其他输入退出" prompt := "本机管理:1 重置超级管理员密码;2 只读核查部署事务;其他输入退出"
if client.DispositionAvailable() { if client.DispositionAvailable() {
prompt = "本机管理:1 重置超级管理员密码;2 只读核查部署事务;3 人工处置部署事务;4 查询人工处置结果;其他输入退出" prompt = "本机管理:1 重置超级管理员密码;2 只读核查部署事务;3 人工处置部署事务;4 查询人工处置结果;5 重新核查不确定处置;其他输入退出"
} }
choice, err := screen.Read(ctx, prompt, false) choice, err := screen.Read(ctx, prompt, false)
if err != nil { if err != nil {
@@ -74,13 +74,16 @@ func runConnected(ctx context.Context, conn net.Conn, screen terminal) error {
return err return err
} }
return nil return nil
case "3", "4": case "3", "4", "5":
if !client.DispositionAvailable() { if !client.DispositionAvailable() {
return localreset.ErrUnavailable return localreset.ErrUnavailable
} }
if strings.TrimSpace(choice) == "4" { if strings.TrimSpace(choice) == "4" {
return recoveryResultFlow(ctx, client, screen) return recoveryResultFlow(ctx, client, screen)
} }
if strings.TrimSpace(choice) == "5" {
return recoveryReconciliationFlow(ctx, client, screen)
}
return recoveryDispositionFlow(ctx, client, screen) return recoveryDispositionFlow(ctx, client, screen)
default: default:
return nil return nil
@@ -130,12 +130,22 @@ func showRecoveryDisposition(screen terminal, result deployment.RecoveryDisposit
message = "本次人工提交核验及结束审计完成;不代表项目全部生产验收通过。" message = "本次人工提交核验及结束审计完成;不代表项目全部生产验收通过。"
case deployment.RecoveryActionRollback: case deployment.RecoveryActionRollback:
message = "本次恢复基线核验及结束审计完成;不代表项目全部生产验收通过。" message = "本次恢复基线核验及结束审计完成;不代表项目全部生产验收通过。"
case deployment.RecoveryActionVerifyCommit, deployment.RecoveryActionVerifyRollback:
message = "重新核查与新审计已完成,占用已安全释放;原不确定处置历史保留,本次未重放远端动作。"
} }
case "rejected": case "rejected":
message = "处置已拒绝,未执行本次远端变更;请重新核查拒绝原因。" message = "处置已拒绝,未执行本次远端变更;请重新核查拒绝原因。"
if result.SourceDispositionID != "" {
message = "重新核查未通过,原占用继续保留;本次未执行远端变更,不得手动删锁或重放原动作。"
}
case "uncertain": case "uncertain":
message = "处置结果不确定,可能已修改远端;占用继续保留,禁止重复执行或手动删锁。" message = "处置结果不确定,可能已修改远端;占用继续保留,禁止重复执行或手动删锁。"
} }
if result.SourceDispositionID != "" {
if err := screen.Show("关联原处置:" + terminalValue(result.SourceDispositionID)); err != nil {
return err
}
}
if err := screen.Show(fmt.Sprintf("处置:%s;事务:%s;服务器:%s;动作:%s", terminalValue(result.ID), terminalValue(result.TransactionID), terminalValue(result.ServerID), dispositionActionLabel(result.Action))); err != nil { if err := screen.Show(fmt.Sprintf("处置:%s;事务:%s;服务器:%s;动作:%s", terminalValue(result.ID), terminalValue(result.TransactionID), terminalValue(result.ServerID), dispositionActionLabel(result.Action))); err != nil {
return err return err
} }
@@ -150,6 +160,10 @@ func dispositionActionLabel(action deployment.RecoveryAction) string {
return "完成提交" return "完成提交"
case deployment.RecoveryActionRollback: case deployment.RecoveryActionRollback:
return "恢复基线" return "恢复基线"
case deployment.RecoveryActionVerifyCommit:
return "重新核查提交结果"
case deployment.RecoveryActionVerifyRollback:
return "重新核查回滚结果"
} }
return "未知动作" return "未知动作"
} }
@@ -0,0 +1,63 @@
package adminconsole
import (
"context"
"fmt"
"strings"
"time"
"git.pcux.cn/xuans/Scert/backend/internal/localreset"
)
type recoveryReconciliationClient interface {
recoveryDispositionClient
PrepareRecoveryReconciliation(string) (localreset.RecoveryPreparation, error)
}
func recoveryReconciliationFlow(ctx context.Context, client recoveryReconciliationClient, screen terminal) error {
sourceID, err := screen.Read(ctx, "请输入结果不确定的原处置编号;本操作只重新核验,不重放文件修改或服务重载", false)
if err != nil {
return err
}
sourceID = strings.TrimSpace(sourceID)
if err := ctx.Err(); err != nil {
return err
}
prepared, err := client.PrepareRecoveryReconciliation(sourceID)
if err != nil {
_ = screen.Show("无法准备重新核查:主服务可能尚未支持,或原处置未结束、历史证据不足、现场不满足收尾条件。占用仍保留。")
return err
}
defer func() { prepared.Token = "" }()
summary := prepared.Summary
if summary.Validate() != nil || summary.SourceDispositionID != sourceID || !time.Now().Before(summary.ExpiresAt) {
return localreset.ErrProtocol
}
for _, line := range []string{
"原处置:" + terminalValue(sourceID),
"新核查编号:" + terminalValue(prepared.DispositionID) + "(请保留,用于断连后查询)",
fmt.Sprintf("事务:%s;服务器:%s;动作:%s", terminalValue(summary.TransactionID), terminalValue(summary.ServerID), dispositionActionLabel(summary.Action)),
"本次只核验原方向,不能改选提交或回滚;不写远端文件、不删锁、不重载。文件、残留、严格 HTTPS 和关闭检查全部通过后,才以新审计结束原事务并释放占用。",
"原不确定记录永久保留;任何核验或审计失败均不释放占用,运行中处置不会因等待超时被接管。",
} {
if err := screen.Show(line); err != nil {
return err
}
}
expected := "确认重新核查 " + sourceID + " " + dispositionActionLabel(summary.Action)
answer, err := screen.Read(ctx, "如确认,请完整输入:"+expected+";其他输入取消", false)
if err != nil {
return err
}
if answer != expected || ctx.Err() != nil || !time.Now().Before(summary.ExpiresAt) {
return screen.Show("已取消或确认过期,未提交重新核查。")
}
result, err := client.ExecuteRecovery(prepared.DispositionID, prepared.Token)
prepared.Token = ""
if err != nil || result.Validate() != nil || result.ID != prepared.DispositionID || result.SourceDispositionID != sourceID ||
result.TransactionID != summary.TransactionID || result.ServerID != summary.ServerID || result.Action != summary.Action {
_ = screen.Show("未取得可靠结果,请重新连接后按新核查编号查询;不要重复执行或直接删除占用。")
return localreset.ErrUnavailable
}
return showRecoveryDisposition(screen, result)
}
@@ -0,0 +1,62 @@
package adminconsole
import (
"strings"
"testing"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
"git.pcux.cn/xuans/Scert/backend/internal/localreset"
)
type reconciliationConsoleClient struct {
dispositionConsoleClient
sourceID string
mismatch bool
}
func (c *reconciliationConsoleClient) PrepareRecoveryReconciliation(sourceID string) (localreset.RecoveryPreparation, error) {
c.sourceID = sourceID
value, err := c.PrepareRecovery("tx-1", deployment.RecoveryActionVerifyCommit)
value.Summary.SourceDispositionID = sourceID
value.Summary.Phase = deployment.TransactionPhaseCommitVerified
return value, err
}
func (c *reconciliationConsoleClient) ExecuteRecovery(id, token string) (deployment.RecoveryDispositionSummary, error) {
value, err := c.dispositionConsoleClient.ExecuteRecovery(id, token)
if err != nil { return value, err }
value.SourceDispositionID = c.sourceID
if value.Status == "succeeded" { value.ResultCode = "committed" }
if c.mismatch { value.SourceDispositionID = strings.Repeat("d", 64) }
return value, nil
}
func TestRecoveryReconciliationConsoleConfirmation(t *testing.T) {
for _, mode := range []string{"success", "rejected", "cancelled", "expired", "disconnect", "mismatch"} {
t.Run(mode, func(t *testing.T) {
sourceID := strings.Repeat("a", 64)
client := &reconciliationConsoleClient{}
answer := "确认重新核查 " + sourceID + " " + dispositionActionLabel(deployment.RecoveryActionVerifyCommit)
switch mode {
case "rejected": client.status = "rejected"
case "cancelled": answer = "确认"
case "expired": client.expired = true
case "disconnect": client.executeError = true
case "mismatch": client.mismatch = true
}
screen := &recoveryTestScreen{testTerminalInput: testTerminalInput{values: []string{sourceID, answer}}}
err := recoveryReconciliationFlow(t.Context(), client, screen)
output := strings.Join(screen.lines, "|")
if client.prepares != 1 || client.queries != 0 || strings.Contains(output, strings.Repeat("c", 64)) { t.Fatal("终端重复调用或泄露令牌") }
if mode == "cancelled" || mode == "expired" {
if client.executes != 0 { t.Fatal("取消或过期仍执行核查") }
return
}
if client.executes != 1 || !strings.Contains(output, "原不确定记录永久保留") { t.Fatal("缺少明确警告或重复执行") }
if mode == "disconnect" || mode == "mismatch" {
if err == nil || !strings.Contains(output, "新核查编号查询") { t.Fatal("失去可靠响应后误报成功") }
} else if err != nil { t.Fatal(err) }
if mode == "success" && !strings.Contains(output, "占用已安全释放") { t.Fatal("缺少完成结果") }
if mode == "rejected" && !strings.Contains(output, "原占用继续保留") { t.Fatal("错误显示拒绝结果") }
})
}
}
@@ -134,6 +134,10 @@ func TestCreateBackupDoesNotChangeExistingParentPermissions(t *testing.T) {
if err := os.Mkdir(parent, 0o755); err != nil { if err := os.Mkdir(parent, 0o755); err != nil {
t.Fatal(err) t.Fatal(err)
} }
// 显式建立权限基线,避免宿主 umask 将创建权限收紧后造成误报。
if err := os.Chmod(parent, 0o755); err != nil {
t.Fatal(err)
}
cfg := isolatedCLIConfig(filepath.Join(base, "scert")) cfg := isolatedCLIConfig(filepath.Join(base, "scert"))
for _, directory := range []string{cfg.Paths.Config, cfg.Paths.Data} { for _, directory := range []string{cfg.Paths.Config, cfg.Paths.Data} {
if err := os.MkdirAll(directory, 0o700); err != nil { if err := os.MkdirAll(directory, 0o700); err != nil {
+12 -1
View File
@@ -25,6 +25,7 @@ import (
//go:embed migrations/sqlite/0022_recovery_dispositions.sql migrations/mysql/0022_recovery_dispositions.sql migrations/postgres/0022_recovery_dispositions.sql //go:embed migrations/sqlite/0022_recovery_dispositions.sql migrations/mysql/0022_recovery_dispositions.sql migrations/postgres/0022_recovery_dispositions.sql
//go:embed migrations/sqlite/0023_recovery_claims.sql migrations/mysql/0023_recovery_claims.sql migrations/postgres/0023_recovery_claims.sql //go:embed migrations/sqlite/0023_recovery_claims.sql migrations/mysql/0023_recovery_claims.sql migrations/postgres/0023_recovery_claims.sql
//go:embed migrations/sqlite/0024_recovery_events.sql migrations/mysql/0024_recovery_events.sql migrations/postgres/0024_recovery_events.sql //go:embed migrations/sqlite/0024_recovery_events.sql migrations/mysql/0024_recovery_events.sql migrations/postgres/0024_recovery_events.sql
//go:embed migrations/sqlite/0025_recovery_reconciliation.sql migrations/mysql/0025_recovery_reconciliation.sql migrations/postgres/0025_recovery_reconciliation.sql
var migrationFiles embed.FS var migrationFiles embed.FS
func Migrate(ctx context.Context, db *sql.DB, driver string) error { func Migrate(ctx context.Context, db *sql.DB, driver string) error {
@@ -49,7 +50,7 @@ func Migrate(ctx context.Context, db *sql.DB, driver string) error {
return nil return nil
} }
const latestMigrationVersion = 24 const latestMigrationVersion = 25
// currentMigrationVersion 在执行业务迁移前拒绝新版数据库及不连续的迁移历史。 // currentMigrationVersion 在执行业务迁移前拒绝新版数据库及不连续的迁移历史。
func currentMigrationVersion(ctx context.Context, db *sql.DB) (int, error) { func currentMigrationVersion(ctx context.Context, db *sql.DB) (int, error) {
@@ -114,6 +115,14 @@ func applyMigration(ctx context.Context, db *sql.DB, driver string, version int)
return err return err
} }
} }
if version == 25 {
var compatible int
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = DATABASE()
AND table_name = 'recovery_dispositions' AND column_name = 'source_disposition_id'
AND data_type = 'varchar' AND character_maximum_length = 64 AND is_nullable = 'NO' AND column_default = ''`).Scan(&compatible); err != nil || compatible != 1 {
return fmt.Errorf("人工处置来源字段与预期不一致,拒绝记录迁移版本")
}
}
if _, err := db.ExecContext(ctx, "INSERT INTO schema_migrations (version) VALUES (?)", version); err != nil { if _, err := db.ExecContext(ctx, "INSERT INTO schema_migrations (version) VALUES (?)", version); err != nil {
return fmt.Errorf("记录迁移版本失败:%w", err) return fmt.Errorf("记录迁移版本失败:%w", err)
} }
@@ -252,6 +261,8 @@ func migrationName(version int) string {
return "recovery_claims" return "recovery_claims"
case 24: case 24:
return "recovery_events" return "recovery_events"
case 25:
return "recovery_reconciliation"
default: default:
return "unknown" return "unknown"
} }
@@ -0,0 +1 @@
ALTER TABLE recovery_dispositions ADD COLUMN source_disposition_id VARCHAR(64) NOT NULL DEFAULT '';
@@ -0,0 +1 @@
ALTER TABLE recovery_dispositions ADD COLUMN source_disposition_id VARCHAR(64) NOT NULL DEFAULT '';
@@ -0,0 +1 @@
ALTER TABLE recovery_dispositions ADD COLUMN source_disposition_id TEXT NOT NULL DEFAULT '';
@@ -25,6 +25,10 @@ func TestMySQLRealRecoveryDispositionMigrationRecovery(t *testing.T) {
`ALTER TABLE recovery_claims DROP PRIMARY KEY`}, `ALTER TABLE recovery_claims DROP PRIMARY KEY`},
{24, "recovery_events", `INSERT INTO recovery_events VALUES ('kept', 'started', '', NOW())`, {24, "recovery_events", `INSERT INTO recovery_events VALUES ('kept', 'started', '', NOW())`,
`ALTER TABLE recovery_events DROP PRIMARY KEY, ADD PRIMARY KEY (event_type, disposition_id)`}, `ALTER TABLE recovery_events DROP PRIMARY KEY, ADD PRIMARY KEY (event_type, disposition_id)`},
{25, "recovery_dispositions", `INSERT INTO recovery_dispositions (id, transaction_id, server_id, operator_id, action, request_key,
record_digest, server_digest, observation_digest, status, result_code, created_at, updated_at) VALUES
('kept', 'transaction', 'server', 'unix:0', 'finish_commit', 'request', 'record', 'server', 'observation', 'uncertain', 'uncertain', NOW(), NOW())`,
`ALTER TABLE recovery_dispositions MODIFY COLUMN source_disposition_id VARCHAR(8) NOT NULL DEFAULT ''`},
} { } {
for _, boundary := range []string{"before_create", "after_create", "with_record", "conflict"} { for _, boundary := range []string{"before_create", "after_create", "with_record", "conflict"} {
t.Run(fmt.Sprintf("%d/%s", item.version, boundary), func(t *testing.T) { t.Run(fmt.Sprintf("%d/%s", item.version, boundary), func(t *testing.T) {
@@ -9,7 +9,7 @@ import (
func TestPostgresRealRecoveryDispositionMigration(t *testing.T) { func TestPostgresRealRecoveryDispositionMigration(t *testing.T) {
server := postgrestest.New(t) server := postgrestest.New(t)
for version := 22; version <= 24; version++ { for version := 22; version <= 25; version++ {
t.Run(fmt.Sprint(version), func(t *testing.T) { t.Run(fmt.Sprint(version), func(t *testing.T) {
cfg := server.Database(t) cfg := server.Database(t)
db, err := Open(t.Context(), cfg) db, err := Open(t.Context(), cfg)
@@ -17,15 +17,16 @@ var (
// RecoveryDispositionIntent 仅保存绑定摘要;不能包含历史证据、凭据或确认令牌原文。 // RecoveryDispositionIntent 仅保存绑定摘要;不能包含历史证据、凭据或确认令牌原文。
type RecoveryDispositionIntent struct { type RecoveryDispositionIntent struct {
ID string SourceDispositionID string
TransactionID string ID string
ServerID string TransactionID string
Operator string ServerID string
Action RecoveryAction Operator string
RequestKey string Action RecoveryAction
RecordDigest string RequestKey string
ServerDigest string RecordDigest string
ObservationDigest string ServerDigest string
ObservationDigest string
} }
type RecoveryDisposition struct { type RecoveryDisposition struct {
@@ -58,7 +59,8 @@ func (r RecoveryReview) DispositionIntent(id, requestKey string) (RecoveryDispos
return RecoveryDispositionIntent{}, ErrRecoveryReviewExpired return RecoveryDispositionIntent{}, ErrRecoveryReviewExpired
} }
intent := RecoveryDispositionIntent{ intent := RecoveryDispositionIntent{
ID: id, TransactionID: r.summary.TransactionID, ServerID: r.summary.ServerID, SourceDispositionID: r.summary.SourceDispositionID,
ID: id, TransactionID: r.summary.TransactionID, ServerID: r.summary.ServerID,
Operator: r.operator, Action: r.summary.Action, RequestKey: requestKey, Operator: r.operator, Action: r.summary.Action, RequestKey: requestKey,
RecordDigest: hex.EncodeToString(r.recordDigest[:]), RecordDigest: hex.EncodeToString(r.recordDigest[:]),
ServerDigest: hex.EncodeToString(r.serverDigest[:]), ServerDigest: hex.EncodeToString(r.serverDigest[:]),
@@ -68,6 +70,9 @@ func (r RecoveryReview) DispositionIntent(id, requestKey string) (RecoveryDispos
} }
func (i RecoveryDispositionIntent) Validate() error { func (i RecoveryDispositionIntent) Validate() error {
if isRecoveryVerification(i.Action) != (i.SourceDispositionID != "") || (i.SourceDispositionID != "" && (!validDispositionID(i.SourceDispositionID) || i.SourceDispositionID == i.ID)) {
return ErrRecoveryDispositionInvalid
}
if !validInspectionID(i.TransactionID) || !validRecoverySummary(i.ServerID) || if !validInspectionID(i.TransactionID) || !validRecoverySummary(i.ServerID) ||
!validReviewOperator(i.Operator) || !validRecoveryAction(i.Action) { !validReviewOperator(i.Operator) || !validRecoveryAction(i.Action) {
return ErrRecoveryDispositionInvalid return ErrRecoveryDispositionInvalid
@@ -106,9 +111,9 @@ func (i RecoveryDispositionIntent) AllowsResult(result string) bool {
case RecoveryResultObserved: case RecoveryResultObserved:
return i.Action == RecoveryActionRecord return i.Action == RecoveryActionRecord
case RecoveryResultCommitted: case RecoveryResultCommitted:
return i.Action == RecoveryActionCommit return i.Action == RecoveryActionCommit || i.Action == RecoveryActionVerifyCommit
case RecoveryResultRolledBack: case RecoveryResultRolledBack:
return i.Action == RecoveryActionRollback return i.Action == RecoveryActionRollback || i.Action == RecoveryActionVerifyRollback
} }
return false return false
} }
@@ -100,6 +100,12 @@ func (s *Service) executeRecoveryAction(ctx context.Context, review RecoveryRevi
case RecoveryActionRollback: case RecoveryActionRollback:
err = s.executeRecoveryRollback(ctx, &record, server, client, &mutated) err = s.executeRecoveryRollback(ctx, &record, server, client, &mutated)
result = RecoveryResultRolledBack result = RecoveryResultRolledBack
case RecoveryActionVerifyCommit, RecoveryActionVerifyRollback:
err = s.verifyRecoveryCompletion(ctx, record, server, client, review.summary.Action)
result = RecoveryResultCommitted
if review.summary.Action == RecoveryActionVerifyRollback {
result = RecoveryResultRolledBack
}
default: default:
return return
} }
@@ -117,6 +123,13 @@ func validateRecoveryAction(record TransactionRecord, action RecoveryAction, ver
return ErrRecoveryActionRejected return ErrRecoveryActionRejected
} }
switch action { switch action {
case RecoveryActionVerifyCommit, RecoveryActionVerifyRollback:
if !verificationPhase(record.Phase, action) ||
(action == RecoveryActionVerifyCommit && record.Evidence.Baseline.ExpectedFingerprint == "") ||
(action == RecoveryActionVerifyRollback && record.Evidence.Baseline.PreviousFingerprint == "") {
return ErrRecoveryActionRejected
}
return nil
case RecoveryActionCommit: case RecoveryActionCommit:
if record.Evidence.Baseline.ExpectedFingerprint == "" { if record.Evidence.Baseline.ExpectedFingerprint == "" {
return ErrRecoveryActionRejected return ErrRecoveryActionRejected
@@ -0,0 +1,113 @@
package deployment
import (
"context"
"encoding/hex"
"time"
)
func isRecoveryVerification(action RecoveryAction) bool {
return action == RecoveryActionVerifyCommit || action == RecoveryActionVerifyRollback
}
// Reconciles 仅允许已持久化结束的不确定原动作;不接管运行中处置或再次执行文件动作。
func (i RecoveryDispositionIntent) Reconciles(source RecoveryDisposition) bool {
if i.SourceDispositionID == "" || i.SourceDispositionID != source.Intent.ID || source.Intent.Validate() != nil ||
source.Status != RecoveryResultUncertain || source.ResultCode != RecoveryResultUncertain ||
i.TransactionID != source.Intent.TransactionID || i.ServerID != source.Intent.ServerID || i.ServerDigest != source.Intent.ServerDigest {
return false
}
return (i.Action == RecoveryActionVerifyCommit && source.Intent.Action == RecoveryActionCommit) ||
(i.Action == RecoveryActionVerifyRollback && source.Intent.Action == RecoveryActionRollback)
}
// PrepareRecoveryReconciliation 只供受保护本机通道调用,必须明确指定原处置编号并重新确认。
func (s *Service) PrepareRecoveryReconciliation(ctx context.Context, sourceID, operator string, resolve func(context.Context, string) (Server, error)) (RecoveryConfirmation, string, error) {
review, err := s.prepareRecoveryReconciliation(ctx, sourceID, operator, resolve)
if err != nil {
return RecoveryConfirmation{}, "", err
}
return s.newRecoveryConfirmation(ctx, review)
}
func (s *Service) prepareRecoveryReconciliation(ctx context.Context, sourceID, operator string, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
if s == nil || !validDispositionID(sourceID) || !validReviewOperator(operator) {
return RecoveryReview{}, ErrRecoveryReviewInvalid
}
journal, ok := s.journal.(RecoveryDispositionJournal)
if !ok {
return RecoveryReview{}, ErrRecoveryDispositionUnavailable
}
source, err := journal.GetRecoveryDisposition(ctx, sourceID)
if err != nil || source.Intent.ID != sourceID || source.Intent.Validate() != nil || source.Summary().Validate() != nil ||
source.Status != RecoveryResultUncertain || source.ResultCode != RecoveryResultUncertain {
return RecoveryReview{}, ErrRecoveryActionRejected
}
var action RecoveryAction
switch source.Intent.Action {
case RecoveryActionCommit:
action = RecoveryActionVerifyCommit
case RecoveryActionRollback:
action = RecoveryActionVerifyRollback
default:
return RecoveryReview{}, ErrRecoveryActionRejected
}
review, err := s.prepareRecoveryReview(ctx, source.Intent.TransactionID, operator, action, resolve)
if err != nil {
return RecoveryReview{}, err
}
review.summary.SourceDispositionID = sourceID
intent := RecoveryDispositionIntent{SourceDispositionID: sourceID, TransactionID: review.summary.TransactionID,
ServerID: review.summary.ServerID, ServerDigest: hex.EncodeToString(review.serverDigest[:]), Action: action}
if !intent.Reconciles(source) || !verificationPhase(review.summary.Phase, action) || s.VerifyRecoveryTarget == nil {
return RecoveryReview{}, ErrRecoveryActionRejected
}
return review, nil
}
func verificationPhase(phase string, action RecoveryAction) bool {
return (action == RecoveryActionVerifyCommit && phase == TransactionPhaseCommitVerified) ||
(action == RecoveryActionVerifyRollback && phase == TransactionPhaseRollbackVerified)
}
// validateRecoveryCompletion 不删除任何远端文件,残留锁或备份均要求继续保留占用。
func validateRecoveryCompletion(ctx context.Context, client Client, record TransactionRecord, committed bool) error {
for _, lock := range record.LockPaths {
if exists, err := client.Exists(ctx, lock); err != nil || exists {
return ErrRecoveryActionRejected
}
}
for _, backup := range record.Backups {
if committed {
if err := checkFileDigest(ctx, client, backup.Original, record.Evidence.Files[backup.Original]); err != nil {
return err
}
} else if backup.OriginalExists {
if err := checkFileDigest(ctx, client, backup.Original, backup.OriginalDigest); err != nil {
return err
}
} else if exists, err := client.Exists(ctx, backup.Original); err != nil || exists {
return ErrRecoveryActionRejected
}
for _, path := range []string{backup.Saved, backup.Original + ".scert-tmp-" + record.ID} {
if exists, err := client.Exists(ctx, path); err != nil || exists {
return ErrRecoveryActionRejected
}
}
}
return ctx.Err()
}
func (s *Service) verifyRecoveryCompletion(ctx context.Context, record TransactionRecord, server Server, client Client, action RecoveryAction) error {
committed := action == RecoveryActionVerifyCommit
if err := validateRecoveryCompletion(ctx, client, record, committed); err != nil {
return err
}
if err := s.VerifyRecoveryTarget(ctx, server, record.Evidence.Baseline, committed); err != nil {
return err
}
// HTTPS 检查结束后重新读取现场,避免检查期间发生的文件变化被忽略。
return validateRecoveryCompletion(ctx, client, record, committed)
}
+14 -9
View File
@@ -8,19 +8,21 @@ import (
// RecoveryDispositionSummary 是本机查询的明确输出边界,不包含请求键、绑定摘要或操作者凭据。 // RecoveryDispositionSummary 是本机查询的明确输出边界,不包含请求键、绑定摘要或操作者凭据。
type RecoveryDispositionSummary struct { type RecoveryDispositionSummary struct {
ID string `json:"id"` SourceDispositionID string `json:"source_disposition_id,omitempty"`
TransactionID string `json:"transaction_id"` ID string `json:"id"`
ServerID string `json:"server_id"` TransactionID string `json:"transaction_id"`
Action RecoveryAction `json:"action"` ServerID string `json:"server_id"`
Status string `json:"status"` Action RecoveryAction `json:"action"`
ResultCode string `json:"result_code"` Status string `json:"status"`
CreatedAt time.Time `json:"created_at"` ResultCode string `json:"result_code"`
UpdatedAt time.Time `json:"updated_at"` CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
} }
func (r RecoveryDisposition) Summary() RecoveryDispositionSummary { func (r RecoveryDisposition) Summary() RecoveryDispositionSummary {
return RecoveryDispositionSummary{ return RecoveryDispositionSummary{
ID: r.Intent.ID, TransactionID: r.Intent.TransactionID, ServerID: r.Intent.ServerID, SourceDispositionID: r.Intent.SourceDispositionID,
ID: r.Intent.ID, TransactionID: r.Intent.TransactionID, ServerID: r.Intent.ServerID,
Action: r.Intent.Action, Status: r.Status, ResultCode: r.ResultCode, Action: r.Intent.Action, Status: r.Status, ResultCode: r.ResultCode,
CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt, CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt,
} }
@@ -31,6 +33,9 @@ func validDispositionID(id string) bool {
} }
func (r RecoveryDispositionSummary) Validate() error { func (r RecoveryDispositionSummary) Validate() error {
if isRecoveryVerification(r.Action) != (r.SourceDispositionID != "") || (r.SourceDispositionID != "" && (!validDispositionID(r.SourceDispositionID) || r.SourceDispositionID == r.ID)) {
return ErrRecoveryDispositionInvalid
}
if !validDispositionID(r.ID) || !validInspectionID(r.TransactionID) || !validRecoverySummary(r.ServerID) || if !validDispositionID(r.ID) || !validInspectionID(r.TransactionID) || !validRecoverySummary(r.ServerID) ||
!validRecoveryAction(r.Action) || r.CreatedAt.IsZero() || r.UpdatedAt.IsZero() || r.UpdatedAt.Before(r.CreatedAt) { !validRecoveryAction(r.Action) || r.CreatedAt.IsZero() || r.UpdatedAt.IsZero() || r.UpdatedAt.Before(r.CreatedAt) {
return ErrRecoveryDispositionInvalid return ErrRecoveryDispositionInvalid
+34 -14
View File
@@ -13,10 +13,12 @@ import (
type RecoveryAction string type RecoveryAction string
const ( const (
RecoveryActionRecord RecoveryAction = "record_inspection" RecoveryActionRecord RecoveryAction = "record_inspection"
RecoveryActionCommit RecoveryAction = "finish_commit" RecoveryActionCommit RecoveryAction = "finish_commit"
RecoveryActionRollback RecoveryAction = "restore_baseline" RecoveryActionRollback RecoveryAction = "restore_baseline"
RecoveryReviewLifetime = time.Minute RecoveryActionVerifyCommit RecoveryAction = "verify_commit"
RecoveryActionVerifyRollback RecoveryAction = "verify_rollback"
RecoveryReviewLifetime = time.Minute
) )
var ( var (
@@ -27,13 +29,14 @@ var (
) )
type RecoveryReviewSummary struct { type RecoveryReviewSummary struct {
TransactionID string `json:"transaction_id"` SourceDispositionID string `json:"source_disposition_id,omitempty"`
ServerID string `json:"server_id"` TransactionID string `json:"transaction_id"`
Phase string `json:"phase"` ServerID string `json:"server_id"`
Action RecoveryAction `json:"action"` Phase string `json:"phase"`
HistoricalEvidence bool `json:"historical_evidence"` Action RecoveryAction `json:"action"`
ObservedAt time.Time `json:"observed_at"` HistoricalEvidence bool `json:"historical_evidence"`
ExpiresAt time.Time `json:"expires_at"` ObservedAt time.Time `json:"observed_at"`
ExpiresAt time.Time `json:"expires_at"`
} }
// Validate 与服务端历史记录使用同一摘要边界,允许旧记录没有阶段,但不允许控制字符或超长字段。 // Validate 与服务端历史记录使用同一摘要边界,允许旧记录没有阶段,但不允许控制字符或超长字段。
@@ -44,6 +47,9 @@ func (r RecoveryReviewSummary) Validate() error {
(r.Action != RecoveryActionRecord && !r.HistoricalEvidence) { (r.Action != RecoveryActionRecord && !r.HistoricalEvidence) {
return ErrRecoveryReviewInvalid return ErrRecoveryReviewInvalid
} }
if isRecoveryVerification(r.Action) != (r.SourceDispositionID != "") || (r.SourceDispositionID != "" && !validDispositionID(r.SourceDispositionID)) {
return ErrRecoveryReviewInvalid
}
return nil return nil
} }
@@ -65,12 +71,20 @@ func validReviewOperator(operator string) bool {
} }
func validRecoveryAction(action RecoveryAction) bool { func validRecoveryAction(action RecoveryAction) bool {
return action == RecoveryActionRecord || action == RecoveryActionCommit || action == RecoveryActionRollback return action == RecoveryActionRecord || action == RecoveryActionCommit || action == RecoveryActionRollback || isRecoveryVerification(action)
} }
// PrepareRecoveryReview 在同一操作上限内读取前后记录,拒绝采样期间发生的配置或事务变化。 // PrepareRecoveryReview 在同一操作上限内读取前后记录,拒绝采样期间发生的配置或事务变化。
// 当前采样摘要只绑定本次准备,不补写到历史 Evidence,也不改变事务终态。 // 当前采样摘要只绑定本次准备,不补写到历史 Evidence,也不改变事务终态。
func (s *Service) PrepareRecoveryReview(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) { func (s *Service) PrepareRecoveryReview(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) {
// 重新核查必须通过独立入口绑定原处置,不能作为普通动作绕过来源检查。
if isRecoveryVerification(action) {
return RecoveryReview{}, ErrRecoveryReviewInvalid
}
return s.prepareRecoveryReview(ctx, id, operator, action, resolve)
}
func (s *Service) prepareRecoveryReview(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second) ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel() defer cancel()
if !validReviewOperator(operator) || !validRecoveryAction(action) || !validInspectionID(id) { if !validReviewOperator(operator) || !validRecoveryAction(action) || !validInspectionID(id) {
@@ -140,14 +154,20 @@ func (s *Service) RecheckRecoveryReview(ctx context.Context, review RecoveryRevi
if s.now == nil || !s.now().Before(review.summary.ExpiresAt) { if s.now == nil || !s.now().Before(review.summary.ExpiresAt) {
return ErrRecoveryReviewExpired return ErrRecoveryReviewExpired
} }
current, err := s.PrepareRecoveryReview(ctx, review.summary.TransactionID, operator, review.summary.Action, resolve) var current RecoveryReview
var err error
if isRecoveryVerification(review.summary.Action) {
current, err = s.prepareRecoveryReconciliation(ctx, review.summary.SourceDispositionID, operator, resolve)
} else {
current, err = s.PrepareRecoveryReview(ctx, review.summary.TransactionID, operator, review.summary.Action, resolve)
}
if err != nil { if err != nil {
return err return err
} }
if !s.now().Before(review.summary.ExpiresAt) { if !s.now().Before(review.summary.ExpiresAt) {
return ErrRecoveryReviewExpired return ErrRecoveryReviewExpired
} }
if current.recordDigest != review.recordDigest || current.serverDigest != review.serverDigest || current.observationDigest != review.observationDigest { if current.recordDigest != review.recordDigest || current.serverDigest != review.serverDigest || current.observationDigest != review.observationDigest || current.summary.SourceDispositionID != review.summary.SourceDispositionID || current.summary.Action != review.summary.Action {
return ErrRecoveryReviewChanged return ErrRecoveryReviewChanged
} }
return ctx.Err() return ctx.Err()
@@ -87,3 +87,26 @@ func (r *localResetSession) GetRecoveryDisposition(ctx context.Context, id strin
} }
var _ localreset.RecoveryDispositionSession = (*localResetSession)(nil) var _ localreset.RecoveryDispositionSession = (*localResetSession)(nil)
type localRecoveryReconciler interface {
PrepareRecoveryReconciliation(context.Context, string, string, func(context.Context, string) (deployment.Server, error)) (deployment.RecoveryConfirmation, string, error)
}
func (r *localResetSession) PrepareRecoveryReconciliation(ctx context.Context, sourceID string) (localreset.RecoveryPreparation, error) {
r.mu.Lock()
defer r.mu.Unlock()
r.recoveryConfirmation = deployment.RecoveryConfirmation{}
executor, ok := r.server.recoveryInspection.(localRecoveryReconciler)
if !r.enterRecovery(ctx) || !ok || !r.DispositionAvailable() {
return localreset.RecoveryPreparation{}, localreset.ErrProtocol
}
confirmation, token, err := executor.PrepareRecoveryReconciliation(ctx, sourceID, r.peer.Identity, r.recoveryServer)
if err != nil || !r.current(ctx) || confirmation.Summary().SourceDispositionID != sourceID {
r.stage = -1
return localreset.RecoveryPreparation{}, localreset.ErrUnavailable
}
r.recoveryConfirmation = confirmation
return localreset.RecoveryPreparation{DispositionID: confirmation.DispositionID(), Token: token, Summary: confirmation.Summary()}, nil
}
var _ localreset.RecoveryReconciliationSession = (*localResetSession)(nil)
@@ -0,0 +1,88 @@
package httpapi
import (
"context"
"strings"
"testing"
"time"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
)
type localReconciliationJournal struct {
localDispositionJournal
source deployment.RecoveryDisposition
}
func (j *localReconciliationJournal) GetRecoveryDisposition(ctx context.Context, id string) (deployment.RecoveryDisposition, error) {
if id == j.source.Intent.ID { return j.source, nil }
return j.localDispositionJournal.GetRecoveryDisposition(ctx, id)
}
func localReconciliationFixture(t *testing.T) (*Server, *localResetSession, *localReconciliationJournal) {
t.Helper()
server, _, original := localDispositionFixture(t)
target := server.servers.(localDispositionServerRepo).server
journal := &localReconciliationJournal{}
journal.record = original.record
journal.record.Phase = deployment.TransactionPhaseCommitVerified
journal.record.Evidence = &deployment.RecoveryEvidence{Version: 1, LockToken: strings.Repeat("d", 64), Address: target.Address,
Port: target.Port, HostKeyFingerprint: target.HostKeyFingerprint, ValidationCommand: target.ValidationCommand, ReloadCommand: target.ReloadCommand,
Files: map[string]string{}, TemporaryFiles: map[string]string{}, Baseline: deployment.RecoveryBaseline{Endpoint: "https://example.test", Domains: []string{"example.test"}, ExpectedFingerprint: "new"}}
for _, path := range journal.record.Paths {
journal.record.LockPaths = append(journal.record.LockPaths, path+".scert-lock")
journal.record.Backups = append(journal.record.Backups, deployment.TransactionBackup{Original: path, Saved: path+".scert-backup-"+journal.record.ID})
journal.record.Evidence.Files[path] = strings.Repeat("e", 64)
}
service := deployment.NewServiceWithJournal(localDispositionFactory{}, journal)
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error { return nil }
resolve := func(context.Context, string) (deployment.Server, error) { return target, nil }
review, err := service.PrepareRecoveryReview(t.Context(), journal.record.ID, "uid:0", deployment.RecoveryActionCommit, resolve)
if err != nil { t.Fatal(err) }
intent, err := review.DispositionIntent(strings.Repeat("a", 64), strings.Repeat("b", 64))
if err != nil { t.Fatal(err) }
now := time.Now().UTC()
journal.source = deployment.RecoveryDisposition{Intent: intent, Status: "uncertain", ResultCode: "uncertain", CreatedAt: now, UpdatedAt: now}
server.recoveryInspection = service
return server, localDispositionNewSession(t, server), journal
}
func TestLocalReconciliationRejectsInvalidSession(t *testing.T) {
for _, mode := range []string{"expired", "cancelled", "changed-admin", "closed", "reset-stage", "wrong-id", "wrong-token", "changed-peer", "foreign-session", "replaced-confirmation"} {
t.Run(mode, func(t *testing.T) {
server, session, journal := localReconciliationFixture(t)
prepared, err := session.PrepareRecoveryReconciliation(t.Context(), journal.source.Intent.ID)
if err != nil { t.Fatal(err) }
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
switch mode {
case "expired": session.expires = time.Time{}
case "cancelled": cancel()
case "changed-admin": server.config.Admin.Email = "changed@example.test"
case "closed": session.Close()
case "reset-stage": session.stage = 1
case "wrong-id": prepared.DispositionID = journal.source.Intent.ID
case "wrong-token": prepared.Token = strings.Repeat("a", 64)
case "changed-peer": session.peer.Identity = "uid:123"
case "foreign-session": session = localDispositionNewSession(t, server)
case "replaced-confirmation":
if _, err := session.PrepareRecoveryReconciliation(ctx, journal.source.Intent.ID); err != nil { t.Fatal(err) }
}
if _, err := session.ExecuteRecovery(ctx, prepared.DispositionID, prepared.Token); err == nil { t.Fatal("无效会话执行了重新核查") }
if _, err := session.ExecuteRecovery(t.Context(), prepared.DispositionID, prepared.Token); err == nil { t.Fatal("失败确认被重放") }
if journal.begins != 0 || journal.finishes != 0 || session.recoveryConfirmation.DispositionID() != "" { t.Fatal("无效会话写入了审计或保留了确认") }
})
}
}
func TestLocalReconciliationRejectsMissingFilesAndQueriesNewAudit(t *testing.T) {
server, session, journal := localReconciliationFixture(t)
prepared, err := session.PrepareRecoveryReconciliation(t.Context(), journal.source.Intent.ID)
if err != nil { t.Fatal(err) }
result, err := session.ExecuteRecovery(t.Context(), prepared.DispositionID, prepared.Token)
if err != nil || result.Status != "rejected" || result.SourceDispositionID != journal.source.Intent.ID { t.Fatalf("缺失文件仍被放行:%+v %v", result, err) }
if journal.item.Intent.Operator != "uid:0" || journal.begins != 1 || journal.finishes != 1 || journal.source.Status != "uncertain" { t.Fatal("身份或历史记录被改变") }
next := localDispositionNewSession(t, server)
queried, err := next.GetRecoveryDisposition(t.Context(), result.ID)
if err != nil || queried != result { t.Fatal("重连未查到关联审计", err) }
}
+1 -1
View File
@@ -30,7 +30,7 @@ func validRecoveryRequest(request Request, nonce string) bool {
return request.Cursor == "" && validRecoveryID(request.TransactionID) && validRecoveryAction(request.RecoveryAction) && request.DispositionID == "" && request.ConfirmationToken == "" return request.Cursor == "" && validRecoveryID(request.TransactionID) && validRecoveryAction(request.RecoveryAction) && request.DispositionID == "" && request.ConfirmationToken == ""
case "recovery_execute": case "recovery_execute":
return request.Cursor == "" && request.TransactionID == "" && request.RecoveryAction == "" && validRecoverySecret(request.DispositionID) && validRecoverySecret(request.ConfirmationToken) return request.Cursor == "" && request.TransactionID == "" && request.RecoveryAction == "" && validRecoverySecret(request.DispositionID) && validRecoverySecret(request.ConfirmationToken)
case "recovery_result": case "recovery_result", "recovery_reconcile_prepare":
return request.Cursor == "" && request.TransactionID == "" && request.RecoveryAction == "" && validRecoverySecret(request.DispositionID) && request.ConfirmationToken == "" return request.Cursor == "" && request.TransactionID == "" && request.RecoveryAction == "" && validRecoverySecret(request.DispositionID) && request.ConfirmationToken == ""
} }
return false return false
@@ -21,6 +21,10 @@ type RecoveryDispositionSession interface {
GetRecoveryDisposition(context.Context, string) (deployment.RecoveryDispositionSummary, error) GetRecoveryDisposition(context.Context, string) (deployment.RecoveryDispositionSummary, error)
} }
type RecoveryReconciliationSession interface {
PrepareRecoveryReconciliation(context.Context, string) (RecoveryPreparation, error)
}
func validRecoveryAction(action deployment.RecoveryAction) bool { func validRecoveryAction(action deployment.RecoveryAction) bool {
return action == deployment.RecoveryActionRecord || action == deployment.RecoveryActionCommit || action == deployment.RecoveryActionRollback return action == deployment.RecoveryActionRecord || action == deployment.RecoveryActionCommit || action == deployment.RecoveryActionRollback
} }
@@ -60,6 +64,11 @@ func validDispositionResponse(request Request, response Response) bool {
return response.Disposition == nil && validPreparation(response.Preparation) && return response.Disposition == nil && validPreparation(response.Preparation) &&
response.Preparation.Summary.TransactionID == request.TransactionID && response.Preparation.Summary.Action == request.RecoveryAction response.Preparation.Summary.TransactionID == request.TransactionID && response.Preparation.Summary.Action == request.RecoveryAction
} }
if request.Action == "recovery_reconcile_prepare" {
return response.Disposition == nil && validPreparation(response.Preparation) &&
response.Preparation.Summary.SourceDispositionID == request.DispositionID &&
response.Preparation.DispositionID != request.DispositionID
}
return response.Preparation == nil && response.Disposition != nil && response.Disposition.Validate() == nil && response.Disposition.ID == request.DispositionID return response.Preparation == nil && response.Disposition != nil && response.Disposition.Validate() == nil && response.Disposition.ID == request.DispositionID
} }
@@ -70,6 +79,14 @@ func dispatchRecoveryDisposition(ctx context.Context, session RecoverySession, r
} }
var err error var err error
switch request.Action { switch request.Action {
case "recovery_reconcile_prepare":
reconciler, ok := session.(RecoveryReconciliationSession)
if !ok {
return ErrUnavailable
}
var value RecoveryPreparation
value, err = reconciler.PrepareRecoveryReconciliation(ctx, request.DispositionID)
response.Preparation = &value
case "recovery_prepare": case "recovery_prepare":
var value RecoveryPreparation var value RecoveryPreparation
value, err = actions.PrepareRecovery(ctx, request.TransactionID, request.RecoveryAction) value, err = actions.PrepareRecovery(ctx, request.TransactionID, request.RecoveryAction)
@@ -101,6 +118,14 @@ func (c *Client) PrepareRecovery(id string, action deployment.RecoveryAction) (R
return *response.Preparation, nil return *response.Preparation, nil
} }
func (c *Client) PrepareRecoveryReconciliation(sourceID string) (RecoveryPreparation, error) {
response, err := c.recoveryExchange(Request{Action: "recovery_reconcile_prepare", DispositionID: sourceID})
if err != nil {
return RecoveryPreparation{}, err
}
return *response.Preparation, nil
}
func (c *Client) ExecuteRecovery(id, token string) (deployment.RecoveryDispositionSummary, error) { func (c *Client) ExecuteRecovery(id, token string) (deployment.RecoveryDispositionSummary, error) {
response, err := c.recoveryExchange(Request{Action: "recovery_execute", DispositionID: id, ConfirmationToken: token}) response, err := c.recoveryExchange(Request{Action: "recovery_execute", DispositionID: id, ConfirmationToken: token})
if err != nil { if err != nil {
@@ -89,6 +89,7 @@ func TestDispositionRequestFieldIsolation(t *testing.T) {
{Action: "recovery_prepare", TransactionID: "tx-1", RecoveryAction: deployment.RecoveryActionRecord}, {Action: "recovery_prepare", TransactionID: "tx-1", RecoveryAction: deployment.RecoveryActionRecord},
{Action: "recovery_execute", DispositionID: strings.Repeat("b", 64), ConfirmationToken: strings.Repeat("c", 64)}, {Action: "recovery_execute", DispositionID: strings.Repeat("b", 64), ConfirmationToken: strings.Repeat("c", 64)},
{Action: "recovery_result", DispositionID: strings.Repeat("b", 64)}, {Action: "recovery_result", DispositionID: strings.Repeat("b", 64)},
{Action: "recovery_reconcile_prepare", DispositionID: strings.Repeat("b", 64)},
} }
fields := []struct { fields := []struct {
name string name string
@@ -101,7 +102,7 @@ func TestDispositionRequestFieldIsolation(t *testing.T) {
{"cursor", func(r *Request) { r.Cursor = "tx-0" }, map[string]bool{"recovery_list": true}}, {"cursor", func(r *Request) { r.Cursor = "tx-0" }, map[string]bool{"recovery_list": true}},
{"transaction", func(r *Request) { r.TransactionID = "tx-1" }, map[string]bool{"recovery_inspect": true, "recovery_prepare": true}}, {"transaction", func(r *Request) { r.TransactionID = "tx-1" }, map[string]bool{"recovery_inspect": true, "recovery_prepare": true}},
{"action", func(r *Request) { r.RecoveryAction = deployment.RecoveryActionRecord }, map[string]bool{"recovery_prepare": true}}, {"action", func(r *Request) { r.RecoveryAction = deployment.RecoveryActionRecord }, map[string]bool{"recovery_prepare": true}},
{"id", func(r *Request) { r.DispositionID = strings.Repeat("b", 64) }, map[string]bool{"recovery_execute": true, "recovery_result": true}}, {"id", func(r *Request) { r.DispositionID = strings.Repeat("b", 64) }, map[string]bool{"recovery_execute": true, "recovery_result": true, "recovery_reconcile_prepare": true}},
{"token", func(r *Request) { r.ConfirmationToken = strings.Repeat("c", 64) }, map[string]bool{"recovery_execute": true}}, {"token", func(r *Request) { r.ConfirmationToken = strings.Repeat("c", 64) }, map[string]bool{"recovery_execute": true}},
} }
for _, request := range requests { for _, request := range requests {
@@ -0,0 +1,82 @@
package localreset
import (
"context"
"strings"
"testing"
"time"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
)
type reconciliationSession struct { dispositionTestSession }
func (s *reconciliationSession) PrepareRecoveryReconciliation(ctx context.Context, sourceID string) (RecoveryPreparation, error) {
value, err := s.PrepareRecovery(ctx, "tx-1", deployment.RecoveryActionVerifyCommit)
value.Summary.SourceDispositionID = sourceID
value.Summary.Phase = deployment.TransactionPhaseCommitVerified
s.prepared = value
return value, err
}
func (s *reconciliationSession) ExecuteRecovery(ctx context.Context, id, token string) (deployment.RecoveryDispositionSummary, error) {
sourceID := s.prepared.Summary.SourceDispositionID
value, err := s.dispositionTestSession.ExecuteRecovery(ctx, id, token)
if err != nil { return value, err }
value.SourceDispositionID, value.ResultCode = sourceID, deployment.RecoveryResultCommitted
s.result = value
return value, nil
}
func TestRecoveryReconciliationProtocolAndQuery(t *testing.T) {
session := &reconciliationSession{}
_, conn, done := recoveryConnection(t, session)
client, err := Begin(t.Context(), conn)
if err != nil { t.Fatal(err) }
sourceID := strings.Repeat("a", 64)
prepared, err := client.PrepareRecoveryReconciliation(sourceID)
if err != nil || prepared.Summary.SourceDispositionID != sourceID { t.Fatal("原处置绑定丢失", err) }
result, err := client.ExecuteRecovery(prepared.DispositionID, prepared.Token)
if err != nil || result.SourceDispositionID != sourceID || result.Status != "succeeded" { t.Fatal("重新核查结果无效", err) }
queried, err := client.GetRecoveryDisposition(result.ID)
if err != nil || queried != result { t.Fatal("查询改变了关联结果", err) }
if _, err := client.ExecuteRecovery(prepared.DispositionID, prepared.Token); err == nil { t.Fatal("旧令牌被重放") }
_ = client.Close()
<-done
if session.prepares.Load() != 1 || session.executes.Load() != 2 || session.queries.Load() != 1 || session.resets.Load() != 0 { t.Fatal("调用了错误业务入口") }
}
func TestRecoveryReconciliationRejectsUnboundResponse(t *testing.T) {
sourceID := strings.Repeat("a", 64)
session := &reconciliationSession{}
prepared, err := session.PrepareRecoveryReconciliation(t.Context(), sourceID)
if err != nil { t.Fatal(err) }
request := Request{Action: "recovery_reconcile_prepare", DispositionID: sourceID}
for _, mode := range []string{"source", "action", "same-id", "transaction", "mixed", "expired-range"} {
t.Run(mode, func(t *testing.T) {
value := prepared
response := Response{Preparation: &value}
switch mode {
case "source": value.Summary.SourceDispositionID = strings.Repeat("d", 64)
case "action": value.Summary.Action = deployment.RecoveryActionCommit
case "same-id": value.DispositionID = sourceID
case "transaction": value.Summary.TransactionID = "../bad"
case "mixed": response.Disposition = &deployment.RecoveryDispositionSummary{}
case "expired-range": value.Summary.ExpiresAt = value.Summary.ObservedAt.Add(2*time.Minute)
}
if validDispositionResponse(request, response) { t.Fatal("接受了混合或无绑定准备响应") }
})
}
if validRecoveryAction(deployment.RecoveryActionVerifyCommit) || validRecoveryAction(deployment.RecoveryActionVerifyRollback) { t.Fatal("普通准备允许绕过原处置绑定") }
}
func TestRecoveryReconciliationOldServerFailsClosed(t *testing.T) {
session := &dispositionTestSession{}
_, conn, done := recoveryConnection(t, session)
client, err := Begin(t.Context(), conn)
if err != nil { t.Fatal(err) }
if _, err := client.PrepareRecoveryReconciliation(strings.Repeat("a", 64)); err == nil { t.Fatal("旧服务错误地支持了重新核查") }
_ = client.Close()
<-done
if session.prepares.Load() != 0 || session.executes.Load() != 0 { t.Fatal("旧服务错误地执行了原动作") }
}
@@ -25,7 +25,7 @@ func (s *Server) handleRecovery(ctx context.Context, conn net.Conn, reader *bufi
case "recovery_inspect": case "recovery_inspect":
inspection, readErr := session.InspectRecovery(ctx, request.TransactionID) inspection, readErr := session.InspectRecovery(ctx, request.TransactionID)
err, response.Inspection = readErr, &inspection err, response.Inspection = readErr, &inspection
case "recovery_prepare", "recovery_execute", "recovery_result": case "recovery_prepare", "recovery_execute", "recovery_result", "recovery_reconcile_prepare":
err = dispatchRecoveryDisposition(ctx, session, request, &response) err = dispatchRecoveryDisposition(ctx, session, request, &response)
default: default:
err = ErrProtocol err = ErrProtocol
@@ -43,6 +43,11 @@ func (r *SQLDeploymentTransactionRepository) FinishRecoveryDisposition(ctx conte
if err := tx.QueryRowContext(ctx, r.query(`SELECT event_type FROM recovery_events WHERE disposition_id = ? AND event_type = 'started'`), id).Scan(&start); err != nil { if err := tx.QueryRowContext(ctx, r.query(`SELECT event_type FROM recovery_events WHERE disposition_id = ? AND event_type = 'started'`), id).Scan(&start); err != nil {
return deployment.RecoveryDisposition{}, dispositionError(ctx) return deployment.RecoveryDisposition{}, dispositionError(ctx)
} }
if item.Intent.SourceDispositionID != "" {
if err := r.validateReconciliationSource(ctx, tx, item.Intent); err != nil {
return deployment.RecoveryDisposition{}, err
}
}
now := time.Now().UTC() now := time.Now().UTC()
if err := r.finishRecoveryOriginal(ctx, tx, item.Intent, resultCode, now); err != nil { if err := r.finishRecoveryOriginal(ctx, tx, item.Intent, resultCode, now); err != nil {
return deployment.RecoveryDisposition{}, err return deployment.RecoveryDisposition{}, err
@@ -58,7 +63,13 @@ func (r *SQLDeploymentTransactionRepository) FinishRecoveryDisposition(ctx conte
if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_events (disposition_id, event_type, result_code, created_at) VALUES (?, 'finished', ?, ?)`), id, resultCode, now); err != nil { if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_events (disposition_id, event_type, result_code, created_at) VALUES (?, 'finished', ?, ?)`), id, resultCode, now); err != nil {
return deployment.RecoveryDisposition{}, dispositionError(ctx) return deployment.RecoveryDisposition{}, dispositionError(ctx)
} }
if resultCode != deployment.RecoveryResultUncertain { if item.Intent.SourceDispositionID != "" && (resultCode == deployment.RecoveryResultRejected || resultCode == deployment.RecoveryResultUncertain) {
// 只读核查未通过时归还原占用,保留两次审计;绝不解锁原事务。
result, err := tx.ExecContext(ctx, r.query(`UPDATE recovery_claims SET disposition_id = ?, created_at = ? WHERE transaction_id = ? AND disposition_id = ?`), item.Intent.SourceDispositionID, now, item.Intent.TransactionID, id)
if err != nil || !oneRecoveryRow(result) {
return deployment.RecoveryDisposition{}, dispositionError(ctx)
}
} else if resultCode != deployment.RecoveryResultUncertain {
result, err := tx.ExecContext(ctx, r.query(`DELETE FROM recovery_claims WHERE transaction_id = ? AND disposition_id = ?`), item.Intent.TransactionID, id) result, err := tx.ExecContext(ctx, r.query(`DELETE FROM recovery_claims WHERE transaction_id = ? AND disposition_id = ?`), item.Intent.TransactionID, id)
if err != nil || !oneRecoveryRow(result) { if err != nil || !oneRecoveryRow(result) {
return deployment.RecoveryDisposition{}, dispositionError(ctx) return deployment.RecoveryDisposition{}, dispositionError(ctx)
@@ -96,6 +107,9 @@ func (r *SQLDeploymentTransactionRepository) finishRecoveryOriginal(ctx context.
(record.Status != deployment.TransactionStatusPending && record.Status != deployment.TransactionStatusRecoveryRequired) { (record.Status != deployment.TransactionStatusPending && record.Status != deployment.TransactionStatusRecoveryRequired) {
return deployment.ErrRecoveryDispositionConflict return deployment.ErrRecoveryDispositionConflict
} }
if intent.SourceDispositionID != "" && !intent.MatchesRecord(record) {
return deployment.ErrRecoveryReviewChanged
}
result, err := tx.ExecContext(ctx, r.query(`UPDATE deployment_transactions SET status = ?, phase = ?, message = ?, updated_at = ? result, err := tx.ExecContext(ctx, r.query(`UPDATE deployment_transactions SET status = ?, phase = ?, message = ?, updated_at = ?
WHERE id = ? AND phase = ? AND status IN ('pending', 'recovery_required')`), status, status, message, now, intent.TransactionID, phase) WHERE id = ? AND phase = ? AND status IN ('pending', 'recovery_required')`), status, status, message, now, intent.TransactionID, phase)
if err != nil || !oneRecoveryRow(result) { if err != nil || !oneRecoveryRow(result) {
@@ -12,13 +12,13 @@ import (
var _ deployment.RecoveryDispositionJournal = (*SQLDeploymentTransactionRepository)(nil) var _ deployment.RecoveryDispositionJournal = (*SQLDeploymentTransactionRepository)(nil)
const recoveryDispositionColumns = `id, transaction_id, server_id, operator_id, action, request_key, const recoveryDispositionColumns = `id, transaction_id, server_id, operator_id, action, request_key,
record_digest, server_digest, observation_digest, status, result_code, created_at, updated_at` record_digest, server_digest, observation_digest, status, result_code, created_at, updated_at, source_disposition_id`
func scanRecoveryDisposition(row deploymentTransactionScanner) (deployment.RecoveryDisposition, error) { func scanRecoveryDisposition(row deploymentTransactionScanner) (deployment.RecoveryDisposition, error) {
var item deployment.RecoveryDisposition var item deployment.RecoveryDisposition
i := &item.Intent i := &item.Intent
err := row.Scan(&i.ID, &i.TransactionID, &i.ServerID, &i.Operator, &i.Action, &i.RequestKey, err := row.Scan(&i.ID, &i.TransactionID, &i.ServerID, &i.Operator, &i.Action, &i.RequestKey,
&i.RecordDigest, &i.ServerDigest, &i.ObservationDigest, &item.Status, &item.ResultCode, &item.CreatedAt, &item.UpdatedAt) &i.RecordDigest, &i.ServerDigest, &i.ObservationDigest, &item.Status, &item.ResultCode, &item.CreatedAt, &item.UpdatedAt, &i.SourceDispositionID)
return item, err return item, err
} }
@@ -75,9 +75,9 @@ func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx contex
defer tx.Rollback() defer tx.Rollback()
now := time.Now().UTC() now := time.Now().UTC()
item := deployment.RecoveryDisposition{Intent: intent, Status: "running", CreatedAt: now, UpdatedAt: now} item := deployment.RecoveryDisposition{Intent: intent, Status: "running", CreatedAt: now, UpdatedAt: now}
_, err = tx.ExecContext(ctx, r.query(`INSERT INTO recovery_dispositions (`+recoveryDispositionColumns+`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`), _, err = tx.ExecContext(ctx, r.query(`INSERT INTO recovery_dispositions (`+recoveryDispositionColumns+`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`),
intent.ID, intent.TransactionID, intent.ServerID, intent.Operator, intent.Action, intent.RequestKey, intent.ID, intent.TransactionID, intent.ServerID, intent.Operator, intent.Action, intent.RequestKey,
intent.RecordDigest, intent.ServerDigest, intent.ObservationDigest, item.Status, "", now, now) intent.RecordDigest, intent.ServerDigest, intent.ObservationDigest, item.Status, "", now, now, intent.SourceDispositionID)
if err != nil { if err != nil {
_ = tx.Rollback() _ = tx.Rollback()
if existing, readErr := r.findRecoveryRequest(ctx, intent.RequestKey); readErr == nil { if existing, readErr := r.findRecoveryRequest(ctx, intent.RequestKey); readErr == nil {
@@ -85,7 +85,7 @@ func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx contex
} }
return deployment.RecoveryDisposition{}, false, dispositionError(ctx) return deployment.RecoveryDisposition{}, false, dispositionError(ctx)
} }
if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_claims (transaction_id, disposition_id, created_at) VALUES (?, ?, ?)`), intent.TransactionID, intent.ID, now); err != nil { if err := r.claimRecoveryDisposition(ctx, tx, intent, now); err != nil {
_ = tx.Rollback() _ = tx.Rollback()
if claimed, readErr := r.HasRecoveryClaim(ctx, intent.TransactionID); readErr == nil && claimed { if claimed, readErr := r.HasRecoveryClaim(ctx, intent.TransactionID); readErr == nil && claimed {
return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionBusy return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionBusy
@@ -109,6 +109,39 @@ func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx contex
return item, true, nil return item, true, nil
} }
func (r *SQLDeploymentTransactionRepository) claimRecoveryDisposition(ctx context.Context, tx *sql.Tx, intent deployment.RecoveryDispositionIntent, now time.Time) error {
if intent.SourceDispositionID == "" {
_, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_claims (transaction_id, disposition_id, created_at) VALUES (?, ?, ?)`), intent.TransactionID, intent.ID, now)
return err
}
if err := r.validateReconciliationSource(ctx, tx, intent); err != nil {
return err
}
// 只从指定的已结束处置接管占用;运行中、其他处置或丢失的占用均不接管。
result, err := tx.ExecContext(ctx, r.query(`UPDATE recovery_claims SET disposition_id = ?, created_at = ? WHERE transaction_id = ? AND disposition_id = ?`), intent.ID, now, intent.TransactionID, intent.SourceDispositionID)
if err != nil || !oneRecoveryRow(result) {
return deployment.ErrRecoveryDispositionBusy
}
return nil
}
func (r *SQLDeploymentTransactionRepository) validateReconciliationSource(ctx context.Context, tx *sql.Tx, intent deployment.RecoveryDispositionIntent) error {
query := `SELECT ` + recoveryDispositionColumns + ` FROM recovery_dispositions WHERE id = ?`
if r.driver != "sqlite" {
query += " FOR UPDATE"
}
source, err := scanRecoveryDisposition(tx.QueryRowContext(ctx, r.query(query), intent.SourceDispositionID))
if err != nil || !intent.Reconciles(source) {
return deployment.ErrRecoveryDispositionConflict
}
var count int
if err := tx.QueryRowContext(ctx, r.query(`SELECT COUNT(*) FROM recovery_events WHERE disposition_id = ? AND
((event_type = 'started' AND result_code = '') OR (event_type = 'finished' AND result_code = 'uncertain'))`), intent.SourceDispositionID).Scan(&count); err != nil || count != 2 {
return deployment.ErrRecoveryDispositionConflict
}
return nil
}
func recoveryRetry(existing deployment.RecoveryDisposition, intent deployment.RecoveryDispositionIntent) (deployment.RecoveryDisposition, bool, error) { func recoveryRetry(existing deployment.RecoveryDisposition, intent deployment.RecoveryDispositionIntent) (deployment.RecoveryDisposition, bool, error) {
if !existing.Intent.SameRequest(intent) { if !existing.Intent.SameRequest(intent) {
return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionConflict return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionConflict
@@ -68,6 +68,16 @@ func openRecoveryExternal(t *testing.T, cfg config.DatabaseConfig) *sql.DB {
// 只替换远端文件实现,准备、确认、执行、迁移、占用和审计均走实际产品代码。 // 只替换远端文件实现,准备、确认、执行、迁移、占用和审计均走实际产品代码。
func runExternalRecovery(t *testing.T, fixture recoveryExternalFixture) { func runExternalRecovery(t *testing.T, fixture recoveryExternalFixture) {
t.Helper() t.Helper()
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
t.Run("reconciliation/"+string(action), func(t *testing.T) {
cfg := fixture.database(t)
db := openRecoveryExternal(t, cfg)
if err := database.Migrate(t.Context(), db, cfg.Driver); err != nil {
t.Fatal(err)
}
runReconciliationSuccess(t, NewSQLDeploymentTransactionRepository(db, cfg.Driver), db, action)
})
}
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionRecord, deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} { for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionRecord, deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
for _, stage := range []string{"none", "started", "finished"} { for _, stage := range []string{"none", "started", "finished"} {
t.Run(string(action)+"/"+stage, func(t *testing.T) { t.Run(string(action)+"/"+stage, func(t *testing.T) {
@@ -0,0 +1,250 @@
package repository
import (
"context"
"database/sql"
"errors"
"maps"
"reflect"
"testing"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
)
// 远端文件是测试替身;事务、关联、占用交接与审计均经过真实数据库和产品执行器。
type reconciliationClient struct {
*executorIntegrationClient
failClosed, readOnly bool
connections, writes int
}
func (c *reconciliationClient) Connect(context.Context, deployment.Server) (deployment.Client, error) {
c.connections++
return c, nil
}
func (c *reconciliationClient) Close() error {
if c.failClosed {
if _, exists := c.files["/cert.pem.scert-lock"]; !exists {
return errors.New("注入动作完成后的连接关闭失败")
}
}
return nil
}
func (c *reconciliationClient) writable() error {
c.writes++
if c.readOnly {
return errors.New("重新核查不得执行远端写操作")
}
return nil
}
func (c *reconciliationClient) Remove(ctx context.Context, path string) error {
if err := c.writable(); err != nil {
return err
}
return c.executorIntegrationClient.Remove(ctx, path)
}
func (c *reconciliationClient) Rename(ctx context.Context, from, to string) error {
if err := c.writable(); err != nil {
return err
}
return c.executorIntegrationClient.Rename(ctx, from, to)
}
func (c *reconciliationClient) Unlock(ctx context.Context, path string) error {
if err := c.writable(); err != nil {
return err
}
return c.executorIntegrationClient.Unlock(ctx, path)
}
func (c *reconciliationClient) Run(ctx context.Context, command []string) (deployment.CommandResult, error) {
if err := c.writable(); err != nil {
return deployment.CommandResult{}, err
}
return c.executorIntegrationClient.Run(ctx, command)
}
func seedUncertainReconciliation(t *testing.T, repo *SQLDeploymentTransactionRepository, db *sql.DB, action deployment.RecoveryAction) (*deployment.Service, *reconciliationClient, deployment.Server, deployment.TransactionRecord, deployment.RecoveryDisposition) {
t.Helper()
_, _, _, remote, server, record := seedRecoveryExecutor(t, repo, db)
client := &reconciliationClient{executorIntegrationClient: remote, failClosed: true}
service := deployment.NewServiceWithJournal(client, repo)
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error { return nil }
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
if err != nil {
t.Fatal(err)
}
source, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if !errors.Is(err, deployment.ErrRecoveryActionUncertain) || source.Status != deployment.RecoveryResultUncertain {
t.Fatalf("未形成真实执行器不确定结果:%+v %v", source, err)
}
client.failClosed, client.readOnly = false, true
client.writes = 0
return service, client, server, record, source
}
func assertReconciliationSource(t *testing.T, repo *SQLDeploymentTransactionRepository, source deployment.RecoveryDisposition) {
t.Helper()
stored, err := repo.GetRecoveryDisposition(t.Context(), source.Intent.ID)
if err != nil || !reflect.DeepEqual(stored, source) {
t.Fatalf("重新核查改写了原不确定历史:%+v %v", stored, err)
}
}
func runReconciliationSuccess(t *testing.T, repo *SQLDeploymentTransactionRepository, db *sql.DB, action deployment.RecoveryAction) {
t.Helper()
service, client, server, record, source := seedUncertainReconciliation(t, repo, db, action)
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
files := maps.Clone(client.files)
verified := false
service.VerifyRecoveryTarget = func(_ context.Context, _ deployment.Server, _ deployment.RecoveryBaseline, committed bool) error {
if committed != (action == deployment.RecoveryActionCommit) {
t.Fatal("重新核查擅自切换方向")
}
verified = true
return nil
}
confirmation, token, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve)
if err != nil || confirmation.Summary().Validate() != nil {
t.Fatal("准备重新核查失败", err)
}
other, otherToken, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve)
if err != nil {
t.Fatal(err)
}
result, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if err != nil || result.Status != "succeeded" || result.Intent.SourceDispositionID != source.Intent.ID || !verified {
t.Fatalf("重新核查失败:%+v %v", result, err)
}
if client.writes != 0 || !reflect.DeepEqual(files, client.files) {
t.Fatal("只读重新核查修改了远端")
}
assertReconciliationSource(t, repo, source)
recoveryCount(t, db, "recovery_dispositions", 2)
recoveryCount(t, db, "recovery_events", 4)
recoveryCount(t, db, "recovery_claims", 0)
pending, err := repo.ListPendingDeploymentTransactions(t.Context())
if err != nil || len(pending) != 0 {
t.Fatal("原事务未原子结束", err)
}
view, err := service.GetRecoveryDisposition(t.Context(), result.Intent.ID)
if err != nil || view.SourceDispositionID != source.Intent.ID || view.Validate() != nil {
t.Fatal("查询缺少可信关联", err)
}
connections := client.connections
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, deployment.ErrRecoveryConfirmation) {
t.Fatal("重新核查确认被重放", err)
}
if _, err := service.ExecuteRecoveryDisposition(t.Context(), other, "unix:0", otherToken, resolve); err == nil {
t.Fatal("其他旧确认再次接管已完成的事务")
}
if client.connections != connections {
t.Fatal("完成后仍使用旧确认连接远端")
}
if record.ID != result.Intent.TransactionID {
t.Fatal("结束了其他事务")
}
}
func TestRecoveryReconciliationSQLiteSuccess(t *testing.T) {
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
t.Run(string(action), func(t *testing.T) {
_, db := newTaskRepositoryTest(t)
t.Cleanup(func() { _ = db.Close() })
runReconciliationSuccess(t, NewSQLDeploymentTransactionRepository(db), db, action)
})
}
}
func TestRecoveryReconciliationSQLiteRejectsUnsafeCompletion(t *testing.T) {
for _, failure := range []string{"file", "backup", "temporary", "lock", "https", "during-https", "close", "changed-after-prepare", "started-audit", "finished-audit", "identity", "phase", "running", "direction"} {
t.Run(failure, func(t *testing.T) {
_, db := newTaskRepositoryTest(t)
t.Cleanup(func() { _ = db.Close() })
repo := NewSQLDeploymentTransactionRepository(db)
service, client, server, record, source := seedUncertainReconciliation(t, repo, db, deployment.RecoveryActionCommit)
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
switch failure {
case "file":
client.files[record.Paths[0]] = "tampered"
case "backup":
client.files[record.Backups[0].Saved] = "old"
case "temporary":
client.files[record.Paths[0]+".scert-tmp-"+record.ID] = "unexpected"
case "lock":
client.files[record.LockPaths[0]] = ""
case "https":
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error {
return errors.New("严格 HTTPS 未通过")
}
case "during-https":
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error {
client.files[record.Paths[0]] = "changed"
return nil
}
case "identity":
server.PasswordRef = "secret://ssh/password/changed"
case "phase":
if _, err := db.ExecContext(t.Context(), `UPDATE deployment_transactions SET phase = 'reload_attempted'`); err != nil {
t.Fatal(err)
}
case "running":
if _, err := db.ExecContext(t.Context(), `UPDATE recovery_dispositions SET status = 'running', result_code = ''`); err != nil {
t.Fatal(err)
}
case "direction":
if _, err := db.ExecContext(t.Context(), `UPDATE recovery_dispositions SET action = 'restore_baseline'`); err != nil {
t.Fatal(err)
}
}
confirmation, token, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve)
if failure == "identity" || failure == "phase" || failure == "running" || failure == "direction" {
if err == nil {
t.Fatal("危险来源允许重新核查")
}
recoveryCount(t, db, "recovery_claims", 1)
return
}
if err != nil {
t.Fatal("准备失败", err)
}
if failure == "close" {
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error {
client.failClosed = true
return nil
}
}
if failure == "changed-after-prepare" {
client.files[record.Paths[1]] = "changed"
}
if failure == "started-audit" || failure == "finished-audit" {
stage := "started"
if failure == "finished-audit" {
stage = "finished"
}
if _, err := db.ExecContext(t.Context(), `CREATE TRIGGER reject_recheck BEFORE INSERT ON recovery_events WHEN NEW.event_type = '`+stage+`' BEGIN SELECT RAISE(ABORT, 'injected'); END`); err != nil {
t.Fatal(err)
}
}
result, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if err == nil || result.Status == "succeeded" || client.writes != 0 {
t.Fatalf("危险收尾未拒绝或执行了写操作:%+v %v", result, err)
}
assertReconciliationSource(t, repo, source)
var owner string
if err := db.QueryRowContext(t.Context(), `SELECT disposition_id FROM recovery_claims WHERE transaction_id = ?`, record.ID).Scan(&owner); err != nil {
t.Fatal(err)
}
expected := source.Intent.ID
if failure == "finished-audit" {
expected = confirmation.DispositionID()
}
if owner != expected {
t.Fatal("失败后占用归属不正确", owner, expected)
}
pending, err := repo.ListPendingDeploymentTransactions(t.Context())
if err != nil || len(pending) != 1 || pending[0].Status != deployment.TransactionStatusRecoveryRequired {
t.Fatal("失败改变了原事务", err)
}
})
}
}
+1 -1
View File
@@ -6,7 +6,7 @@
- 第一轮任务 `b795be80-e086-46b4-a2ba-e7b51c8662f0` 因旧 systemd 不支持 `--kill-whom`,注入失败并正常完成;不能计为崩溃测试通过。确认本机帮助后改用 `--kill-who=main`。 - 第一轮任务 `b795be80-e086-46b4-a2ba-e7b51c8662f0` 因旧 systemd 不支持 `--kill-whom`,注入失败并正常完成;不能计为崩溃测试通过。确认本机帮助后改用 `--kill-who=main`。
- 22:57:25 对正式 Scert 主进程发送 SIGKILL,命中任务 `c68e3956-1b6d-42c7-ba9a-753ef2d1a44b` 第二目标事务的 reload_attempted 阶段。 - 22:57:25 对正式 Scert 主进程发送 SIGKILL,命中任务 `c68e3956-1b6d-42c7-ba9a-753ef2d1a44b` 第二目标事务的 reload_attempted 阶段。
- systemd 的 MainPID 从 130794 变为 134595,NRestarts=1;23:58 前已核实服务恢复 active、回环严格 TLS 就绪检查通过。此处实际核查时间为 22:58:16。 - systemd 的 MainPID 从 130794 变为 134595,NRestarts=1;22:58:16 已核实服务恢复 active、回环严格 TLS 就绪检查通过。
- 事务 `20261003-145716.121859533-09a2a331489a` 与 `20261003-145720.641188477-c769c8d3e153` 均 rolled_back;任务为 failed/service_restart,没有覆盖为成功。 - 事务 `20261003-145716.121859533-09a2a331489a` 与 `20261003-145720.641188477-c769c8d3e153` 均 rolled_back;任务为 failed/service_restart,没有覆盖为成功。
- 双机 cert/fullchain 摘要保持 `7e8b56227b345ed9b66659e428f9bd4421604f11253ab88d65b3ed61737f35cf`,未发现该域名遗留事务文件。第二台正常 DNS HTTPS 200、peer=45.136.13.241、TLS 校验结果零。 - 双机 cert/fullchain 摘要保持 `7e8b56227b345ed9b66659e428f9bd4421604f11253ab88d65b3ed61737f35cf`,未发现该域名遗留事务文件。第二台正常 DNS HTTPS 200、peer=45.136.13.241、TLS 校验结果零。
@@ -1,5 +1,17 @@
# 部署事务人工核查任务清单 # 部署事务人工核查任务清单
## 2026年10月3日23:30 不确定处置重新核查任务
沿用已确认的 Linux 范围,不增加网络入口,不直接修改现场事务或删除占用。
- [✓] 上轮静态检查通过;后端全量唯一失败已定位:宿主 umask 为 0077,测试创建的父目录从开始就是 0700;原代码在显式 umask 0022 下通过,不能归因于备份修改权限。
- [x] 测试显式建立 0755 权限基线,不修改进程 umask 或产品权限策略,等待 Linux 严格 umask 回归。
- [~] R6-A 独立重新核查动作:绑定原 uncertain 处置编号、原方向、服务器身份和本次现场摘要;只允许已完成审计的 uncertain,不以超时接管 running。
- [ ] R6-B 持久化关联与占用原子交接:新增处置审计,保留原 uncertain;拒绝或核查失败恢复原占用,结束审计失败保留新占用;成功才原子结束原事务并释放占用。
- [ ] R6-C 只读远端重新核验:仅限原方向对应的 verified 阶段,核验全部文件、临时文件与备份不存在、锁不存在、严格 HTTPS 和关闭结果;不得重载、删锁、重写或重放原动作。
- [ ] R6-D 受保护本机终端单独入口、完整确认、一次性绑定、断连查询及三数据库回归。
- [ ] R6-E 正式 Linux 包升级和当前真实 uncertain 处置收尾;其他不满足严格收尾条件的历史事务继续保留,不能用本功能冒充完成。
## 2026年10月3日23:08 实机关闭误判整改任务 ## 2026年10月3日23:08 实机关闭误判整改任务
- [✓] 真实 SIGKILL 中断部署后,两条事务自动回滚,第二台正常 DNS 严格 HTTPS 仍通过。 - [✓] 真实 SIGKILL 中断部署后,两条事务自动回滚,第二台正常 DNS 严格 HTTPS 仍通过。