保存不确定部署处置重新核查实现及测试检查点

This commit is contained in:
xuans committed 2026-10-03 23:49:31 +08:00
1 parent 7fa5e9b3d2
commit f9cd19f631
29 files changed
+877 -19

No files matched your search

+5 -2
View File
@@ -60,7 +60,7 @@ func runConnected(ctx context.Context, conn net.Conn, screen terminal) error {
if client.RecoveryAvailable() { if client.RecoveryAvailable() {
prompt := "本机管理:1 重置超级管理员密码;2 只读核查部署事务;其他输入退出" prompt := "本机管理:1 重置超级管理员密码;2 只读核查部署事务;其他输入退出"
if client.DispositionAvailable() { if client.DispositionAvailable() {
prompt = "本机管理:1 重置超级管理员密码;2 只读核查部署事务;3 人工处置部署事务;4 查询人工处置结果;其他输入退出" prompt = "本机管理:1 重置超级管理员密码;2 只读核查部署事务;3 人工处置部署事务;4 查询人工处置结果;5 重新核查不确定处置;其他输入退出"
} }
choice, err := screen.Read(ctx, prompt, false) choice, err := screen.Read(ctx, prompt, false)
if err != nil { if err != nil {
@@ -74,13 +74,16 @@ func runConnected(ctx context.Context, conn net.Conn, screen terminal) error {
return err return err
} }
return nil return nil
case "3", "4": case "3", "4", "5":
if !client.DispositionAvailable() { if !client.DispositionAvailable() {
return localreset.ErrUnavailable return localreset.ErrUnavailable
} }
if strings.TrimSpace(choice) == "4" { if strings.TrimSpace(choice) == "4" {
return recoveryResultFlow(ctx, client, screen) return recoveryResultFlow(ctx, client, screen)
} }
if strings.TrimSpace(choice) == "5" {
return recoveryReconciliationFlow(ctx, client, screen)
}
return recoveryDispositionFlow(ctx, client, screen) return recoveryDispositionFlow(ctx, client, screen)
default: default:
return nil return nil
@@ -130,12 +130,22 @@ func showRecoveryDisposition(screen terminal, result deployment.RecoveryDisposit
message = "本次人工提交核验及结束审计完成;不代表项目全部生产验收通过。" message = "本次人工提交核验及结束审计完成;不代表项目全部生产验收通过。"
case deployment.RecoveryActionRollback: case deployment.RecoveryActionRollback:
message = "本次恢复基线核验及结束审计完成;不代表项目全部生产验收通过。" message = "本次恢复基线核验及结束审计完成;不代表项目全部生产验收通过。"
case deployment.RecoveryActionVerifyCommit, deployment.RecoveryActionVerifyRollback:
message = "重新核查与新审计已完成,占用已安全释放;原不确定处置历史保留,本次未重放远端动作。"
} }
case "rejected": case "rejected":
message = "处置已拒绝,未执行本次远端变更;请重新核查拒绝原因。" message = "处置已拒绝,未执行本次远端变更;请重新核查拒绝原因。"
if result.SourceDispositionID != "" {
message = "重新核查未通过,原占用继续保留;本次未执行远端变更,不得手动删锁或重放原动作。"
}
case "uncertain": case "uncertain":
message = "处置结果不确定,可能已修改远端;占用继续保留,禁止重复执行或手动删锁。" message = "处置结果不确定,可能已修改远端;占用继续保留,禁止重复执行或手动删锁。"
} }
if result.SourceDispositionID != "" {
if err := screen.Show("关联原处置:" + terminalValue(result.SourceDispositionID)); err != nil {
return err
}
}
if err := screen.Show(fmt.Sprintf("处置:%s;事务:%s;服务器:%s;动作:%s", terminalValue(result.ID), terminalValue(result.TransactionID), terminalValue(result.ServerID), dispositionActionLabel(result.Action))); err != nil { if err := screen.Show(fmt.Sprintf("处置:%s;事务:%s;服务器:%s;动作:%s", terminalValue(result.ID), terminalValue(result.TransactionID), terminalValue(result.ServerID), dispositionActionLabel(result.Action))); err != nil {
return err return err
} }
@@ -150,6 +160,10 @@ func dispositionActionLabel(action deployment.RecoveryAction) string {
return "完成提交" return "完成提交"
case deployment.RecoveryActionRollback: case deployment.RecoveryActionRollback:
return "恢复基线" return "恢复基线"
case deployment.RecoveryActionVerifyCommit:
return "重新核查提交结果"
case deployment.RecoveryActionVerifyRollback:
return "重新核查回滚结果"
} }
return "未知动作" return "未知动作"
} }
@@ -0,0 +1,63 @@
package adminconsole
import (
"context"
"fmt"
"strings"
"time"
"git.pcux.cn/xuans/Scert/backend/internal/localreset"
)
type recoveryReconciliationClient interface {
recoveryDispositionClient
PrepareRecoveryReconciliation(string) (localreset.RecoveryPreparation, error)
}
func recoveryReconciliationFlow(ctx context.Context, client recoveryReconciliationClient, screen terminal) error {
sourceID, err := screen.Read(ctx, "请输入结果不确定的原处置编号;本操作只重新核验,不重放文件修改或服务重载", false)
if err != nil {
return err
}
sourceID = strings.TrimSpace(sourceID)
if err := ctx.Err(); err != nil {
return err
}
prepared, err := client.PrepareRecoveryReconciliation(sourceID)
if err != nil {
_ = screen.Show("无法准备重新核查:主服务可能尚未支持,或原处置未结束、历史证据不足、现场不满足收尾条件。占用仍保留。")
return err
}
defer func() { prepared.Token = "" }()
summary := prepared.Summary
if summary.Validate() != nil || summary.SourceDispositionID != sourceID || !time.Now().Before(summary.ExpiresAt) {
return localreset.ErrProtocol
}
for _, line := range []string{
"原处置:" + terminalValue(sourceID),
"新核查编号:" + terminalValue(prepared.DispositionID) + "(请保留,用于断连后查询)",
fmt.Sprintf("事务:%s;服务器:%s;动作:%s", terminalValue(summary.TransactionID), terminalValue(summary.ServerID), dispositionActionLabel(summary.Action)),
"本次只核验原方向,不能改选提交或回滚;不写远端文件、不删锁、不重载。文件、残留、严格 HTTPS 和关闭检查全部通过后,才以新审计结束原事务并释放占用。",
"原不确定记录永久保留;任何核验或审计失败均不释放占用,运行中处置不会因等待超时被接管。",
} {
if err := screen.Show(line); err != nil {
return err
}
}
expected := "确认重新核查 " + sourceID + " " + dispositionActionLabel(summary.Action)
answer, err := screen.Read(ctx, "如确认,请完整输入:"+expected+";其他输入取消", false)
if err != nil {
return err
}
if answer != expected || ctx.Err() != nil || !time.Now().Before(summary.ExpiresAt) {
return screen.Show("已取消或确认过期,未提交重新核查。")
}
result, err := client.ExecuteRecovery(prepared.DispositionID, prepared.Token)
prepared.Token = ""
if err != nil || result.Validate() != nil || result.ID != prepared.DispositionID || result.SourceDispositionID != sourceID ||
result.TransactionID != summary.TransactionID || result.ServerID != summary.ServerID || result.Action != summary.Action {
_ = screen.Show("未取得可靠结果,请重新连接后按新核查编号查询;不要重复执行或直接删除占用。")
return localreset.ErrUnavailable
}
return showRecoveryDisposition(screen, result)
}
@@ -0,0 +1,62 @@
package adminconsole
import (
"strings"
"testing"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
"git.pcux.cn/xuans/Scert/backend/internal/localreset"
)
type reconciliationConsoleClient struct {
dispositionConsoleClient
sourceID string
mismatch bool
}
func (c *reconciliationConsoleClient) PrepareRecoveryReconciliation(sourceID string) (localreset.RecoveryPreparation, error) {
c.sourceID = sourceID
value, err := c.PrepareRecovery("tx-1", deployment.RecoveryActionVerifyCommit)
value.Summary.SourceDispositionID = sourceID
value.Summary.Phase = deployment.TransactionPhaseCommitVerified
return value, err
}
func (c *reconciliationConsoleClient) ExecuteRecovery(id, token string) (deployment.RecoveryDispositionSummary, error) {
value, err := c.dispositionConsoleClient.ExecuteRecovery(id, token)
if err != nil { return value, err }
value.SourceDispositionID = c.sourceID
if value.Status == "succeeded" { value.ResultCode = "committed" }
if c.mismatch { value.SourceDispositionID = strings.Repeat("d", 64) }
return value, nil
}
func TestRecoveryReconciliationConsoleConfirmation(t *testing.T) {
for _, mode := range []string{"success", "rejected", "cancelled", "expired", "disconnect", "mismatch"} {
t.Run(mode, func(t *testing.T) {
sourceID := strings.Repeat("a", 64)
client := &reconciliationConsoleClient{}
answer := "确认重新核查 " + sourceID + " " + dispositionActionLabel(deployment.RecoveryActionVerifyCommit)
switch mode {
case "rejected": client.status = "rejected"
case "cancelled": answer = "确认"
case "expired": client.expired = true
case "disconnect": client.executeError = true
case "mismatch": client.mismatch = true
}
screen := &recoveryTestScreen{testTerminalInput: testTerminalInput{values: []string{sourceID, answer}}}
err := recoveryReconciliationFlow(t.Context(), client, screen)
output := strings.Join(screen.lines, "|")
if client.prepares != 1 || client.queries != 0 || strings.Contains(output, strings.Repeat("c", 64)) { t.Fatal("终端重复调用或泄露令牌") }
if mode == "cancelled" || mode == "expired" {
if client.executes != 0 { t.Fatal("取消或过期仍执行核查") }
return
}
if client.executes != 1 || !strings.Contains(output, "原不确定记录永久保留") { t.Fatal("缺少明确警告或重复执行") }
if mode == "disconnect" || mode == "mismatch" {
if err == nil || !strings.Contains(output, "新核查编号查询") { t.Fatal("失去可靠响应后误报成功") }
} else if err != nil { t.Fatal(err) }
if mode == "success" && !strings.Contains(output, "占用已安全释放") { t.Fatal("缺少完成结果") }
if mode == "rejected" && !strings.Contains(output, "原占用继续保留") { t.Fatal("错误显示拒绝结果") }
})
}
}
@@ -134,6 +134,10 @@ func TestCreateBackupDoesNotChangeExistingParentPermissions(t *testing.T) {
if err := os.Mkdir(parent, 0o755); err != nil { if err := os.Mkdir(parent, 0o755); err != nil {
t.Fatal(err) t.Fatal(err)
} }
// 显式建立权限基线,避免宿主 umask 将创建权限收紧后造成误报。
if err := os.Chmod(parent, 0o755); err != nil {
t.Fatal(err)
}
cfg := isolatedCLIConfig(filepath.Join(base, "scert")) cfg := isolatedCLIConfig(filepath.Join(base, "scert"))
for _, directory := range []string{cfg.Paths.Config, cfg.Paths.Data} { for _, directory := range []string{cfg.Paths.Config, cfg.Paths.Data} {
if err := os.MkdirAll(directory, 0o700); err != nil { if err := os.MkdirAll(directory, 0o700); err != nil {
+12 -1
View File
@@ -25,6 +25,7 @@ import (
//go:embed migrations/sqlite/0022_recovery_dispositions.sql migrations/mysql/0022_recovery_dispositions.sql migrations/postgres/0022_recovery_dispositions.sql //go:embed migrations/sqlite/0022_recovery_dispositions.sql migrations/mysql/0022_recovery_dispositions.sql migrations/postgres/0022_recovery_dispositions.sql
//go:embed migrations/sqlite/0023_recovery_claims.sql migrations/mysql/0023_recovery_claims.sql migrations/postgres/0023_recovery_claims.sql //go:embed migrations/sqlite/0023_recovery_claims.sql migrations/mysql/0023_recovery_claims.sql migrations/postgres/0023_recovery_claims.sql
//go:embed migrations/sqlite/0024_recovery_events.sql migrations/mysql/0024_recovery_events.sql migrations/postgres/0024_recovery_events.sql //go:embed migrations/sqlite/0024_recovery_events.sql migrations/mysql/0024_recovery_events.sql migrations/postgres/0024_recovery_events.sql
//go:embed migrations/sqlite/0025_recovery_reconciliation.sql migrations/mysql/0025_recovery_reconciliation.sql migrations/postgres/0025_recovery_reconciliation.sql
var migrationFiles embed.FS var migrationFiles embed.FS
func Migrate(ctx context.Context, db *sql.DB, driver string) error { func Migrate(ctx context.Context, db *sql.DB, driver string) error {
@@ -49,7 +50,7 @@ func Migrate(ctx context.Context, db *sql.DB, driver string) error {
return nil return nil
} }
const latestMigrationVersion = 24 const latestMigrationVersion = 25
// currentMigrationVersion 在执行业务迁移前拒绝新版数据库及不连续的迁移历史。 // currentMigrationVersion 在执行业务迁移前拒绝新版数据库及不连续的迁移历史。
func currentMigrationVersion(ctx context.Context, db *sql.DB) (int, error) { func currentMigrationVersion(ctx context.Context, db *sql.DB) (int, error) {
@@ -114,6 +115,14 @@ func applyMigration(ctx context.Context, db *sql.DB, driver string, version int)
return err return err
} }
} }
if version == 25 {
var compatible int
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = DATABASE()
AND table_name = 'recovery_dispositions' AND column_name = 'source_disposition_id'
AND data_type = 'varchar' AND character_maximum_length = 64 AND is_nullable = 'NO' AND column_default = ''`).Scan(&compatible); err != nil || compatible != 1 {
return fmt.Errorf("人工处置来源字段与预期不一致,拒绝记录迁移版本")
}
}
if _, err := db.ExecContext(ctx, "INSERT INTO schema_migrations (version) VALUES (?)", version); err != nil { if _, err := db.ExecContext(ctx, "INSERT INTO schema_migrations (version) VALUES (?)", version); err != nil {
return fmt.Errorf("记录迁移版本失败:%w", err) return fmt.Errorf("记录迁移版本失败:%w", err)
} }
@@ -252,6 +261,8 @@ func migrationName(version int) string {
return "recovery_claims" return "recovery_claims"
case 24: case 24:
return "recovery_events" return "recovery_events"
case 25:
return "recovery_reconciliation"
default: default:
return "unknown" return "unknown"
} }
@@ -0,0 +1 @@
ALTER TABLE recovery_dispositions ADD COLUMN source_disposition_id VARCHAR(64) NOT NULL DEFAULT '';
@@ -0,0 +1 @@
ALTER TABLE recovery_dispositions ADD COLUMN source_disposition_id VARCHAR(64) NOT NULL DEFAULT '';
@@ -0,0 +1 @@
ALTER TABLE recovery_dispositions ADD COLUMN source_disposition_id TEXT NOT NULL DEFAULT '';
@@ -25,6 +25,10 @@ func TestMySQLRealRecoveryDispositionMigrationRecovery(t *testing.T) {
`ALTER TABLE recovery_claims DROP PRIMARY KEY`}, `ALTER TABLE recovery_claims DROP PRIMARY KEY`},
{24, "recovery_events", `INSERT INTO recovery_events VALUES ('kept', 'started', '', NOW())`, {24, "recovery_events", `INSERT INTO recovery_events VALUES ('kept', 'started', '', NOW())`,
`ALTER TABLE recovery_events DROP PRIMARY KEY, ADD PRIMARY KEY (event_type, disposition_id)`}, `ALTER TABLE recovery_events DROP PRIMARY KEY, ADD PRIMARY KEY (event_type, disposition_id)`},
{25, "recovery_dispositions", `INSERT INTO recovery_dispositions (id, transaction_id, server_id, operator_id, action, request_key,
record_digest, server_digest, observation_digest, status, result_code, created_at, updated_at) VALUES
('kept', 'transaction', 'server', 'unix:0', 'finish_commit', 'request', 'record', 'server', 'observation', 'uncertain', 'uncertain', NOW(), NOW())`,
`ALTER TABLE recovery_dispositions MODIFY COLUMN source_disposition_id VARCHAR(8) NOT NULL DEFAULT ''`},
} { } {
for _, boundary := range []string{"before_create", "after_create", "with_record", "conflict"} { for _, boundary := range []string{"before_create", "after_create", "with_record", "conflict"} {
t.Run(fmt.Sprintf("%d/%s", item.version, boundary), func(t *testing.T) { t.Run(fmt.Sprintf("%d/%s", item.version, boundary), func(t *testing.T) {
@@ -9,7 +9,7 @@ import (
func TestPostgresRealRecoveryDispositionMigration(t *testing.T) { func TestPostgresRealRecoveryDispositionMigration(t *testing.T) {
server := postgrestest.New(t) server := postgrestest.New(t)
for version := 22; version <= 24; version++ { for version := 22; version <= 25; version++ {
t.Run(fmt.Sprint(version), func(t *testing.T) { t.Run(fmt.Sprint(version), func(t *testing.T) {
cfg := server.Database(t) cfg := server.Database(t)
db, err := Open(t.Context(), cfg) db, err := Open(t.Context(), cfg)
@@ -17,6 +17,7 @@ var (
// RecoveryDispositionIntent 仅保存绑定摘要;不能包含历史证据、凭据或确认令牌原文。 // RecoveryDispositionIntent 仅保存绑定摘要;不能包含历史证据、凭据或确认令牌原文。
type RecoveryDispositionIntent struct { type RecoveryDispositionIntent struct {
SourceDispositionID string
ID string ID string
TransactionID string TransactionID string
ServerID string ServerID string
@@ -58,6 +59,7 @@ func (r RecoveryReview) DispositionIntent(id, requestKey string) (RecoveryDispos
return RecoveryDispositionIntent{}, ErrRecoveryReviewExpired return RecoveryDispositionIntent{}, ErrRecoveryReviewExpired
} }
intent := RecoveryDispositionIntent{ intent := RecoveryDispositionIntent{
SourceDispositionID: r.summary.SourceDispositionID,
ID: id, TransactionID: r.summary.TransactionID, ServerID: r.summary.ServerID, ID: id, TransactionID: r.summary.TransactionID, ServerID: r.summary.ServerID,
Operator: r.operator, Action: r.summary.Action, RequestKey: requestKey, Operator: r.operator, Action: r.summary.Action, RequestKey: requestKey,
RecordDigest: hex.EncodeToString(r.recordDigest[:]), RecordDigest: hex.EncodeToString(r.recordDigest[:]),
@@ -68,6 +70,9 @@ func (r RecoveryReview) DispositionIntent(id, requestKey string) (RecoveryDispos
} }
func (i RecoveryDispositionIntent) Validate() error { func (i RecoveryDispositionIntent) Validate() error {
if isRecoveryVerification(i.Action) != (i.SourceDispositionID != "") || (i.SourceDispositionID != "" && (!validDispositionID(i.SourceDispositionID) || i.SourceDispositionID == i.ID)) {
return ErrRecoveryDispositionInvalid
}
if !validInspectionID(i.TransactionID) || !validRecoverySummary(i.ServerID) || if !validInspectionID(i.TransactionID) || !validRecoverySummary(i.ServerID) ||
!validReviewOperator(i.Operator) || !validRecoveryAction(i.Action) { !validReviewOperator(i.Operator) || !validRecoveryAction(i.Action) {
return ErrRecoveryDispositionInvalid return ErrRecoveryDispositionInvalid
@@ -106,9 +111,9 @@ func (i RecoveryDispositionIntent) AllowsResult(result string) bool {
case RecoveryResultObserved: case RecoveryResultObserved:
return i.Action == RecoveryActionRecord return i.Action == RecoveryActionRecord
case RecoveryResultCommitted: case RecoveryResultCommitted:
return i.Action == RecoveryActionCommit return i.Action == RecoveryActionCommit || i.Action == RecoveryActionVerifyCommit
case RecoveryResultRolledBack: case RecoveryResultRolledBack:
return i.Action == RecoveryActionRollback return i.Action == RecoveryActionRollback || i.Action == RecoveryActionVerifyRollback
} }
return false return false
} }
@@ -100,6 +100,12 @@ func (s *Service) executeRecoveryAction(ctx context.Context, review RecoveryRevi
case RecoveryActionRollback: case RecoveryActionRollback:
err = s.executeRecoveryRollback(ctx, &record, server, client, &mutated) err = s.executeRecoveryRollback(ctx, &record, server, client, &mutated)
result = RecoveryResultRolledBack result = RecoveryResultRolledBack
case RecoveryActionVerifyCommit, RecoveryActionVerifyRollback:
err = s.verifyRecoveryCompletion(ctx, record, server, client, review.summary.Action)
result = RecoveryResultCommitted
if review.summary.Action == RecoveryActionVerifyRollback {
result = RecoveryResultRolledBack
}
default: default:
return return
} }
@@ -117,6 +123,13 @@ func validateRecoveryAction(record TransactionRecord, action RecoveryAction, ver
return ErrRecoveryActionRejected return ErrRecoveryActionRejected
} }
switch action { switch action {
case RecoveryActionVerifyCommit, RecoveryActionVerifyRollback:
if !verificationPhase(record.Phase, action) ||
(action == RecoveryActionVerifyCommit && record.Evidence.Baseline.ExpectedFingerprint == "") ||
(action == RecoveryActionVerifyRollback && record.Evidence.Baseline.PreviousFingerprint == "") {
return ErrRecoveryActionRejected
}
return nil
case RecoveryActionCommit: case RecoveryActionCommit:
if record.Evidence.Baseline.ExpectedFingerprint == "" { if record.Evidence.Baseline.ExpectedFingerprint == "" {
return ErrRecoveryActionRejected return ErrRecoveryActionRejected
@@ -0,0 +1,113 @@
package deployment
import (
"context"
"encoding/hex"
"time"
)
func isRecoveryVerification(action RecoveryAction) bool {
return action == RecoveryActionVerifyCommit || action == RecoveryActionVerifyRollback
}
// Reconciles 仅允许已持久化结束的不确定原动作;不接管运行中处置或再次执行文件动作。
func (i RecoveryDispositionIntent) Reconciles(source RecoveryDisposition) bool {
if i.SourceDispositionID == "" || i.SourceDispositionID != source.Intent.ID || source.Intent.Validate() != nil ||
source.Status != RecoveryResultUncertain || source.ResultCode != RecoveryResultUncertain ||
i.TransactionID != source.Intent.TransactionID || i.ServerID != source.Intent.ServerID || i.ServerDigest != source.Intent.ServerDigest {
return false
}
return (i.Action == RecoveryActionVerifyCommit && source.Intent.Action == RecoveryActionCommit) ||
(i.Action == RecoveryActionVerifyRollback && source.Intent.Action == RecoveryActionRollback)
}
// PrepareRecoveryReconciliation 只供受保护本机通道调用,必须明确指定原处置编号并重新确认。
func (s *Service) PrepareRecoveryReconciliation(ctx context.Context, sourceID, operator string, resolve func(context.Context, string) (Server, error)) (RecoveryConfirmation, string, error) {
review, err := s.prepareRecoveryReconciliation(ctx, sourceID, operator, resolve)
if err != nil {
return RecoveryConfirmation{}, "", err
}
return s.newRecoveryConfirmation(ctx, review)
}
func (s *Service) prepareRecoveryReconciliation(ctx context.Context, sourceID, operator string, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
if s == nil || !validDispositionID(sourceID) || !validReviewOperator(operator) {
return RecoveryReview{}, ErrRecoveryReviewInvalid
}
journal, ok := s.journal.(RecoveryDispositionJournal)
if !ok {
return RecoveryReview{}, ErrRecoveryDispositionUnavailable
}
source, err := journal.GetRecoveryDisposition(ctx, sourceID)
if err != nil || source.Intent.ID != sourceID || source.Intent.Validate() != nil || source.Summary().Validate() != nil ||
source.Status != RecoveryResultUncertain || source.ResultCode != RecoveryResultUncertain {
return RecoveryReview{}, ErrRecoveryActionRejected
}
var action RecoveryAction
switch source.Intent.Action {
case RecoveryActionCommit:
action = RecoveryActionVerifyCommit
case RecoveryActionRollback:
action = RecoveryActionVerifyRollback
default:
return RecoveryReview{}, ErrRecoveryActionRejected
}
review, err := s.prepareRecoveryReview(ctx, source.Intent.TransactionID, operator, action, resolve)
if err != nil {
return RecoveryReview{}, err
}
review.summary.SourceDispositionID = sourceID
intent := RecoveryDispositionIntent{SourceDispositionID: sourceID, TransactionID: review.summary.TransactionID,
ServerID: review.summary.ServerID, ServerDigest: hex.EncodeToString(review.serverDigest[:]), Action: action}
if !intent.Reconciles(source) || !verificationPhase(review.summary.Phase, action) || s.VerifyRecoveryTarget == nil {
return RecoveryReview{}, ErrRecoveryActionRejected
}
return review, nil
}
func verificationPhase(phase string, action RecoveryAction) bool {
return (action == RecoveryActionVerifyCommit && phase == TransactionPhaseCommitVerified) ||
(action == RecoveryActionVerifyRollback && phase == TransactionPhaseRollbackVerified)
}
// validateRecoveryCompletion 不删除任何远端文件,残留锁或备份均要求继续保留占用。
func validateRecoveryCompletion(ctx context.Context, client Client, record TransactionRecord, committed bool) error {
for _, lock := range record.LockPaths {
if exists, err := client.Exists(ctx, lock); err != nil || exists {
return ErrRecoveryActionRejected
}
}
for _, backup := range record.Backups {
if committed {
if err := checkFileDigest(ctx, client, backup.Original, record.Evidence.Files[backup.Original]); err != nil {
return err
}
} else if backup.OriginalExists {
if err := checkFileDigest(ctx, client, backup.Original, backup.OriginalDigest); err != nil {
return err
}
} else if exists, err := client.Exists(ctx, backup.Original); err != nil || exists {
return ErrRecoveryActionRejected
}
for _, path := range []string{backup.Saved, backup.Original + ".scert-tmp-" + record.ID} {
if exists, err := client.Exists(ctx, path); err != nil || exists {
return ErrRecoveryActionRejected
}
}
}
return ctx.Err()
}
func (s *Service) verifyRecoveryCompletion(ctx context.Context, record TransactionRecord, server Server, client Client, action RecoveryAction) error {
committed := action == RecoveryActionVerifyCommit
if err := validateRecoveryCompletion(ctx, client, record, committed); err != nil {
return err
}
if err := s.VerifyRecoveryTarget(ctx, server, record.Evidence.Baseline, committed); err != nil {
return err
}
// HTTPS 检查结束后重新读取现场,避免检查期间发生的文件变化被忽略。
return validateRecoveryCompletion(ctx, client, record, committed)
}
@@ -8,6 +8,7 @@ import (
// RecoveryDispositionSummary 是本机查询的明确输出边界,不包含请求键、绑定摘要或操作者凭据。 // RecoveryDispositionSummary 是本机查询的明确输出边界,不包含请求键、绑定摘要或操作者凭据。
type RecoveryDispositionSummary struct { type RecoveryDispositionSummary struct {
SourceDispositionID string `json:"source_disposition_id,omitempty"`
ID string `json:"id"` ID string `json:"id"`
TransactionID string `json:"transaction_id"` TransactionID string `json:"transaction_id"`
ServerID string `json:"server_id"` ServerID string `json:"server_id"`
@@ -20,6 +21,7 @@ type RecoveryDispositionSummary struct {
func (r RecoveryDisposition) Summary() RecoveryDispositionSummary { func (r RecoveryDisposition) Summary() RecoveryDispositionSummary {
return RecoveryDispositionSummary{ return RecoveryDispositionSummary{
SourceDispositionID: r.Intent.SourceDispositionID,
ID: r.Intent.ID, TransactionID: r.Intent.TransactionID, ServerID: r.Intent.ServerID, ID: r.Intent.ID, TransactionID: r.Intent.TransactionID, ServerID: r.Intent.ServerID,
Action: r.Intent.Action, Status: r.Status, ResultCode: r.ResultCode, Action: r.Intent.Action, Status: r.Status, ResultCode: r.ResultCode,
CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt, CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt,
@@ -31,6 +33,9 @@ func validDispositionID(id string) bool {
} }
func (r RecoveryDispositionSummary) Validate() error { func (r RecoveryDispositionSummary) Validate() error {
if isRecoveryVerification(r.Action) != (r.SourceDispositionID != "") || (r.SourceDispositionID != "" && (!validDispositionID(r.SourceDispositionID) || r.SourceDispositionID == r.ID)) {
return ErrRecoveryDispositionInvalid
}
if !validDispositionID(r.ID) || !validInspectionID(r.TransactionID) || !validRecoverySummary(r.ServerID) || if !validDispositionID(r.ID) || !validInspectionID(r.TransactionID) || !validRecoverySummary(r.ServerID) ||
!validRecoveryAction(r.Action) || r.CreatedAt.IsZero() || r.UpdatedAt.IsZero() || r.UpdatedAt.Before(r.CreatedAt) { !validRecoveryAction(r.Action) || r.CreatedAt.IsZero() || r.UpdatedAt.IsZero() || r.UpdatedAt.Before(r.CreatedAt) {
return ErrRecoveryDispositionInvalid return ErrRecoveryDispositionInvalid
+23 -3
View File
@@ -16,6 +16,8 @@ const (
RecoveryActionRecord RecoveryAction = "record_inspection" RecoveryActionRecord RecoveryAction = "record_inspection"
RecoveryActionCommit RecoveryAction = "finish_commit" RecoveryActionCommit RecoveryAction = "finish_commit"
RecoveryActionRollback RecoveryAction = "restore_baseline" RecoveryActionRollback RecoveryAction = "restore_baseline"
RecoveryActionVerifyCommit RecoveryAction = "verify_commit"
RecoveryActionVerifyRollback RecoveryAction = "verify_rollback"
RecoveryReviewLifetime = time.Minute RecoveryReviewLifetime = time.Minute
) )
@@ -27,6 +29,7 @@ var (
) )
type RecoveryReviewSummary struct { type RecoveryReviewSummary struct {
SourceDispositionID string `json:"source_disposition_id,omitempty"`
TransactionID string `json:"transaction_id"` TransactionID string `json:"transaction_id"`
ServerID string `json:"server_id"` ServerID string `json:"server_id"`
Phase string `json:"phase"` Phase string `json:"phase"`
@@ -44,6 +47,9 @@ func (r RecoveryReviewSummary) Validate() error {
(r.Action != RecoveryActionRecord && !r.HistoricalEvidence) { (r.Action != RecoveryActionRecord && !r.HistoricalEvidence) {
return ErrRecoveryReviewInvalid return ErrRecoveryReviewInvalid
} }
if isRecoveryVerification(r.Action) != (r.SourceDispositionID != "") || (r.SourceDispositionID != "" && !validDispositionID(r.SourceDispositionID)) {
return ErrRecoveryReviewInvalid
}
return nil return nil
} }
@@ -65,12 +71,20 @@ func validReviewOperator(operator string) bool {
} }
func validRecoveryAction(action RecoveryAction) bool { func validRecoveryAction(action RecoveryAction) bool {
return action == RecoveryActionRecord || action == RecoveryActionCommit || action == RecoveryActionRollback return action == RecoveryActionRecord || action == RecoveryActionCommit || action == RecoveryActionRollback || isRecoveryVerification(action)
} }
// PrepareRecoveryReview 在同一操作上限内读取前后记录,拒绝采样期间发生的配置或事务变化。 // PrepareRecoveryReview 在同一操作上限内读取前后记录,拒绝采样期间发生的配置或事务变化。
// 当前采样摘要只绑定本次准备,不补写到历史 Evidence,也不改变事务终态。 // 当前采样摘要只绑定本次准备,不补写到历史 Evidence,也不改变事务终态。
func (s *Service) PrepareRecoveryReview(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) { func (s *Service) PrepareRecoveryReview(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) {
// 重新核查必须通过独立入口绑定原处置,不能作为普通动作绕过来源检查。
if isRecoveryVerification(action) {
return RecoveryReview{}, ErrRecoveryReviewInvalid
}
return s.prepareRecoveryReview(ctx, id, operator, action, resolve)
}
func (s *Service) prepareRecoveryReview(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second) ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel() defer cancel()
if !validReviewOperator(operator) || !validRecoveryAction(action) || !validInspectionID(id) { if !validReviewOperator(operator) || !validRecoveryAction(action) || !validInspectionID(id) {
@@ -140,14 +154,20 @@ func (s *Service) RecheckRecoveryReview(ctx context.Context, review RecoveryRevi
if s.now == nil || !s.now().Before(review.summary.ExpiresAt) { if s.now == nil || !s.now().Before(review.summary.ExpiresAt) {
return ErrRecoveryReviewExpired return ErrRecoveryReviewExpired
} }
current, err := s.PrepareRecoveryReview(ctx, review.summary.TransactionID, operator, review.summary.Action, resolve) var current RecoveryReview
var err error
if isRecoveryVerification(review.summary.Action) {
current, err = s.prepareRecoveryReconciliation(ctx, review.summary.SourceDispositionID, operator, resolve)
} else {
current, err = s.PrepareRecoveryReview(ctx, review.summary.TransactionID, operator, review.summary.Action, resolve)
}
if err != nil { if err != nil {
return err return err
} }
if !s.now().Before(review.summary.ExpiresAt) { if !s.now().Before(review.summary.ExpiresAt) {
return ErrRecoveryReviewExpired return ErrRecoveryReviewExpired
} }
if current.recordDigest != review.recordDigest || current.serverDigest != review.serverDigest || current.observationDigest != review.observationDigest { if current.recordDigest != review.recordDigest || current.serverDigest != review.serverDigest || current.observationDigest != review.observationDigest || current.summary.SourceDispositionID != review.summary.SourceDispositionID || current.summary.Action != review.summary.Action {
return ErrRecoveryReviewChanged return ErrRecoveryReviewChanged
} }
return ctx.Err() return ctx.Err()
@@ -87,3 +87,26 @@ func (r *localResetSession) GetRecoveryDisposition(ctx context.Context, id strin
} }
var _ localreset.RecoveryDispositionSession = (*localResetSession)(nil) var _ localreset.RecoveryDispositionSession = (*localResetSession)(nil)
type localRecoveryReconciler interface {
PrepareRecoveryReconciliation(context.Context, string, string, func(context.Context, string) (deployment.Server, error)) (deployment.RecoveryConfirmation, string, error)
}
func (r *localResetSession) PrepareRecoveryReconciliation(ctx context.Context, sourceID string) (localreset.RecoveryPreparation, error) {
r.mu.Lock()
defer r.mu.Unlock()
r.recoveryConfirmation = deployment.RecoveryConfirmation{}
executor, ok := r.server.recoveryInspection.(localRecoveryReconciler)
if !r.enterRecovery(ctx) || !ok || !r.DispositionAvailable() {
return localreset.RecoveryPreparation{}, localreset.ErrProtocol
}
confirmation, token, err := executor.PrepareRecoveryReconciliation(ctx, sourceID, r.peer.Identity, r.recoveryServer)
if err != nil || !r.current(ctx) || confirmation.Summary().SourceDispositionID != sourceID {
r.stage = -1
return localreset.RecoveryPreparation{}, localreset.ErrUnavailable
}
r.recoveryConfirmation = confirmation
return localreset.RecoveryPreparation{DispositionID: confirmation.DispositionID(), Token: token, Summary: confirmation.Summary()}, nil
}
var _ localreset.RecoveryReconciliationSession = (*localResetSession)(nil)
@@ -0,0 +1,88 @@
package httpapi
import (
"context"
"strings"
"testing"
"time"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
)
type localReconciliationJournal struct {
localDispositionJournal
source deployment.RecoveryDisposition
}
func (j *localReconciliationJournal) GetRecoveryDisposition(ctx context.Context, id string) (deployment.RecoveryDisposition, error) {
if id == j.source.Intent.ID { return j.source, nil }
return j.localDispositionJournal.GetRecoveryDisposition(ctx, id)
}
func localReconciliationFixture(t *testing.T) (*Server, *localResetSession, *localReconciliationJournal) {
t.Helper()
server, _, original := localDispositionFixture(t)
target := server.servers.(localDispositionServerRepo).server
journal := &localReconciliationJournal{}
journal.record = original.record
journal.record.Phase = deployment.TransactionPhaseCommitVerified
journal.record.Evidence = &deployment.RecoveryEvidence{Version: 1, LockToken: strings.Repeat("d", 64), Address: target.Address,
Port: target.Port, HostKeyFingerprint: target.HostKeyFingerprint, ValidationCommand: target.ValidationCommand, ReloadCommand: target.ReloadCommand,
Files: map[string]string{}, TemporaryFiles: map[string]string{}, Baseline: deployment.RecoveryBaseline{Endpoint: "https://example.test", Domains: []string{"example.test"}, ExpectedFingerprint: "new"}}
for _, path := range journal.record.Paths {
journal.record.LockPaths = append(journal.record.LockPaths, path+".scert-lock")
journal.record.Backups = append(journal.record.Backups, deployment.TransactionBackup{Original: path, Saved: path+".scert-backup-"+journal.record.ID})
journal.record.Evidence.Files[path] = strings.Repeat("e", 64)
}
service := deployment.NewServiceWithJournal(localDispositionFactory{}, journal)
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error { return nil }
resolve := func(context.Context, string) (deployment.Server, error) { return target, nil }
review, err := service.PrepareRecoveryReview(t.Context(), journal.record.ID, "uid:0", deployment.RecoveryActionCommit, resolve)
if err != nil { t.Fatal(err) }
intent, err := review.DispositionIntent(strings.Repeat("a", 64), strings.Repeat("b", 64))
if err != nil { t.Fatal(err) }
now := time.Now().UTC()
journal.source = deployment.RecoveryDisposition{Intent: intent, Status: "uncertain", ResultCode: "uncertain", CreatedAt: now, UpdatedAt: now}
server.recoveryInspection = service
return server, localDispositionNewSession(t, server), journal
}
func TestLocalReconciliationRejectsInvalidSession(t *testing.T) {
for _, mode := range []string{"expired", "cancelled", "changed-admin", "closed", "reset-stage", "wrong-id", "wrong-token", "changed-peer", "foreign-session", "replaced-confirmation"} {
t.Run(mode, func(t *testing.T) {
server, session, journal := localReconciliationFixture(t)
prepared, err := session.PrepareRecoveryReconciliation(t.Context(), journal.source.Intent.ID)
if err != nil { t.Fatal(err) }
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
switch mode {
case "expired": session.expires = time.Time{}
case "cancelled": cancel()
case "changed-admin": server.config.Admin.Email = "changed@example.test"
case "closed": session.Close()
case "reset-stage": session.stage = 1
case "wrong-id": prepared.DispositionID = journal.source.Intent.ID
case "wrong-token": prepared.Token = strings.Repeat("a", 64)
case "changed-peer": session.peer.Identity = "uid:123"
case "foreign-session": session = localDispositionNewSession(t, server)
case "replaced-confirmation":
if _, err := session.PrepareRecoveryReconciliation(ctx, journal.source.Intent.ID); err != nil { t.Fatal(err) }
}
if _, err := session.ExecuteRecovery(ctx, prepared.DispositionID, prepared.Token); err == nil { t.Fatal("无效会话执行了重新核查") }
if _, err := session.ExecuteRecovery(t.Context(), prepared.DispositionID, prepared.Token); err == nil { t.Fatal("失败确认被重放") }
if journal.begins != 0 || journal.finishes != 0 || session.recoveryConfirmation.DispositionID() != "" { t.Fatal("无效会话写入了审计或保留了确认") }
})
}
}
func TestLocalReconciliationRejectsMissingFilesAndQueriesNewAudit(t *testing.T) {
server, session, journal := localReconciliationFixture(t)
prepared, err := session.PrepareRecoveryReconciliation(t.Context(), journal.source.Intent.ID)
if err != nil { t.Fatal(err) }
result, err := session.ExecuteRecovery(t.Context(), prepared.DispositionID, prepared.Token)
if err != nil || result.Status != "rejected" || result.SourceDispositionID != journal.source.Intent.ID { t.Fatalf("缺失文件仍被放行:%+v %v", result, err) }
if journal.item.Intent.Operator != "uid:0" || journal.begins != 1 || journal.finishes != 1 || journal.source.Status != "uncertain" { t.Fatal("身份或历史记录被改变") }
next := localDispositionNewSession(t, server)
queried, err := next.GetRecoveryDisposition(t.Context(), result.ID)
if err != nil || queried != result { t.Fatal("重连未查到关联审计", err) }
}
+1 -1
View File
@@ -30,7 +30,7 @@ func validRecoveryRequest(request Request, nonce string) bool {
return request.Cursor == "" && validRecoveryID(request.TransactionID) && validRecoveryAction(request.RecoveryAction) && request.DispositionID == "" && request.ConfirmationToken == "" return request.Cursor == "" && validRecoveryID(request.TransactionID) && validRecoveryAction(request.RecoveryAction) && request.DispositionID == "" && request.ConfirmationToken == ""
case "recovery_execute": case "recovery_execute":
return request.Cursor == "" && request.TransactionID == "" && request.RecoveryAction == "" && validRecoverySecret(request.DispositionID) && validRecoverySecret(request.ConfirmationToken) return request.Cursor == "" && request.TransactionID == "" && request.RecoveryAction == "" && validRecoverySecret(request.DispositionID) && validRecoverySecret(request.ConfirmationToken)
case "recovery_result": case "recovery_result", "recovery_reconcile_prepare":
return request.Cursor == "" && request.TransactionID == "" && request.RecoveryAction == "" && validRecoverySecret(request.DispositionID) && request.ConfirmationToken == "" return request.Cursor == "" && request.TransactionID == "" && request.RecoveryAction == "" && validRecoverySecret(request.DispositionID) && request.ConfirmationToken == ""
} }
return false return false
@@ -21,6 +21,10 @@ type RecoveryDispositionSession interface {
GetRecoveryDisposition(context.Context, string) (deployment.RecoveryDispositionSummary, error) GetRecoveryDisposition(context.Context, string) (deployment.RecoveryDispositionSummary, error)
} }
type RecoveryReconciliationSession interface {
PrepareRecoveryReconciliation(context.Context, string) (RecoveryPreparation, error)
}
func validRecoveryAction(action deployment.RecoveryAction) bool { func validRecoveryAction(action deployment.RecoveryAction) bool {
return action == deployment.RecoveryActionRecord || action == deployment.RecoveryActionCommit || action == deployment.RecoveryActionRollback return action == deployment.RecoveryActionRecord || action == deployment.RecoveryActionCommit || action == deployment.RecoveryActionRollback
} }
@@ -60,6 +64,11 @@ func validDispositionResponse(request Request, response Response) bool {
return response.Disposition == nil && validPreparation(response.Preparation) && return response.Disposition == nil && validPreparation(response.Preparation) &&
response.Preparation.Summary.TransactionID == request.TransactionID && response.Preparation.Summary.Action == request.RecoveryAction response.Preparation.Summary.TransactionID == request.TransactionID && response.Preparation.Summary.Action == request.RecoveryAction
} }
if request.Action == "recovery_reconcile_prepare" {
return response.Disposition == nil && validPreparation(response.Preparation) &&
response.Preparation.Summary.SourceDispositionID == request.DispositionID &&
response.Preparation.DispositionID != request.DispositionID
}
return response.Preparation == nil && response.Disposition != nil && response.Disposition.Validate() == nil && response.Disposition.ID == request.DispositionID return response.Preparation == nil && response.Disposition != nil && response.Disposition.Validate() == nil && response.Disposition.ID == request.DispositionID
} }
@@ -70,6 +79,14 @@ func dispatchRecoveryDisposition(ctx context.Context, session RecoverySession, r
} }
var err error var err error
switch request.Action { switch request.Action {
case "recovery_reconcile_prepare":
reconciler, ok := session.(RecoveryReconciliationSession)
if !ok {
return ErrUnavailable
}
var value RecoveryPreparation
value, err = reconciler.PrepareRecoveryReconciliation(ctx, request.DispositionID)
response.Preparation = &value
case "recovery_prepare": case "recovery_prepare":
var value RecoveryPreparation var value RecoveryPreparation
value, err = actions.PrepareRecovery(ctx, request.TransactionID, request.RecoveryAction) value, err = actions.PrepareRecovery(ctx, request.TransactionID, request.RecoveryAction)
@@ -101,6 +118,14 @@ func (c *Client) PrepareRecovery(id string, action deployment.RecoveryAction) (R
return *response.Preparation, nil return *response.Preparation, nil
} }
func (c *Client) PrepareRecoveryReconciliation(sourceID string) (RecoveryPreparation, error) {
response, err := c.recoveryExchange(Request{Action: "recovery_reconcile_prepare", DispositionID: sourceID})
if err != nil {
return RecoveryPreparation{}, err
}
return *response.Preparation, nil
}
func (c *Client) ExecuteRecovery(id, token string) (deployment.RecoveryDispositionSummary, error) { func (c *Client) ExecuteRecovery(id, token string) (deployment.RecoveryDispositionSummary, error) {
response, err := c.recoveryExchange(Request{Action: "recovery_execute", DispositionID: id, ConfirmationToken: token}) response, err := c.recoveryExchange(Request{Action: "recovery_execute", DispositionID: id, ConfirmationToken: token})
if err != nil { if err != nil {
@@ -89,6 +89,7 @@ func TestDispositionRequestFieldIsolation(t *testing.T) {
{Action: "recovery_prepare", TransactionID: "tx-1", RecoveryAction: deployment.RecoveryActionRecord}, {Action: "recovery_prepare", TransactionID: "tx-1", RecoveryAction: deployment.RecoveryActionRecord},
{Action: "recovery_execute", DispositionID: strings.Repeat("b", 64), ConfirmationToken: strings.Repeat("c", 64)}, {Action: "recovery_execute", DispositionID: strings.Repeat("b", 64), ConfirmationToken: strings.Repeat("c", 64)},
{Action: "recovery_result", DispositionID: strings.Repeat("b", 64)}, {Action: "recovery_result", DispositionID: strings.Repeat("b", 64)},
{Action: "recovery_reconcile_prepare", DispositionID: strings.Repeat("b", 64)},
} }
fields := []struct { fields := []struct {
name string name string
@@ -101,7 +102,7 @@ func TestDispositionRequestFieldIsolation(t *testing.T) {
{"cursor", func(r *Request) { r.Cursor = "tx-0" }, map[string]bool{"recovery_list": true}}, {"cursor", func(r *Request) { r.Cursor = "tx-0" }, map[string]bool{"recovery_list": true}},
{"transaction", func(r *Request) { r.TransactionID = "tx-1" }, map[string]bool{"recovery_inspect": true, "recovery_prepare": true}}, {"transaction", func(r *Request) { r.TransactionID = "tx-1" }, map[string]bool{"recovery_inspect": true, "recovery_prepare": true}},
{"action", func(r *Request) { r.RecoveryAction = deployment.RecoveryActionRecord }, map[string]bool{"recovery_prepare": true}}, {"action", func(r *Request) { r.RecoveryAction = deployment.RecoveryActionRecord }, map[string]bool{"recovery_prepare": true}},
{"id", func(r *Request) { r.DispositionID = strings.Repeat("b", 64) }, map[string]bool{"recovery_execute": true, "recovery_result": true}}, {"id", func(r *Request) { r.DispositionID = strings.Repeat("b", 64) }, map[string]bool{"recovery_execute": true, "recovery_result": true, "recovery_reconcile_prepare": true}},
{"token", func(r *Request) { r.ConfirmationToken = strings.Repeat("c", 64) }, map[string]bool{"recovery_execute": true}}, {"token", func(r *Request) { r.ConfirmationToken = strings.Repeat("c", 64) }, map[string]bool{"recovery_execute": true}},
} }
for _, request := range requests { for _, request := range requests {
@@ -0,0 +1,82 @@
package localreset
import (
"context"
"strings"
"testing"
"time"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
)
type reconciliationSession struct { dispositionTestSession }
func (s *reconciliationSession) PrepareRecoveryReconciliation(ctx context.Context, sourceID string) (RecoveryPreparation, error) {
value, err := s.PrepareRecovery(ctx, "tx-1", deployment.RecoveryActionVerifyCommit)
value.Summary.SourceDispositionID = sourceID
value.Summary.Phase = deployment.TransactionPhaseCommitVerified
s.prepared = value
return value, err
}
func (s *reconciliationSession) ExecuteRecovery(ctx context.Context, id, token string) (deployment.RecoveryDispositionSummary, error) {
sourceID := s.prepared.Summary.SourceDispositionID
value, err := s.dispositionTestSession.ExecuteRecovery(ctx, id, token)
if err != nil { return value, err }
value.SourceDispositionID, value.ResultCode = sourceID, deployment.RecoveryResultCommitted
s.result = value
return value, nil
}
func TestRecoveryReconciliationProtocolAndQuery(t *testing.T) {
session := &reconciliationSession{}
_, conn, done := recoveryConnection(t, session)
client, err := Begin(t.Context(), conn)
if err != nil { t.Fatal(err) }
sourceID := strings.Repeat("a", 64)
prepared, err := client.PrepareRecoveryReconciliation(sourceID)
if err != nil || prepared.Summary.SourceDispositionID != sourceID { t.Fatal("原处置绑定丢失", err) }
result, err := client.ExecuteRecovery(prepared.DispositionID, prepared.Token)
if err != nil || result.SourceDispositionID != sourceID || result.Status != "succeeded" { t.Fatal("重新核查结果无效", err) }
queried, err := client.GetRecoveryDisposition(result.ID)
if err != nil || queried != result { t.Fatal("查询改变了关联结果", err) }
if _, err := client.ExecuteRecovery(prepared.DispositionID, prepared.Token); err == nil { t.Fatal("旧令牌被重放") }
_ = client.Close()
<-done
if session.prepares.Load() != 1 || session.executes.Load() != 2 || session.queries.Load() != 1 || session.resets.Load() != 0 { t.Fatal("调用了错误业务入口") }
}
func TestRecoveryReconciliationRejectsUnboundResponse(t *testing.T) {
sourceID := strings.Repeat("a", 64)
session := &reconciliationSession{}
prepared, err := session.PrepareRecoveryReconciliation(t.Context(), sourceID)
if err != nil { t.Fatal(err) }
request := Request{Action: "recovery_reconcile_prepare", DispositionID: sourceID}
for _, mode := range []string{"source", "action", "same-id", "transaction", "mixed", "expired-range"} {
t.Run(mode, func(t *testing.T) {
value := prepared
response := Response{Preparation: &value}
switch mode {
case "source": value.Summary.SourceDispositionID = strings.Repeat("d", 64)
case "action": value.Summary.Action = deployment.RecoveryActionCommit
case "same-id": value.DispositionID = sourceID
case "transaction": value.Summary.TransactionID = "../bad"
case "mixed": response.Disposition = &deployment.RecoveryDispositionSummary{}
case "expired-range": value.Summary.ExpiresAt = value.Summary.ObservedAt.Add(2*time.Minute)
}
if validDispositionResponse(request, response) { t.Fatal("接受了混合或无绑定准备响应") }
})
}
if validRecoveryAction(deployment.RecoveryActionVerifyCommit) || validRecoveryAction(deployment.RecoveryActionVerifyRollback) { t.Fatal("普通准备允许绕过原处置绑定") }
}
func TestRecoveryReconciliationOldServerFailsClosed(t *testing.T) {
session := &dispositionTestSession{}
_, conn, done := recoveryConnection(t, session)
client, err := Begin(t.Context(), conn)
if err != nil { t.Fatal(err) }
if _, err := client.PrepareRecoveryReconciliation(strings.Repeat("a", 64)); err == nil { t.Fatal("旧服务错误地支持了重新核查") }
_ = client.Close()
<-done
if session.prepares.Load() != 0 || session.executes.Load() != 0 { t.Fatal("旧服务错误地执行了原动作") }
}
@@ -25,7 +25,7 @@ func (s *Server) handleRecovery(ctx context.Context, conn net.Conn, reader *bufi
case "recovery_inspect": case "recovery_inspect":
inspection, readErr := session.InspectRecovery(ctx, request.TransactionID) inspection, readErr := session.InspectRecovery(ctx, request.TransactionID)
err, response.Inspection = readErr, &inspection err, response.Inspection = readErr, &inspection
case "recovery_prepare", "recovery_execute", "recovery_result": case "recovery_prepare", "recovery_execute", "recovery_result", "recovery_reconcile_prepare":
err = dispatchRecoveryDisposition(ctx, session, request, &response) err = dispatchRecoveryDisposition(ctx, session, request, &response)
default: default:
err = ErrProtocol err = ErrProtocol
@@ -43,6 +43,11 @@ func (r *SQLDeploymentTransactionRepository) FinishRecoveryDisposition(ctx conte
if err := tx.QueryRowContext(ctx, r.query(`SELECT event_type FROM recovery_events WHERE disposition_id = ? AND event_type = 'started'`), id).Scan(&start); err != nil { if err := tx.QueryRowContext(ctx, r.query(`SELECT event_type FROM recovery_events WHERE disposition_id = ? AND event_type = 'started'`), id).Scan(&start); err != nil {
return deployment.RecoveryDisposition{}, dispositionError(ctx) return deployment.RecoveryDisposition{}, dispositionError(ctx)
} }
if item.Intent.SourceDispositionID != "" {
if err := r.validateReconciliationSource(ctx, tx, item.Intent); err != nil {
return deployment.RecoveryDisposition{}, err
}
}
now := time.Now().UTC() now := time.Now().UTC()
if err := r.finishRecoveryOriginal(ctx, tx, item.Intent, resultCode, now); err != nil { if err := r.finishRecoveryOriginal(ctx, tx, item.Intent, resultCode, now); err != nil {
return deployment.RecoveryDisposition{}, err return deployment.RecoveryDisposition{}, err
@@ -58,7 +63,13 @@ func (r *SQLDeploymentTransactionRepository) FinishRecoveryDisposition(ctx conte
if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_events (disposition_id, event_type, result_code, created_at) VALUES (?, 'finished', ?, ?)`), id, resultCode, now); err != nil { if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_events (disposition_id, event_type, result_code, created_at) VALUES (?, 'finished', ?, ?)`), id, resultCode, now); err != nil {
return deployment.RecoveryDisposition{}, dispositionError(ctx) return deployment.RecoveryDisposition{}, dispositionError(ctx)
} }
if resultCode != deployment.RecoveryResultUncertain { if item.Intent.SourceDispositionID != "" && (resultCode == deployment.RecoveryResultRejected || resultCode == deployment.RecoveryResultUncertain) {
// 只读核查未通过时归还原占用,保留两次审计;绝不解锁原事务。
result, err := tx.ExecContext(ctx, r.query(`UPDATE recovery_claims SET disposition_id = ?, created_at = ? WHERE transaction_id = ? AND disposition_id = ?`), item.Intent.SourceDispositionID, now, item.Intent.TransactionID, id)
if err != nil || !oneRecoveryRow(result) {
return deployment.RecoveryDisposition{}, dispositionError(ctx)
}
} else if resultCode != deployment.RecoveryResultUncertain {
result, err := tx.ExecContext(ctx, r.query(`DELETE FROM recovery_claims WHERE transaction_id = ? AND disposition_id = ?`), item.Intent.TransactionID, id) result, err := tx.ExecContext(ctx, r.query(`DELETE FROM recovery_claims WHERE transaction_id = ? AND disposition_id = ?`), item.Intent.TransactionID, id)
if err != nil || !oneRecoveryRow(result) { if err != nil || !oneRecoveryRow(result) {
return deployment.RecoveryDisposition{}, dispositionError(ctx) return deployment.RecoveryDisposition{}, dispositionError(ctx)
@@ -96,6 +107,9 @@ func (r *SQLDeploymentTransactionRepository) finishRecoveryOriginal(ctx context.
(record.Status != deployment.TransactionStatusPending && record.Status != deployment.TransactionStatusRecoveryRequired) { (record.Status != deployment.TransactionStatusPending && record.Status != deployment.TransactionStatusRecoveryRequired) {
return deployment.ErrRecoveryDispositionConflict return deployment.ErrRecoveryDispositionConflict
} }
if intent.SourceDispositionID != "" && !intent.MatchesRecord(record) {
return deployment.ErrRecoveryReviewChanged
}
result, err := tx.ExecContext(ctx, r.query(`UPDATE deployment_transactions SET status = ?, phase = ?, message = ?, updated_at = ? result, err := tx.ExecContext(ctx, r.query(`UPDATE deployment_transactions SET status = ?, phase = ?, message = ?, updated_at = ?
WHERE id = ? AND phase = ? AND status IN ('pending', 'recovery_required')`), status, status, message, now, intent.TransactionID, phase) WHERE id = ? AND phase = ? AND status IN ('pending', 'recovery_required')`), status, status, message, now, intent.TransactionID, phase)
if err != nil || !oneRecoveryRow(result) { if err != nil || !oneRecoveryRow(result) {
@@ -12,13 +12,13 @@ import (
var _ deployment.RecoveryDispositionJournal = (*SQLDeploymentTransactionRepository)(nil) var _ deployment.RecoveryDispositionJournal = (*SQLDeploymentTransactionRepository)(nil)
const recoveryDispositionColumns = `id, transaction_id, server_id, operator_id, action, request_key, const recoveryDispositionColumns = `id, transaction_id, server_id, operator_id, action, request_key,
record_digest, server_digest, observation_digest, status, result_code, created_at, updated_at` record_digest, server_digest, observation_digest, status, result_code, created_at, updated_at, source_disposition_id`
func scanRecoveryDisposition(row deploymentTransactionScanner) (deployment.RecoveryDisposition, error) { func scanRecoveryDisposition(row deploymentTransactionScanner) (deployment.RecoveryDisposition, error) {
var item deployment.RecoveryDisposition var item deployment.RecoveryDisposition
i := &item.Intent i := &item.Intent
err := row.Scan(&i.ID, &i.TransactionID, &i.ServerID, &i.Operator, &i.Action, &i.RequestKey, err := row.Scan(&i.ID, &i.TransactionID, &i.ServerID, &i.Operator, &i.Action, &i.RequestKey,
&i.RecordDigest, &i.ServerDigest, &i.ObservationDigest, &item.Status, &item.ResultCode, &item.CreatedAt, &item.UpdatedAt) &i.RecordDigest, &i.ServerDigest, &i.ObservationDigest, &item.Status, &item.ResultCode, &item.CreatedAt, &item.UpdatedAt, &i.SourceDispositionID)
return item, err return item, err
} }
@@ -75,9 +75,9 @@ func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx contex
defer tx.Rollback() defer tx.Rollback()
now := time.Now().UTC() now := time.Now().UTC()
item := deployment.RecoveryDisposition{Intent: intent, Status: "running", CreatedAt: now, UpdatedAt: now} item := deployment.RecoveryDisposition{Intent: intent, Status: "running", CreatedAt: now, UpdatedAt: now}
_, err = tx.ExecContext(ctx, r.query(`INSERT INTO recovery_dispositions (`+recoveryDispositionColumns+`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`), _, err = tx.ExecContext(ctx, r.query(`INSERT INTO recovery_dispositions (`+recoveryDispositionColumns+`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`),
intent.ID, intent.TransactionID, intent.ServerID, intent.Operator, intent.Action, intent.RequestKey, intent.ID, intent.TransactionID, intent.ServerID, intent.Operator, intent.Action, intent.RequestKey,
intent.RecordDigest, intent.ServerDigest, intent.ObservationDigest, item.Status, "", now, now) intent.RecordDigest, intent.ServerDigest, intent.ObservationDigest, item.Status, "", now, now, intent.SourceDispositionID)
if err != nil { if err != nil {
_ = tx.Rollback() _ = tx.Rollback()
if existing, readErr := r.findRecoveryRequest(ctx, intent.RequestKey); readErr == nil { if existing, readErr := r.findRecoveryRequest(ctx, intent.RequestKey); readErr == nil {
@@ -85,7 +85,7 @@ func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx contex
} }
return deployment.RecoveryDisposition{}, false, dispositionError(ctx) return deployment.RecoveryDisposition{}, false, dispositionError(ctx)
} }
if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_claims (transaction_id, disposition_id, created_at) VALUES (?, ?, ?)`), intent.TransactionID, intent.ID, now); err != nil { if err := r.claimRecoveryDisposition(ctx, tx, intent, now); err != nil {
_ = tx.Rollback() _ = tx.Rollback()
if claimed, readErr := r.HasRecoveryClaim(ctx, intent.TransactionID); readErr == nil && claimed { if claimed, readErr := r.HasRecoveryClaim(ctx, intent.TransactionID); readErr == nil && claimed {
return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionBusy return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionBusy
@@ -109,6 +109,39 @@ func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx contex
return item, true, nil return item, true, nil
} }
func (r *SQLDeploymentTransactionRepository) claimRecoveryDisposition(ctx context.Context, tx *sql.Tx, intent deployment.RecoveryDispositionIntent, now time.Time) error {
if intent.SourceDispositionID == "" {
_, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_claims (transaction_id, disposition_id, created_at) VALUES (?, ?, ?)`), intent.TransactionID, intent.ID, now)
return err
}
if err := r.validateReconciliationSource(ctx, tx, intent); err != nil {
return err
}
// 只从指定的已结束处置接管占用;运行中、其他处置或丢失的占用均不接管。
result, err := tx.ExecContext(ctx, r.query(`UPDATE recovery_claims SET disposition_id = ?, created_at = ? WHERE transaction_id = ? AND disposition_id = ?`), intent.ID, now, intent.TransactionID, intent.SourceDispositionID)
if err != nil || !oneRecoveryRow(result) {
return deployment.ErrRecoveryDispositionBusy
}
return nil
}
func (r *SQLDeploymentTransactionRepository) validateReconciliationSource(ctx context.Context, tx *sql.Tx, intent deployment.RecoveryDispositionIntent) error {
query := `SELECT ` + recoveryDispositionColumns + ` FROM recovery_dispositions WHERE id = ?`
if r.driver != "sqlite" {
query += " FOR UPDATE"
}
source, err := scanRecoveryDisposition(tx.QueryRowContext(ctx, r.query(query), intent.SourceDispositionID))
if err != nil || !intent.Reconciles(source) {
return deployment.ErrRecoveryDispositionConflict
}
var count int
if err := tx.QueryRowContext(ctx, r.query(`SELECT COUNT(*) FROM recovery_events WHERE disposition_id = ? AND
((event_type = 'started' AND result_code = '') OR (event_type = 'finished' AND result_code = 'uncertain'))`), intent.SourceDispositionID).Scan(&count); err != nil || count != 2 {
return deployment.ErrRecoveryDispositionConflict
}
return nil
}
func recoveryRetry(existing deployment.RecoveryDisposition, intent deployment.RecoveryDispositionIntent) (deployment.RecoveryDisposition, bool, error) { func recoveryRetry(existing deployment.RecoveryDisposition, intent deployment.RecoveryDispositionIntent) (deployment.RecoveryDisposition, bool, error) {
if !existing.Intent.SameRequest(intent) { if !existing.Intent.SameRequest(intent) {
return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionConflict return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionConflict
@@ -68,6 +68,16 @@ func openRecoveryExternal(t *testing.T, cfg config.DatabaseConfig) *sql.DB {
// 只替换远端文件实现,准备、确认、执行、迁移、占用和审计均走实际产品代码。 // 只替换远端文件实现,准备、确认、执行、迁移、占用和审计均走实际产品代码。
func runExternalRecovery(t *testing.T, fixture recoveryExternalFixture) { func runExternalRecovery(t *testing.T, fixture recoveryExternalFixture) {
t.Helper() t.Helper()
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
t.Run("reconciliation/"+string(action), func(t *testing.T) {
cfg := fixture.database(t)
db := openRecoveryExternal(t, cfg)
if err := database.Migrate(t.Context(), db, cfg.Driver); err != nil {
t.Fatal(err)
}
runReconciliationSuccess(t, NewSQLDeploymentTransactionRepository(db, cfg.Driver), db, action)
})
}
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionRecord, deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} { for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionRecord, deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
for _, stage := range []string{"none", "started", "finished"} { for _, stage := range []string{"none", "started", "finished"} {
t.Run(string(action)+"/"+stage, func(t *testing.T) { t.Run(string(action)+"/"+stage, func(t *testing.T) {
@@ -0,0 +1,250 @@
package repository
import (
"context"
"database/sql"
"errors"
"maps"
"reflect"
"testing"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
)
// 远端文件是测试替身;事务、关联、占用交接与审计均经过真实数据库和产品执行器。
type reconciliationClient struct {
*executorIntegrationClient
failClosed, readOnly bool
connections, writes int
}
func (c *reconciliationClient) Connect(context.Context, deployment.Server) (deployment.Client, error) {
c.connections++
return c, nil
}
func (c *reconciliationClient) Close() error {
if c.failClosed {
if _, exists := c.files["/cert.pem.scert-lock"]; !exists {
return errors.New("注入动作完成后的连接关闭失败")
}
}
return nil
}
func (c *reconciliationClient) writable() error {
c.writes++
if c.readOnly {
return errors.New("重新核查不得执行远端写操作")
}
return nil
}
func (c *reconciliationClient) Remove(ctx context.Context, path string) error {
if err := c.writable(); err != nil {
return err
}
return c.executorIntegrationClient.Remove(ctx, path)
}
func (c *reconciliationClient) Rename(ctx context.Context, from, to string) error {
if err := c.writable(); err != nil {
return err
}
return c.executorIntegrationClient.Rename(ctx, from, to)
}
func (c *reconciliationClient) Unlock(ctx context.Context, path string) error {
if err := c.writable(); err != nil {
return err
}
return c.executorIntegrationClient.Unlock(ctx, path)
}
func (c *reconciliationClient) Run(ctx context.Context, command []string) (deployment.CommandResult, error) {
if err := c.writable(); err != nil {
return deployment.CommandResult{}, err
}
return c.executorIntegrationClient.Run(ctx, command)
}
func seedUncertainReconciliation(t *testing.T, repo *SQLDeploymentTransactionRepository, db *sql.DB, action deployment.RecoveryAction) (*deployment.Service, *reconciliationClient, deployment.Server, deployment.TransactionRecord, deployment.RecoveryDisposition) {
t.Helper()
_, _, _, remote, server, record := seedRecoveryExecutor(t, repo, db)
client := &reconciliationClient{executorIntegrationClient: remote, failClosed: true}
service := deployment.NewServiceWithJournal(client, repo)
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error { return nil }
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
if err != nil {
t.Fatal(err)
}
source, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if !errors.Is(err, deployment.ErrRecoveryActionUncertain) || source.Status != deployment.RecoveryResultUncertain {
t.Fatalf("未形成真实执行器不确定结果:%+v %v", source, err)
}
client.failClosed, client.readOnly = false, true
client.writes = 0
return service, client, server, record, source
}
func assertReconciliationSource(t *testing.T, repo *SQLDeploymentTransactionRepository, source deployment.RecoveryDisposition) {
t.Helper()
stored, err := repo.GetRecoveryDisposition(t.Context(), source.Intent.ID)
if err != nil || !reflect.DeepEqual(stored, source) {
t.Fatalf("重新核查改写了原不确定历史:%+v %v", stored, err)
}
}
func runReconciliationSuccess(t *testing.T, repo *SQLDeploymentTransactionRepository, db *sql.DB, action deployment.RecoveryAction) {
t.Helper()
service, client, server, record, source := seedUncertainReconciliation(t, repo, db, action)
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
files := maps.Clone(client.files)
verified := false
service.VerifyRecoveryTarget = func(_ context.Context, _ deployment.Server, _ deployment.RecoveryBaseline, committed bool) error {
if committed != (action == deployment.RecoveryActionCommit) {
t.Fatal("重新核查擅自切换方向")
}
verified = true
return nil
}
confirmation, token, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve)
if err != nil || confirmation.Summary().Validate() != nil {
t.Fatal("准备重新核查失败", err)
}
other, otherToken, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve)
if err != nil {
t.Fatal(err)
}
result, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if err != nil || result.Status != "succeeded" || result.Intent.SourceDispositionID != source.Intent.ID || !verified {
t.Fatalf("重新核查失败:%+v %v", result, err)
}
if client.writes != 0 || !reflect.DeepEqual(files, client.files) {
t.Fatal("只读重新核查修改了远端")
}
assertReconciliationSource(t, repo, source)
recoveryCount(t, db, "recovery_dispositions", 2)
recoveryCount(t, db, "recovery_events", 4)
recoveryCount(t, db, "recovery_claims", 0)
pending, err := repo.ListPendingDeploymentTransactions(t.Context())
if err != nil || len(pending) != 0 {
t.Fatal("原事务未原子结束", err)
}
view, err := service.GetRecoveryDisposition(t.Context(), result.Intent.ID)
if err != nil || view.SourceDispositionID != source.Intent.ID || view.Validate() != nil {
t.Fatal("查询缺少可信关联", err)
}
connections := client.connections
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, deployment.ErrRecoveryConfirmation) {
t.Fatal("重新核查确认被重放", err)
}
if _, err := service.ExecuteRecoveryDisposition(t.Context(), other, "unix:0", otherToken, resolve); err == nil {
t.Fatal("其他旧确认再次接管已完成的事务")
}
if client.connections != connections {
t.Fatal("完成后仍使用旧确认连接远端")
}
if record.ID != result.Intent.TransactionID {
t.Fatal("结束了其他事务")
}
}
func TestRecoveryReconciliationSQLiteSuccess(t *testing.T) {
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
t.Run(string(action), func(t *testing.T) {
_, db := newTaskRepositoryTest(t)
t.Cleanup(func() { _ = db.Close() })
runReconciliationSuccess(t, NewSQLDeploymentTransactionRepository(db), db, action)
})
}
}
func TestRecoveryReconciliationSQLiteRejectsUnsafeCompletion(t *testing.T) {
for _, failure := range []string{"file", "backup", "temporary", "lock", "https", "during-https", "close", "changed-after-prepare", "started-audit", "finished-audit", "identity", "phase", "running", "direction"} {
t.Run(failure, func(t *testing.T) {
_, db := newTaskRepositoryTest(t)
t.Cleanup(func() { _ = db.Close() })
repo := NewSQLDeploymentTransactionRepository(db)
service, client, server, record, source := seedUncertainReconciliation(t, repo, db, deployment.RecoveryActionCommit)
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
switch failure {
case "file":
client.files[record.Paths[0]] = "tampered"
case "backup":
client.files[record.Backups[0].Saved] = "old"
case "temporary":
client.files[record.Paths[0]+".scert-tmp-"+record.ID] = "unexpected"
case "lock":
client.files[record.LockPaths[0]] = ""
case "https":
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error {
return errors.New("严格 HTTPS 未通过")
}
case "during-https":
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error {
client.files[record.Paths[0]] = "changed"
return nil
}
case "identity":
server.PasswordRef = "secret://ssh/password/changed"
case "phase":
if _, err := db.ExecContext(t.Context(), `UPDATE deployment_transactions SET phase = 'reload_attempted'`); err != nil {
t.Fatal(err)
}
case "running":
if _, err := db.ExecContext(t.Context(), `UPDATE recovery_dispositions SET status = 'running', result_code = ''`); err != nil {
t.Fatal(err)
}
case "direction":
if _, err := db.ExecContext(t.Context(), `UPDATE recovery_dispositions SET action = 'restore_baseline'`); err != nil {
t.Fatal(err)
}
}
confirmation, token, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve)
if failure == "identity" || failure == "phase" || failure == "running" || failure == "direction" {
if err == nil {
t.Fatal("危险来源允许重新核查")
}
recoveryCount(t, db, "recovery_claims", 1)
return
}
if err != nil {
t.Fatal("准备失败", err)
}
if failure == "close" {
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error {
client.failClosed = true
return nil
}
}
if failure == "changed-after-prepare" {
client.files[record.Paths[1]] = "changed"
}
if failure == "started-audit" || failure == "finished-audit" {
stage := "started"
if failure == "finished-audit" {
stage = "finished"
}
if _, err := db.ExecContext(t.Context(), `CREATE TRIGGER reject_recheck BEFORE INSERT ON recovery_events WHEN NEW.event_type = '`+stage+`' BEGIN SELECT RAISE(ABORT, 'injected'); END`); err != nil {
t.Fatal(err)
}
}
result, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if err == nil || result.Status == "succeeded" || client.writes != 0 {
t.Fatalf("危险收尾未拒绝或执行了写操作:%+v %v", result, err)
}
assertReconciliationSource(t, repo, source)
var owner string
if err := db.QueryRowContext(t.Context(), `SELECT disposition_id FROM recovery_claims WHERE transaction_id = ?`, record.ID).Scan(&owner); err != nil {
t.Fatal(err)
}
expected := source.Intent.ID
if failure == "finished-audit" {
expected = confirmation.DispositionID()
}
if owner != expected {
t.Fatal("失败后占用归属不正确", owner, expected)
}
pending, err := repo.ListPendingDeploymentTransactions(t.Context())
if err != nil || len(pending) != 1 || pending[0].Status != deployment.TransactionStatusRecoveryRequired {
t.Fatal("失败改变了原事务", err)
}
})
}
}
+1 -1
View File
@@ -6,7 +6,7 @@
- 第一轮任务 `b795be80-e086-46b4-a2ba-e7b51c8662f0` 因旧 systemd 不支持 `--kill-whom`,注入失败并正常完成;不能计为崩溃测试通过。确认本机帮助后改用 `--kill-who=main`。 - 第一轮任务 `b795be80-e086-46b4-a2ba-e7b51c8662f0` 因旧 systemd 不支持 `--kill-whom`,注入失败并正常完成;不能计为崩溃测试通过。确认本机帮助后改用 `--kill-who=main`。
- 22:57:25 对正式 Scert 主进程发送 SIGKILL,命中任务 `c68e3956-1b6d-42c7-ba9a-753ef2d1a44b` 第二目标事务的 reload_attempted 阶段。 - 22:57:25 对正式 Scert 主进程发送 SIGKILL,命中任务 `c68e3956-1b6d-42c7-ba9a-753ef2d1a44b` 第二目标事务的 reload_attempted 阶段。
- systemd 的 MainPID 从 130794 变为 134595,NRestarts=1;23:58 前已核实服务恢复 active、回环严格 TLS 就绪检查通过。此处实际核查时间为 22:58:16。 - systemd 的 MainPID 从 130794 变为 134595,NRestarts=1;22:58:16 已核实服务恢复 active、回环严格 TLS 就绪检查通过。
- 事务 `20261003-145716.121859533-09a2a331489a` 与 `20261003-145720.641188477-c769c8d3e153` 均 rolled_back;任务为 failed/service_restart,没有覆盖为成功。 - 事务 `20261003-145716.121859533-09a2a331489a` 与 `20261003-145720.641188477-c769c8d3e153` 均 rolled_back;任务为 failed/service_restart,没有覆盖为成功。
- 双机 cert/fullchain 摘要保持 `7e8b56227b345ed9b66659e428f9bd4421604f11253ab88d65b3ed61737f35cf`,未发现该域名遗留事务文件。第二台正常 DNS HTTPS 200、peer=45.136.13.241、TLS 校验结果零。 - 双机 cert/fullchain 摘要保持 `7e8b56227b345ed9b66659e428f9bd4421604f11253ab88d65b3ed61737f35cf`,未发现该域名遗留事务文件。第二台正常 DNS HTTPS 200、peer=45.136.13.241、TLS 校验结果零。
@@ -1,5 +1,17 @@
# 部署事务人工核查任务清单 # 部署事务人工核查任务清单
## 2026年10月3日23:30 不确定处置重新核查任务
沿用已确认的 Linux 范围,不增加网络入口,不直接修改现场事务或删除占用。
- [✓] 上轮静态检查通过;后端全量唯一失败已定位:宿主 umask 为 0077,测试创建的父目录从开始就是 0700;原代码在显式 umask 0022 下通过,不能归因于备份修改权限。
- [x] 测试显式建立 0755 权限基线,不修改进程 umask 或产品权限策略,等待 Linux 严格 umask 回归。
- [~] R6-A 独立重新核查动作:绑定原 uncertain 处置编号、原方向、服务器身份和本次现场摘要;只允许已完成审计的 uncertain,不以超时接管 running。
- [ ] R6-B 持久化关联与占用原子交接:新增处置审计,保留原 uncertain;拒绝或核查失败恢复原占用,结束审计失败保留新占用;成功才原子结束原事务并释放占用。
- [ ] R6-C 只读远端重新核验:仅限原方向对应的 verified 阶段,核验全部文件、临时文件与备份不存在、锁不存在、严格 HTTPS 和关闭结果;不得重载、删锁、重写或重放原动作。
- [ ] R6-D 受保护本机终端单独入口、完整确认、一次性绑定、断连查询及三数据库回归。
- [ ] R6-E 正式 Linux 包升级和当前真实 uncertain 处置收尾;其他不满足严格收尾条件的历史事务继续保留,不能用本功能冒充完成。
## 2026年10月3日23:08 实机关闭误判整改任务 ## 2026年10月3日23:08 实机关闭误判整改任务
- [✓] 真实 SIGKILL 中断部署后,两条事务自动回滚,第二台正常 DNS 严格 HTTPS 仍通过。 - [✓] 真实 SIGKILL 中断部署后,两条事务自动回滚,第二台正常 DNS 严格 HTTPS 仍通过。