保存不确定部署处置重新核查实现及测试检查点
This commit is contained in:
1 parent
7fa5e9b3d2
commit
f9cd19f631
29 files changed
+907
-49
No files matched your search
@@ -60,7 +60,7 @@ func runConnected(ctx context.Context, conn net.Conn, screen terminal) error {
|
|||||||
if client.RecoveryAvailable() {
|
if client.RecoveryAvailable() {
|
||||||
prompt := "本机管理:1 重置超级管理员密码;2 只读核查部署事务;其他输入退出"
|
prompt := "本机管理:1 重置超级管理员密码;2 只读核查部署事务;其他输入退出"
|
||||||
if client.DispositionAvailable() {
|
if client.DispositionAvailable() {
|
||||||
prompt = "本机管理:1 重置超级管理员密码;2 只读核查部署事务;3 人工处置部署事务;4 查询人工处置结果;其他输入退出"
|
prompt = "本机管理:1 重置超级管理员密码;2 只读核查部署事务;3 人工处置部署事务;4 查询人工处置结果;5 重新核查不确定处置;其他输入退出"
|
||||||
}
|
}
|
||||||
choice, err := screen.Read(ctx, prompt, false)
|
choice, err := screen.Read(ctx, prompt, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -74,13 +74,16 @@ func runConnected(ctx context.Context, conn net.Conn, screen terminal) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
case "3", "4":
|
case "3", "4", "5":
|
||||||
if !client.DispositionAvailable() {
|
if !client.DispositionAvailable() {
|
||||||
return localreset.ErrUnavailable
|
return localreset.ErrUnavailable
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(choice) == "4" {
|
if strings.TrimSpace(choice) == "4" {
|
||||||
return recoveryResultFlow(ctx, client, screen)
|
return recoveryResultFlow(ctx, client, screen)
|
||||||
}
|
}
|
||||||
|
if strings.TrimSpace(choice) == "5" {
|
||||||
|
return recoveryReconciliationFlow(ctx, client, screen)
|
||||||
|
}
|
||||||
return recoveryDispositionFlow(ctx, client, screen)
|
return recoveryDispositionFlow(ctx, client, screen)
|
||||||
default:
|
default:
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -130,12 +130,22 @@ func showRecoveryDisposition(screen terminal, result deployment.RecoveryDisposit
|
|||||||
message = "本次人工提交核验及结束审计完成;不代表项目全部生产验收通过。"
|
message = "本次人工提交核验及结束审计完成;不代表项目全部生产验收通过。"
|
||||||
case deployment.RecoveryActionRollback:
|
case deployment.RecoveryActionRollback:
|
||||||
message = "本次恢复基线核验及结束审计完成;不代表项目全部生产验收通过。"
|
message = "本次恢复基线核验及结束审计完成;不代表项目全部生产验收通过。"
|
||||||
|
case deployment.RecoveryActionVerifyCommit, deployment.RecoveryActionVerifyRollback:
|
||||||
|
message = "重新核查与新审计已完成,占用已安全释放;原不确定处置历史保留,本次未重放远端动作。"
|
||||||
}
|
}
|
||||||
case "rejected":
|
case "rejected":
|
||||||
message = "处置已拒绝,未执行本次远端变更;请重新核查拒绝原因。"
|
message = "处置已拒绝,未执行本次远端变更;请重新核查拒绝原因。"
|
||||||
|
if result.SourceDispositionID != "" {
|
||||||
|
message = "重新核查未通过,原占用继续保留;本次未执行远端变更,不得手动删锁或重放原动作。"
|
||||||
|
}
|
||||||
case "uncertain":
|
case "uncertain":
|
||||||
message = "处置结果不确定,可能已修改远端;占用继续保留,禁止重复执行或手动删锁。"
|
message = "处置结果不确定,可能已修改远端;占用继续保留,禁止重复执行或手动删锁。"
|
||||||
}
|
}
|
||||||
|
if result.SourceDispositionID != "" {
|
||||||
|
if err := screen.Show("关联原处置:" + terminalValue(result.SourceDispositionID)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
if err := screen.Show(fmt.Sprintf("处置:%s;事务:%s;服务器:%s;动作:%s", terminalValue(result.ID), terminalValue(result.TransactionID), terminalValue(result.ServerID), dispositionActionLabel(result.Action))); err != nil {
|
if err := screen.Show(fmt.Sprintf("处置:%s;事务:%s;服务器:%s;动作:%s", terminalValue(result.ID), terminalValue(result.TransactionID), terminalValue(result.ServerID), dispositionActionLabel(result.Action))); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -150,6 +160,10 @@ func dispositionActionLabel(action deployment.RecoveryAction) string {
|
|||||||
return "完成提交"
|
return "完成提交"
|
||||||
case deployment.RecoveryActionRollback:
|
case deployment.RecoveryActionRollback:
|
||||||
return "恢复基线"
|
return "恢复基线"
|
||||||
|
case deployment.RecoveryActionVerifyCommit:
|
||||||
|
return "重新核查提交结果"
|
||||||
|
case deployment.RecoveryActionVerifyRollback:
|
||||||
|
return "重新核查回滚结果"
|
||||||
}
|
}
|
||||||
return "未知动作"
|
return "未知动作"
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
package adminconsole
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.pcux.cn/xuans/Scert/backend/internal/localreset"
|
||||||
|
)
|
||||||
|
|
||||||
|
type recoveryReconciliationClient interface {
|
||||||
|
recoveryDispositionClient
|
||||||
|
PrepareRecoveryReconciliation(string) (localreset.RecoveryPreparation, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func recoveryReconciliationFlow(ctx context.Context, client recoveryReconciliationClient, screen terminal) error {
|
||||||
|
sourceID, err := screen.Read(ctx, "请输入结果不确定的原处置编号;本操作只重新核验,不重放文件修改或服务重载", false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sourceID = strings.TrimSpace(sourceID)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
prepared, err := client.PrepareRecoveryReconciliation(sourceID)
|
||||||
|
if err != nil {
|
||||||
|
_ = screen.Show("无法准备重新核查:主服务可能尚未支持,或原处置未结束、历史证据不足、现场不满足收尾条件。占用仍保留。")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { prepared.Token = "" }()
|
||||||
|
summary := prepared.Summary
|
||||||
|
if summary.Validate() != nil || summary.SourceDispositionID != sourceID || !time.Now().Before(summary.ExpiresAt) {
|
||||||
|
return localreset.ErrProtocol
|
||||||
|
}
|
||||||
|
for _, line := range []string{
|
||||||
|
"原处置:" + terminalValue(sourceID),
|
||||||
|
"新核查编号:" + terminalValue(prepared.DispositionID) + "(请保留,用于断连后查询)",
|
||||||
|
fmt.Sprintf("事务:%s;服务器:%s;动作:%s", terminalValue(summary.TransactionID), terminalValue(summary.ServerID), dispositionActionLabel(summary.Action)),
|
||||||
|
"本次只核验原方向,不能改选提交或回滚;不写远端文件、不删锁、不重载。文件、残留、严格 HTTPS 和关闭检查全部通过后,才以新审计结束原事务并释放占用。",
|
||||||
|
"原不确定记录永久保留;任何核验或审计失败均不释放占用,运行中处置不会因等待超时被接管。",
|
||||||
|
} {
|
||||||
|
if err := screen.Show(line); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
expected := "确认重新核查 " + sourceID + " " + dispositionActionLabel(summary.Action)
|
||||||
|
answer, err := screen.Read(ctx, "如确认,请完整输入:"+expected+";其他输入取消", false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if answer != expected || ctx.Err() != nil || !time.Now().Before(summary.ExpiresAt) {
|
||||||
|
return screen.Show("已取消或确认过期,未提交重新核查。")
|
||||||
|
}
|
||||||
|
result, err := client.ExecuteRecovery(prepared.DispositionID, prepared.Token)
|
||||||
|
prepared.Token = ""
|
||||||
|
if err != nil || result.Validate() != nil || result.ID != prepared.DispositionID || result.SourceDispositionID != sourceID ||
|
||||||
|
result.TransactionID != summary.TransactionID || result.ServerID != summary.ServerID || result.Action != summary.Action {
|
||||||
|
_ = screen.Show("未取得可靠结果,请重新连接后按新核查编号查询;不要重复执行或直接删除占用。")
|
||||||
|
return localreset.ErrUnavailable
|
||||||
|
}
|
||||||
|
return showRecoveryDisposition(screen, result)
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
package adminconsole
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||||
|
"git.pcux.cn/xuans/Scert/backend/internal/localreset"
|
||||||
|
)
|
||||||
|
|
||||||
|
type reconciliationConsoleClient struct {
|
||||||
|
dispositionConsoleClient
|
||||||
|
sourceID string
|
||||||
|
mismatch bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *reconciliationConsoleClient) PrepareRecoveryReconciliation(sourceID string) (localreset.RecoveryPreparation, error) {
|
||||||
|
c.sourceID = sourceID
|
||||||
|
value, err := c.PrepareRecovery("tx-1", deployment.RecoveryActionVerifyCommit)
|
||||||
|
value.Summary.SourceDispositionID = sourceID
|
||||||
|
value.Summary.Phase = deployment.TransactionPhaseCommitVerified
|
||||||
|
return value, err
|
||||||
|
}
|
||||||
|
func (c *reconciliationConsoleClient) ExecuteRecovery(id, token string) (deployment.RecoveryDispositionSummary, error) {
|
||||||
|
value, err := c.dispositionConsoleClient.ExecuteRecovery(id, token)
|
||||||
|
if err != nil { return value, err }
|
||||||
|
value.SourceDispositionID = c.sourceID
|
||||||
|
if value.Status == "succeeded" { value.ResultCode = "committed" }
|
||||||
|
if c.mismatch { value.SourceDispositionID = strings.Repeat("d", 64) }
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecoveryReconciliationConsoleConfirmation(t *testing.T) {
|
||||||
|
for _, mode := range []string{"success", "rejected", "cancelled", "expired", "disconnect", "mismatch"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
sourceID := strings.Repeat("a", 64)
|
||||||
|
client := &reconciliationConsoleClient{}
|
||||||
|
answer := "确认重新核查 " + sourceID + " " + dispositionActionLabel(deployment.RecoveryActionVerifyCommit)
|
||||||
|
switch mode {
|
||||||
|
case "rejected": client.status = "rejected"
|
||||||
|
case "cancelled": answer = "确认"
|
||||||
|
case "expired": client.expired = true
|
||||||
|
case "disconnect": client.executeError = true
|
||||||
|
case "mismatch": client.mismatch = true
|
||||||
|
}
|
||||||
|
screen := &recoveryTestScreen{testTerminalInput: testTerminalInput{values: []string{sourceID, answer}}}
|
||||||
|
err := recoveryReconciliationFlow(t.Context(), client, screen)
|
||||||
|
output := strings.Join(screen.lines, "|")
|
||||||
|
if client.prepares != 1 || client.queries != 0 || strings.Contains(output, strings.Repeat("c", 64)) { t.Fatal("终端重复调用或泄露令牌") }
|
||||||
|
if mode == "cancelled" || mode == "expired" {
|
||||||
|
if client.executes != 0 { t.Fatal("取消或过期仍执行核查") }
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if client.executes != 1 || !strings.Contains(output, "原不确定记录永久保留") { t.Fatal("缺少明确警告或重复执行") }
|
||||||
|
if mode == "disconnect" || mode == "mismatch" {
|
||||||
|
if err == nil || !strings.Contains(output, "新核查编号查询") { t.Fatal("失去可靠响应后误报成功") }
|
||||||
|
} else if err != nil { t.Fatal(err) }
|
||||||
|
if mode == "success" && !strings.Contains(output, "占用已安全释放") { t.Fatal("缺少完成结果") }
|
||||||
|
if mode == "rejected" && !strings.Contains(output, "原占用继续保留") { t.Fatal("错误显示拒绝结果") }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -134,6 +134,10 @@ func TestCreateBackupDoesNotChangeExistingParentPermissions(t *testing.T) {
|
|||||||
if err := os.Mkdir(parent, 0o755); err != nil {
|
if err := os.Mkdir(parent, 0o755); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
// 显式建立权限基线,避免宿主 umask 将创建权限收紧后造成误报。
|
||||||
|
if err := os.Chmod(parent, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
cfg := isolatedCLIConfig(filepath.Join(base, "scert"))
|
cfg := isolatedCLIConfig(filepath.Join(base, "scert"))
|
||||||
for _, directory := range []string{cfg.Paths.Config, cfg.Paths.Data} {
|
for _, directory := range []string{cfg.Paths.Config, cfg.Paths.Data} {
|
||||||
if err := os.MkdirAll(directory, 0o700); err != nil {
|
if err := os.MkdirAll(directory, 0o700); err != nil {
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ import (
|
|||||||
//go:embed migrations/sqlite/0022_recovery_dispositions.sql migrations/mysql/0022_recovery_dispositions.sql migrations/postgres/0022_recovery_dispositions.sql
|
//go:embed migrations/sqlite/0022_recovery_dispositions.sql migrations/mysql/0022_recovery_dispositions.sql migrations/postgres/0022_recovery_dispositions.sql
|
||||||
//go:embed migrations/sqlite/0023_recovery_claims.sql migrations/mysql/0023_recovery_claims.sql migrations/postgres/0023_recovery_claims.sql
|
//go:embed migrations/sqlite/0023_recovery_claims.sql migrations/mysql/0023_recovery_claims.sql migrations/postgres/0023_recovery_claims.sql
|
||||||
//go:embed migrations/sqlite/0024_recovery_events.sql migrations/mysql/0024_recovery_events.sql migrations/postgres/0024_recovery_events.sql
|
//go:embed migrations/sqlite/0024_recovery_events.sql migrations/mysql/0024_recovery_events.sql migrations/postgres/0024_recovery_events.sql
|
||||||
|
//go:embed migrations/sqlite/0025_recovery_reconciliation.sql migrations/mysql/0025_recovery_reconciliation.sql migrations/postgres/0025_recovery_reconciliation.sql
|
||||||
var migrationFiles embed.FS
|
var migrationFiles embed.FS
|
||||||
|
|
||||||
func Migrate(ctx context.Context, db *sql.DB, driver string) error {
|
func Migrate(ctx context.Context, db *sql.DB, driver string) error {
|
||||||
@@ -49,7 +50,7 @@ func Migrate(ctx context.Context, db *sql.DB, driver string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
const latestMigrationVersion = 24
|
const latestMigrationVersion = 25
|
||||||
|
|
||||||
// currentMigrationVersion 在执行业务迁移前拒绝新版数据库及不连续的迁移历史。
|
// currentMigrationVersion 在执行业务迁移前拒绝新版数据库及不连续的迁移历史。
|
||||||
func currentMigrationVersion(ctx context.Context, db *sql.DB) (int, error) {
|
func currentMigrationVersion(ctx context.Context, db *sql.DB) (int, error) {
|
||||||
@@ -114,6 +115,14 @@ func applyMigration(ctx context.Context, db *sql.DB, driver string, version int)
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if version == 25 {
|
||||||
|
var compatible int
|
||||||
|
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = DATABASE()
|
||||||
|
AND table_name = 'recovery_dispositions' AND column_name = 'source_disposition_id'
|
||||||
|
AND data_type = 'varchar' AND character_maximum_length = 64 AND is_nullable = 'NO' AND column_default = ''`).Scan(&compatible); err != nil || compatible != 1 {
|
||||||
|
return fmt.Errorf("人工处置来源字段与预期不一致,拒绝记录迁移版本")
|
||||||
|
}
|
||||||
|
}
|
||||||
if _, err := db.ExecContext(ctx, "INSERT INTO schema_migrations (version) VALUES (?)", version); err != nil {
|
if _, err := db.ExecContext(ctx, "INSERT INTO schema_migrations (version) VALUES (?)", version); err != nil {
|
||||||
return fmt.Errorf("记录迁移版本失败:%w", err)
|
return fmt.Errorf("记录迁移版本失败:%w", err)
|
||||||
}
|
}
|
||||||
@@ -252,6 +261,8 @@ func migrationName(version int) string {
|
|||||||
return "recovery_claims"
|
return "recovery_claims"
|
||||||
case 24:
|
case 24:
|
||||||
return "recovery_events"
|
return "recovery_events"
|
||||||
|
case 25:
|
||||||
|
return "recovery_reconciliation"
|
||||||
default:
|
default:
|
||||||
return "unknown"
|
return "unknown"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE recovery_dispositions ADD COLUMN source_disposition_id VARCHAR(64) NOT NULL DEFAULT '';
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE recovery_dispositions ADD COLUMN source_disposition_id VARCHAR(64) NOT NULL DEFAULT '';
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE recovery_dispositions ADD COLUMN source_disposition_id TEXT NOT NULL DEFAULT '';
|
||||||
@@ -25,6 +25,10 @@ func TestMySQLRealRecoveryDispositionMigrationRecovery(t *testing.T) {
|
|||||||
`ALTER TABLE recovery_claims DROP PRIMARY KEY`},
|
`ALTER TABLE recovery_claims DROP PRIMARY KEY`},
|
||||||
{24, "recovery_events", `INSERT INTO recovery_events VALUES ('kept', 'started', '', NOW())`,
|
{24, "recovery_events", `INSERT INTO recovery_events VALUES ('kept', 'started', '', NOW())`,
|
||||||
`ALTER TABLE recovery_events DROP PRIMARY KEY, ADD PRIMARY KEY (event_type, disposition_id)`},
|
`ALTER TABLE recovery_events DROP PRIMARY KEY, ADD PRIMARY KEY (event_type, disposition_id)`},
|
||||||
|
{25, "recovery_dispositions", `INSERT INTO recovery_dispositions (id, transaction_id, server_id, operator_id, action, request_key,
|
||||||
|
record_digest, server_digest, observation_digest, status, result_code, created_at, updated_at) VALUES
|
||||||
|
('kept', 'transaction', 'server', 'unix:0', 'finish_commit', 'request', 'record', 'server', 'observation', 'uncertain', 'uncertain', NOW(), NOW())`,
|
||||||
|
`ALTER TABLE recovery_dispositions MODIFY COLUMN source_disposition_id VARCHAR(8) NOT NULL DEFAULT ''`},
|
||||||
} {
|
} {
|
||||||
for _, boundary := range []string{"before_create", "after_create", "with_record", "conflict"} {
|
for _, boundary := range []string{"before_create", "after_create", "with_record", "conflict"} {
|
||||||
t.Run(fmt.Sprintf("%d/%s", item.version, boundary), func(t *testing.T) {
|
t.Run(fmt.Sprintf("%d/%s", item.version, boundary), func(t *testing.T) {
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
|
|
||||||
func TestPostgresRealRecoveryDispositionMigration(t *testing.T) {
|
func TestPostgresRealRecoveryDispositionMigration(t *testing.T) {
|
||||||
server := postgrestest.New(t)
|
server := postgrestest.New(t)
|
||||||
for version := 22; version <= 24; version++ {
|
for version := 22; version <= 25; version++ {
|
||||||
t.Run(fmt.Sprint(version), func(t *testing.T) {
|
t.Run(fmt.Sprint(version), func(t *testing.T) {
|
||||||
cfg := server.Database(t)
|
cfg := server.Database(t)
|
||||||
db, err := Open(t.Context(), cfg)
|
db, err := Open(t.Context(), cfg)
|
||||||
|
|||||||
@@ -17,15 +17,16 @@ var (
|
|||||||
|
|
||||||
// RecoveryDispositionIntent 仅保存绑定摘要;不能包含历史证据、凭据或确认令牌原文。
|
// RecoveryDispositionIntent 仅保存绑定摘要;不能包含历史证据、凭据或确认令牌原文。
|
||||||
type RecoveryDispositionIntent struct {
|
type RecoveryDispositionIntent struct {
|
||||||
ID string
|
SourceDispositionID string
|
||||||
TransactionID string
|
ID string
|
||||||
ServerID string
|
TransactionID string
|
||||||
Operator string
|
ServerID string
|
||||||
Action RecoveryAction
|
Operator string
|
||||||
RequestKey string
|
Action RecoveryAction
|
||||||
RecordDigest string
|
RequestKey string
|
||||||
ServerDigest string
|
RecordDigest string
|
||||||
ObservationDigest string
|
ServerDigest string
|
||||||
|
ObservationDigest string
|
||||||
}
|
}
|
||||||
|
|
||||||
type RecoveryDisposition struct {
|
type RecoveryDisposition struct {
|
||||||
@@ -58,7 +59,8 @@ func (r RecoveryReview) DispositionIntent(id, requestKey string) (RecoveryDispos
|
|||||||
return RecoveryDispositionIntent{}, ErrRecoveryReviewExpired
|
return RecoveryDispositionIntent{}, ErrRecoveryReviewExpired
|
||||||
}
|
}
|
||||||
intent := RecoveryDispositionIntent{
|
intent := RecoveryDispositionIntent{
|
||||||
ID: id, TransactionID: r.summary.TransactionID, ServerID: r.summary.ServerID,
|
SourceDispositionID: r.summary.SourceDispositionID,
|
||||||
|
ID: id, TransactionID: r.summary.TransactionID, ServerID: r.summary.ServerID,
|
||||||
Operator: r.operator, Action: r.summary.Action, RequestKey: requestKey,
|
Operator: r.operator, Action: r.summary.Action, RequestKey: requestKey,
|
||||||
RecordDigest: hex.EncodeToString(r.recordDigest[:]),
|
RecordDigest: hex.EncodeToString(r.recordDigest[:]),
|
||||||
ServerDigest: hex.EncodeToString(r.serverDigest[:]),
|
ServerDigest: hex.EncodeToString(r.serverDigest[:]),
|
||||||
@@ -68,6 +70,9 @@ func (r RecoveryReview) DispositionIntent(id, requestKey string) (RecoveryDispos
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (i RecoveryDispositionIntent) Validate() error {
|
func (i RecoveryDispositionIntent) Validate() error {
|
||||||
|
if isRecoveryVerification(i.Action) != (i.SourceDispositionID != "") || (i.SourceDispositionID != "" && (!validDispositionID(i.SourceDispositionID) || i.SourceDispositionID == i.ID)) {
|
||||||
|
return ErrRecoveryDispositionInvalid
|
||||||
|
}
|
||||||
if !validInspectionID(i.TransactionID) || !validRecoverySummary(i.ServerID) ||
|
if !validInspectionID(i.TransactionID) || !validRecoverySummary(i.ServerID) ||
|
||||||
!validReviewOperator(i.Operator) || !validRecoveryAction(i.Action) {
|
!validReviewOperator(i.Operator) || !validRecoveryAction(i.Action) {
|
||||||
return ErrRecoveryDispositionInvalid
|
return ErrRecoveryDispositionInvalid
|
||||||
@@ -106,9 +111,9 @@ func (i RecoveryDispositionIntent) AllowsResult(result string) bool {
|
|||||||
case RecoveryResultObserved:
|
case RecoveryResultObserved:
|
||||||
return i.Action == RecoveryActionRecord
|
return i.Action == RecoveryActionRecord
|
||||||
case RecoveryResultCommitted:
|
case RecoveryResultCommitted:
|
||||||
return i.Action == RecoveryActionCommit
|
return i.Action == RecoveryActionCommit || i.Action == RecoveryActionVerifyCommit
|
||||||
case RecoveryResultRolledBack:
|
case RecoveryResultRolledBack:
|
||||||
return i.Action == RecoveryActionRollback
|
return i.Action == RecoveryActionRollback || i.Action == RecoveryActionVerifyRollback
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
@@ -100,6 +100,12 @@ func (s *Service) executeRecoveryAction(ctx context.Context, review RecoveryRevi
|
|||||||
case RecoveryActionRollback:
|
case RecoveryActionRollback:
|
||||||
err = s.executeRecoveryRollback(ctx, &record, server, client, &mutated)
|
err = s.executeRecoveryRollback(ctx, &record, server, client, &mutated)
|
||||||
result = RecoveryResultRolledBack
|
result = RecoveryResultRolledBack
|
||||||
|
case RecoveryActionVerifyCommit, RecoveryActionVerifyRollback:
|
||||||
|
err = s.verifyRecoveryCompletion(ctx, record, server, client, review.summary.Action)
|
||||||
|
result = RecoveryResultCommitted
|
||||||
|
if review.summary.Action == RecoveryActionVerifyRollback {
|
||||||
|
result = RecoveryResultRolledBack
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -117,6 +123,13 @@ func validateRecoveryAction(record TransactionRecord, action RecoveryAction, ver
|
|||||||
return ErrRecoveryActionRejected
|
return ErrRecoveryActionRejected
|
||||||
}
|
}
|
||||||
switch action {
|
switch action {
|
||||||
|
case RecoveryActionVerifyCommit, RecoveryActionVerifyRollback:
|
||||||
|
if !verificationPhase(record.Phase, action) ||
|
||||||
|
(action == RecoveryActionVerifyCommit && record.Evidence.Baseline.ExpectedFingerprint == "") ||
|
||||||
|
(action == RecoveryActionVerifyRollback && record.Evidence.Baseline.PreviousFingerprint == "") {
|
||||||
|
return ErrRecoveryActionRejected
|
||||||
|
}
|
||||||
|
return nil
|
||||||
case RecoveryActionCommit:
|
case RecoveryActionCommit:
|
||||||
if record.Evidence.Baseline.ExpectedFingerprint == "" {
|
if record.Evidence.Baseline.ExpectedFingerprint == "" {
|
||||||
return ErrRecoveryActionRejected
|
return ErrRecoveryActionRejected
|
||||||
|
|||||||
@@ -0,0 +1,113 @@
|
|||||||
|
package deployment
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/hex"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func isRecoveryVerification(action RecoveryAction) bool {
|
||||||
|
return action == RecoveryActionVerifyCommit || action == RecoveryActionVerifyRollback
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reconciles 仅允许已持久化结束的不确定原动作;不接管运行中处置或再次执行文件动作。
|
||||||
|
func (i RecoveryDispositionIntent) Reconciles(source RecoveryDisposition) bool {
|
||||||
|
if i.SourceDispositionID == "" || i.SourceDispositionID != source.Intent.ID || source.Intent.Validate() != nil ||
|
||||||
|
source.Status != RecoveryResultUncertain || source.ResultCode != RecoveryResultUncertain ||
|
||||||
|
i.TransactionID != source.Intent.TransactionID || i.ServerID != source.Intent.ServerID || i.ServerDigest != source.Intent.ServerDigest {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return (i.Action == RecoveryActionVerifyCommit && source.Intent.Action == RecoveryActionCommit) ||
|
||||||
|
(i.Action == RecoveryActionVerifyRollback && source.Intent.Action == RecoveryActionRollback)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PrepareRecoveryReconciliation 只供受保护本机通道调用,必须明确指定原处置编号并重新确认。
|
||||||
|
func (s *Service) PrepareRecoveryReconciliation(ctx context.Context, sourceID, operator string, resolve func(context.Context, string) (Server, error)) (RecoveryConfirmation, string, error) {
|
||||||
|
review, err := s.prepareRecoveryReconciliation(ctx, sourceID, operator, resolve)
|
||||||
|
if err != nil {
|
||||||
|
return RecoveryConfirmation{}, "", err
|
||||||
|
}
|
||||||
|
return s.newRecoveryConfirmation(ctx, review)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) prepareRecoveryReconciliation(ctx context.Context, sourceID, operator string, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if s == nil || !validDispositionID(sourceID) || !validReviewOperator(operator) {
|
||||||
|
return RecoveryReview{}, ErrRecoveryReviewInvalid
|
||||||
|
}
|
||||||
|
journal, ok := s.journal.(RecoveryDispositionJournal)
|
||||||
|
if !ok {
|
||||||
|
return RecoveryReview{}, ErrRecoveryDispositionUnavailable
|
||||||
|
}
|
||||||
|
source, err := journal.GetRecoveryDisposition(ctx, sourceID)
|
||||||
|
if err != nil || source.Intent.ID != sourceID || source.Intent.Validate() != nil || source.Summary().Validate() != nil ||
|
||||||
|
source.Status != RecoveryResultUncertain || source.ResultCode != RecoveryResultUncertain {
|
||||||
|
return RecoveryReview{}, ErrRecoveryActionRejected
|
||||||
|
}
|
||||||
|
var action RecoveryAction
|
||||||
|
switch source.Intent.Action {
|
||||||
|
case RecoveryActionCommit:
|
||||||
|
action = RecoveryActionVerifyCommit
|
||||||
|
case RecoveryActionRollback:
|
||||||
|
action = RecoveryActionVerifyRollback
|
||||||
|
default:
|
||||||
|
return RecoveryReview{}, ErrRecoveryActionRejected
|
||||||
|
}
|
||||||
|
review, err := s.prepareRecoveryReview(ctx, source.Intent.TransactionID, operator, action, resolve)
|
||||||
|
if err != nil {
|
||||||
|
return RecoveryReview{}, err
|
||||||
|
}
|
||||||
|
review.summary.SourceDispositionID = sourceID
|
||||||
|
intent := RecoveryDispositionIntent{SourceDispositionID: sourceID, TransactionID: review.summary.TransactionID,
|
||||||
|
ServerID: review.summary.ServerID, ServerDigest: hex.EncodeToString(review.serverDigest[:]), Action: action}
|
||||||
|
if !intent.Reconciles(source) || !verificationPhase(review.summary.Phase, action) || s.VerifyRecoveryTarget == nil {
|
||||||
|
return RecoveryReview{}, ErrRecoveryActionRejected
|
||||||
|
}
|
||||||
|
return review, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func verificationPhase(phase string, action RecoveryAction) bool {
|
||||||
|
return (action == RecoveryActionVerifyCommit && phase == TransactionPhaseCommitVerified) ||
|
||||||
|
(action == RecoveryActionVerifyRollback && phase == TransactionPhaseRollbackVerified)
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateRecoveryCompletion 不删除任何远端文件,残留锁或备份均要求继续保留占用。
|
||||||
|
func validateRecoveryCompletion(ctx context.Context, client Client, record TransactionRecord, committed bool) error {
|
||||||
|
for _, lock := range record.LockPaths {
|
||||||
|
if exists, err := client.Exists(ctx, lock); err != nil || exists {
|
||||||
|
return ErrRecoveryActionRejected
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, backup := range record.Backups {
|
||||||
|
if committed {
|
||||||
|
if err := checkFileDigest(ctx, client, backup.Original, record.Evidence.Files[backup.Original]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else if backup.OriginalExists {
|
||||||
|
if err := checkFileDigest(ctx, client, backup.Original, backup.OriginalDigest); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else if exists, err := client.Exists(ctx, backup.Original); err != nil || exists {
|
||||||
|
return ErrRecoveryActionRejected
|
||||||
|
}
|
||||||
|
for _, path := range []string{backup.Saved, backup.Original + ".scert-tmp-" + record.ID} {
|
||||||
|
if exists, err := client.Exists(ctx, path); err != nil || exists {
|
||||||
|
return ErrRecoveryActionRejected
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) verifyRecoveryCompletion(ctx context.Context, record TransactionRecord, server Server, client Client, action RecoveryAction) error {
|
||||||
|
committed := action == RecoveryActionVerifyCommit
|
||||||
|
if err := validateRecoveryCompletion(ctx, client, record, committed); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := s.VerifyRecoveryTarget(ctx, server, record.Evidence.Baseline, committed); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// HTTPS 检查结束后重新读取现场,避免检查期间发生的文件变化被忽略。
|
||||||
|
return validateRecoveryCompletion(ctx, client, record, committed)
|
||||||
|
}
|
||||||
@@ -8,19 +8,21 @@ import (
|
|||||||
|
|
||||||
// RecoveryDispositionSummary 是本机查询的明确输出边界,不包含请求键、绑定摘要或操作者凭据。
|
// RecoveryDispositionSummary 是本机查询的明确输出边界,不包含请求键、绑定摘要或操作者凭据。
|
||||||
type RecoveryDispositionSummary struct {
|
type RecoveryDispositionSummary struct {
|
||||||
ID string `json:"id"`
|
SourceDispositionID string `json:"source_disposition_id,omitempty"`
|
||||||
TransactionID string `json:"transaction_id"`
|
ID string `json:"id"`
|
||||||
ServerID string `json:"server_id"`
|
TransactionID string `json:"transaction_id"`
|
||||||
Action RecoveryAction `json:"action"`
|
ServerID string `json:"server_id"`
|
||||||
Status string `json:"status"`
|
Action RecoveryAction `json:"action"`
|
||||||
ResultCode string `json:"result_code"`
|
Status string `json:"status"`
|
||||||
CreatedAt time.Time `json:"created_at"`
|
ResultCode string `json:"result_code"`
|
||||||
UpdatedAt time.Time `json:"updated_at"`
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r RecoveryDisposition) Summary() RecoveryDispositionSummary {
|
func (r RecoveryDisposition) Summary() RecoveryDispositionSummary {
|
||||||
return RecoveryDispositionSummary{
|
return RecoveryDispositionSummary{
|
||||||
ID: r.Intent.ID, TransactionID: r.Intent.TransactionID, ServerID: r.Intent.ServerID,
|
SourceDispositionID: r.Intent.SourceDispositionID,
|
||||||
|
ID: r.Intent.ID, TransactionID: r.Intent.TransactionID, ServerID: r.Intent.ServerID,
|
||||||
Action: r.Intent.Action, Status: r.Status, ResultCode: r.ResultCode,
|
Action: r.Intent.Action, Status: r.Status, ResultCode: r.ResultCode,
|
||||||
CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt,
|
CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt,
|
||||||
}
|
}
|
||||||
@@ -31,6 +33,9 @@ func validDispositionID(id string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r RecoveryDispositionSummary) Validate() error {
|
func (r RecoveryDispositionSummary) Validate() error {
|
||||||
|
if isRecoveryVerification(r.Action) != (r.SourceDispositionID != "") || (r.SourceDispositionID != "" && (!validDispositionID(r.SourceDispositionID) || r.SourceDispositionID == r.ID)) {
|
||||||
|
return ErrRecoveryDispositionInvalid
|
||||||
|
}
|
||||||
if !validDispositionID(r.ID) || !validInspectionID(r.TransactionID) || !validRecoverySummary(r.ServerID) ||
|
if !validDispositionID(r.ID) || !validInspectionID(r.TransactionID) || !validRecoverySummary(r.ServerID) ||
|
||||||
!validRecoveryAction(r.Action) || r.CreatedAt.IsZero() || r.UpdatedAt.IsZero() || r.UpdatedAt.Before(r.CreatedAt) {
|
!validRecoveryAction(r.Action) || r.CreatedAt.IsZero() || r.UpdatedAt.IsZero() || r.UpdatedAt.Before(r.CreatedAt) {
|
||||||
return ErrRecoveryDispositionInvalid
|
return ErrRecoveryDispositionInvalid
|
||||||
|
|||||||
@@ -13,10 +13,12 @@ import (
|
|||||||
type RecoveryAction string
|
type RecoveryAction string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
RecoveryActionRecord RecoveryAction = "record_inspection"
|
RecoveryActionRecord RecoveryAction = "record_inspection"
|
||||||
RecoveryActionCommit RecoveryAction = "finish_commit"
|
RecoveryActionCommit RecoveryAction = "finish_commit"
|
||||||
RecoveryActionRollback RecoveryAction = "restore_baseline"
|
RecoveryActionRollback RecoveryAction = "restore_baseline"
|
||||||
RecoveryReviewLifetime = time.Minute
|
RecoveryActionVerifyCommit RecoveryAction = "verify_commit"
|
||||||
|
RecoveryActionVerifyRollback RecoveryAction = "verify_rollback"
|
||||||
|
RecoveryReviewLifetime = time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -27,13 +29,14 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type RecoveryReviewSummary struct {
|
type RecoveryReviewSummary struct {
|
||||||
TransactionID string `json:"transaction_id"`
|
SourceDispositionID string `json:"source_disposition_id,omitempty"`
|
||||||
ServerID string `json:"server_id"`
|
TransactionID string `json:"transaction_id"`
|
||||||
Phase string `json:"phase"`
|
ServerID string `json:"server_id"`
|
||||||
Action RecoveryAction `json:"action"`
|
Phase string `json:"phase"`
|
||||||
HistoricalEvidence bool `json:"historical_evidence"`
|
Action RecoveryAction `json:"action"`
|
||||||
ObservedAt time.Time `json:"observed_at"`
|
HistoricalEvidence bool `json:"historical_evidence"`
|
||||||
ExpiresAt time.Time `json:"expires_at"`
|
ObservedAt time.Time `json:"observed_at"`
|
||||||
|
ExpiresAt time.Time `json:"expires_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate 与服务端历史记录使用同一摘要边界,允许旧记录没有阶段,但不允许控制字符或超长字段。
|
// Validate 与服务端历史记录使用同一摘要边界,允许旧记录没有阶段,但不允许控制字符或超长字段。
|
||||||
@@ -44,6 +47,9 @@ func (r RecoveryReviewSummary) Validate() error {
|
|||||||
(r.Action != RecoveryActionRecord && !r.HistoricalEvidence) {
|
(r.Action != RecoveryActionRecord && !r.HistoricalEvidence) {
|
||||||
return ErrRecoveryReviewInvalid
|
return ErrRecoveryReviewInvalid
|
||||||
}
|
}
|
||||||
|
if isRecoveryVerification(r.Action) != (r.SourceDispositionID != "") || (r.SourceDispositionID != "" && !validDispositionID(r.SourceDispositionID)) {
|
||||||
|
return ErrRecoveryReviewInvalid
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -65,12 +71,20 @@ func validReviewOperator(operator string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validRecoveryAction(action RecoveryAction) bool {
|
func validRecoveryAction(action RecoveryAction) bool {
|
||||||
return action == RecoveryActionRecord || action == RecoveryActionCommit || action == RecoveryActionRollback
|
return action == RecoveryActionRecord || action == RecoveryActionCommit || action == RecoveryActionRollback || isRecoveryVerification(action)
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrepareRecoveryReview 在同一操作上限内读取前后记录,拒绝采样期间发生的配置或事务变化。
|
// PrepareRecoveryReview 在同一操作上限内读取前后记录,拒绝采样期间发生的配置或事务变化。
|
||||||
// 当前采样摘要只绑定本次准备,不补写到历史 Evidence,也不改变事务终态。
|
// 当前采样摘要只绑定本次准备,不补写到历史 Evidence,也不改变事务终态。
|
||||||
func (s *Service) PrepareRecoveryReview(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) {
|
func (s *Service) PrepareRecoveryReview(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) {
|
||||||
|
// 重新核查必须通过独立入口绑定原处置,不能作为普通动作绕过来源检查。
|
||||||
|
if isRecoveryVerification(action) {
|
||||||
|
return RecoveryReview{}, ErrRecoveryReviewInvalid
|
||||||
|
}
|
||||||
|
return s.prepareRecoveryReview(ctx, id, operator, action, resolve)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) prepareRecoveryReview(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) {
|
||||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
if !validReviewOperator(operator) || !validRecoveryAction(action) || !validInspectionID(id) {
|
if !validReviewOperator(operator) || !validRecoveryAction(action) || !validInspectionID(id) {
|
||||||
@@ -140,14 +154,20 @@ func (s *Service) RecheckRecoveryReview(ctx context.Context, review RecoveryRevi
|
|||||||
if s.now == nil || !s.now().Before(review.summary.ExpiresAt) {
|
if s.now == nil || !s.now().Before(review.summary.ExpiresAt) {
|
||||||
return ErrRecoveryReviewExpired
|
return ErrRecoveryReviewExpired
|
||||||
}
|
}
|
||||||
current, err := s.PrepareRecoveryReview(ctx, review.summary.TransactionID, operator, review.summary.Action, resolve)
|
var current RecoveryReview
|
||||||
|
var err error
|
||||||
|
if isRecoveryVerification(review.summary.Action) {
|
||||||
|
current, err = s.prepareRecoveryReconciliation(ctx, review.summary.SourceDispositionID, operator, resolve)
|
||||||
|
} else {
|
||||||
|
current, err = s.PrepareRecoveryReview(ctx, review.summary.TransactionID, operator, review.summary.Action, resolve)
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if !s.now().Before(review.summary.ExpiresAt) {
|
if !s.now().Before(review.summary.ExpiresAt) {
|
||||||
return ErrRecoveryReviewExpired
|
return ErrRecoveryReviewExpired
|
||||||
}
|
}
|
||||||
if current.recordDigest != review.recordDigest || current.serverDigest != review.serverDigest || current.observationDigest != review.observationDigest {
|
if current.recordDigest != review.recordDigest || current.serverDigest != review.serverDigest || current.observationDigest != review.observationDigest || current.summary.SourceDispositionID != review.summary.SourceDispositionID || current.summary.Action != review.summary.Action {
|
||||||
return ErrRecoveryReviewChanged
|
return ErrRecoveryReviewChanged
|
||||||
}
|
}
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
|
|||||||
@@ -87,3 +87,26 @@ func (r *localResetSession) GetRecoveryDisposition(ctx context.Context, id strin
|
|||||||
}
|
}
|
||||||
|
|
||||||
var _ localreset.RecoveryDispositionSession = (*localResetSession)(nil)
|
var _ localreset.RecoveryDispositionSession = (*localResetSession)(nil)
|
||||||
|
|
||||||
|
type localRecoveryReconciler interface {
|
||||||
|
PrepareRecoveryReconciliation(context.Context, string, string, func(context.Context, string) (deployment.Server, error)) (deployment.RecoveryConfirmation, string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *localResetSession) PrepareRecoveryReconciliation(ctx context.Context, sourceID string) (localreset.RecoveryPreparation, error) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
r.recoveryConfirmation = deployment.RecoveryConfirmation{}
|
||||||
|
executor, ok := r.server.recoveryInspection.(localRecoveryReconciler)
|
||||||
|
if !r.enterRecovery(ctx) || !ok || !r.DispositionAvailable() {
|
||||||
|
return localreset.RecoveryPreparation{}, localreset.ErrProtocol
|
||||||
|
}
|
||||||
|
confirmation, token, err := executor.PrepareRecoveryReconciliation(ctx, sourceID, r.peer.Identity, r.recoveryServer)
|
||||||
|
if err != nil || !r.current(ctx) || confirmation.Summary().SourceDispositionID != sourceID {
|
||||||
|
r.stage = -1
|
||||||
|
return localreset.RecoveryPreparation{}, localreset.ErrUnavailable
|
||||||
|
}
|
||||||
|
r.recoveryConfirmation = confirmation
|
||||||
|
return localreset.RecoveryPreparation{DispositionID: confirmation.DispositionID(), Token: token, Summary: confirmation.Summary()}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ localreset.RecoveryReconciliationSession = (*localResetSession)(nil)
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||||
|
)
|
||||||
|
|
||||||
|
type localReconciliationJournal struct {
|
||||||
|
localDispositionJournal
|
||||||
|
source deployment.RecoveryDisposition
|
||||||
|
}
|
||||||
|
|
||||||
|
func (j *localReconciliationJournal) GetRecoveryDisposition(ctx context.Context, id string) (deployment.RecoveryDisposition, error) {
|
||||||
|
if id == j.source.Intent.ID { return j.source, nil }
|
||||||
|
return j.localDispositionJournal.GetRecoveryDisposition(ctx, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
func localReconciliationFixture(t *testing.T) (*Server, *localResetSession, *localReconciliationJournal) {
|
||||||
|
t.Helper()
|
||||||
|
server, _, original := localDispositionFixture(t)
|
||||||
|
target := server.servers.(localDispositionServerRepo).server
|
||||||
|
journal := &localReconciliationJournal{}
|
||||||
|
journal.record = original.record
|
||||||
|
journal.record.Phase = deployment.TransactionPhaseCommitVerified
|
||||||
|
journal.record.Evidence = &deployment.RecoveryEvidence{Version: 1, LockToken: strings.Repeat("d", 64), Address: target.Address,
|
||||||
|
Port: target.Port, HostKeyFingerprint: target.HostKeyFingerprint, ValidationCommand: target.ValidationCommand, ReloadCommand: target.ReloadCommand,
|
||||||
|
Files: map[string]string{}, TemporaryFiles: map[string]string{}, Baseline: deployment.RecoveryBaseline{Endpoint: "https://example.test", Domains: []string{"example.test"}, ExpectedFingerprint: "new"}}
|
||||||
|
for _, path := range journal.record.Paths {
|
||||||
|
journal.record.LockPaths = append(journal.record.LockPaths, path+".scert-lock")
|
||||||
|
journal.record.Backups = append(journal.record.Backups, deployment.TransactionBackup{Original: path, Saved: path+".scert-backup-"+journal.record.ID})
|
||||||
|
journal.record.Evidence.Files[path] = strings.Repeat("e", 64)
|
||||||
|
}
|
||||||
|
service := deployment.NewServiceWithJournal(localDispositionFactory{}, journal)
|
||||||
|
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error { return nil }
|
||||||
|
resolve := func(context.Context, string) (deployment.Server, error) { return target, nil }
|
||||||
|
review, err := service.PrepareRecoveryReview(t.Context(), journal.record.ID, "uid:0", deployment.RecoveryActionCommit, resolve)
|
||||||
|
if err != nil { t.Fatal(err) }
|
||||||
|
intent, err := review.DispositionIntent(strings.Repeat("a", 64), strings.Repeat("b", 64))
|
||||||
|
if err != nil { t.Fatal(err) }
|
||||||
|
now := time.Now().UTC()
|
||||||
|
journal.source = deployment.RecoveryDisposition{Intent: intent, Status: "uncertain", ResultCode: "uncertain", CreatedAt: now, UpdatedAt: now}
|
||||||
|
server.recoveryInspection = service
|
||||||
|
return server, localDispositionNewSession(t, server), journal
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLocalReconciliationRejectsInvalidSession(t *testing.T) {
|
||||||
|
for _, mode := range []string{"expired", "cancelled", "changed-admin", "closed", "reset-stage", "wrong-id", "wrong-token", "changed-peer", "foreign-session", "replaced-confirmation"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
server, session, journal := localReconciliationFixture(t)
|
||||||
|
prepared, err := session.PrepareRecoveryReconciliation(t.Context(), journal.source.Intent.ID)
|
||||||
|
if err != nil { t.Fatal(err) }
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
defer cancel()
|
||||||
|
switch mode {
|
||||||
|
case "expired": session.expires = time.Time{}
|
||||||
|
case "cancelled": cancel()
|
||||||
|
case "changed-admin": server.config.Admin.Email = "changed@example.test"
|
||||||
|
case "closed": session.Close()
|
||||||
|
case "reset-stage": session.stage = 1
|
||||||
|
case "wrong-id": prepared.DispositionID = journal.source.Intent.ID
|
||||||
|
case "wrong-token": prepared.Token = strings.Repeat("a", 64)
|
||||||
|
case "changed-peer": session.peer.Identity = "uid:123"
|
||||||
|
case "foreign-session": session = localDispositionNewSession(t, server)
|
||||||
|
case "replaced-confirmation":
|
||||||
|
if _, err := session.PrepareRecoveryReconciliation(ctx, journal.source.Intent.ID); err != nil { t.Fatal(err) }
|
||||||
|
}
|
||||||
|
if _, err := session.ExecuteRecovery(ctx, prepared.DispositionID, prepared.Token); err == nil { t.Fatal("无效会话执行了重新核查") }
|
||||||
|
if _, err := session.ExecuteRecovery(t.Context(), prepared.DispositionID, prepared.Token); err == nil { t.Fatal("失败确认被重放") }
|
||||||
|
if journal.begins != 0 || journal.finishes != 0 || session.recoveryConfirmation.DispositionID() != "" { t.Fatal("无效会话写入了审计或保留了确认") }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLocalReconciliationRejectsMissingFilesAndQueriesNewAudit(t *testing.T) {
|
||||||
|
server, session, journal := localReconciliationFixture(t)
|
||||||
|
prepared, err := session.PrepareRecoveryReconciliation(t.Context(), journal.source.Intent.ID)
|
||||||
|
if err != nil { t.Fatal(err) }
|
||||||
|
result, err := session.ExecuteRecovery(t.Context(), prepared.DispositionID, prepared.Token)
|
||||||
|
if err != nil || result.Status != "rejected" || result.SourceDispositionID != journal.source.Intent.ID { t.Fatalf("缺失文件仍被放行:%+v %v", result, err) }
|
||||||
|
if journal.item.Intent.Operator != "uid:0" || journal.begins != 1 || journal.finishes != 1 || journal.source.Status != "uncertain" { t.Fatal("身份或历史记录被改变") }
|
||||||
|
next := localDispositionNewSession(t, server)
|
||||||
|
queried, err := next.GetRecoveryDisposition(t.Context(), result.ID)
|
||||||
|
if err != nil || queried != result { t.Fatal("重连未查到关联审计", err) }
|
||||||
|
}
|
||||||
@@ -30,7 +30,7 @@ func validRecoveryRequest(request Request, nonce string) bool {
|
|||||||
return request.Cursor == "" && validRecoveryID(request.TransactionID) && validRecoveryAction(request.RecoveryAction) && request.DispositionID == "" && request.ConfirmationToken == ""
|
return request.Cursor == "" && validRecoveryID(request.TransactionID) && validRecoveryAction(request.RecoveryAction) && request.DispositionID == "" && request.ConfirmationToken == ""
|
||||||
case "recovery_execute":
|
case "recovery_execute":
|
||||||
return request.Cursor == "" && request.TransactionID == "" && request.RecoveryAction == "" && validRecoverySecret(request.DispositionID) && validRecoverySecret(request.ConfirmationToken)
|
return request.Cursor == "" && request.TransactionID == "" && request.RecoveryAction == "" && validRecoverySecret(request.DispositionID) && validRecoverySecret(request.ConfirmationToken)
|
||||||
case "recovery_result":
|
case "recovery_result", "recovery_reconcile_prepare":
|
||||||
return request.Cursor == "" && request.TransactionID == "" && request.RecoveryAction == "" && validRecoverySecret(request.DispositionID) && request.ConfirmationToken == ""
|
return request.Cursor == "" && request.TransactionID == "" && request.RecoveryAction == "" && validRecoverySecret(request.DispositionID) && request.ConfirmationToken == ""
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
|
|||||||
@@ -21,6 +21,10 @@ type RecoveryDispositionSession interface {
|
|||||||
GetRecoveryDisposition(context.Context, string) (deployment.RecoveryDispositionSummary, error)
|
GetRecoveryDisposition(context.Context, string) (deployment.RecoveryDispositionSummary, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type RecoveryReconciliationSession interface {
|
||||||
|
PrepareRecoveryReconciliation(context.Context, string) (RecoveryPreparation, error)
|
||||||
|
}
|
||||||
|
|
||||||
func validRecoveryAction(action deployment.RecoveryAction) bool {
|
func validRecoveryAction(action deployment.RecoveryAction) bool {
|
||||||
return action == deployment.RecoveryActionRecord || action == deployment.RecoveryActionCommit || action == deployment.RecoveryActionRollback
|
return action == deployment.RecoveryActionRecord || action == deployment.RecoveryActionCommit || action == deployment.RecoveryActionRollback
|
||||||
}
|
}
|
||||||
@@ -60,6 +64,11 @@ func validDispositionResponse(request Request, response Response) bool {
|
|||||||
return response.Disposition == nil && validPreparation(response.Preparation) &&
|
return response.Disposition == nil && validPreparation(response.Preparation) &&
|
||||||
response.Preparation.Summary.TransactionID == request.TransactionID && response.Preparation.Summary.Action == request.RecoveryAction
|
response.Preparation.Summary.TransactionID == request.TransactionID && response.Preparation.Summary.Action == request.RecoveryAction
|
||||||
}
|
}
|
||||||
|
if request.Action == "recovery_reconcile_prepare" {
|
||||||
|
return response.Disposition == nil && validPreparation(response.Preparation) &&
|
||||||
|
response.Preparation.Summary.SourceDispositionID == request.DispositionID &&
|
||||||
|
response.Preparation.DispositionID != request.DispositionID
|
||||||
|
}
|
||||||
return response.Preparation == nil && response.Disposition != nil && response.Disposition.Validate() == nil && response.Disposition.ID == request.DispositionID
|
return response.Preparation == nil && response.Disposition != nil && response.Disposition.Validate() == nil && response.Disposition.ID == request.DispositionID
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -70,6 +79,14 @@ func dispatchRecoveryDisposition(ctx context.Context, session RecoverySession, r
|
|||||||
}
|
}
|
||||||
var err error
|
var err error
|
||||||
switch request.Action {
|
switch request.Action {
|
||||||
|
case "recovery_reconcile_prepare":
|
||||||
|
reconciler, ok := session.(RecoveryReconciliationSession)
|
||||||
|
if !ok {
|
||||||
|
return ErrUnavailable
|
||||||
|
}
|
||||||
|
var value RecoveryPreparation
|
||||||
|
value, err = reconciler.PrepareRecoveryReconciliation(ctx, request.DispositionID)
|
||||||
|
response.Preparation = &value
|
||||||
case "recovery_prepare":
|
case "recovery_prepare":
|
||||||
var value RecoveryPreparation
|
var value RecoveryPreparation
|
||||||
value, err = actions.PrepareRecovery(ctx, request.TransactionID, request.RecoveryAction)
|
value, err = actions.PrepareRecovery(ctx, request.TransactionID, request.RecoveryAction)
|
||||||
@@ -101,6 +118,14 @@ func (c *Client) PrepareRecovery(id string, action deployment.RecoveryAction) (R
|
|||||||
return *response.Preparation, nil
|
return *response.Preparation, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Client) PrepareRecoveryReconciliation(sourceID string) (RecoveryPreparation, error) {
|
||||||
|
response, err := c.recoveryExchange(Request{Action: "recovery_reconcile_prepare", DispositionID: sourceID})
|
||||||
|
if err != nil {
|
||||||
|
return RecoveryPreparation{}, err
|
||||||
|
}
|
||||||
|
return *response.Preparation, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Client) ExecuteRecovery(id, token string) (deployment.RecoveryDispositionSummary, error) {
|
func (c *Client) ExecuteRecovery(id, token string) (deployment.RecoveryDispositionSummary, error) {
|
||||||
response, err := c.recoveryExchange(Request{Action: "recovery_execute", DispositionID: id, ConfirmationToken: token})
|
response, err := c.recoveryExchange(Request{Action: "recovery_execute", DispositionID: id, ConfirmationToken: token})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -89,6 +89,7 @@ func TestDispositionRequestFieldIsolation(t *testing.T) {
|
|||||||
{Action: "recovery_prepare", TransactionID: "tx-1", RecoveryAction: deployment.RecoveryActionRecord},
|
{Action: "recovery_prepare", TransactionID: "tx-1", RecoveryAction: deployment.RecoveryActionRecord},
|
||||||
{Action: "recovery_execute", DispositionID: strings.Repeat("b", 64), ConfirmationToken: strings.Repeat("c", 64)},
|
{Action: "recovery_execute", DispositionID: strings.Repeat("b", 64), ConfirmationToken: strings.Repeat("c", 64)},
|
||||||
{Action: "recovery_result", DispositionID: strings.Repeat("b", 64)},
|
{Action: "recovery_result", DispositionID: strings.Repeat("b", 64)},
|
||||||
|
{Action: "recovery_reconcile_prepare", DispositionID: strings.Repeat("b", 64)},
|
||||||
}
|
}
|
||||||
fields := []struct {
|
fields := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -101,7 +102,7 @@ func TestDispositionRequestFieldIsolation(t *testing.T) {
|
|||||||
{"cursor", func(r *Request) { r.Cursor = "tx-0" }, map[string]bool{"recovery_list": true}},
|
{"cursor", func(r *Request) { r.Cursor = "tx-0" }, map[string]bool{"recovery_list": true}},
|
||||||
{"transaction", func(r *Request) { r.TransactionID = "tx-1" }, map[string]bool{"recovery_inspect": true, "recovery_prepare": true}},
|
{"transaction", func(r *Request) { r.TransactionID = "tx-1" }, map[string]bool{"recovery_inspect": true, "recovery_prepare": true}},
|
||||||
{"action", func(r *Request) { r.RecoveryAction = deployment.RecoveryActionRecord }, map[string]bool{"recovery_prepare": true}},
|
{"action", func(r *Request) { r.RecoveryAction = deployment.RecoveryActionRecord }, map[string]bool{"recovery_prepare": true}},
|
||||||
{"id", func(r *Request) { r.DispositionID = strings.Repeat("b", 64) }, map[string]bool{"recovery_execute": true, "recovery_result": true}},
|
{"id", func(r *Request) { r.DispositionID = strings.Repeat("b", 64) }, map[string]bool{"recovery_execute": true, "recovery_result": true, "recovery_reconcile_prepare": true}},
|
||||||
{"token", func(r *Request) { r.ConfirmationToken = strings.Repeat("c", 64) }, map[string]bool{"recovery_execute": true}},
|
{"token", func(r *Request) { r.ConfirmationToken = strings.Repeat("c", 64) }, map[string]bool{"recovery_execute": true}},
|
||||||
}
|
}
|
||||||
for _, request := range requests {
|
for _, request := range requests {
|
||||||
|
|||||||
@@ -0,0 +1,82 @@
|
|||||||
|
package localreset
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||||
|
)
|
||||||
|
|
||||||
|
type reconciliationSession struct { dispositionTestSession }
|
||||||
|
|
||||||
|
func (s *reconciliationSession) PrepareRecoveryReconciliation(ctx context.Context, sourceID string) (RecoveryPreparation, error) {
|
||||||
|
value, err := s.PrepareRecovery(ctx, "tx-1", deployment.RecoveryActionVerifyCommit)
|
||||||
|
value.Summary.SourceDispositionID = sourceID
|
||||||
|
value.Summary.Phase = deployment.TransactionPhaseCommitVerified
|
||||||
|
s.prepared = value
|
||||||
|
return value, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *reconciliationSession) ExecuteRecovery(ctx context.Context, id, token string) (deployment.RecoveryDispositionSummary, error) {
|
||||||
|
sourceID := s.prepared.Summary.SourceDispositionID
|
||||||
|
value, err := s.dispositionTestSession.ExecuteRecovery(ctx, id, token)
|
||||||
|
if err != nil { return value, err }
|
||||||
|
value.SourceDispositionID, value.ResultCode = sourceID, deployment.RecoveryResultCommitted
|
||||||
|
s.result = value
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecoveryReconciliationProtocolAndQuery(t *testing.T) {
|
||||||
|
session := &reconciliationSession{}
|
||||||
|
_, conn, done := recoveryConnection(t, session)
|
||||||
|
client, err := Begin(t.Context(), conn)
|
||||||
|
if err != nil { t.Fatal(err) }
|
||||||
|
sourceID := strings.Repeat("a", 64)
|
||||||
|
prepared, err := client.PrepareRecoveryReconciliation(sourceID)
|
||||||
|
if err != nil || prepared.Summary.SourceDispositionID != sourceID { t.Fatal("原处置绑定丢失", err) }
|
||||||
|
result, err := client.ExecuteRecovery(prepared.DispositionID, prepared.Token)
|
||||||
|
if err != nil || result.SourceDispositionID != sourceID || result.Status != "succeeded" { t.Fatal("重新核查结果无效", err) }
|
||||||
|
queried, err := client.GetRecoveryDisposition(result.ID)
|
||||||
|
if err != nil || queried != result { t.Fatal("查询改变了关联结果", err) }
|
||||||
|
if _, err := client.ExecuteRecovery(prepared.DispositionID, prepared.Token); err == nil { t.Fatal("旧令牌被重放") }
|
||||||
|
_ = client.Close()
|
||||||
|
<-done
|
||||||
|
if session.prepares.Load() != 1 || session.executes.Load() != 2 || session.queries.Load() != 1 || session.resets.Load() != 0 { t.Fatal("调用了错误业务入口") }
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecoveryReconciliationRejectsUnboundResponse(t *testing.T) {
|
||||||
|
sourceID := strings.Repeat("a", 64)
|
||||||
|
session := &reconciliationSession{}
|
||||||
|
prepared, err := session.PrepareRecoveryReconciliation(t.Context(), sourceID)
|
||||||
|
if err != nil { t.Fatal(err) }
|
||||||
|
request := Request{Action: "recovery_reconcile_prepare", DispositionID: sourceID}
|
||||||
|
for _, mode := range []string{"source", "action", "same-id", "transaction", "mixed", "expired-range"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
value := prepared
|
||||||
|
response := Response{Preparation: &value}
|
||||||
|
switch mode {
|
||||||
|
case "source": value.Summary.SourceDispositionID = strings.Repeat("d", 64)
|
||||||
|
case "action": value.Summary.Action = deployment.RecoveryActionCommit
|
||||||
|
case "same-id": value.DispositionID = sourceID
|
||||||
|
case "transaction": value.Summary.TransactionID = "../bad"
|
||||||
|
case "mixed": response.Disposition = &deployment.RecoveryDispositionSummary{}
|
||||||
|
case "expired-range": value.Summary.ExpiresAt = value.Summary.ObservedAt.Add(2*time.Minute)
|
||||||
|
}
|
||||||
|
if validDispositionResponse(request, response) { t.Fatal("接受了混合或无绑定准备响应") }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if validRecoveryAction(deployment.RecoveryActionVerifyCommit) || validRecoveryAction(deployment.RecoveryActionVerifyRollback) { t.Fatal("普通准备允许绕过原处置绑定") }
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecoveryReconciliationOldServerFailsClosed(t *testing.T) {
|
||||||
|
session := &dispositionTestSession{}
|
||||||
|
_, conn, done := recoveryConnection(t, session)
|
||||||
|
client, err := Begin(t.Context(), conn)
|
||||||
|
if err != nil { t.Fatal(err) }
|
||||||
|
if _, err := client.PrepareRecoveryReconciliation(strings.Repeat("a", 64)); err == nil { t.Fatal("旧服务错误地支持了重新核查") }
|
||||||
|
_ = client.Close()
|
||||||
|
<-done
|
||||||
|
if session.prepares.Load() != 0 || session.executes.Load() != 0 { t.Fatal("旧服务错误地执行了原动作") }
|
||||||
|
}
|
||||||
@@ -25,7 +25,7 @@ func (s *Server) handleRecovery(ctx context.Context, conn net.Conn, reader *bufi
|
|||||||
case "recovery_inspect":
|
case "recovery_inspect":
|
||||||
inspection, readErr := session.InspectRecovery(ctx, request.TransactionID)
|
inspection, readErr := session.InspectRecovery(ctx, request.TransactionID)
|
||||||
err, response.Inspection = readErr, &inspection
|
err, response.Inspection = readErr, &inspection
|
||||||
case "recovery_prepare", "recovery_execute", "recovery_result":
|
case "recovery_prepare", "recovery_execute", "recovery_result", "recovery_reconcile_prepare":
|
||||||
err = dispatchRecoveryDisposition(ctx, session, request, &response)
|
err = dispatchRecoveryDisposition(ctx, session, request, &response)
|
||||||
default:
|
default:
|
||||||
err = ErrProtocol
|
err = ErrProtocol
|
||||||
|
|||||||
@@ -43,6 +43,11 @@ func (r *SQLDeploymentTransactionRepository) FinishRecoveryDisposition(ctx conte
|
|||||||
if err := tx.QueryRowContext(ctx, r.query(`SELECT event_type FROM recovery_events WHERE disposition_id = ? AND event_type = 'started'`), id).Scan(&start); err != nil {
|
if err := tx.QueryRowContext(ctx, r.query(`SELECT event_type FROM recovery_events WHERE disposition_id = ? AND event_type = 'started'`), id).Scan(&start); err != nil {
|
||||||
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||||
}
|
}
|
||||||
|
if item.Intent.SourceDispositionID != "" {
|
||||||
|
if err := r.validateReconciliationSource(ctx, tx, item.Intent); err != nil {
|
||||||
|
return deployment.RecoveryDisposition{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
if err := r.finishRecoveryOriginal(ctx, tx, item.Intent, resultCode, now); err != nil {
|
if err := r.finishRecoveryOriginal(ctx, tx, item.Intent, resultCode, now); err != nil {
|
||||||
return deployment.RecoveryDisposition{}, err
|
return deployment.RecoveryDisposition{}, err
|
||||||
@@ -58,7 +63,13 @@ func (r *SQLDeploymentTransactionRepository) FinishRecoveryDisposition(ctx conte
|
|||||||
if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_events (disposition_id, event_type, result_code, created_at) VALUES (?, 'finished', ?, ?)`), id, resultCode, now); err != nil {
|
if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_events (disposition_id, event_type, result_code, created_at) VALUES (?, 'finished', ?, ?)`), id, resultCode, now); err != nil {
|
||||||
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||||
}
|
}
|
||||||
if resultCode != deployment.RecoveryResultUncertain {
|
if item.Intent.SourceDispositionID != "" && (resultCode == deployment.RecoveryResultRejected || resultCode == deployment.RecoveryResultUncertain) {
|
||||||
|
// 只读核查未通过时归还原占用,保留两次审计;绝不解锁原事务。
|
||||||
|
result, err := tx.ExecContext(ctx, r.query(`UPDATE recovery_claims SET disposition_id = ?, created_at = ? WHERE transaction_id = ? AND disposition_id = ?`), item.Intent.SourceDispositionID, now, item.Intent.TransactionID, id)
|
||||||
|
if err != nil || !oneRecoveryRow(result) {
|
||||||
|
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||||
|
}
|
||||||
|
} else if resultCode != deployment.RecoveryResultUncertain {
|
||||||
result, err := tx.ExecContext(ctx, r.query(`DELETE FROM recovery_claims WHERE transaction_id = ? AND disposition_id = ?`), item.Intent.TransactionID, id)
|
result, err := tx.ExecContext(ctx, r.query(`DELETE FROM recovery_claims WHERE transaction_id = ? AND disposition_id = ?`), item.Intent.TransactionID, id)
|
||||||
if err != nil || !oneRecoveryRow(result) {
|
if err != nil || !oneRecoveryRow(result) {
|
||||||
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||||
@@ -96,6 +107,9 @@ func (r *SQLDeploymentTransactionRepository) finishRecoveryOriginal(ctx context.
|
|||||||
(record.Status != deployment.TransactionStatusPending && record.Status != deployment.TransactionStatusRecoveryRequired) {
|
(record.Status != deployment.TransactionStatusPending && record.Status != deployment.TransactionStatusRecoveryRequired) {
|
||||||
return deployment.ErrRecoveryDispositionConflict
|
return deployment.ErrRecoveryDispositionConflict
|
||||||
}
|
}
|
||||||
|
if intent.SourceDispositionID != "" && !intent.MatchesRecord(record) {
|
||||||
|
return deployment.ErrRecoveryReviewChanged
|
||||||
|
}
|
||||||
result, err := tx.ExecContext(ctx, r.query(`UPDATE deployment_transactions SET status = ?, phase = ?, message = ?, updated_at = ?
|
result, err := tx.ExecContext(ctx, r.query(`UPDATE deployment_transactions SET status = ?, phase = ?, message = ?, updated_at = ?
|
||||||
WHERE id = ? AND phase = ? AND status IN ('pending', 'recovery_required')`), status, status, message, now, intent.TransactionID, phase)
|
WHERE id = ? AND phase = ? AND status IN ('pending', 'recovery_required')`), status, status, message, now, intent.TransactionID, phase)
|
||||||
if err != nil || !oneRecoveryRow(result) {
|
if err != nil || !oneRecoveryRow(result) {
|
||||||
|
|||||||
@@ -12,13 +12,13 @@ import (
|
|||||||
var _ deployment.RecoveryDispositionJournal = (*SQLDeploymentTransactionRepository)(nil)
|
var _ deployment.RecoveryDispositionJournal = (*SQLDeploymentTransactionRepository)(nil)
|
||||||
|
|
||||||
const recoveryDispositionColumns = `id, transaction_id, server_id, operator_id, action, request_key,
|
const recoveryDispositionColumns = `id, transaction_id, server_id, operator_id, action, request_key,
|
||||||
record_digest, server_digest, observation_digest, status, result_code, created_at, updated_at`
|
record_digest, server_digest, observation_digest, status, result_code, created_at, updated_at, source_disposition_id`
|
||||||
|
|
||||||
func scanRecoveryDisposition(row deploymentTransactionScanner) (deployment.RecoveryDisposition, error) {
|
func scanRecoveryDisposition(row deploymentTransactionScanner) (deployment.RecoveryDisposition, error) {
|
||||||
var item deployment.RecoveryDisposition
|
var item deployment.RecoveryDisposition
|
||||||
i := &item.Intent
|
i := &item.Intent
|
||||||
err := row.Scan(&i.ID, &i.TransactionID, &i.ServerID, &i.Operator, &i.Action, &i.RequestKey,
|
err := row.Scan(&i.ID, &i.TransactionID, &i.ServerID, &i.Operator, &i.Action, &i.RequestKey,
|
||||||
&i.RecordDigest, &i.ServerDigest, &i.ObservationDigest, &item.Status, &item.ResultCode, &item.CreatedAt, &item.UpdatedAt)
|
&i.RecordDigest, &i.ServerDigest, &i.ObservationDigest, &item.Status, &item.ResultCode, &item.CreatedAt, &item.UpdatedAt, &i.SourceDispositionID)
|
||||||
return item, err
|
return item, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -75,9 +75,9 @@ func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx contex
|
|||||||
defer tx.Rollback()
|
defer tx.Rollback()
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
item := deployment.RecoveryDisposition{Intent: intent, Status: "running", CreatedAt: now, UpdatedAt: now}
|
item := deployment.RecoveryDisposition{Intent: intent, Status: "running", CreatedAt: now, UpdatedAt: now}
|
||||||
_, err = tx.ExecContext(ctx, r.query(`INSERT INTO recovery_dispositions (`+recoveryDispositionColumns+`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`),
|
_, err = tx.ExecContext(ctx, r.query(`INSERT INTO recovery_dispositions (`+recoveryDispositionColumns+`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`),
|
||||||
intent.ID, intent.TransactionID, intent.ServerID, intent.Operator, intent.Action, intent.RequestKey,
|
intent.ID, intent.TransactionID, intent.ServerID, intent.Operator, intent.Action, intent.RequestKey,
|
||||||
intent.RecordDigest, intent.ServerDigest, intent.ObservationDigest, item.Status, "", now, now)
|
intent.RecordDigest, intent.ServerDigest, intent.ObservationDigest, item.Status, "", now, now, intent.SourceDispositionID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = tx.Rollback()
|
_ = tx.Rollback()
|
||||||
if existing, readErr := r.findRecoveryRequest(ctx, intent.RequestKey); readErr == nil {
|
if existing, readErr := r.findRecoveryRequest(ctx, intent.RequestKey); readErr == nil {
|
||||||
@@ -85,7 +85,7 @@ func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx contex
|
|||||||
}
|
}
|
||||||
return deployment.RecoveryDisposition{}, false, dispositionError(ctx)
|
return deployment.RecoveryDisposition{}, false, dispositionError(ctx)
|
||||||
}
|
}
|
||||||
if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_claims (transaction_id, disposition_id, created_at) VALUES (?, ?, ?)`), intent.TransactionID, intent.ID, now); err != nil {
|
if err := r.claimRecoveryDisposition(ctx, tx, intent, now); err != nil {
|
||||||
_ = tx.Rollback()
|
_ = tx.Rollback()
|
||||||
if claimed, readErr := r.HasRecoveryClaim(ctx, intent.TransactionID); readErr == nil && claimed {
|
if claimed, readErr := r.HasRecoveryClaim(ctx, intent.TransactionID); readErr == nil && claimed {
|
||||||
return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionBusy
|
return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionBusy
|
||||||
@@ -109,6 +109,39 @@ func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx contex
|
|||||||
return item, true, nil
|
return item, true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *SQLDeploymentTransactionRepository) claimRecoveryDisposition(ctx context.Context, tx *sql.Tx, intent deployment.RecoveryDispositionIntent, now time.Time) error {
|
||||||
|
if intent.SourceDispositionID == "" {
|
||||||
|
_, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_claims (transaction_id, disposition_id, created_at) VALUES (?, ?, ?)`), intent.TransactionID, intent.ID, now)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := r.validateReconciliationSource(ctx, tx, intent); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// 只从指定的已结束处置接管占用;运行中、其他处置或丢失的占用均不接管。
|
||||||
|
result, err := tx.ExecContext(ctx, r.query(`UPDATE recovery_claims SET disposition_id = ?, created_at = ? WHERE transaction_id = ? AND disposition_id = ?`), intent.ID, now, intent.TransactionID, intent.SourceDispositionID)
|
||||||
|
if err != nil || !oneRecoveryRow(result) {
|
||||||
|
return deployment.ErrRecoveryDispositionBusy
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *SQLDeploymentTransactionRepository) validateReconciliationSource(ctx context.Context, tx *sql.Tx, intent deployment.RecoveryDispositionIntent) error {
|
||||||
|
query := `SELECT ` + recoveryDispositionColumns + ` FROM recovery_dispositions WHERE id = ?`
|
||||||
|
if r.driver != "sqlite" {
|
||||||
|
query += " FOR UPDATE"
|
||||||
|
}
|
||||||
|
source, err := scanRecoveryDisposition(tx.QueryRowContext(ctx, r.query(query), intent.SourceDispositionID))
|
||||||
|
if err != nil || !intent.Reconciles(source) {
|
||||||
|
return deployment.ErrRecoveryDispositionConflict
|
||||||
|
}
|
||||||
|
var count int
|
||||||
|
if err := tx.QueryRowContext(ctx, r.query(`SELECT COUNT(*) FROM recovery_events WHERE disposition_id = ? AND
|
||||||
|
((event_type = 'started' AND result_code = '') OR (event_type = 'finished' AND result_code = 'uncertain'))`), intent.SourceDispositionID).Scan(&count); err != nil || count != 2 {
|
||||||
|
return deployment.ErrRecoveryDispositionConflict
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func recoveryRetry(existing deployment.RecoveryDisposition, intent deployment.RecoveryDispositionIntent) (deployment.RecoveryDisposition, bool, error) {
|
func recoveryRetry(existing deployment.RecoveryDisposition, intent deployment.RecoveryDispositionIntent) (deployment.RecoveryDisposition, bool, error) {
|
||||||
if !existing.Intent.SameRequest(intent) {
|
if !existing.Intent.SameRequest(intent) {
|
||||||
return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionConflict
|
return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionConflict
|
||||||
|
|||||||
@@ -68,6 +68,16 @@ func openRecoveryExternal(t *testing.T, cfg config.DatabaseConfig) *sql.DB {
|
|||||||
// 只替换远端文件实现,准备、确认、执行、迁移、占用和审计均走实际产品代码。
|
// 只替换远端文件实现,准备、确认、执行、迁移、占用和审计均走实际产品代码。
|
||||||
func runExternalRecovery(t *testing.T, fixture recoveryExternalFixture) {
|
func runExternalRecovery(t *testing.T, fixture recoveryExternalFixture) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
|
||||||
|
t.Run("reconciliation/"+string(action), func(t *testing.T) {
|
||||||
|
cfg := fixture.database(t)
|
||||||
|
db := openRecoveryExternal(t, cfg)
|
||||||
|
if err := database.Migrate(t.Context(), db, cfg.Driver); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
runReconciliationSuccess(t, NewSQLDeploymentTransactionRepository(db, cfg.Driver), db, action)
|
||||||
|
})
|
||||||
|
}
|
||||||
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionRecord, deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
|
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionRecord, deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
|
||||||
for _, stage := range []string{"none", "started", "finished"} {
|
for _, stage := range []string{"none", "started", "finished"} {
|
||||||
t.Run(string(action)+"/"+stage, func(t *testing.T) {
|
t.Run(string(action)+"/"+stage, func(t *testing.T) {
|
||||||
|
|||||||
@@ -0,0 +1,250 @@
|
|||||||
|
package repository
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"errors"
|
||||||
|
"maps"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 远端文件是测试替身;事务、关联、占用交接与审计均经过真实数据库和产品执行器。
|
||||||
|
type reconciliationClient struct {
|
||||||
|
*executorIntegrationClient
|
||||||
|
failClosed, readOnly bool
|
||||||
|
connections, writes int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *reconciliationClient) Connect(context.Context, deployment.Server) (deployment.Client, error) {
|
||||||
|
c.connections++
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
func (c *reconciliationClient) Close() error {
|
||||||
|
if c.failClosed {
|
||||||
|
if _, exists := c.files["/cert.pem.scert-lock"]; !exists {
|
||||||
|
return errors.New("注入动作完成后的连接关闭失败")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (c *reconciliationClient) writable() error {
|
||||||
|
c.writes++
|
||||||
|
if c.readOnly {
|
||||||
|
return errors.New("重新核查不得执行远端写操作")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (c *reconciliationClient) Remove(ctx context.Context, path string) error {
|
||||||
|
if err := c.writable(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return c.executorIntegrationClient.Remove(ctx, path)
|
||||||
|
}
|
||||||
|
func (c *reconciliationClient) Rename(ctx context.Context, from, to string) error {
|
||||||
|
if err := c.writable(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return c.executorIntegrationClient.Rename(ctx, from, to)
|
||||||
|
}
|
||||||
|
func (c *reconciliationClient) Unlock(ctx context.Context, path string) error {
|
||||||
|
if err := c.writable(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return c.executorIntegrationClient.Unlock(ctx, path)
|
||||||
|
}
|
||||||
|
func (c *reconciliationClient) Run(ctx context.Context, command []string) (deployment.CommandResult, error) {
|
||||||
|
if err := c.writable(); err != nil {
|
||||||
|
return deployment.CommandResult{}, err
|
||||||
|
}
|
||||||
|
return c.executorIntegrationClient.Run(ctx, command)
|
||||||
|
}
|
||||||
|
|
||||||
|
func seedUncertainReconciliation(t *testing.T, repo *SQLDeploymentTransactionRepository, db *sql.DB, action deployment.RecoveryAction) (*deployment.Service, *reconciliationClient, deployment.Server, deployment.TransactionRecord, deployment.RecoveryDisposition) {
|
||||||
|
t.Helper()
|
||||||
|
_, _, _, remote, server, record := seedRecoveryExecutor(t, repo, db)
|
||||||
|
client := &reconciliationClient{executorIntegrationClient: remote, failClosed: true}
|
||||||
|
service := deployment.NewServiceWithJournal(client, repo)
|
||||||
|
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error { return nil }
|
||||||
|
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
|
||||||
|
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
source, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
|
||||||
|
if !errors.Is(err, deployment.ErrRecoveryActionUncertain) || source.Status != deployment.RecoveryResultUncertain {
|
||||||
|
t.Fatalf("未形成真实执行器不确定结果:%+v %v", source, err)
|
||||||
|
}
|
||||||
|
client.failClosed, client.readOnly = false, true
|
||||||
|
client.writes = 0
|
||||||
|
return service, client, server, record, source
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertReconciliationSource(t *testing.T, repo *SQLDeploymentTransactionRepository, source deployment.RecoveryDisposition) {
|
||||||
|
t.Helper()
|
||||||
|
stored, err := repo.GetRecoveryDisposition(t.Context(), source.Intent.ID)
|
||||||
|
if err != nil || !reflect.DeepEqual(stored, source) {
|
||||||
|
t.Fatalf("重新核查改写了原不确定历史:%+v %v", stored, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runReconciliationSuccess(t *testing.T, repo *SQLDeploymentTransactionRepository, db *sql.DB, action deployment.RecoveryAction) {
|
||||||
|
t.Helper()
|
||||||
|
service, client, server, record, source := seedUncertainReconciliation(t, repo, db, action)
|
||||||
|
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
|
||||||
|
files := maps.Clone(client.files)
|
||||||
|
verified := false
|
||||||
|
service.VerifyRecoveryTarget = func(_ context.Context, _ deployment.Server, _ deployment.RecoveryBaseline, committed bool) error {
|
||||||
|
if committed != (action == deployment.RecoveryActionCommit) {
|
||||||
|
t.Fatal("重新核查擅自切换方向")
|
||||||
|
}
|
||||||
|
verified = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
confirmation, token, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve)
|
||||||
|
if err != nil || confirmation.Summary().Validate() != nil {
|
||||||
|
t.Fatal("准备重新核查失败", err)
|
||||||
|
}
|
||||||
|
other, otherToken, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
result, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
|
||||||
|
if err != nil || result.Status != "succeeded" || result.Intent.SourceDispositionID != source.Intent.ID || !verified {
|
||||||
|
t.Fatalf("重新核查失败:%+v %v", result, err)
|
||||||
|
}
|
||||||
|
if client.writes != 0 || !reflect.DeepEqual(files, client.files) {
|
||||||
|
t.Fatal("只读重新核查修改了远端")
|
||||||
|
}
|
||||||
|
assertReconciliationSource(t, repo, source)
|
||||||
|
recoveryCount(t, db, "recovery_dispositions", 2)
|
||||||
|
recoveryCount(t, db, "recovery_events", 4)
|
||||||
|
recoveryCount(t, db, "recovery_claims", 0)
|
||||||
|
pending, err := repo.ListPendingDeploymentTransactions(t.Context())
|
||||||
|
if err != nil || len(pending) != 0 {
|
||||||
|
t.Fatal("原事务未原子结束", err)
|
||||||
|
}
|
||||||
|
view, err := service.GetRecoveryDisposition(t.Context(), result.Intent.ID)
|
||||||
|
if err != nil || view.SourceDispositionID != source.Intent.ID || view.Validate() != nil {
|
||||||
|
t.Fatal("查询缺少可信关联", err)
|
||||||
|
}
|
||||||
|
connections := client.connections
|
||||||
|
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, deployment.ErrRecoveryConfirmation) {
|
||||||
|
t.Fatal("重新核查确认被重放", err)
|
||||||
|
}
|
||||||
|
if _, err := service.ExecuteRecoveryDisposition(t.Context(), other, "unix:0", otherToken, resolve); err == nil {
|
||||||
|
t.Fatal("其他旧确认再次接管已完成的事务")
|
||||||
|
}
|
||||||
|
if client.connections != connections {
|
||||||
|
t.Fatal("完成后仍使用旧确认连接远端")
|
||||||
|
}
|
||||||
|
if record.ID != result.Intent.TransactionID {
|
||||||
|
t.Fatal("结束了其他事务")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecoveryReconciliationSQLiteSuccess(t *testing.T) {
|
||||||
|
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
|
||||||
|
t.Run(string(action), func(t *testing.T) {
|
||||||
|
_, db := newTaskRepositoryTest(t)
|
||||||
|
t.Cleanup(func() { _ = db.Close() })
|
||||||
|
runReconciliationSuccess(t, NewSQLDeploymentTransactionRepository(db), db, action)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecoveryReconciliationSQLiteRejectsUnsafeCompletion(t *testing.T) {
|
||||||
|
for _, failure := range []string{"file", "backup", "temporary", "lock", "https", "during-https", "close", "changed-after-prepare", "started-audit", "finished-audit", "identity", "phase", "running", "direction"} {
|
||||||
|
t.Run(failure, func(t *testing.T) {
|
||||||
|
_, db := newTaskRepositoryTest(t)
|
||||||
|
t.Cleanup(func() { _ = db.Close() })
|
||||||
|
repo := NewSQLDeploymentTransactionRepository(db)
|
||||||
|
service, client, server, record, source := seedUncertainReconciliation(t, repo, db, deployment.RecoveryActionCommit)
|
||||||
|
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
|
||||||
|
switch failure {
|
||||||
|
case "file":
|
||||||
|
client.files[record.Paths[0]] = "tampered"
|
||||||
|
case "backup":
|
||||||
|
client.files[record.Backups[0].Saved] = "old"
|
||||||
|
case "temporary":
|
||||||
|
client.files[record.Paths[0]+".scert-tmp-"+record.ID] = "unexpected"
|
||||||
|
case "lock":
|
||||||
|
client.files[record.LockPaths[0]] = ""
|
||||||
|
case "https":
|
||||||
|
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error {
|
||||||
|
return errors.New("严格 HTTPS 未通过")
|
||||||
|
}
|
||||||
|
case "during-https":
|
||||||
|
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error {
|
||||||
|
client.files[record.Paths[0]] = "changed"
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
case "identity":
|
||||||
|
server.PasswordRef = "secret://ssh/password/changed"
|
||||||
|
case "phase":
|
||||||
|
if _, err := db.ExecContext(t.Context(), `UPDATE deployment_transactions SET phase = 'reload_attempted'`); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case "running":
|
||||||
|
if _, err := db.ExecContext(t.Context(), `UPDATE recovery_dispositions SET status = 'running', result_code = ''`); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case "direction":
|
||||||
|
if _, err := db.ExecContext(t.Context(), `UPDATE recovery_dispositions SET action = 'restore_baseline'`); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
confirmation, token, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve)
|
||||||
|
if failure == "identity" || failure == "phase" || failure == "running" || failure == "direction" {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("危险来源允许重新核查")
|
||||||
|
}
|
||||||
|
recoveryCount(t, db, "recovery_claims", 1)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("准备失败", err)
|
||||||
|
}
|
||||||
|
if failure == "close" {
|
||||||
|
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error {
|
||||||
|
client.failClosed = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if failure == "changed-after-prepare" {
|
||||||
|
client.files[record.Paths[1]] = "changed"
|
||||||
|
}
|
||||||
|
if failure == "started-audit" || failure == "finished-audit" {
|
||||||
|
stage := "started"
|
||||||
|
if failure == "finished-audit" {
|
||||||
|
stage = "finished"
|
||||||
|
}
|
||||||
|
if _, err := db.ExecContext(t.Context(), `CREATE TRIGGER reject_recheck BEFORE INSERT ON recovery_events WHEN NEW.event_type = '`+stage+`' BEGIN SELECT RAISE(ABORT, 'injected'); END`); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
|
||||||
|
if err == nil || result.Status == "succeeded" || client.writes != 0 {
|
||||||
|
t.Fatalf("危险收尾未拒绝或执行了写操作:%+v %v", result, err)
|
||||||
|
}
|
||||||
|
assertReconciliationSource(t, repo, source)
|
||||||
|
var owner string
|
||||||
|
if err := db.QueryRowContext(t.Context(), `SELECT disposition_id FROM recovery_claims WHERE transaction_id = ?`, record.ID).Scan(&owner); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
expected := source.Intent.ID
|
||||||
|
if failure == "finished-audit" {
|
||||||
|
expected = confirmation.DispositionID()
|
||||||
|
}
|
||||||
|
if owner != expected {
|
||||||
|
t.Fatal("失败后占用归属不正确", owner, expected)
|
||||||
|
}
|
||||||
|
pending, err := repo.ListPendingDeploymentTransactions(t.Context())
|
||||||
|
if err != nil || len(pending) != 1 || pending[0].Status != deployment.TransactionStatusRecoveryRequired {
|
||||||
|
t.Fatal("失败改变了原事务", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
- 第一轮任务 `b795be80-e086-46b4-a2ba-e7b51c8662f0` 因旧 systemd 不支持 `--kill-whom`,注入失败并正常完成;不能计为崩溃测试通过。确认本机帮助后改用 `--kill-who=main`。
|
- 第一轮任务 `b795be80-e086-46b4-a2ba-e7b51c8662f0` 因旧 systemd 不支持 `--kill-whom`,注入失败并正常完成;不能计为崩溃测试通过。确认本机帮助后改用 `--kill-who=main`。
|
||||||
- 22:57:25 对正式 Scert 主进程发送 SIGKILL,命中任务 `c68e3956-1b6d-42c7-ba9a-753ef2d1a44b` 第二目标事务的 reload_attempted 阶段。
|
- 22:57:25 对正式 Scert 主进程发送 SIGKILL,命中任务 `c68e3956-1b6d-42c7-ba9a-753ef2d1a44b` 第二目标事务的 reload_attempted 阶段。
|
||||||
- systemd 的 MainPID 从 130794 变为 134595,NRestarts=1;23:58 前已核实服务恢复 active、回环严格 TLS 就绪检查通过。此处实际核查时间为 22:58:16。
|
- systemd 的 MainPID 从 130794 变为 134595,NRestarts=1;22:58:16 已核实服务恢复 active、回环严格 TLS 就绪检查通过。
|
||||||
- 事务 `20261003-145716.121859533-09a2a331489a` 与 `20261003-145720.641188477-c769c8d3e153` 均 rolled_back;任务为 failed/service_restart,没有覆盖为成功。
|
- 事务 `20261003-145716.121859533-09a2a331489a` 与 `20261003-145720.641188477-c769c8d3e153` 均 rolled_back;任务为 failed/service_restart,没有覆盖为成功。
|
||||||
- 双机 cert/fullchain 摘要保持 `7e8b56227b345ed9b66659e428f9bd4421604f11253ab88d65b3ed61737f35cf`,未发现该域名遗留事务文件。第二台正常 DNS HTTPS 200、peer=45.136.13.241、TLS 校验结果零。
|
- 双机 cert/fullchain 摘要保持 `7e8b56227b345ed9b66659e428f9bd4421604f11253ab88d65b3ed61737f35cf`,未发现该域名遗留事务文件。第二台正常 DNS HTTPS 200、peer=45.136.13.241、TLS 校验结果零。
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,17 @@
|
|||||||
# 部署事务人工核查任务清单
|
# 部署事务人工核查任务清单
|
||||||
|
|
||||||
|
## 2026年10月3日23:30 不确定处置重新核查任务
|
||||||
|
|
||||||
|
沿用已确认的 Linux 范围,不增加网络入口,不直接修改现场事务或删除占用。
|
||||||
|
|
||||||
|
- [✓] 上轮静态检查通过;后端全量唯一失败已定位:宿主 umask 为 0077,测试创建的父目录从开始就是 0700;原代码在显式 umask 0022 下通过,不能归因于备份修改权限。
|
||||||
|
- [x] 测试显式建立 0755 权限基线,不修改进程 umask 或产品权限策略,等待 Linux 严格 umask 回归。
|
||||||
|
- [~] R6-A 独立重新核查动作:绑定原 uncertain 处置编号、原方向、服务器身份和本次现场摘要;只允许已完成审计的 uncertain,不以超时接管 running。
|
||||||
|
- [ ] R6-B 持久化关联与占用原子交接:新增处置审计,保留原 uncertain;拒绝或核查失败恢复原占用,结束审计失败保留新占用;成功才原子结束原事务并释放占用。
|
||||||
|
- [ ] R6-C 只读远端重新核验:仅限原方向对应的 verified 阶段,核验全部文件、临时文件与备份不存在、锁不存在、严格 HTTPS 和关闭结果;不得重载、删锁、重写或重放原动作。
|
||||||
|
- [ ] R6-D 受保护本机终端单独入口、完整确认、一次性绑定、断连查询及三数据库回归。
|
||||||
|
- [ ] R6-E 正式 Linux 包升级和当前真实 uncertain 处置收尾;其他不满足严格收尾条件的历史事务继续保留,不能用本功能冒充完成。
|
||||||
|
|
||||||
## 2026年10月3日23:08 实机关闭误判整改任务
|
## 2026年10月3日23:08 实机关闭误判整改任务
|
||||||
|
|
||||||
- [✓] 真实 SIGKILL 中断部署后,两条事务自动回滚,第二台正常 DNS 严格 HTTPS 仍通过。
|
- [✓] 真实 SIGKILL 中断部署后,两条事务自动回滚,第二台正常 DNS 严格 HTTPS 仍通过。
|
||||||
|
|||||||
Reference in new issue
Block a user