From 17eb2fc29507acabd5d3e49098aa029f4ef65ade Mon Sep 17 00:00:00 2001 From: xuans Date: Sat, 3 Oct 2026 19:49:36 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=9D=E5=AD=98=E4=BA=BA=E5=B7=A5=E5=A4=84?= =?UTF-8?q?=E7=BD=AE=E6=8C=81=E4=B9=85=E5=8C=96=E4=B8=8E=E5=AE=9E=E5=BA=93?= =?UTF-8?q?=E5=9B=9E=E5=BD=92=E6=A3=80=E6=9F=A5=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/internal/database/migrate.go | 13 +- .../mysql/0022_recovery_dispositions.sql | 17 ++ .../migrations/mysql/0023_recovery_claims.sql | 7 + .../migrations/mysql/0024_recovery_events.sql | 7 + .../postgres/0022_recovery_dispositions.sql | 15 + .../postgres/0023_recovery_claims.sql | 5 + .../postgres/0024_recovery_events.sql | 7 + .../sqlite/0022_recovery_dispositions.sql | 15 + .../sqlite/0023_recovery_claims.sql | 5 + .../sqlite/0024_recovery_events.sql | 7 + .../mysql_deployment_migration_test.go | 2 +- ...sql_recovery_disposition_migration_test.go | 58 ++++ .../deployment/recovery_disposition.go | 114 ++++++++ .../deployment/recovery_disposition_test.go | 51 ++++ .../recovery_disposition_completion.go | 105 +++++++ .../recovery_disposition_completion_test.go | 87 ++++++ .../recovery_disposition_mysql_test.go | 97 +++++++ .../recovery_disposition_repository.go | 133 +++++++++ .../recovery_disposition_repository_test.go | 257 ++++++++++++++++++ doc/Linux生产流程测试记录.md | 14 + doc/Scert开发任务清单.md | 10 + doc/Scert整改任务执行清单.md | 10 + doc/Scert整改合并优化清单.md | 14 + doc/部署事务人工处置设计.md | 13 + doc/部署事务人工核查任务清单.md | 10 + 25 files changed, 1070 insertions(+), 3 deletions(-) create mode 100644 backend/internal/database/migrations/mysql/0022_recovery_dispositions.sql create mode 100644 backend/internal/database/migrations/mysql/0023_recovery_claims.sql create mode 100644 backend/internal/database/migrations/mysql/0024_recovery_events.sql create mode 100644 backend/internal/database/migrations/postgres/0022_recovery_dispositions.sql create mode 100644 backend/internal/database/migrations/postgres/0023_recovery_claims.sql create mode 100644 backend/internal/database/migrations/postgres/0024_recovery_events.sql create mode 100644 backend/internal/database/migrations/sqlite/0022_recovery_dispositions.sql create mode 100644 backend/internal/database/migrations/sqlite/0023_recovery_claims.sql create mode 100644 backend/internal/database/migrations/sqlite/0024_recovery_events.sql create mode 100644 backend/internal/database/mysql_recovery_disposition_migration_test.go create mode 100644 backend/internal/deployment/recovery_disposition.go create mode 100644 backend/internal/deployment/recovery_disposition_test.go create mode 100644 backend/internal/repository/recovery_disposition_completion.go create mode 100644 backend/internal/repository/recovery_disposition_completion_test.go create mode 100644 backend/internal/repository/recovery_disposition_mysql_test.go create mode 100644 backend/internal/repository/recovery_disposition_repository.go create mode 100644 backend/internal/repository/recovery_disposition_repository_test.go diff --git a/backend/internal/database/migrate.go b/backend/internal/database/migrate.go index cdeaee1..947385c 100644 --- a/backend/internal/database/migrate.go +++ b/backend/internal/database/migrate.go @@ -22,6 +22,9 @@ import ( //go:embed migrations/sqlite/0018_recovery_evidence.sql migrations/mysql/0018_recovery_evidence.sql migrations/postgres/0018_recovery_evidence.sql migrations/sqlite/0019_password_auth_only.sql migrations/mysql/0019_password_auth_only.sql migrations/postgres/0019_password_auth_only.sql //go:embed migrations/sqlite/0020_task_deployment_reports.sql migrations/mysql/0020_task_deployment_reports.sql migrations/postgres/0020_task_deployment_reports.sql //go:embed migrations/sqlite/0021_task_validation_results.sql migrations/mysql/0021_task_validation_results.sql migrations/postgres/0021_task_validation_results.sql +//go:embed migrations/sqlite/0022_recovery_dispositions.sql migrations/mysql/0022_recovery_dispositions.sql migrations/postgres/0022_recovery_dispositions.sql +//go:embed migrations/sqlite/0023_recovery_claims.sql migrations/mysql/0023_recovery_claims.sql migrations/postgres/0023_recovery_claims.sql +//go:embed migrations/sqlite/0024_recovery_events.sql migrations/mysql/0024_recovery_events.sql migrations/postgres/0024_recovery_events.sql var migrationFiles embed.FS func Migrate(ctx context.Context, db *sql.DB, driver string) error { @@ -46,7 +49,7 @@ func Migrate(ctx context.Context, db *sql.DB, driver string) error { return nil } -const latestMigrationVersion = 21 +const latestMigrationVersion = 24 // currentMigrationVersion 在执行业务迁移前拒绝新版数据库及不连续的迁移历史。 func currentMigrationVersion(ctx context.Context, db *sql.DB) (int, error) { @@ -86,7 +89,7 @@ func applyMigration(ctx context.Context, db *sql.DB, driver string, version int) return applySQLiteTaskMigration(ctx, db, statements, version) } if driver == "mysql" { - if (version >= 15 && version <= 17) || version == 20 || version == 21 { + if (version >= 15 && version <= 17) || (version >= 20 && version <= 24) { return applyMySQLDeploymentMigration(ctx, db, version, statements) } if version == 14 { @@ -243,6 +246,12 @@ func migrationName(version int) string { return "task_deployment_reports" case 21: return "task_validation_results" + case 22: + return "recovery_dispositions" + case 23: + return "recovery_claims" + case 24: + return "recovery_events" default: return "unknown" } diff --git a/backend/internal/database/migrations/mysql/0022_recovery_dispositions.sql b/backend/internal/database/migrations/mysql/0022_recovery_dispositions.sql new file mode 100644 index 0000000..830ecdb --- /dev/null +++ b/backend/internal/database/migrations/mysql/0022_recovery_dispositions.sql @@ -0,0 +1,17 @@ +CREATE TABLE recovery_dispositions ( + id VARCHAR(64) NOT NULL, + transaction_id VARCHAR(160) NOT NULL, + server_id VARCHAR(160) NOT NULL, + operator_id VARCHAR(160) NOT NULL, + action VARCHAR(32) NOT NULL, + request_key VARCHAR(64) NOT NULL, + record_digest VARCHAR(64) NOT NULL, + server_digest VARCHAR(64) NOT NULL, + observation_digest VARCHAR(64) NOT NULL, + status VARCHAR(32) NOT NULL, + result_code VARCHAR(64) NOT NULL, + created_at DATETIME NOT NULL, + updated_at DATETIME NOT NULL, + PRIMARY KEY (id), + UNIQUE KEY uq_recovery_request (request_key) +); diff --git a/backend/internal/database/migrations/mysql/0023_recovery_claims.sql b/backend/internal/database/migrations/mysql/0023_recovery_claims.sql new file mode 100644 index 0000000..435a004 --- /dev/null +++ b/backend/internal/database/migrations/mysql/0023_recovery_claims.sql @@ -0,0 +1,7 @@ +CREATE TABLE recovery_claims ( + transaction_id VARCHAR(160) NOT NULL, + disposition_id VARCHAR(64) NOT NULL, + created_at DATETIME NOT NULL, + PRIMARY KEY (transaction_id), + UNIQUE KEY uq_recovery_claim (disposition_id) +); diff --git a/backend/internal/database/migrations/mysql/0024_recovery_events.sql b/backend/internal/database/migrations/mysql/0024_recovery_events.sql new file mode 100644 index 0000000..3f78c0a --- /dev/null +++ b/backend/internal/database/migrations/mysql/0024_recovery_events.sql @@ -0,0 +1,7 @@ +CREATE TABLE recovery_events ( + disposition_id VARCHAR(64) NOT NULL, + event_type VARCHAR(32) NOT NULL, + result_code VARCHAR(64) NOT NULL, + created_at DATETIME NOT NULL, + PRIMARY KEY (disposition_id, event_type) +); diff --git a/backend/internal/database/migrations/postgres/0022_recovery_dispositions.sql b/backend/internal/database/migrations/postgres/0022_recovery_dispositions.sql new file mode 100644 index 0000000..8e307ce --- /dev/null +++ b/backend/internal/database/migrations/postgres/0022_recovery_dispositions.sql @@ -0,0 +1,15 @@ +CREATE TABLE recovery_dispositions ( + id TEXT NOT NULL PRIMARY KEY, + transaction_id TEXT NOT NULL, + server_id TEXT NOT NULL, + operator_id TEXT NOT NULL, + action TEXT NOT NULL, + request_key TEXT NOT NULL UNIQUE, + record_digest TEXT NOT NULL, + server_digest TEXT NOT NULL, + observation_digest TEXT NOT NULL, + status TEXT NOT NULL, + result_code TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); diff --git a/backend/internal/database/migrations/postgres/0023_recovery_claims.sql b/backend/internal/database/migrations/postgres/0023_recovery_claims.sql new file mode 100644 index 0000000..b06a8e4 --- /dev/null +++ b/backend/internal/database/migrations/postgres/0023_recovery_claims.sql @@ -0,0 +1,5 @@ +CREATE TABLE recovery_claims ( + transaction_id TEXT NOT NULL PRIMARY KEY, + disposition_id TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL +); diff --git a/backend/internal/database/migrations/postgres/0024_recovery_events.sql b/backend/internal/database/migrations/postgres/0024_recovery_events.sql new file mode 100644 index 0000000..755fce2 --- /dev/null +++ b/backend/internal/database/migrations/postgres/0024_recovery_events.sql @@ -0,0 +1,7 @@ +CREATE TABLE recovery_events ( + disposition_id TEXT NOT NULL, + event_type TEXT NOT NULL, + result_code TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (disposition_id, event_type) +); diff --git a/backend/internal/database/migrations/sqlite/0022_recovery_dispositions.sql b/backend/internal/database/migrations/sqlite/0022_recovery_dispositions.sql new file mode 100644 index 0000000..8ff3f46 --- /dev/null +++ b/backend/internal/database/migrations/sqlite/0022_recovery_dispositions.sql @@ -0,0 +1,15 @@ +CREATE TABLE recovery_dispositions ( + id TEXT NOT NULL PRIMARY KEY, + transaction_id TEXT NOT NULL, + server_id TEXT NOT NULL, + operator_id TEXT NOT NULL, + action TEXT NOT NULL, + request_key TEXT NOT NULL UNIQUE, + record_digest TEXT NOT NULL, + server_digest TEXT NOT NULL, + observation_digest TEXT NOT NULL, + status TEXT NOT NULL, + result_code TEXT NOT NULL, + created_at DATETIME NOT NULL, + updated_at DATETIME NOT NULL +); diff --git a/backend/internal/database/migrations/sqlite/0023_recovery_claims.sql b/backend/internal/database/migrations/sqlite/0023_recovery_claims.sql new file mode 100644 index 0000000..0d80d40 --- /dev/null +++ b/backend/internal/database/migrations/sqlite/0023_recovery_claims.sql @@ -0,0 +1,5 @@ +CREATE TABLE recovery_claims ( + transaction_id TEXT NOT NULL PRIMARY KEY, + disposition_id TEXT NOT NULL UNIQUE, + created_at DATETIME NOT NULL +); diff --git a/backend/internal/database/migrations/sqlite/0024_recovery_events.sql b/backend/internal/database/migrations/sqlite/0024_recovery_events.sql new file mode 100644 index 0000000..2caa635 --- /dev/null +++ b/backend/internal/database/migrations/sqlite/0024_recovery_events.sql @@ -0,0 +1,7 @@ +CREATE TABLE recovery_events ( + disposition_id TEXT NOT NULL, + event_type TEXT NOT NULL, + result_code TEXT NOT NULL, + created_at DATETIME NOT NULL, + PRIMARY KEY (disposition_id, event_type) +); diff --git a/backend/internal/database/mysql_deployment_migration_test.go b/backend/internal/database/mysql_deployment_migration_test.go index 506f964..faaf197 100644 --- a/backend/internal/database/mysql_deployment_migration_test.go +++ b/backend/internal/database/mysql_deployment_migration_test.go @@ -8,7 +8,7 @@ import ( ) func TestMySQLDeploymentShapeDefinitions(t *testing.T) { - for _, version := range []int{15, 16, 17, 20, 21} { + for _, version := range []int{15, 16, 17, 20, 21, 22, 23, 24} { statements := mysqlDeploymentStatements(t, version) shape, err := parseMySQLDeploymentShape(statements[0]) if err != nil || len(shape.Columns) == 0 || len(shape.Indexes) == 0 { diff --git a/backend/internal/database/mysql_recovery_disposition_migration_test.go b/backend/internal/database/mysql_recovery_disposition_migration_test.go new file mode 100644 index 0000000..31d00b6 --- /dev/null +++ b/backend/internal/database/mysql_recovery_disposition_migration_test.go @@ -0,0 +1,58 @@ +package database + +import ( + "fmt" + "testing" +) + +func TestMySQLRealRecoveryDispositionMigrationRecovery(t *testing.T) { + f := startRealMySQL(t) + baseline := f.database(t) + if err := Migrate(t.Context(), baseline.DB, "mysql"); err != nil { + t.Fatal(err) + } + expected := mysqlSchemaSnapshot(t, baseline.DB) + for _, item := range []struct { + version int + table string + seed string + conflict string + }{ + {22, "recovery_dispositions", `INSERT INTO recovery_dispositions VALUES + ('kept', 'transaction', 'server', 'unix:0', 'record_inspection', 'request', 'record', 'server', 'observation', 'running', '', NOW(), NOW())`, + `ALTER TABLE recovery_dispositions DROP INDEX uq_recovery_request`}, + {23, "recovery_claims", `INSERT INTO recovery_claims VALUES ('transaction', 'kept', NOW())`, + `ALTER TABLE recovery_claims DROP PRIMARY KEY`}, + {24, "recovery_events", `INSERT INTO recovery_events VALUES ('kept', 'started', '', NOW())`, + `ALTER TABLE recovery_events DROP PRIMARY KEY, ADD PRIMARY KEY (event_type, disposition_id)`}, + } { + for _, boundary := range []string{"before_create", "after_create", "with_record", "conflict"} { + t.Run(fmt.Sprintf("%d/%s", item.version, boundary), func(t *testing.T) { + db := f.database(t) + mysqlPrepareVersion(t, db.DB, item.version-1) + if boundary != "before_create" { + mysqlDeploymentExec(t, db.DB, mysqlDeploymentStatements(t, item.version)[0]) + } + if boundary == "with_record" || boundary == "conflict" { + mysqlDeploymentExec(t, db.DB, item.seed) + } + if boundary == "conflict" { + mysqlDeploymentExec(t, db.DB, item.conflict) + before := fmt.Sprint(mysqlSchemaSnapshot(t, db.DB)) + assertMySQLDeploymentMigrationRejected(t, db.DB, item.version-1) + if after := fmt.Sprint(mysqlSchemaSnapshot(t, db.DB)); after != before { + t.Fatal("结构冲突不得自动修复或覆盖") + } + } else { + mysqlAssertUpgrade(t, db.DB, expected) + } + if boundary == "with_record" || boundary == "conflict" { + var count int + if err := db.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM "+item.table).Scan(&count); err != nil || count != 1 { + t.Fatal("迁移或冲突拒绝丢失已有处置记录", err) + } + } + }) + } + } +} diff --git a/backend/internal/deployment/recovery_disposition.go b/backend/internal/deployment/recovery_disposition.go new file mode 100644 index 0000000..5af683c --- /dev/null +++ b/backend/internal/deployment/recovery_disposition.go @@ -0,0 +1,114 @@ +package deployment + +import ( + "context" + "encoding/hex" + "errors" + "strings" + "time" +) + +var ( + ErrRecoveryDispositionInvalid = errors.New("人工处置记录参数无效") + ErrRecoveryDispositionConflict = errors.New("人工处置请求与已有记录不一致") + ErrRecoveryDispositionBusy = errors.New("该事务已有未释放的人工处置占用") + ErrRecoveryDispositionUnavailable = errors.New("人工处置记录暂不可用，未授权执行远端动作") +) + +// RecoveryDispositionIntent 仅保存绑定摘要；不能包含历史证据、凭据或确认令牌原文。 +type RecoveryDispositionIntent struct { + ID string + TransactionID string + ServerID string + Operator string + Action RecoveryAction + RequestKey string + RecordDigest string + ServerDigest string + ObservationDigest string +} + +type RecoveryDisposition struct { + Intent RecoveryDispositionIntent `json:"-"` + Status string `json:"status"` + ResultCode string `json:"result_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +const ( + RecoveryResultObserved = "observed" + RecoveryResultCommitted = "committed" + RecoveryResultRolledBack = "rolled_back" + RecoveryResultRejected = "rejected" + RecoveryResultUncertain = "uncertain" +) + +// RecoveryDispositionJournal 的开始成功且新建返回值为真，才允许执行器继续核验。 +// 记录本身不是远端写入授权；执行器仍需一次性确认、完整互斥及动作专属核验。 +type RecoveryDispositionJournal interface { + BeginRecoveryDisposition(context.Context, RecoveryDispositionIntent) (RecoveryDisposition, bool, error) + FinishRecoveryDisposition(context.Context, string, string) (RecoveryDisposition, error) + GetRecoveryDisposition(context.Context, string) (RecoveryDisposition, error) + HasRecoveryClaim(context.Context, string) (bool, error) +} + +func (r RecoveryReview) DispositionIntent(id, requestKey string) (RecoveryDispositionIntent, error) { + if r.service == nil || r.service.now == nil || !r.service.now().Before(r.summary.ExpiresAt) { + return RecoveryDispositionIntent{}, ErrRecoveryReviewExpired + } + intent := RecoveryDispositionIntent{ + ID: id, TransactionID: r.summary.TransactionID, ServerID: r.summary.ServerID, + Operator: r.operator, Action: r.summary.Action, RequestKey: requestKey, + RecordDigest: hex.EncodeToString(r.recordDigest[:]), + ServerDigest: hex.EncodeToString(r.serverDigest[:]), + ObservationDigest: hex.EncodeToString(r.observationDigest[:]), + } + return intent, intent.Validate() +} + +func (i RecoveryDispositionIntent) Validate() error { + if !validInspectionID(i.TransactionID) || !validRecoverySummary(i.ServerID) || + !validReviewOperator(i.Operator) || !validRecoveryAction(i.Action) { + return ErrRecoveryDispositionInvalid + } + for _, value := range []string{i.ID, i.RequestKey, i.RecordDigest, i.ServerDigest, i.ObservationDigest} { + if len(value) != 64 || value != strings.ToLower(value) || value == strings.Repeat("0", 64) { + return ErrRecoveryDispositionInvalid + } + if _, err := hex.DecodeString(value); err != nil { + return ErrRecoveryDispositionInvalid + } + } + return nil +} + +// SameRequest 忽略重试时新生成的记录标识，但必须匹配原请求的全部绑定值。 +func (i RecoveryDispositionIntent) SameRequest(other RecoveryDispositionIntent) bool { + i.ID, other.ID = "", "" + return i == other +} + +func (i RecoveryDispositionIntent) MatchesRecord(record TransactionRecord) bool { + if record.ID != i.TransactionID || record.ServerID != i.ServerID || + (record.Status != TransactionStatusPending && record.Status != TransactionStatusRecoveryRequired) || + (i.Action != RecoveryActionRecord && record.Evidence == nil) { + return false + } + digest, err := reviewDigest(record) + return err == nil && hex.EncodeToString(digest[:]) == i.RecordDigest +} + +func (i RecoveryDispositionIntent) AllowsResult(result string) bool { + switch result { + case RecoveryResultRejected, RecoveryResultUncertain: + return validRecoveryAction(i.Action) + case RecoveryResultObserved: + return i.Action == RecoveryActionRecord + case RecoveryResultCommitted: + return i.Action == RecoveryActionCommit + case RecoveryResultRolledBack: + return i.Action == RecoveryActionRollback + } + return false +} diff --git a/backend/internal/deployment/recovery_disposition_test.go b/backend/internal/deployment/recovery_disposition_test.go new file mode 100644 index 0000000..7fd3ac9 --- /dev/null +++ b/backend/internal/deployment/recovery_disposition_test.go @@ -0,0 +1,51 @@ +package deployment + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" +) + +func TestRecoveryReviewDispositionIntent(t *testing.T) { + server, memory, record := recoveryPreflightFixture() + journal := &memoryTransactionJournal{pending: []TransactionRecord{record}} + service := NewServiceWithJournal(fixedClientFactory{&inspectionClient{memoryClient: memory}}, journal) + resolve := func(context.Context, string) (Server, error) { return server, nil } + review, err := service.PrepareRecoveryReview(t.Context(), record.ID, "unix:0", RecoveryActionCommit, resolve) + if err != nil { + t.Fatal(err) + } + intent, err := review.DispositionIntent(strings.Repeat("a", 64), strings.Repeat("b", 64)) + if err != nil || intent.Validate() != nil || !intent.MatchesRecord(record) { + t.Fatal("review bindings not retained", err) + } + if !intent.AllowsResult(RecoveryResultCommitted) || intent.AllowsResult(RecoveryResultRolledBack) || intent.AllowsResult(RecoveryResultObserved) { + t.Fatal("action result isolation failed") + } + encoded, _ := json.Marshal(RecoveryDisposition{Intent: intent}) + if strings.Contains(string(encoded), intent.RequestKey) || strings.Contains(string(encoded), intent.RecordDigest) || strings.Contains(string(encoded), "unix:0") { + t.Fatal("internal bindings exposed in public JSON") + } + for _, value := range []string{"", "token-secret", strings.Repeat("A", 64), strings.Repeat("0", 64), strings.Repeat("a", 63), strings.Repeat("g", 64)} { + if _, err := review.DispositionIntent(value, strings.Repeat("b", 64)); !errors.Is(err, ErrRecoveryDispositionInvalid) { + t.Fatal("invalid identifier accepted") + } + if _, err := review.DispositionIntent(strings.Repeat("a", 64), value); !errors.Is(err, ErrRecoveryDispositionInvalid) { + t.Fatal("invalid request key accepted") + } + } + record.Message = "changed" + if intent.MatchesRecord(record) { + t.Fatal("changed transaction accepted") + } + service.now = func() time.Time { return review.Summary().ExpiresAt } + if _, err := review.DispositionIntent(strings.Repeat("a", 64), strings.Repeat("b", 64)); !errors.Is(err, ErrRecoveryReviewExpired) { + t.Fatal("expired preparation accepted") + } + if _, err := (RecoveryReview{}).DispositionIntent(strings.Repeat("a", 64), strings.Repeat("b", 64)); err == nil { + t.Fatal("zero review accepted") + } +} diff --git a/backend/internal/repository/recovery_disposition_completion.go b/backend/internal/repository/recovery_disposition_completion.go new file mode 100644 index 0000000..6323bc0 --- /dev/null +++ b/backend/internal/repository/recovery_disposition_completion.go @@ -0,0 +1,105 @@ +package repository + +import ( + "context" + "database/sql" + "time" + + "git.pcux.cn/xuans/Scert/backend/internal/deployment" +) + +// FinishRecoveryDisposition 仅由完成现场核验的内部执行器调用，不接受公网客户端结果。 +// 不确定结果保留占用；成功结果、必要的原事务终态与结束审计在同一事务内提交。 +func (r *SQLDeploymentTransactionRepository) FinishRecoveryDisposition(ctx context.Context, id, resultCode string) (deployment.RecoveryDisposition, error) { + if r == nil || r.db == nil { + return deployment.RecoveryDisposition{}, dispositionError(ctx) + } + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return deployment.RecoveryDisposition{}, dispositionError(ctx) + } + defer tx.Rollback() + query := `SELECT ` + recoveryDispositionColumns + ` FROM recovery_dispositions WHERE id = ?` + if r.driver != "sqlite" { + query += " FOR UPDATE" + } + item, err := scanRecoveryDisposition(tx.QueryRowContext(ctx, r.query(query), id)) + if err != nil { + return deployment.RecoveryDisposition{}, dispositionError(ctx) + } + if !item.Intent.AllowsResult(resultCode) { + return deployment.RecoveryDisposition{}, deployment.ErrRecoveryDispositionInvalid + } + if item.Status != "running" { + if item.ResultCode == resultCode { + return item, nil + } + return deployment.RecoveryDisposition{}, deployment.ErrRecoveryDispositionConflict + } + var claim, start string + if err := tx.QueryRowContext(ctx, r.query(`SELECT disposition_id FROM recovery_claims WHERE transaction_id = ?`), item.Intent.TransactionID).Scan(&claim); err != nil || claim != id { + return deployment.RecoveryDisposition{}, deployment.ErrRecoveryDispositionConflict + } + if err := tx.QueryRowContext(ctx, r.query(`SELECT event_type FROM recovery_events WHERE disposition_id = ? AND event_type = 'started'`), id).Scan(&start); err != nil { + return deployment.RecoveryDisposition{}, dispositionError(ctx) + } + now := time.Now().UTC() + if err := r.finishRecoveryOriginal(ctx, tx, item.Intent, resultCode, now); err != nil { + return deployment.RecoveryDisposition{}, err + } + item.Status = "succeeded" + if resultCode == deployment.RecoveryResultRejected || resultCode == deployment.RecoveryResultUncertain { + item.Status = resultCode + } + result, err := tx.ExecContext(ctx, r.query(`UPDATE recovery_dispositions SET status = ?, result_code = ?, updated_at = ? WHERE id = ? AND status = 'running'`), item.Status, resultCode, now, id) + if err != nil || !oneRecoveryRow(result) { + return deployment.RecoveryDisposition{}, dispositionError(ctx) + } + if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_events (disposition_id, event_type, result_code, created_at) VALUES (?, 'finished', ?, ?)`), id, resultCode, now); err != nil { + return deployment.RecoveryDisposition{}, dispositionError(ctx) + } + if resultCode != deployment.RecoveryResultUncertain { + result, err := tx.ExecContext(ctx, r.query(`DELETE FROM recovery_claims WHERE transaction_id = ? AND disposition_id = ?`), item.Intent.TransactionID, id) + if err != nil || !oneRecoveryRow(result) { + return deployment.RecoveryDisposition{}, dispositionError(ctx) + } + } + if err := tx.Commit(); err != nil { + return deployment.RecoveryDisposition{}, dispositionError(ctx) + } + item.ResultCode, item.UpdatedAt = resultCode, now + return item, nil +} + +func oneRecoveryRow(result sql.Result) bool { + if result == nil { + return false + } + count, err := result.RowsAffected() + return err == nil && count == 1 +} + +func (r *SQLDeploymentTransactionRepository) finishRecoveryOriginal(ctx context.Context, tx *sql.Tx, intent deployment.RecoveryDispositionIntent, resultCode string, now time.Time) error { + if resultCode != deployment.RecoveryResultCommitted && resultCode != deployment.RecoveryResultRolledBack { + // 只记录核查、核验拒绝与不确定结果均不改变原事务；不能以已阅读代替恢复成功。 + return nil + } + record, err := r.lockRecoveryTransaction(ctx, tx, intent.TransactionID) + if err != nil { + return dispositionError(ctx) + } + phase, status, message := deployment.TransactionPhaseCommitVerified, deployment.TransactionStatusCommitted, "人工处置已完成提交核验" + if resultCode == deployment.RecoveryResultRolledBack { + phase, status, message = deployment.TransactionPhaseRollbackVerified, deployment.TransactionStatusRolledBack, "人工处置已完成回滚核验" + } + if record.ServerID != intent.ServerID || record.Evidence == nil || record.Phase != phase || + (record.Status != deployment.TransactionStatusPending && record.Status != deployment.TransactionStatusRecoveryRequired) { + return deployment.ErrRecoveryDispositionConflict + } + result, err := tx.ExecContext(ctx, r.query(`UPDATE deployment_transactions SET status = ?, phase = ?, message = ?, updated_at = ? + WHERE id = ? AND phase = ? AND status IN ('pending', 'recovery_required')`), status, status, message, now, intent.TransactionID, phase) + if err != nil || !oneRecoveryRow(result) { + return dispositionError(ctx) + } + return nil +} diff --git a/backend/internal/repository/recovery_disposition_completion_test.go b/backend/internal/repository/recovery_disposition_completion_test.go new file mode 100644 index 0000000..7cb0fd0 --- /dev/null +++ b/backend/internal/repository/recovery_disposition_completion_test.go @@ -0,0 +1,87 @@ +package repository + +import ( + "errors" + "testing" + + "git.pcux.cn/xuans/Scert/backend/internal/deployment" +) + +func TestRecoveryDispositionVerifiedCompletionAtomicity(t *testing.T) { + for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} { + for _, failure := range []string{"none", "original", "result", "audit", "release"} { + t.Run(string(action)+"/"+failure, func(t *testing.T) { + repo, db, intent := recoveryDispositionFixture(t, action) + if _, _, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil { + t.Fatal(err) + } + phase, resultCode := deployment.TransactionPhaseCommitVerified, deployment.RecoveryResultCommitted + if action == deployment.RecoveryActionRollback { + phase, resultCode = deployment.TransactionPhaseRollbackVerified, deployment.RecoveryResultRolledBack + } + if _, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, resultCode); !errors.Is(err, deployment.ErrRecoveryDispositionConflict) { + t.Fatal("unverified original finalized", err) + } + if _, err := db.ExecContext(t.Context(), `UPDATE deployment_transactions SET phase = ?`, phase); err != nil { + t.Fatal(err) + } + triggers := map[string]string{ + "original": "BEFORE UPDATE ON deployment_transactions", + "result": "BEFORE UPDATE ON recovery_dispositions", + "audit": "BEFORE INSERT ON recovery_events", + "release": "BEFORE DELETE ON recovery_claims", + } + if trigger, ok := triggers[failure]; ok { + if _, err := db.ExecContext(t.Context(), `CREATE TRIGGER reject_finish `+trigger+` BEGIN SELECT RAISE(ABORT, 'injected'); END`); err != nil { + t.Fatal(err) + } + } + item, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, resultCode) + var status, storedPhase string + if readErr := db.QueryRowContext(t.Context(), `SELECT status, phase FROM deployment_transactions WHERE id = ?`, intent.TransactionID).Scan(&status, &storedPhase); readErr != nil { + t.Fatal(readErr) + } + if failure == "none" { + if err != nil || item.Status != "succeeded" || status != resultCode || storedPhase != resultCode { + t.Fatal("atomic completion failed", err) + } + recoveryCount(t, db, "recovery_claims", 0) + recoveryCount(t, db, "recovery_events", 2) + if _, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, resultCode); err != nil { + t.Fatal(err) + } + } else { + if err == nil || status != deployment.TransactionStatusRecoveryRequired || storedPhase != phase { + t.Fatal("failed completion changed original") + } + stored, err := repo.GetRecoveryDisposition(t.Context(), intent.ID) + if err != nil || stored.Status != "running" || stored.ResultCode != "" { + t.Fatal("partial disposition outcome saved", err) + } + recoveryCount(t, db, "recovery_claims", 1) + recoveryCount(t, db, "recovery_events", 1) + } + }) + } + } +} + +func TestRecoveryDispositionRejectsCrossActionCompletion(t *testing.T) { + repo, db, intent := recoveryDispositionFixture(t, deployment.RecoveryActionRecord) + if _, _, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil { + t.Fatal(err) + } + for _, result := range []string{deployment.RecoveryResultCommitted, deployment.RecoveryResultRolledBack, "unknown", ""} { + if _, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, result); !errors.Is(err, deployment.ErrRecoveryDispositionInvalid) { + t.Fatal("invalid result accepted", err) + } + } + if _, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, deployment.RecoveryResultRejected); err != nil { + t.Fatal(err) + } + recoveryCount(t, db, "recovery_claims", 0) + pending, err := repo.ListPendingDeploymentTransactions(t.Context()) + if err != nil || len(pending) != 1 || pending[0].Evidence != nil { + t.Fatal("rejected review changed original", err) + } +} diff --git a/backend/internal/repository/recovery_disposition_mysql_test.go b/backend/internal/repository/recovery_disposition_mysql_test.go new file mode 100644 index 0000000..0e31045 --- /dev/null +++ b/backend/internal/repository/recovery_disposition_mysql_test.go @@ -0,0 +1,97 @@ +package repository + +import ( + "errors" + "strings" + "testing" + + "git.pcux.cn/xuans/Scert/backend/internal/database" + "git.pcux.cn/xuans/Scert/backend/internal/deployment" + "git.pcux.cn/xuans/Scert/backend/internal/testsupport/mysqltest" +) + +func TestRecoveryDispositionMySQLReal(t *testing.T) { + server := mysqltest.New(t) + for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionRecord, deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} { + for _, auditFailure := range []bool{false, true} { + name := string(action) + if auditFailure { + name += "/audit_failure" + } + t.Run(name, func(t *testing.T) { + cfg := server.Database(t) + db, err := database.Open(t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + if err := database.Migrate(t.Context(), db.DB, cfg.Driver); err != nil { + t.Fatal(err) + } + repo, raw, intent := seedRecoveryDisposition(t, NewSQLDeploymentTransactionRepository(db.DB, cfg.Driver), db.DB, action) + if _, created, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil || !created { + t.Fatal("start failed", err) + } + if _, created, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil || created { + t.Fatal("retry authorized replay", err) + } + changed := intent + changed.Operator = "unix:1" + if _, _, err := repo.BeginRecoveryDisposition(t.Context(), changed); !errors.Is(err, deployment.ErrRecoveryDispositionConflict) { + t.Fatal("conflicting replay accepted", err) + } + changed = intent + changed.ID, changed.RequestKey = strings.Repeat("e", 64), strings.Repeat("f", 64) + if _, _, err := repo.BeginRecoveryDisposition(t.Context(), changed); !errors.Is(err, deployment.ErrRecoveryDispositionBusy) { + t.Fatal("duplicate claim accepted", err) + } + result, phase := deployment.RecoveryResultObserved, deployment.TransactionPhaseInstalled + if action == deployment.RecoveryActionCommit { + result, phase = deployment.RecoveryResultCommitted, deployment.TransactionPhaseCommitVerified + } + if action == deployment.RecoveryActionRollback { + result, phase = deployment.RecoveryResultRolledBack, deployment.TransactionPhaseRollbackVerified + } + if _, err := raw.ExecContext(t.Context(), `UPDATE deployment_transactions SET phase = ? WHERE id = ?`, phase, intent.TransactionID); err != nil { + t.Fatal(err) + } + if auditFailure { + if _, err := raw.ExecContext(t.Context(), `CREATE TRIGGER reject_finish BEFORE INSERT ON recovery_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'injected'`); err != nil { + t.Fatal(err) + } + } + item, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, result) + var status, storedPhase string + if readErr := raw.QueryRowContext(t.Context(), `SELECT status, phase FROM deployment_transactions WHERE id = ?`, intent.TransactionID).Scan(&status, &storedPhase); readErr != nil { + t.Fatal(readErr) + } + if auditFailure { + if err == nil || status != deployment.TransactionStatusRecoveryRequired || storedPhase != phase { + t.Fatal("audit failure committed partial outcome") + } + stored, err := repo.GetRecoveryDisposition(t.Context(), intent.ID) + if err != nil || stored.Status != "running" { + t.Fatal("failed completion changed disposition", err) + } + recoveryCount(t, raw, "recovery_events", 1) + recoveryCount(t, raw, "recovery_claims", 1) + } else { + if err != nil || item.Status != "succeeded" { + t.Fatal("completion failed", err) + } + if action != deployment.RecoveryActionRecord && (status != result || storedPhase != result) { + t.Fatal("original terminal state missing") + } + if action == deployment.RecoveryActionRecord && (status != deployment.TransactionStatusRecoveryRequired || storedPhase != phase) { + t.Fatal("observation changed original") + } + if _, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, result); err != nil { + t.Fatal("completion retry failed", err) + } + recoveryCount(t, raw, "recovery_events", 2) + recoveryCount(t, raw, "recovery_claims", 0) + } + }) + } + } +} diff --git a/backend/internal/repository/recovery_disposition_repository.go b/backend/internal/repository/recovery_disposition_repository.go new file mode 100644 index 0000000..2290df6 --- /dev/null +++ b/backend/internal/repository/recovery_disposition_repository.go @@ -0,0 +1,133 @@ +package repository + +import ( + "context" + "database/sql" + "errors" + "time" + + "git.pcux.cn/xuans/Scert/backend/internal/deployment" +) + +var _ deployment.RecoveryDispositionJournal = (*SQLDeploymentTransactionRepository)(nil) + +const recoveryDispositionColumns = `id, transaction_id, server_id, operator_id, action, request_key, + record_digest, server_digest, observation_digest, status, result_code, created_at, updated_at` + +func scanRecoveryDisposition(row deploymentTransactionScanner) (deployment.RecoveryDisposition, error) { + var item deployment.RecoveryDisposition + i := &item.Intent + err := row.Scan(&i.ID, &i.TransactionID, &i.ServerID, &i.Operator, &i.Action, &i.RequestKey, + &i.RecordDigest, &i.ServerDigest, &i.ObservationDigest, &item.Status, &item.ResultCode, &item.CreatedAt, &item.UpdatedAt) + return item, err +} + +func dispositionError(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + return deployment.ErrRecoveryDispositionUnavailable +} + +func (r *SQLDeploymentTransactionRepository) GetRecoveryDisposition(ctx context.Context, id string) (deployment.RecoveryDisposition, error) { + if r == nil || r.db == nil { + return deployment.RecoveryDisposition{}, dispositionError(ctx) + } + item, err := scanRecoveryDisposition(r.db.QueryRowContext(ctx, r.query(`SELECT `+recoveryDispositionColumns+` FROM recovery_dispositions WHERE id = ?`), id)) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + err = dispositionError(ctx) + } + return item, err +} + +func (r *SQLDeploymentTransactionRepository) HasRecoveryClaim(ctx context.Context, transactionID string) (bool, error) { + if r == nil || r.db == nil { + return false, dispositionError(ctx) + } + var id string + err := r.db.QueryRowContext(ctx, r.query(`SELECT disposition_id FROM recovery_claims WHERE transaction_id = ?`), transactionID).Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, dispositionError(ctx) + } + return true, nil +} + +func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx context.Context, intent deployment.RecoveryDispositionIntent) (deployment.RecoveryDisposition, bool, error) { + if err := intent.Validate(); err != nil { + return deployment.RecoveryDisposition{}, false, err + } + if r == nil || r.db == nil { + return deployment.RecoveryDisposition{}, false, dispositionError(ctx) + } + // 重试先读取既有结果；唯一约束负责并发首次请求，不能靠进程内缓存判定幂等。 + if existing, err := r.findRecoveryRequest(ctx, intent.RequestKey); err == nil { + return recoveryRetry(existing, intent) + } else if !errors.Is(err, sql.ErrNoRows) { + return deployment.RecoveryDisposition{}, false, dispositionError(ctx) + } + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return deployment.RecoveryDisposition{}, false, dispositionError(ctx) + } + defer tx.Rollback() + now := time.Now().UTC() + item := deployment.RecoveryDisposition{Intent: intent, Status: "running", CreatedAt: now, UpdatedAt: now} + _, err = tx.ExecContext(ctx, r.query(`INSERT INTO recovery_dispositions (`+recoveryDispositionColumns+`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`), + intent.ID, intent.TransactionID, intent.ServerID, intent.Operator, intent.Action, intent.RequestKey, + intent.RecordDigest, intent.ServerDigest, intent.ObservationDigest, item.Status, "", now, now) + if err != nil { + _ = tx.Rollback() + if existing, readErr := r.findRecoveryRequest(ctx, intent.RequestKey); readErr == nil { + return recoveryRetry(existing, intent) + } + return deployment.RecoveryDisposition{}, false, dispositionError(ctx) + } + if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_claims (transaction_id, disposition_id, created_at) VALUES (?, ?, ?)`), intent.TransactionID, intent.ID, now); err != nil { + _ = tx.Rollback() + if claimed, readErr := r.HasRecoveryClaim(ctx, intent.TransactionID); readErr == nil && claimed { + return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionBusy + } + return deployment.RecoveryDisposition{}, false, dispositionError(ctx) + } + record, err := r.lockRecoveryTransaction(ctx, tx, intent.TransactionID) + if err != nil { + return deployment.RecoveryDisposition{}, false, dispositionError(ctx) + } + if !intent.MatchesRecord(record) { + return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryReviewChanged + } + if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_events (disposition_id, event_type, result_code, created_at) VALUES (?, 'started', '', ?)`), intent.ID, now); err != nil { + return deployment.RecoveryDisposition{}, false, dispositionError(ctx) + } + if err := tx.Commit(); err != nil { + // 提交结果不确定时不得授权动作；下次凭同一请求键读取持久记录。 + return deployment.RecoveryDisposition{}, false, dispositionError(ctx) + } + return item, true, nil +} + +func recoveryRetry(existing deployment.RecoveryDisposition, intent deployment.RecoveryDispositionIntent) (deployment.RecoveryDisposition, bool, error) { + if !existing.Intent.SameRequest(intent) { + return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionConflict + } + return existing, false, nil +} + +func (r *SQLDeploymentTransactionRepository) findRecoveryRequest(ctx context.Context, key string) (deployment.RecoveryDisposition, error) { + return scanRecoveryDisposition(r.db.QueryRowContext(ctx, r.query(`SELECT `+recoveryDispositionColumns+` FROM recovery_dispositions WHERE request_key = ?`), key)) +} + +func (r *SQLDeploymentTransactionRepository) lockRecoveryTransaction(ctx context.Context, tx *sql.Tx, id string) (deployment.TransactionRecord, error) { + query := `SELECT id, task_id, attempt_id, attempt_number, result_index, + server_id, server_name, certificate_path, certificate_chain_path, private_key_path, + temporary_paths_json, installed_paths_json, backups_json, lock_paths_json, + reload_attempted, phase, status, message, updated_at, recovery_evidence_json + FROM deployment_transactions WHERE id = ?` + if r.driver != "sqlite" { + query += " FOR UPDATE" + } + return scanDeploymentTransaction(tx.QueryRowContext(ctx, r.query(query), id)) +} diff --git a/backend/internal/repository/recovery_disposition_repository_test.go b/backend/internal/repository/recovery_disposition_repository_test.go new file mode 100644 index 0000000..62f23ed --- /dev/null +++ b/backend/internal/repository/recovery_disposition_repository_test.go @@ -0,0 +1,257 @@ +package repository + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + + "git.pcux.cn/xuans/Scert/backend/internal/deployment" +) + +func recoveryDispositionFixture(t *testing.T, action deployment.RecoveryAction) (*SQLDeploymentTransactionRepository, *sql.DB, deployment.RecoveryDispositionIntent) { + t.Helper() + _, db := newTaskRepositoryTest(t) + t.Cleanup(func() { _ = db.Close() }) + repo := NewSQLDeploymentTransactionRepository(db) + return seedRecoveryDisposition(t, repo, db, action) +} + +func seedRecoveryDisposition(t *testing.T, repo *SQLDeploymentTransactionRepository, db *sql.DB, action deployment.RecoveryAction) (*SQLDeploymentTransactionRepository, *sql.DB, deployment.RecoveryDispositionIntent) { + t.Helper() + record := deployment.TransactionRecord{ + ID: "recovery-transaction", TaskID: "recovery-task", AttemptID: "recovery-attempt", AttemptNumber: 1, + ServerID: "recovery-server", ServerName: "server", Paths: []string{"/cert.pem", "/chain.pem", "/key.pem"}, + Phase: deployment.TransactionPhaseInstalled, Status: deployment.TransactionStatusRecoveryRequired, + } + if action != deployment.RecoveryActionRecord { + record.Evidence = &deployment.RecoveryEvidence{Version: 1, LockToken: "fixture-secret-not-for-disposition"} + } + if err := repo.CreateDeploymentTransaction(t.Context(), record); err != nil { + t.Fatal(err) + } + stored, err := repo.ListPendingDeploymentTransactions(t.Context()) + if err != nil || len(stored) != 1 { + t.Fatal("missing fixture transaction", err) + } + encoded, err := json.Marshal(stored[0]) + if err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(encoded) + intent := deployment.RecoveryDispositionIntent{ + ID: strings.Repeat("a", 64), TransactionID: record.ID, ServerID: record.ServerID, Operator: "unix:0", Action: action, + RequestKey: strings.Repeat("b", 64), RecordDigest: hex.EncodeToString(digest[:]), + ServerDigest: strings.Repeat("c", 64), ObservationDigest: strings.Repeat("d", 64), + } + return repo, db, intent +} + +func recoveryCount(t *testing.T, db *sql.DB, table string, want int) { + t.Helper() + var count int + if err := db.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM "+table).Scan(&count); err != nil || count != want { + t.Fatalf("%s rows: got %d want %d: %v", table, count, want, err) + } +} + +func TestRecoveryDispositionRecordLifecycle(t *testing.T) { + repo, db, intent := recoveryDispositionFixture(t, deployment.RecoveryActionRecord) + before, _ := repo.ListPendingDeploymentTransactions(t.Context()) + item, created, err := repo.BeginRecoveryDisposition(t.Context(), intent) + if err != nil || !created || item.Status != "running" { + t.Fatal("begin failed", err) + } + for _, table := range []string{"recovery_dispositions", "recovery_claims", "recovery_events"} { + recoveryCount(t, db, table, 1) + } + reopened := NewSQLDeploymentTransactionRepository(db) + if claimed, err := reopened.HasRecoveryClaim(t.Context(), intent.TransactionID); err != nil || !claimed { + t.Fatal("claim lost across repository restart", err) + } + if item, created, err := reopened.BeginRecoveryDisposition(t.Context(), intent); err != nil || created || item.Intent.ID != intent.ID { + t.Fatal("retry may not authorize a second execution", err) + } + for range 2 { + item, err = reopened.FinishRecoveryDisposition(t.Context(), intent.ID, deployment.RecoveryResultObserved) + if err != nil || item.Status != "succeeded" || item.ResultCode != deployment.RecoveryResultObserved { + t.Fatal("completion/retry failed", err) + } + } + recoveryCount(t, db, "recovery_claims", 0) + recoveryCount(t, db, "recovery_events", 2) + after, _ := repo.ListPendingDeploymentTransactions(t.Context()) + beforeJSON, _ := json.Marshal(before) + afterJSON, _ := json.Marshal(after) + if string(beforeJSON) != string(afterJSON) { + t.Fatal("recording observation modified original transaction or manufactured evidence") + } + if _, created, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil || created { + t.Fatal("completed request replayed", err) + } + if _, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, deployment.RecoveryResultUncertain); !errors.Is(err, deployment.ErrRecoveryDispositionConflict) { + t.Fatal("changed terminal outcome accepted", err) + } +} + +func TestRecoveryDispositionRequestBinding(t *testing.T) { + for _, change := range []string{"transaction", "server", "operator", "action", "record", "configuration", "observation"} { + t.Run(change, func(t *testing.T) { + repo, db, intent := recoveryDispositionFixture(t, deployment.RecoveryActionRecord) + if _, _, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil { + t.Fatal(err) + } + changed := intent + switch change { + case "transaction": + changed.TransactionID = "different" + case "server": + changed.ServerID = "different" + case "operator": + changed.Operator = "unix:1" + case "action": + changed.Action = deployment.RecoveryActionCommit + case "record": + changed.RecordDigest = strings.Repeat("e", 64) + case "configuration": + changed.ServerDigest = strings.Repeat("e", 64) + case "observation": + changed.ObservationDigest = strings.Repeat("e", 64) + } + if _, created, err := repo.BeginRecoveryDisposition(t.Context(), changed); !errors.Is(err, deployment.ErrRecoveryDispositionConflict) || created { + t.Fatal("conflicting idempotency key accepted", err) + } + changed = intent + changed.ID = strings.Repeat("f", 64) + if item, created, err := repo.BeginRecoveryDisposition(t.Context(), changed); err != nil || created || item.Intent.ID != intent.ID { + t.Fatal("same request must return original ID", err) + } + recoveryCount(t, db, "recovery_dispositions", 1) + }) + } +} + +func TestRecoveryDispositionBeginFailuresAreAtomic(t *testing.T) { + for _, failure := range []string{"recovery_dispositions", "recovery_claims", "recovery_events", "record_changed", "cancelled"} { + t.Run(failure, func(t *testing.T) { + repo, db, intent := recoveryDispositionFixture(t, deployment.RecoveryActionRecord) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + switch failure { + case "record_changed": + if _, err := db.ExecContext(t.Context(), `UPDATE deployment_transactions SET message = 'changed'`); err != nil { + t.Fatal(err) + } + case "cancelled": + cancel() + default: + if _, err := db.ExecContext(t.Context(), `CREATE TRIGGER reject_begin BEFORE INSERT ON `+failure+` BEGIN SELECT RAISE(ABORT, 'injected'); END`); err != nil { + t.Fatal(err) + } + } + if _, created, err := repo.BeginRecoveryDisposition(ctx, intent); err == nil || created { + t.Fatal("failed start authorized execution") + } + for _, table := range []string{"recovery_dispositions", "recovery_claims", "recovery_events"} { + recoveryCount(t, db, table, 0) + } + }) + } +} + +func TestRecoveryDispositionUncertainKeepsClaim(t *testing.T) { + repo, db, intent := recoveryDispositionFixture(t, deployment.RecoveryActionCommit) + if _, _, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil { + t.Fatal(err) + } + if _, err := repo.FinishRecoveryDisposition(t.Context(), intent.ID, deployment.RecoveryResultUncertain); err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(t.Context(), `UPDATE recovery_claims SET created_at = '2000-01-01 00:00:00'`); err != nil { + t.Fatal(err) + } + reopened := NewSQLDeploymentTransactionRepository(db) + intent.ID, intent.RequestKey = strings.Repeat("e", 64), strings.Repeat("f", 64) + if _, created, err := reopened.BeginRecoveryDisposition(t.Context(), intent); !errors.Is(err, deployment.ErrRecoveryDispositionBusy) || created { + t.Fatal("old uncertain claim must not auto-expire", err) + } + recoveryCount(t, db, "recovery_dispositions", 1) + recoveryCount(t, db, "recovery_claims", 1) + recoveryCount(t, db, "recovery_events", 2) +} + +func TestRecoveryDispositionConcurrentRequests(t *testing.T) { + for _, sameKey := range []bool{true, false} { + t.Run(fmt.Sprint(sameKey), func(t *testing.T) { + repo, db, intent := recoveryDispositionFixture(t, deployment.RecoveryActionRecord) + db.SetMaxOpenConns(1) + var started atomic.Int32 + var wg sync.WaitGroup + for i := range 8 { + wg.Add(1) + go func() { + defer wg.Done() + candidate := intent + if !sameKey { + candidate.ID, candidate.RequestKey = fmt.Sprintf("%064x", i+1), fmt.Sprintf("%064x", i+10) + } + _, created, err := repo.BeginRecoveryDisposition(t.Context(), candidate) + if created { + started.Add(1) + } + if sameKey && err != nil { + t.Errorf("same request failed: %v", err) + } + if !sameKey && err != nil && !errors.Is(err, deployment.ErrRecoveryDispositionBusy) { + t.Errorf("unexpected competing request error: %v", err) + } + }() + } + wg.Wait() + if started.Load() != 1 { + t.Fatal("multiple remote executions authorized") + } + for _, table := range []string{"recovery_dispositions", "recovery_claims", "recovery_events"} { + recoveryCount(t, db, table, 1) + } + }) + } +} + +func TestRecoveryDispositionClaimSurvivesDatabaseReopen(t *testing.T) { + repo, db, intent := recoveryDispositionFixture(t, deployment.RecoveryActionCommit) + if _, _, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil { + t.Fatal(err) + } + var sequence int + var name, path string + if err := db.QueryRowContext(t.Context(), "PRAGMA database_list").Scan(&sequence, &name, &path); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + reopened, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + repo = NewSQLDeploymentTransactionRepository(reopened) + if claimed, err := repo.HasRecoveryClaim(t.Context(), intent.TransactionID); err != nil || !claimed { + t.Fatal("incomplete operation lost its persistent claim", err) + } + if existing, created, err := repo.BeginRecoveryDisposition(t.Context(), intent); err != nil || created || existing.Status != "running" { + t.Fatal("reopened request must not authorize replay", err) + } + intent.ID, intent.RequestKey = strings.Repeat("e", 64), strings.Repeat("f", 64) + if _, _, err := repo.BeginRecoveryDisposition(t.Context(), intent); !errors.Is(err, deployment.ErrRecoveryDispositionBusy) { + t.Fatal("restart released unfinished claim", err) + } +} diff --git a/doc/Linux生产流程测试记录.md b/doc/Linux生产流程测试记录.md index c52c66e..625ebe2 100644 --- a/doc/Linux生产流程测试记录.md +++ b/doc/Linux生产流程测试记录.md @@ -1,5 +1,19 @@ # Linux 生产流程测试记录 +## 2026年10月3日19:43 自动回归及实机边界 + +本节覆盖先前仍标运行中的结果；这些是第一台隔离目录中的原生 Linux Go 回归，不是正式候选包验收。未清理生产数据库、未手工改历史事务、未创建辅助验收脚本。 + +- [x] `final-full-http-race.log`：R2 最新后端全量退出码零；当时 HTTP 包四分钟累计超时，不能将当次整组记为通过。 +- [x] `review-race-vet.log`：加入 R3 准备绑定后，部署、本机协议、管理员终端三包竞态、完整 `go vet -p 1 ./...`、HTTP 全包竞态分别退出码零；HTTP 约 374.825 秒；整组 `REVIEW_RUN_EXIT=0`。 +- [x] `disposition-race-vet.log`：仓储约 104.544 秒、数据库约 133.013 秒、部署约 3.103 秒，三包竞态通过；完整静态检查通过；整组退出码零。测试单元限制内存 900 MiB、CPU 75%，以 nobody 运行。 +- [~] 后续新增数据库关闭重开、准备摘要转换及 MySQL 实库专项，尚未取得结果；不能由上一批结果覆盖这些新增测试。 +- [x] 第二台实机复核：根分区 29 GB，使用 2.9 GB，可用 26 GB，使用率 10%；五项既有服务 active，Nginx 配置检查通过。本轮没有扩大删除范围。 +- [ ] 正式候选包、三数据库首次 Web、系统邮件、真实 DNS/ACME、公共 HTTPS、多对多推送和失败恢复仍需单独验收。 + +自动回归证据目录为第一台 `/var/tmp/scert-local-recovery-20261003-1828/`；第二台继续只用于证书推送测试。 + + ## 2026年10月3日19:02 本机核查边界回归 第一台 `8.137.23.95` 隔离目录 `/var/tmp/scert-local-recovery-20261003-1828/`；不使用第二台运行本轮回归，不清理业务数据库或恢复现场。 diff --git a/doc/Scert开发任务清单.md b/doc/Scert开发任务清单.md index 2ab1e99..254620f 100644 --- a/doc/Scert开发任务清单.md +++ b/doc/Scert开发任务清单.md @@ -1,5 +1,15 @@ # Scert 开发任务清单 +## 2026年10月3日19:43 当前开发进度 + +- [x] R3 只读准备与重新核验基础已编码并通过 Linux 三包竞态；HTTP 全包竞态与完整静态检查通过。原后端最新全量也已通过。 +- [x] R4 独立处置记录、唯一占用、幂等请求和原子开始/结束审计已编码；不确定结果保留占用，记录核查不改原事务。 +- [x] 本轮持久化第一轮仓储、数据库、部署三包竞态和完整静态检查通过。 +- [~] 数据库关闭重开、准备摘要转换与 MySQL 隔离实库专项正在补测；三数据库迁移文件已加入，不等于三数据库实库通过。 +- [ ] 一次性本机确认、完整执行互斥、动作执行和本机终端接入仍未完成；R3/R4 整体不能标完成。 +- [ ] 正式 Linux 候选包部署到第一台，第二台仅测试推送；生产门禁继续保留。 + + ## 2026年10月3日19:02 当前开发进度 本节优先于历史章节。第一台用于部署，第二台仅用于推送；仅处理 Linux。 diff --git a/doc/Scert整改任务执行清单.md b/doc/Scert整改任务执行清单.md index b4af5ca..5d076eb 100644 --- a/doc/Scert整改任务执行清单.md +++ b/doc/Scert整改任务执行清单.md @@ -1,5 +1,15 @@ # Scert 整改任务执行清单 +## 2026年10月3日19:43 当前执行状态 + +- [x] 修改前检查点 `ed0c97b` 已提交并推送，根目录 `=` 未暂存、未删除。 +- [x] 最新准备绑定的三包竞态、完整静态检查及 HTTP 全包竞态通过；此前四分钟包累计超时已由十分钟有界回归获得真实通过结果，未跳过用例。 +- [x] 持久化基础及第一轮仓储、数据库、部署三包竞态、完整静态检查通过；原子审计故障、幂等冲突、并发唯一占用及不确定保留占用有回归覆盖。 +- [~] 补测数据库重开与 MySQL DDL 中断和实库处置审计；未把默认跳过的外部实库测试当作通过。 +- [ ] 接入一次性确认、完整互斥与真实动作执行，再推进正式候选包和生产门禁。 +- [~] 第二台已复核可用约 26 GB，未扩大清理；第一台仅运行受限隔离 Go 回归，未修改线上业务数据库或恢复终态。 + + ## 2026年10月3日19:02 当前执行状态 本节优先于历史章节。第一台部署、第二台推送，Linux 范围不变。 diff --git a/doc/Scert整改合并优化清单.md b/doc/Scert整改合并优化清单.md index e212f57..cb8f25a 100644 --- a/doc/Scert整改合并优化清单.md +++ b/doc/Scert整改合并优化清单.md @@ -1,5 +1,19 @@ # Scert 整改、合并与优化清单 +## 2026年10月3日19:43 最新执行进度 + +本节覆盖下方历史状态。范围仍为 Linux；第一台部署测试，第二台仅证书推送，Smail 地址为 `web.cdis.cn`。状态标记：[ ] 未开始；[~] 进行中；[x] 已完成等待复审；[✓] 复审通过；[!] 阻塞。 + +- [x] R2 后端全量通过；R3 准备绑定加入后的部署、本机协议、管理员终端三包竞态、完整静态检查和 HTTP 全包竞态通过。HTTP 约 375 秒，整组退出码零；此前四分钟累计超时不再是当前阻塞。 +- [x] R3 准备与重新核验基础通过原生 Linux 竞态：操作者、动作、事务、配置和现场摘要绑定；一分钟有效期；缺历史证据拒绝写动作准备。它不是执行授权，不能把 R3 整体标完成。 +- [x] R4 持久化基础已编码：独立处置记录、事务唯一占用、开始与结束审计、幂等冲突拒绝；开始和结束分别原子落库。不确定结果保留占用，不按时间自动释放。 +- [x] 第一轮仓储、数据库、部署三包竞态与后端完整静态检查通过；这里只包含隔离自动回归，不代表 MySQL/PostgreSQL 实库或正式安装流程通过。 +- [~] 补充数据库关闭重开、准备摘要转换和 MySQL 实库 DDL 中断、审计失败专项；结果待取得。迁移为 22—24，每版仅一个表，复用 MySQL 严格结构核验，不改旧迁移回填行为。 +- [ ] R3/R4 仍需一次性本机确认、执行器接入、普通部署与恢复的完整互斥、动作专属核验及最终原子结果联动；存储方法不对公网开放。 +- [ ] R5 正式候选包、三数据库首次 Web、系统邮件与密码重置、DNS/ACME、严格公共 HTTPS、多对多部署及失败/重启恢复仍未完成。 +- [~] 修改前检查点 `ed0c97b` 已推送；总体生产验收未通过，不能承诺已具备上线条件。 + + ## 2026年10月3日19:02 最新进度 本节为最新状态，后续章节仅保留历史过程。第一台用于部署测试，第二台仅用于推送测试，Smail 为 `web.cdis.cn`，只处理 Linux。 diff --git a/doc/部署事务人工处置设计.md b/doc/部署事务人工处置设计.md index 8fff0b5..76157ae 100644 --- a/doc/部署事务人工处置设计.md +++ b/doc/部署事务人工处置设计.md @@ -1,5 +1,18 @@ # 部署事务人工处置设计 +## 2026年10月3日19:43 持久化基础实现状态 + +- [x] 准备与重新核验基础已通过原生 Linux 竞态；HTTP 全包竞态与完整静态检查通过，准备对象仍不代表执行授权。 +- [x] 新增迁移 22—24：独立处置记录、事务唯一占用、开始/结束事件。每版仅一个表，兼容现有 MySQL DDL 中断后结构校验机制，不影响迁移 15 的专用回填。 +- [x] 开始操作在同一事务内保存请求绑定、占用和开始事件，并核对原事务摘要。相同请求返回原记录且不重新授权；同一幂等键不同绑定拒绝。 +- [x] 结束操作在同一事务内保存处置结果、必要的原事务终态和结束事件；任一步失败整体回滚。只读核查不改原事务；不确定结果保留占用；仅允许已核验阶段写入对应提交/回滚终态。 +- [x] 第一轮仓储、数据库、部署三包竞态及完整静态检查通过；故障注入覆盖开始写入、结束结果、原事务、审计及占用释放。 +- [~] 数据库关闭重开、准备摘要转换、MySQL 实库迁移和审计测试补测中。 +- [ ] 尚未接入一次性确认、本机协议和完整生命周期互斥；尚未实现动作专属执行器。这些仓储接口仅是内部持久化能力，不是用户可用的提交或回滚入口。 + +本轮没有替代 R3/R4 整体目标：历史阶段只是存储层防误用条件，不能单独证明本次人工处置做过远端检查。执行器必须在持久化占用和完整互斥内重新执行身份、文件、锁及严格 HTTPS 核验；不能直接利用历史已核验阶段跳过现场验证。 + + ## 2026年10月3日 设计与任务拆分 本文件对应《部署事务人工核查任务清单》R3、R4，是待实现的约束和执行清单，不是已完成能力或验收结论。范围仅限 Linux：第一台部署 Scert，第二台作为证书推送目标。 diff --git a/doc/部署事务人工核查任务清单.md b/doc/部署事务人工核查任务清单.md index eb32fb2..2f7ed4a 100644 --- a/doc/部署事务人工核查任务清单.md +++ b/doc/部署事务人工核查任务清单.md @@ -1,5 +1,15 @@ # 部署事务人工核查任务清单 +## 2026年10月3日19:43 最新任务状态 + +- [x] R1/R2 底层及本机入口的最新 Linux 三包竞态、完整静态检查、HTTP 全包竞态通过；仍待正式包复审。 +- [~] R3-B/R3-C：准备绑定及重新核验基础通过；一次性确认凭据、本机终端和动作执行尚未接入。 +- [~] R3-D/R4-A/R4-B：独立处置表、占用表、事件表和仓储已编码；原子开始/结束、幂等冲突、不确定占用保留的第一轮竞态通过。完整部署生命周期互斥尚未接入。 +- [~] R4-C/R4-D：故障注入第一轮通过；补测真实数据库重开、MySQL DDL 中断与实库审计。PostgreSQL 实库尚无本轮证据。 +- [ ] R3-E/R5：真实动作资格、执行链路、正式候选包和双机生产闭环未完成。 +- [~] R6：同步总清单、开发清单、执行清单、设计和 Linux 测试记录；生产验收保持未通过。 + + ## 2026年10月3日19:02 边界整改结果 本节优先于历史记录。