补充重新核查实库持久化测试并同步整改进度

This commit is contained in:
xuans committed 2026-10-04 00:01:25 +08:00
1 parent f9cd19f631
commit c1faebf9ef
12 files changed
+311 -60

No files matched your search

@@ -23,10 +23,16 @@ func (c *reconciliationConsoleClient) PrepareRecoveryReconciliation(sourceID str
}
func (c *reconciliationConsoleClient) ExecuteRecovery(id, token string) (deployment.RecoveryDispositionSummary, error) {
value, err := c.dispositionConsoleClient.ExecuteRecovery(id, token)
if err != nil { return value, err }
if err != nil {
return value, err
}
value.SourceDispositionID = c.sourceID
if value.Status == "succeeded" { value.ResultCode = "committed" }
if c.mismatch { value.SourceDispositionID = strings.Repeat("d", 64) }
if value.Status == "succeeded" {
value.ResultCode = "committed"
}
if c.mismatch {
value.SourceDispositionID = strings.Repeat("d", 64)
}
return value, nil
}
@@ -37,26 +43,45 @@ func TestRecoveryReconciliationConsoleConfirmation(t *testing.T) {
client := &reconciliationConsoleClient{}
answer := "确认重新核查 " + sourceID + " " + dispositionActionLabel(deployment.RecoveryActionVerifyCommit)
switch mode {
case "rejected": client.status = "rejected"
case "cancelled": answer = "确认"
case "expired": client.expired = true
case "disconnect": client.executeError = true
case "mismatch": client.mismatch = true
case "rejected":
client.status = "rejected"
case "cancelled":
answer = "确认"
case "expired":
client.expired = true
case "disconnect":
client.executeError = true
case "mismatch":
client.mismatch = true
}
screen := &recoveryTestScreen{testTerminalInput: testTerminalInput{values: []string{sourceID, answer}}}
err := recoveryReconciliationFlow(t.Context(), client, screen)
output := strings.Join(screen.lines, "|")
if client.prepares != 1 || client.queries != 0 || strings.Contains(output, strings.Repeat("c", 64)) { t.Fatal("终端重复调用或泄露令牌") }
if client.prepares != 1 || client.queries != 0 || strings.Contains(output, strings.Repeat("c", 64)) {
t.Fatal("终端重复调用或泄露令牌")
}
if mode == "cancelled" || mode == "expired" {
if client.executes != 0 { t.Fatal("取消或过期仍执行核查") }
if client.executes != 0 {
t.Fatal("取消或过期仍执行核查")
}
return
}
if client.executes != 1 || !strings.Contains(output, "原不确定记录永久保留") { t.Fatal("缺少明确警告或重复执行") }
if client.executes != 1 || !strings.Contains(output, "原不确定记录永久保留") {
t.Fatal("缺少明确警告或重复执行")
}
if mode == "disconnect" || mode == "mismatch" {
if err == nil || !strings.Contains(output, "新核查编号查询") { t.Fatal("失去可靠响应后误报成功") }
} else if err != nil { t.Fatal(err) }
if mode == "success" && !strings.Contains(output, "占用已安全释放") { t.Fatal("缺少完成结果") }
if mode == "rejected" && !strings.Contains(output, "原占用继续保留") { t.Fatal("错误显示拒绝结果") }
if err == nil || !strings.Contains(output, "新核查编号查询") {
t.Fatal("失去可靠响应后误报成功")
}
} else if err != nil {
t.Fatal(err)
}
if mode == "success" && !strings.Contains(output, "占用已安全释放") {
t.Fatal("缺少完成结果")
}
if mode == "rejected" && !strings.Contains(output, "原占用继续保留") {
t.Fatal("错误显示拒绝结果")
}
})
}
}
@@ -15,7 +15,9 @@ type localReconciliationJournal struct {
}
func (j *localReconciliationJournal) GetRecoveryDisposition(ctx context.Context, id string) (deployment.RecoveryDisposition, error) {
if id == j.source.Intent.ID { return j.source, nil }
if id == j.source.Intent.ID {
return j.source, nil
}
return j.localDispositionJournal.GetRecoveryDisposition(ctx, id)
}
@@ -31,16 +33,20 @@ func localReconciliationFixture(t *testing.T) (*Server, *localResetSession, *loc
Files: map[string]string{}, TemporaryFiles: map[string]string{}, Baseline: deployment.RecoveryBaseline{Endpoint: "https://example.test", Domains: []string{"example.test"}, ExpectedFingerprint: "new"}}
for _, path := range journal.record.Paths {
journal.record.LockPaths = append(journal.record.LockPaths, path+".scert-lock")
journal.record.Backups = append(journal.record.Backups, deployment.TransactionBackup{Original: path, Saved: path+".scert-backup-"+journal.record.ID})
journal.record.Backups = append(journal.record.Backups, deployment.TransactionBackup{Original: path, Saved: path + ".scert-backup-" + journal.record.ID})
journal.record.Evidence.Files[path] = strings.Repeat("e", 64)
}
service := deployment.NewServiceWithJournal(localDispositionFactory{}, journal)
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error { return nil }
resolve := func(context.Context, string) (deployment.Server, error) { return target, nil }
review, err := service.PrepareRecoveryReview(t.Context(), journal.record.ID, "uid:0", deployment.RecoveryActionCommit, resolve)
if err != nil { t.Fatal(err) }
if err != nil {
t.Fatal(err)
}
intent, err := review.DispositionIntent(strings.Repeat("a", 64), strings.Repeat("b", 64))
if err != nil { t.Fatal(err) }
if err != nil {
t.Fatal(err)
}
now := time.Now().UTC()
journal.source = deployment.RecoveryDisposition{Intent: intent, Status: "uncertain", ResultCode: "uncertain", CreatedAt: now, UpdatedAt: now}
server.recoveryInspection = service
@@ -52,25 +58,44 @@ func TestLocalReconciliationRejectsInvalidSession(t *testing.T) {
t.Run(mode, func(t *testing.T) {
server, session, journal := localReconciliationFixture(t)
prepared, err := session.PrepareRecoveryReconciliation(t.Context(), journal.source.Intent.ID)
if err != nil { t.Fatal(err) }
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
switch mode {
case "expired": session.expires = time.Time{}
case "cancelled": cancel()
case "changed-admin": server.config.Admin.Email = "changed@example.test"
case "closed": session.Close()
case "reset-stage": session.stage = 1
case "wrong-id": prepared.DispositionID = journal.source.Intent.ID
case "wrong-token": prepared.Token = strings.Repeat("a", 64)
case "changed-peer": session.peer.Identity = "uid:123"
case "foreign-session": session = localDispositionNewSession(t, server)
case "expired":
session.expires = time.Time{}
case "cancelled":
cancel()
case "changed-admin":
server.config.Admin.Email = "changed@example.test"
case "closed":
session.Close()
case "reset-stage":
session.stage = 1
case "wrong-id":
prepared.DispositionID = journal.source.Intent.ID
case "wrong-token":
prepared.Token = strings.Repeat("a", 64)
case "changed-peer":
session.peer.Identity = "uid:123"
case "foreign-session":
session = localDispositionNewSession(t, server)
case "replaced-confirmation":
if _, err := session.PrepareRecoveryReconciliation(ctx, journal.source.Intent.ID); err != nil { t.Fatal(err) }
if _, err := session.PrepareRecoveryReconciliation(ctx, journal.source.Intent.ID); err != nil {
t.Fatal(err)
}
}
if _, err := session.ExecuteRecovery(ctx, prepared.DispositionID, prepared.Token); err == nil {
t.Fatal("无效会话执行了重新核查")
}
if _, err := session.ExecuteRecovery(t.Context(), prepared.DispositionID, prepared.Token); err == nil {
t.Fatal("失败确认被重放")
}
if journal.begins != 0 || journal.finishes != 0 || session.recoveryConfirmation.DispositionID() != "" {
t.Fatal("无效会话写入了审计或保留了确认")
}
if _, err := session.ExecuteRecovery(ctx, prepared.DispositionID, prepared.Token); err == nil { t.Fatal("无效会话执行了重新核查") }
if _, err := session.ExecuteRecovery(t.Context(), prepared.DispositionID, prepared.Token); err == nil { t.Fatal("失败确认被重放") }
if journal.begins != 0 || journal.finishes != 0 || session.recoveryConfirmation.DispositionID() != "" { t.Fatal("无效会话写入了审计或保留了确认") }
})
}
}
@@ -78,11 +103,19 @@ func TestLocalReconciliationRejectsInvalidSession(t *testing.T) {
func TestLocalReconciliationRejectsMissingFilesAndQueriesNewAudit(t *testing.T) {
server, session, journal := localReconciliationFixture(t)
prepared, err := session.PrepareRecoveryReconciliation(t.Context(), journal.source.Intent.ID)
if err != nil { t.Fatal(err) }
if err != nil {
t.Fatal(err)
}
result, err := session.ExecuteRecovery(t.Context(), prepared.DispositionID, prepared.Token)
if err != nil || result.Status != "rejected" || result.SourceDispositionID != journal.source.Intent.ID { t.Fatalf("缺失文件仍被放行:%+v %v", result, err) }
if journal.item.Intent.Operator != "uid:0" || journal.begins != 1 || journal.finishes != 1 || journal.source.Status != "uncertain" { t.Fatal("身份或历史记录被改变") }
if err != nil || result.Status != "rejected" || result.SourceDispositionID != journal.source.Intent.ID {
t.Fatalf("缺失文件仍被放行:%+v %v", result, err)
}
if journal.item.Intent.Operator != "uid:0" || journal.begins != 1 || journal.finishes != 1 || journal.source.Status != "uncertain" {
t.Fatal("身份或历史记录被改变")
}
next := localDispositionNewSession(t, server)
queried, err := next.GetRecoveryDisposition(t.Context(), result.ID)
if err != nil || queried != result { t.Fatal("重连未查到关联审计", err) }
if err != nil || queried != result {
t.Fatal("重连未查到关联审计", err)
}
}
@@ -9,7 +9,7 @@ import (
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
)
type reconciliationSession struct { dispositionTestSession }
type reconciliationSession struct{ dispositionTestSession }
func (s *reconciliationSession) PrepareRecoveryReconciliation(ctx context.Context, sourceID string) (RecoveryPreparation, error) {
value, err := s.PrepareRecovery(ctx, "tx-1", deployment.RecoveryActionVerifyCommit)
@@ -22,7 +22,9 @@ func (s *reconciliationSession) PrepareRecoveryReconciliation(ctx context.Contex
func (s *reconciliationSession) ExecuteRecovery(ctx context.Context, id, token string) (deployment.RecoveryDispositionSummary, error) {
sourceID := s.prepared.Summary.SourceDispositionID
value, err := s.dispositionTestSession.ExecuteRecovery(ctx, id, token)
if err != nil { return value, err }
if err != nil {
return value, err
}
value.SourceDispositionID, value.ResultCode = sourceID, deployment.RecoveryResultCommitted
s.result = value
return value, nil
@@ -32,51 +34,81 @@ func TestRecoveryReconciliationProtocolAndQuery(t *testing.T) {
session := &reconciliationSession{}
_, conn, done := recoveryConnection(t, session)
client, err := Begin(t.Context(), conn)
if err != nil { t.Fatal(err) }
if err != nil {
t.Fatal(err)
}
sourceID := strings.Repeat("a", 64)
prepared, err := client.PrepareRecoveryReconciliation(sourceID)
if err != nil || prepared.Summary.SourceDispositionID != sourceID { t.Fatal("原处置绑定丢失", err) }
if err != nil || prepared.Summary.SourceDispositionID != sourceID {
t.Fatal("原处置绑定丢失", err)
}
result, err := client.ExecuteRecovery(prepared.DispositionID, prepared.Token)
if err != nil || result.SourceDispositionID != sourceID || result.Status != "succeeded" { t.Fatal("重新核查结果无效", err) }
if err != nil || result.SourceDispositionID != sourceID || result.Status != "succeeded" {
t.Fatal("重新核查结果无效", err)
}
queried, err := client.GetRecoveryDisposition(result.ID)
if err != nil || queried != result { t.Fatal("查询改变了关联结果", err) }
if _, err := client.ExecuteRecovery(prepared.DispositionID, prepared.Token); err == nil { t.Fatal("旧令牌被重放") }
if err != nil || queried != result {
t.Fatal("查询改变了关联结果", err)
}
if _, err := client.ExecuteRecovery(prepared.DispositionID, prepared.Token); err == nil {
t.Fatal("旧令牌被重放")
}
_ = client.Close()
<-done
if session.prepares.Load() != 1 || session.executes.Load() != 2 || session.queries.Load() != 1 || session.resets.Load() != 0 { t.Fatal("调用了错误业务入口") }
if session.prepares.Load() != 1 || session.executes.Load() != 2 || session.queries.Load() != 1 || session.resets.Load() != 0 {
t.Fatal("调用了错误业务入口")
}
}
func TestRecoveryReconciliationRejectsUnboundResponse(t *testing.T) {
sourceID := strings.Repeat("a", 64)
session := &reconciliationSession{}
prepared, err := session.PrepareRecoveryReconciliation(t.Context(), sourceID)
if err != nil { t.Fatal(err) }
if err != nil {
t.Fatal(err)
}
request := Request{Action: "recovery_reconcile_prepare", DispositionID: sourceID}
for _, mode := range []string{"source", "action", "same-id", "transaction", "mixed", "expired-range"} {
t.Run(mode, func(t *testing.T) {
value := prepared
response := Response{Preparation: &value}
switch mode {
case "source": value.Summary.SourceDispositionID = strings.Repeat("d", 64)
case "action": value.Summary.Action = deployment.RecoveryActionCommit
case "same-id": value.DispositionID = sourceID
case "transaction": value.Summary.TransactionID = "../bad"
case "mixed": response.Disposition = &deployment.RecoveryDispositionSummary{}
case "expired-range": value.Summary.ExpiresAt = value.Summary.ObservedAt.Add(2*time.Minute)
case "source":
value.Summary.SourceDispositionID = strings.Repeat("d", 64)
case "action":
value.Summary.Action = deployment.RecoveryActionCommit
case "same-id":
value.DispositionID = sourceID
case "transaction":
value.Summary.TransactionID = "../bad"
case "mixed":
response.Disposition = &deployment.RecoveryDispositionSummary{}
case "expired-range":
value.Summary.ExpiresAt = value.Summary.ObservedAt.Add(2 * time.Minute)
}
if validDispositionResponse(request, response) {
t.Fatal("接受了混合或无绑定准备响应")
}
if validDispositionResponse(request, response) { t.Fatal("接受了混合或无绑定准备响应") }
})
}
if validRecoveryAction(deployment.RecoveryActionVerifyCommit) || validRecoveryAction(deployment.RecoveryActionVerifyRollback) { t.Fatal("普通准备允许绕过原处置绑定") }
if validRecoveryAction(deployment.RecoveryActionVerifyCommit) || validRecoveryAction(deployment.RecoveryActionVerifyRollback) {
t.Fatal("普通准备允许绕过原处置绑定")
}
}
func TestRecoveryReconciliationOldServerFailsClosed(t *testing.T) {
session := &dispositionTestSession{}
_, conn, done := recoveryConnection(t, session)
client, err := Begin(t.Context(), conn)
if err != nil { t.Fatal(err) }
if _, err := client.PrepareRecoveryReconciliation(strings.Repeat("a", 64)); err == nil { t.Fatal("旧服务错误地支持了重新核查") }
if err != nil {
t.Fatal(err)
}
if _, err := client.PrepareRecoveryReconciliation(strings.Repeat("a", 64)); err == nil {
t.Fatal("旧服务错误地支持了重新核查")
}
_ = client.Close()
<-done
if session.prepares.Load() != 0 || session.executes.Load() != 0 { t.Fatal("旧服务错误地执行了原动作") }
if session.prepares.Load() != 0 || session.executes.Load() != 0 {
t.Fatal("旧服务错误地执行了原动作")
}
}
@@ -76,7 +76,23 @@ func runExternalRecovery(t *testing.T, fixture recoveryExternalFixture) {
t.Fatal(err)
}
runReconciliationSuccess(t, NewSQLDeploymentTransactionRepository(db, cfg.Driver), db, action)
if err := db.Close(); err != nil {
t.Fatal(err)
}
db = openRecoveryExternal(t, cfg)
recoveryCount(t, db, "recovery_dispositions", 2)
recoveryCount(t, db, "recovery_events", 4)
recoveryCount(t, db, "recovery_claims", 0)
pending, err := NewSQLDeploymentTransactionRepository(db, cfg.Driver).ListPendingDeploymentTransactions(t.Context())
if err != nil || len(pending) != 0 {
t.Fatal("重新打开实库后成功收尾丢失", err)
}
})
for _, stage := range []string{"started", "finished"} {
t.Run("reconciliation_audit/"+string(action)+"/"+stage, func(t *testing.T) {
runExternalReconciliationAuditFailure(t, fixture, action, stage)
})
}
}
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionRecord, deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
for _, stage := range []string{"none", "started", "finished"} {
@@ -0,0 +1,73 @@
package repository
import (
"context"
"errors"
"maps"
"reflect"
"testing"
"git.pcux.cn/xuans/Scert/backend/internal/database"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
)
// 重开真实隔离数据库,验证审计失败不丢失占用,也不将只读核查变成远端重放。
func runExternalReconciliationAuditFailure(t *testing.T, fixture recoveryExternalFixture, action deployment.RecoveryAction, stage string) {
t.Helper()
cfg := fixture.database(t)
db := openRecoveryExternal(t, cfg)
if err := database.Migrate(t.Context(), db, cfg.Driver); err != nil {
t.Fatal(err)
}
repo := NewSQLDeploymentTransactionRepository(db, cfg.Driver)
service, client, server, record, source := seedUncertainReconciliation(t, repo, db, action)
files := maps.Clone(client.files)
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
confirmation, token, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve)
if err != nil {
t.Fatal(err)
}
fixture.inject(t, db, cfg, stage)
result, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
if !errors.Is(err, deployment.ErrRecoveryDispositionUnavailable) || result.Status == "succeeded" {
t.Fatal("重新核查审计失败发布成功结果", err)
}
if client.writes != 0 || !reflect.DeepEqual(files, client.files) {
t.Fatal("审计失败核查修改远端文件")
}
if err := db.Close(); err != nil {
t.Fatal(err)
}
db = openRecoveryExternal(t, cfg)
repo = NewSQLDeploymentTransactionRepository(db, cfg.Driver)
assertReconciliationSource(t, repo, source)
recoveryCount(t, db, "recovery_claims", 1)
wantOwner := source.Intent.ID
if stage == "finished" {
wantOwner = confirmation.DispositionID()
recoveryCount(t, db, "recovery_dispositions", 2)
recoveryCount(t, db, "recovery_events", 3)
stored, err := repo.GetRecoveryDisposition(t.Context(), wantOwner)
if err != nil || stored.Status != "running" || stored.ResultCode != "" || stored.Intent.SourceDispositionID != source.Intent.ID {
t.Fatal("结束审计失败未持久保留运行中处置", err)
}
} else {
recoveryCount(t, db, "recovery_dispositions", 1)
recoveryCount(t, db, "recovery_events", 2)
}
var owner string
if err := db.QueryRowContext(t.Context(), repo.query("SELECT disposition_id FROM recovery_claims WHERE transaction_id = ?"), record.ID).Scan(&owner); err != nil || owner != wantOwner {
t.Fatal("重开后占用归属错误", err)
}
pending, err := repo.ListPendingDeploymentTransactions(t.Context())
if err != nil || len(pending) != 1 || pending[0].Status != deployment.TransactionStatusRecoveryRequired {
t.Fatal("审计失败改写了原事务终态", err)
}
restarted := deployment.NewServiceWithJournal(nil, repo)
if err := restarted.RecoverPending(t.Context(), func(context.Context, string) (deployment.Server, error) {
t.Error("重开后自动恢复试图重放已占用事务")
return deployment.Server{}, errors.New("禁止连接远端")
}); !errors.Is(err, deployment.ErrRecoveryDispositionBusy) {
t.Fatal("重开后自动恢复未遵守持久占用", err)
}
}