保存不确定部署处置重新核查实现及测试检查点

This commit is contained in:
xuans committed 2026-10-03 23:49:31 +08:00
1 parent 7fa5e9b3d2
commit f9cd19f631
29 files changed
+907 -49

No files matched your search

@@ -17,15 +17,16 @@ var (
// RecoveryDispositionIntent 仅保存绑定摘要;不能包含历史证据、凭据或确认令牌原文。
type RecoveryDispositionIntent struct {
ID string
TransactionID string
ServerID string
Operator string
Action RecoveryAction
RequestKey string
RecordDigest string
ServerDigest string
ObservationDigest string
SourceDispositionID string
ID string
TransactionID string
ServerID string
Operator string
Action RecoveryAction
RequestKey string
RecordDigest string
ServerDigest string
ObservationDigest string
}
type RecoveryDisposition struct {
@@ -58,7 +59,8 @@ func (r RecoveryReview) DispositionIntent(id, requestKey string) (RecoveryDispos
return RecoveryDispositionIntent{}, ErrRecoveryReviewExpired
}
intent := RecoveryDispositionIntent{
ID: id, TransactionID: r.summary.TransactionID, ServerID: r.summary.ServerID,
SourceDispositionID: r.summary.SourceDispositionID,
ID: id, TransactionID: r.summary.TransactionID, ServerID: r.summary.ServerID,
Operator: r.operator, Action: r.summary.Action, RequestKey: requestKey,
RecordDigest: hex.EncodeToString(r.recordDigest[:]),
ServerDigest: hex.EncodeToString(r.serverDigest[:]),
@@ -68,6 +70,9 @@ func (r RecoveryReview) DispositionIntent(id, requestKey string) (RecoveryDispos
}
func (i RecoveryDispositionIntent) Validate() error {
if isRecoveryVerification(i.Action) != (i.SourceDispositionID != "") || (i.SourceDispositionID != "" && (!validDispositionID(i.SourceDispositionID) || i.SourceDispositionID == i.ID)) {
return ErrRecoveryDispositionInvalid
}
if !validInspectionID(i.TransactionID) || !validRecoverySummary(i.ServerID) ||
!validReviewOperator(i.Operator) || !validRecoveryAction(i.Action) {
return ErrRecoveryDispositionInvalid
@@ -106,9 +111,9 @@ func (i RecoveryDispositionIntent) AllowsResult(result string) bool {
case RecoveryResultObserved:
return i.Action == RecoveryActionRecord
case RecoveryResultCommitted:
return i.Action == RecoveryActionCommit
return i.Action == RecoveryActionCommit || i.Action == RecoveryActionVerifyCommit
case RecoveryResultRolledBack:
return i.Action == RecoveryActionRollback
return i.Action == RecoveryActionRollback || i.Action == RecoveryActionVerifyRollback
}
return false
}
@@ -100,6 +100,12 @@ func (s *Service) executeRecoveryAction(ctx context.Context, review RecoveryRevi
case RecoveryActionRollback:
err = s.executeRecoveryRollback(ctx, &record, server, client, &mutated)
result = RecoveryResultRolledBack
case RecoveryActionVerifyCommit, RecoveryActionVerifyRollback:
err = s.verifyRecoveryCompletion(ctx, record, server, client, review.summary.Action)
result = RecoveryResultCommitted
if review.summary.Action == RecoveryActionVerifyRollback {
result = RecoveryResultRolledBack
}
default:
return
}
@@ -117,6 +123,13 @@ func validateRecoveryAction(record TransactionRecord, action RecoveryAction, ver
return ErrRecoveryActionRejected
}
switch action {
case RecoveryActionVerifyCommit, RecoveryActionVerifyRollback:
if !verificationPhase(record.Phase, action) ||
(action == RecoveryActionVerifyCommit && record.Evidence.Baseline.ExpectedFingerprint == "") ||
(action == RecoveryActionVerifyRollback && record.Evidence.Baseline.PreviousFingerprint == "") {
return ErrRecoveryActionRejected
}
return nil
case RecoveryActionCommit:
if record.Evidence.Baseline.ExpectedFingerprint == "" {
return ErrRecoveryActionRejected
@@ -0,0 +1,113 @@
package deployment
import (
"context"
"encoding/hex"
"time"
)
func isRecoveryVerification(action RecoveryAction) bool {
return action == RecoveryActionVerifyCommit || action == RecoveryActionVerifyRollback
}
// Reconciles 仅允许已持久化结束的不确定原动作;不接管运行中处置或再次执行文件动作。
func (i RecoveryDispositionIntent) Reconciles(source RecoveryDisposition) bool {
if i.SourceDispositionID == "" || i.SourceDispositionID != source.Intent.ID || source.Intent.Validate() != nil ||
source.Status != RecoveryResultUncertain || source.ResultCode != RecoveryResultUncertain ||
i.TransactionID != source.Intent.TransactionID || i.ServerID != source.Intent.ServerID || i.ServerDigest != source.Intent.ServerDigest {
return false
}
return (i.Action == RecoveryActionVerifyCommit && source.Intent.Action == RecoveryActionCommit) ||
(i.Action == RecoveryActionVerifyRollback && source.Intent.Action == RecoveryActionRollback)
}
// PrepareRecoveryReconciliation 只供受保护本机通道调用,必须明确指定原处置编号并重新确认。
func (s *Service) PrepareRecoveryReconciliation(ctx context.Context, sourceID, operator string, resolve func(context.Context, string) (Server, error)) (RecoveryConfirmation, string, error) {
review, err := s.prepareRecoveryReconciliation(ctx, sourceID, operator, resolve)
if err != nil {
return RecoveryConfirmation{}, "", err
}
return s.newRecoveryConfirmation(ctx, review)
}
func (s *Service) prepareRecoveryReconciliation(ctx context.Context, sourceID, operator string, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
if s == nil || !validDispositionID(sourceID) || !validReviewOperator(operator) {
return RecoveryReview{}, ErrRecoveryReviewInvalid
}
journal, ok := s.journal.(RecoveryDispositionJournal)
if !ok {
return RecoveryReview{}, ErrRecoveryDispositionUnavailable
}
source, err := journal.GetRecoveryDisposition(ctx, sourceID)
if err != nil || source.Intent.ID != sourceID || source.Intent.Validate() != nil || source.Summary().Validate() != nil ||
source.Status != RecoveryResultUncertain || source.ResultCode != RecoveryResultUncertain {
return RecoveryReview{}, ErrRecoveryActionRejected
}
var action RecoveryAction
switch source.Intent.Action {
case RecoveryActionCommit:
action = RecoveryActionVerifyCommit
case RecoveryActionRollback:
action = RecoveryActionVerifyRollback
default:
return RecoveryReview{}, ErrRecoveryActionRejected
}
review, err := s.prepareRecoveryReview(ctx, source.Intent.TransactionID, operator, action, resolve)
if err != nil {
return RecoveryReview{}, err
}
review.summary.SourceDispositionID = sourceID
intent := RecoveryDispositionIntent{SourceDispositionID: sourceID, TransactionID: review.summary.TransactionID,
ServerID: review.summary.ServerID, ServerDigest: hex.EncodeToString(review.serverDigest[:]), Action: action}
if !intent.Reconciles(source) || !verificationPhase(review.summary.Phase, action) || s.VerifyRecoveryTarget == nil {
return RecoveryReview{}, ErrRecoveryActionRejected
}
return review, nil
}
func verificationPhase(phase string, action RecoveryAction) bool {
return (action == RecoveryActionVerifyCommit && phase == TransactionPhaseCommitVerified) ||
(action == RecoveryActionVerifyRollback && phase == TransactionPhaseRollbackVerified)
}
// validateRecoveryCompletion 不删除任何远端文件,残留锁或备份均要求继续保留占用。
func validateRecoveryCompletion(ctx context.Context, client Client, record TransactionRecord, committed bool) error {
for _, lock := range record.LockPaths {
if exists, err := client.Exists(ctx, lock); err != nil || exists {
return ErrRecoveryActionRejected
}
}
for _, backup := range record.Backups {
if committed {
if err := checkFileDigest(ctx, client, backup.Original, record.Evidence.Files[backup.Original]); err != nil {
return err
}
} else if backup.OriginalExists {
if err := checkFileDigest(ctx, client, backup.Original, backup.OriginalDigest); err != nil {
return err
}
} else if exists, err := client.Exists(ctx, backup.Original); err != nil || exists {
return ErrRecoveryActionRejected
}
for _, path := range []string{backup.Saved, backup.Original + ".scert-tmp-" + record.ID} {
if exists, err := client.Exists(ctx, path); err != nil || exists {
return ErrRecoveryActionRejected
}
}
}
return ctx.Err()
}
func (s *Service) verifyRecoveryCompletion(ctx context.Context, record TransactionRecord, server Server, client Client, action RecoveryAction) error {
committed := action == RecoveryActionVerifyCommit
if err := validateRecoveryCompletion(ctx, client, record, committed); err != nil {
return err
}
if err := s.VerifyRecoveryTarget(ctx, server, record.Evidence.Baseline, committed); err != nil {
return err
}
// HTTPS 检查结束后重新读取现场,避免检查期间发生的文件变化被忽略。
return validateRecoveryCompletion(ctx, client, record, committed)
}
+14 -9
View File
@@ -8,19 +8,21 @@ import (
// RecoveryDispositionSummary 是本机查询的明确输出边界,不包含请求键、绑定摘要或操作者凭据。
type RecoveryDispositionSummary struct {
ID string `json:"id"`
TransactionID string `json:"transaction_id"`
ServerID string `json:"server_id"`
Action RecoveryAction `json:"action"`
Status string `json:"status"`
ResultCode string `json:"result_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
SourceDispositionID string `json:"source_disposition_id,omitempty"`
ID string `json:"id"`
TransactionID string `json:"transaction_id"`
ServerID string `json:"server_id"`
Action RecoveryAction `json:"action"`
Status string `json:"status"`
ResultCode string `json:"result_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
func (r RecoveryDisposition) Summary() RecoveryDispositionSummary {
return RecoveryDispositionSummary{
ID: r.Intent.ID, TransactionID: r.Intent.TransactionID, ServerID: r.Intent.ServerID,
SourceDispositionID: r.Intent.SourceDispositionID,
ID: r.Intent.ID, TransactionID: r.Intent.TransactionID, ServerID: r.Intent.ServerID,
Action: r.Intent.Action, Status: r.Status, ResultCode: r.ResultCode,
CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt,
}
@@ -31,6 +33,9 @@ func validDispositionID(id string) bool {
}
func (r RecoveryDispositionSummary) Validate() error {
if isRecoveryVerification(r.Action) != (r.SourceDispositionID != "") || (r.SourceDispositionID != "" && (!validDispositionID(r.SourceDispositionID) || r.SourceDispositionID == r.ID)) {
return ErrRecoveryDispositionInvalid
}
if !validDispositionID(r.ID) || !validInspectionID(r.TransactionID) || !validRecoverySummary(r.ServerID) ||
!validRecoveryAction(r.Action) || r.CreatedAt.IsZero() || r.UpdatedAt.IsZero() || r.UpdatedAt.Before(r.CreatedAt) {
return ErrRecoveryDispositionInvalid
+34 -14
View File
@@ -13,10 +13,12 @@ import (
type RecoveryAction string
const (
RecoveryActionRecord RecoveryAction = "record_inspection"
RecoveryActionCommit RecoveryAction = "finish_commit"
RecoveryActionRollback RecoveryAction = "restore_baseline"
RecoveryReviewLifetime = time.Minute
RecoveryActionRecord RecoveryAction = "record_inspection"
RecoveryActionCommit RecoveryAction = "finish_commit"
RecoveryActionRollback RecoveryAction = "restore_baseline"
RecoveryActionVerifyCommit RecoveryAction = "verify_commit"
RecoveryActionVerifyRollback RecoveryAction = "verify_rollback"
RecoveryReviewLifetime = time.Minute
)
var (
@@ -27,13 +29,14 @@ var (
)
type RecoveryReviewSummary struct {
TransactionID string `json:"transaction_id"`
ServerID string `json:"server_id"`
Phase string `json:"phase"`
Action RecoveryAction `json:"action"`
HistoricalEvidence bool `json:"historical_evidence"`
ObservedAt time.Time `json:"observed_at"`
ExpiresAt time.Time `json:"expires_at"`
SourceDispositionID string `json:"source_disposition_id,omitempty"`
TransactionID string `json:"transaction_id"`
ServerID string `json:"server_id"`
Phase string `json:"phase"`
Action RecoveryAction `json:"action"`
HistoricalEvidence bool `json:"historical_evidence"`
ObservedAt time.Time `json:"observed_at"`
ExpiresAt time.Time `json:"expires_at"`
}
// Validate 与服务端历史记录使用同一摘要边界,允许旧记录没有阶段,但不允许控制字符或超长字段。
@@ -44,6 +47,9 @@ func (r RecoveryReviewSummary) Validate() error {
(r.Action != RecoveryActionRecord && !r.HistoricalEvidence) {
return ErrRecoveryReviewInvalid
}
if isRecoveryVerification(r.Action) != (r.SourceDispositionID != "") || (r.SourceDispositionID != "" && !validDispositionID(r.SourceDispositionID)) {
return ErrRecoveryReviewInvalid
}
return nil
}
@@ -65,12 +71,20 @@ func validReviewOperator(operator string) bool {
}
func validRecoveryAction(action RecoveryAction) bool {
return action == RecoveryActionRecord || action == RecoveryActionCommit || action == RecoveryActionRollback
return action == RecoveryActionRecord || action == RecoveryActionCommit || action == RecoveryActionRollback || isRecoveryVerification(action)
}
// PrepareRecoveryReview 在同一操作上限内读取前后记录,拒绝采样期间发生的配置或事务变化。
// 当前采样摘要只绑定本次准备,不补写到历史 Evidence,也不改变事务终态。
func (s *Service) PrepareRecoveryReview(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) {
// 重新核查必须通过独立入口绑定原处置,不能作为普通动作绕过来源检查。
if isRecoveryVerification(action) {
return RecoveryReview{}, ErrRecoveryReviewInvalid
}
return s.prepareRecoveryReview(ctx, id, operator, action, resolve)
}
func (s *Service) prepareRecoveryReview(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
if !validReviewOperator(operator) || !validRecoveryAction(action) || !validInspectionID(id) {
@@ -140,14 +154,20 @@ func (s *Service) RecheckRecoveryReview(ctx context.Context, review RecoveryRevi
if s.now == nil || !s.now().Before(review.summary.ExpiresAt) {
return ErrRecoveryReviewExpired
}
current, err := s.PrepareRecoveryReview(ctx, review.summary.TransactionID, operator, review.summary.Action, resolve)
var current RecoveryReview
var err error
if isRecoveryVerification(review.summary.Action) {
current, err = s.prepareRecoveryReconciliation(ctx, review.summary.SourceDispositionID, operator, resolve)
} else {
current, err = s.PrepareRecoveryReview(ctx, review.summary.TransactionID, operator, review.summary.Action, resolve)
}
if err != nil {
return err
}
if !s.now().Before(review.summary.ExpiresAt) {
return ErrRecoveryReviewExpired
}
if current.recordDigest != review.recordDigest || current.serverDigest != review.serverDigest || current.observationDigest != review.observationDigest {
if current.recordDigest != review.recordDigest || current.serverDigest != review.serverDigest || current.observationDigest != review.observationDigest || current.summary.SourceDispositionID != review.summary.SourceDispositionID || current.summary.Action != review.summary.Action {
return ErrRecoveryReviewChanged
}
return ctx.Err()