保存不确定部署处置重新核查实现及测试检查点
This commit is contained in:
1 parent
7fa5e9b3d2
commit
f9cd19f631
29 files changed
+907
-49
No files matched your search
@@ -17,15 +17,16 @@ var (
|
||||
|
||||
// RecoveryDispositionIntent 仅保存绑定摘要;不能包含历史证据、凭据或确认令牌原文。
|
||||
type RecoveryDispositionIntent struct {
|
||||
ID string
|
||||
TransactionID string
|
||||
ServerID string
|
||||
Operator string
|
||||
Action RecoveryAction
|
||||
RequestKey string
|
||||
RecordDigest string
|
||||
ServerDigest string
|
||||
ObservationDigest string
|
||||
SourceDispositionID string
|
||||
ID string
|
||||
TransactionID string
|
||||
ServerID string
|
||||
Operator string
|
||||
Action RecoveryAction
|
||||
RequestKey string
|
||||
RecordDigest string
|
||||
ServerDigest string
|
||||
ObservationDigest string
|
||||
}
|
||||
|
||||
type RecoveryDisposition struct {
|
||||
@@ -58,7 +59,8 @@ func (r RecoveryReview) DispositionIntent(id, requestKey string) (RecoveryDispos
|
||||
return RecoveryDispositionIntent{}, ErrRecoveryReviewExpired
|
||||
}
|
||||
intent := RecoveryDispositionIntent{
|
||||
ID: id, TransactionID: r.summary.TransactionID, ServerID: r.summary.ServerID,
|
||||
SourceDispositionID: r.summary.SourceDispositionID,
|
||||
ID: id, TransactionID: r.summary.TransactionID, ServerID: r.summary.ServerID,
|
||||
Operator: r.operator, Action: r.summary.Action, RequestKey: requestKey,
|
||||
RecordDigest: hex.EncodeToString(r.recordDigest[:]),
|
||||
ServerDigest: hex.EncodeToString(r.serverDigest[:]),
|
||||
@@ -68,6 +70,9 @@ func (r RecoveryReview) DispositionIntent(id, requestKey string) (RecoveryDispos
|
||||
}
|
||||
|
||||
func (i RecoveryDispositionIntent) Validate() error {
|
||||
if isRecoveryVerification(i.Action) != (i.SourceDispositionID != "") || (i.SourceDispositionID != "" && (!validDispositionID(i.SourceDispositionID) || i.SourceDispositionID == i.ID)) {
|
||||
return ErrRecoveryDispositionInvalid
|
||||
}
|
||||
if !validInspectionID(i.TransactionID) || !validRecoverySummary(i.ServerID) ||
|
||||
!validReviewOperator(i.Operator) || !validRecoveryAction(i.Action) {
|
||||
return ErrRecoveryDispositionInvalid
|
||||
@@ -106,9 +111,9 @@ func (i RecoveryDispositionIntent) AllowsResult(result string) bool {
|
||||
case RecoveryResultObserved:
|
||||
return i.Action == RecoveryActionRecord
|
||||
case RecoveryResultCommitted:
|
||||
return i.Action == RecoveryActionCommit
|
||||
return i.Action == RecoveryActionCommit || i.Action == RecoveryActionVerifyCommit
|
||||
case RecoveryResultRolledBack:
|
||||
return i.Action == RecoveryActionRollback
|
||||
return i.Action == RecoveryActionRollback || i.Action == RecoveryActionVerifyRollback
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -100,6 +100,12 @@ func (s *Service) executeRecoveryAction(ctx context.Context, review RecoveryRevi
|
||||
case RecoveryActionRollback:
|
||||
err = s.executeRecoveryRollback(ctx, &record, server, client, &mutated)
|
||||
result = RecoveryResultRolledBack
|
||||
case RecoveryActionVerifyCommit, RecoveryActionVerifyRollback:
|
||||
err = s.verifyRecoveryCompletion(ctx, record, server, client, review.summary.Action)
|
||||
result = RecoveryResultCommitted
|
||||
if review.summary.Action == RecoveryActionVerifyRollback {
|
||||
result = RecoveryResultRolledBack
|
||||
}
|
||||
default:
|
||||
return
|
||||
}
|
||||
@@ -117,6 +123,13 @@ func validateRecoveryAction(record TransactionRecord, action RecoveryAction, ver
|
||||
return ErrRecoveryActionRejected
|
||||
}
|
||||
switch action {
|
||||
case RecoveryActionVerifyCommit, RecoveryActionVerifyRollback:
|
||||
if !verificationPhase(record.Phase, action) ||
|
||||
(action == RecoveryActionVerifyCommit && record.Evidence.Baseline.ExpectedFingerprint == "") ||
|
||||
(action == RecoveryActionVerifyRollback && record.Evidence.Baseline.PreviousFingerprint == "") {
|
||||
return ErrRecoveryActionRejected
|
||||
}
|
||||
return nil
|
||||
case RecoveryActionCommit:
|
||||
if record.Evidence.Baseline.ExpectedFingerprint == "" {
|
||||
return ErrRecoveryActionRejected
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"time"
|
||||
)
|
||||
|
||||
func isRecoveryVerification(action RecoveryAction) bool {
|
||||
return action == RecoveryActionVerifyCommit || action == RecoveryActionVerifyRollback
|
||||
}
|
||||
|
||||
// Reconciles 仅允许已持久化结束的不确定原动作;不接管运行中处置或再次执行文件动作。
|
||||
func (i RecoveryDispositionIntent) Reconciles(source RecoveryDisposition) bool {
|
||||
if i.SourceDispositionID == "" || i.SourceDispositionID != source.Intent.ID || source.Intent.Validate() != nil ||
|
||||
source.Status != RecoveryResultUncertain || source.ResultCode != RecoveryResultUncertain ||
|
||||
i.TransactionID != source.Intent.TransactionID || i.ServerID != source.Intent.ServerID || i.ServerDigest != source.Intent.ServerDigest {
|
||||
return false
|
||||
}
|
||||
return (i.Action == RecoveryActionVerifyCommit && source.Intent.Action == RecoveryActionCommit) ||
|
||||
(i.Action == RecoveryActionVerifyRollback && source.Intent.Action == RecoveryActionRollback)
|
||||
}
|
||||
|
||||
// PrepareRecoveryReconciliation 只供受保护本机通道调用,必须明确指定原处置编号并重新确认。
|
||||
func (s *Service) PrepareRecoveryReconciliation(ctx context.Context, sourceID, operator string, resolve func(context.Context, string) (Server, error)) (RecoveryConfirmation, string, error) {
|
||||
review, err := s.prepareRecoveryReconciliation(ctx, sourceID, operator, resolve)
|
||||
if err != nil {
|
||||
return RecoveryConfirmation{}, "", err
|
||||
}
|
||||
return s.newRecoveryConfirmation(ctx, review)
|
||||
}
|
||||
|
||||
func (s *Service) prepareRecoveryReconciliation(ctx context.Context, sourceID, operator string, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if s == nil || !validDispositionID(sourceID) || !validReviewOperator(operator) {
|
||||
return RecoveryReview{}, ErrRecoveryReviewInvalid
|
||||
}
|
||||
journal, ok := s.journal.(RecoveryDispositionJournal)
|
||||
if !ok {
|
||||
return RecoveryReview{}, ErrRecoveryDispositionUnavailable
|
||||
}
|
||||
source, err := journal.GetRecoveryDisposition(ctx, sourceID)
|
||||
if err != nil || source.Intent.ID != sourceID || source.Intent.Validate() != nil || source.Summary().Validate() != nil ||
|
||||
source.Status != RecoveryResultUncertain || source.ResultCode != RecoveryResultUncertain {
|
||||
return RecoveryReview{}, ErrRecoveryActionRejected
|
||||
}
|
||||
var action RecoveryAction
|
||||
switch source.Intent.Action {
|
||||
case RecoveryActionCommit:
|
||||
action = RecoveryActionVerifyCommit
|
||||
case RecoveryActionRollback:
|
||||
action = RecoveryActionVerifyRollback
|
||||
default:
|
||||
return RecoveryReview{}, ErrRecoveryActionRejected
|
||||
}
|
||||
review, err := s.prepareRecoveryReview(ctx, source.Intent.TransactionID, operator, action, resolve)
|
||||
if err != nil {
|
||||
return RecoveryReview{}, err
|
||||
}
|
||||
review.summary.SourceDispositionID = sourceID
|
||||
intent := RecoveryDispositionIntent{SourceDispositionID: sourceID, TransactionID: review.summary.TransactionID,
|
||||
ServerID: review.summary.ServerID, ServerDigest: hex.EncodeToString(review.serverDigest[:]), Action: action}
|
||||
if !intent.Reconciles(source) || !verificationPhase(review.summary.Phase, action) || s.VerifyRecoveryTarget == nil {
|
||||
return RecoveryReview{}, ErrRecoveryActionRejected
|
||||
}
|
||||
return review, nil
|
||||
}
|
||||
|
||||
func verificationPhase(phase string, action RecoveryAction) bool {
|
||||
return (action == RecoveryActionVerifyCommit && phase == TransactionPhaseCommitVerified) ||
|
||||
(action == RecoveryActionVerifyRollback && phase == TransactionPhaseRollbackVerified)
|
||||
}
|
||||
|
||||
// validateRecoveryCompletion 不删除任何远端文件,残留锁或备份均要求继续保留占用。
|
||||
func validateRecoveryCompletion(ctx context.Context, client Client, record TransactionRecord, committed bool) error {
|
||||
for _, lock := range record.LockPaths {
|
||||
if exists, err := client.Exists(ctx, lock); err != nil || exists {
|
||||
return ErrRecoveryActionRejected
|
||||
}
|
||||
}
|
||||
for _, backup := range record.Backups {
|
||||
if committed {
|
||||
if err := checkFileDigest(ctx, client, backup.Original, record.Evidence.Files[backup.Original]); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if backup.OriginalExists {
|
||||
if err := checkFileDigest(ctx, client, backup.Original, backup.OriginalDigest); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if exists, err := client.Exists(ctx, backup.Original); err != nil || exists {
|
||||
return ErrRecoveryActionRejected
|
||||
}
|
||||
for _, path := range []string{backup.Saved, backup.Original + ".scert-tmp-" + record.ID} {
|
||||
if exists, err := client.Exists(ctx, path); err != nil || exists {
|
||||
return ErrRecoveryActionRejected
|
||||
}
|
||||
}
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func (s *Service) verifyRecoveryCompletion(ctx context.Context, record TransactionRecord, server Server, client Client, action RecoveryAction) error {
|
||||
committed := action == RecoveryActionVerifyCommit
|
||||
if err := validateRecoveryCompletion(ctx, client, record, committed); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.VerifyRecoveryTarget(ctx, server, record.Evidence.Baseline, committed); err != nil {
|
||||
return err
|
||||
}
|
||||
// HTTPS 检查结束后重新读取现场,避免检查期间发生的文件变化被忽略。
|
||||
return validateRecoveryCompletion(ctx, client, record, committed)
|
||||
}
|
||||
@@ -8,19 +8,21 @@ import (
|
||||
|
||||
// RecoveryDispositionSummary 是本机查询的明确输出边界,不包含请求键、绑定摘要或操作者凭据。
|
||||
type RecoveryDispositionSummary struct {
|
||||
ID string `json:"id"`
|
||||
TransactionID string `json:"transaction_id"`
|
||||
ServerID string `json:"server_id"`
|
||||
Action RecoveryAction `json:"action"`
|
||||
Status string `json:"status"`
|
||||
ResultCode string `json:"result_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
SourceDispositionID string `json:"source_disposition_id,omitempty"`
|
||||
ID string `json:"id"`
|
||||
TransactionID string `json:"transaction_id"`
|
||||
ServerID string `json:"server_id"`
|
||||
Action RecoveryAction `json:"action"`
|
||||
Status string `json:"status"`
|
||||
ResultCode string `json:"result_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
func (r RecoveryDisposition) Summary() RecoveryDispositionSummary {
|
||||
return RecoveryDispositionSummary{
|
||||
ID: r.Intent.ID, TransactionID: r.Intent.TransactionID, ServerID: r.Intent.ServerID,
|
||||
SourceDispositionID: r.Intent.SourceDispositionID,
|
||||
ID: r.Intent.ID, TransactionID: r.Intent.TransactionID, ServerID: r.Intent.ServerID,
|
||||
Action: r.Intent.Action, Status: r.Status, ResultCode: r.ResultCode,
|
||||
CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt,
|
||||
}
|
||||
@@ -31,6 +33,9 @@ func validDispositionID(id string) bool {
|
||||
}
|
||||
|
||||
func (r RecoveryDispositionSummary) Validate() error {
|
||||
if isRecoveryVerification(r.Action) != (r.SourceDispositionID != "") || (r.SourceDispositionID != "" && (!validDispositionID(r.SourceDispositionID) || r.SourceDispositionID == r.ID)) {
|
||||
return ErrRecoveryDispositionInvalid
|
||||
}
|
||||
if !validDispositionID(r.ID) || !validInspectionID(r.TransactionID) || !validRecoverySummary(r.ServerID) ||
|
||||
!validRecoveryAction(r.Action) || r.CreatedAt.IsZero() || r.UpdatedAt.IsZero() || r.UpdatedAt.Before(r.CreatedAt) {
|
||||
return ErrRecoveryDispositionInvalid
|
||||
|
||||
@@ -13,10 +13,12 @@ import (
|
||||
type RecoveryAction string
|
||||
|
||||
const (
|
||||
RecoveryActionRecord RecoveryAction = "record_inspection"
|
||||
RecoveryActionCommit RecoveryAction = "finish_commit"
|
||||
RecoveryActionRollback RecoveryAction = "restore_baseline"
|
||||
RecoveryReviewLifetime = time.Minute
|
||||
RecoveryActionRecord RecoveryAction = "record_inspection"
|
||||
RecoveryActionCommit RecoveryAction = "finish_commit"
|
||||
RecoveryActionRollback RecoveryAction = "restore_baseline"
|
||||
RecoveryActionVerifyCommit RecoveryAction = "verify_commit"
|
||||
RecoveryActionVerifyRollback RecoveryAction = "verify_rollback"
|
||||
RecoveryReviewLifetime = time.Minute
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -27,13 +29,14 @@ var (
|
||||
)
|
||||
|
||||
type RecoveryReviewSummary struct {
|
||||
TransactionID string `json:"transaction_id"`
|
||||
ServerID string `json:"server_id"`
|
||||
Phase string `json:"phase"`
|
||||
Action RecoveryAction `json:"action"`
|
||||
HistoricalEvidence bool `json:"historical_evidence"`
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
SourceDispositionID string `json:"source_disposition_id,omitempty"`
|
||||
TransactionID string `json:"transaction_id"`
|
||||
ServerID string `json:"server_id"`
|
||||
Phase string `json:"phase"`
|
||||
Action RecoveryAction `json:"action"`
|
||||
HistoricalEvidence bool `json:"historical_evidence"`
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
}
|
||||
|
||||
// Validate 与服务端历史记录使用同一摘要边界,允许旧记录没有阶段,但不允许控制字符或超长字段。
|
||||
@@ -44,6 +47,9 @@ func (r RecoveryReviewSummary) Validate() error {
|
||||
(r.Action != RecoveryActionRecord && !r.HistoricalEvidence) {
|
||||
return ErrRecoveryReviewInvalid
|
||||
}
|
||||
if isRecoveryVerification(r.Action) != (r.SourceDispositionID != "") || (r.SourceDispositionID != "" && !validDispositionID(r.SourceDispositionID)) {
|
||||
return ErrRecoveryReviewInvalid
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -65,12 +71,20 @@ func validReviewOperator(operator string) bool {
|
||||
}
|
||||
|
||||
func validRecoveryAction(action RecoveryAction) bool {
|
||||
return action == RecoveryActionRecord || action == RecoveryActionCommit || action == RecoveryActionRollback
|
||||
return action == RecoveryActionRecord || action == RecoveryActionCommit || action == RecoveryActionRollback || isRecoveryVerification(action)
|
||||
}
|
||||
|
||||
// PrepareRecoveryReview 在同一操作上限内读取前后记录,拒绝采样期间发生的配置或事务变化。
|
||||
// 当前采样摘要只绑定本次准备,不补写到历史 Evidence,也不改变事务终态。
|
||||
func (s *Service) PrepareRecoveryReview(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) {
|
||||
// 重新核查必须通过独立入口绑定原处置,不能作为普通动作绕过来源检查。
|
||||
if isRecoveryVerification(action) {
|
||||
return RecoveryReview{}, ErrRecoveryReviewInvalid
|
||||
}
|
||||
return s.prepareRecoveryReview(ctx, id, operator, action, resolve)
|
||||
}
|
||||
|
||||
func (s *Service) prepareRecoveryReview(ctx context.Context, id, operator string, action RecoveryAction, resolve func(context.Context, string) (Server, error)) (RecoveryReview, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if !validReviewOperator(operator) || !validRecoveryAction(action) || !validInspectionID(id) {
|
||||
@@ -140,14 +154,20 @@ func (s *Service) RecheckRecoveryReview(ctx context.Context, review RecoveryRevi
|
||||
if s.now == nil || !s.now().Before(review.summary.ExpiresAt) {
|
||||
return ErrRecoveryReviewExpired
|
||||
}
|
||||
current, err := s.PrepareRecoveryReview(ctx, review.summary.TransactionID, operator, review.summary.Action, resolve)
|
||||
var current RecoveryReview
|
||||
var err error
|
||||
if isRecoveryVerification(review.summary.Action) {
|
||||
current, err = s.prepareRecoveryReconciliation(ctx, review.summary.SourceDispositionID, operator, resolve)
|
||||
} else {
|
||||
current, err = s.PrepareRecoveryReview(ctx, review.summary.TransactionID, operator, review.summary.Action, resolve)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !s.now().Before(review.summary.ExpiresAt) {
|
||||
return ErrRecoveryReviewExpired
|
||||
}
|
||||
if current.recordDigest != review.recordDigest || current.serverDigest != review.serverDigest || current.observationDigest != review.observationDigest {
|
||||
if current.recordDigest != review.recordDigest || current.serverDigest != review.serverDigest || current.observationDigest != review.observationDigest || current.summary.SourceDispositionID != review.summary.SourceDispositionID || current.summary.Action != review.summary.Action {
|
||||
return ErrRecoveryReviewChanged
|
||||
}
|
||||
return ctx.Err()
|
||||
|
||||
Reference in new issue
Block a user