接入完整部署互斥与持久化人工占用防护

This commit is contained in:
xuans committed 2026-10-03 20:18:35 +08:00
1 parent d801fb3833
commit b3f27e88df
12 files changed
+430 -2

No files matched your search

+85
View File
@@ -0,0 +1,85 @@
package deployment
import (
"context"
"fmt"
)
// acquireOperation 覆盖整个部署、站点核验及收尾;人工执行器必须共用此入口。
// 它不替代数据库持久化占用和远端文件锁,等待可以取消,不能提前释放。
func (s *Service) acquireOperation(ctx context.Context) (func(), error) {
if s == nil {
return nil, ErrRecoveryDispositionUnavailable
}
if err := ctx.Err(); err != nil {
return nil, err
}
s.operationOnce.Do(func() { s.operationGate = make(chan struct{}, 1) })
select {
case <-ctx.Done():
return nil, ctx.Err()
case s.operationGate <- struct{}{}:
if err := ctx.Err(); err != nil {
<-s.operationGate
return nil, err
}
return func() { <-s.operationGate }, nil
}
}
func operationFailure(targets []Target, err error) Report {
report := Report{Total: len(targets), Failed: len(targets), Status: "failed", Results: make([]Result, 0, len(targets))}
for _, target := range targets {
report.Results = append(report.Results, Result{
ServerID: target.Server.ID, Server: target.Server.Name, Status: "failed",
FailureCode: FailureUnavailable, Message: fmt.Sprintf("未获得部署操作许可:%v", err),
})
}
return report
}
type recoveryClaimReader interface {
HasRecoveryClaim(context.Context, string) (bool, error)
}
func (s *Service) checkRecoveryClaim(ctx context.Context, id string) error {
if err := ctx.Err(); err != nil {
return err
}
reader, ok := s.journal.(recoveryClaimReader)
if !ok {
return nil
}
claimed, err := reader.HasRecoveryClaim(ctx, id)
if err != nil {
return ErrRecoveryDispositionUnavailable
}
if claimed {
return ErrRecoveryDispositionBusy
}
return ctx.Err()
}
// checkDeploymentClaims 在连接远端前拦截仍有人工占用的服务器。
// 同地址同端口的别名也须检查;凭据或路径修改不能绕过原服务器占用。
func (s *Service) checkDeploymentClaims(ctx context.Context, server Server) error {
if _, ok := s.journal.(recoveryClaimReader); !ok {
return nil
}
records, err := s.journal.ListPendingDeploymentTransactions(ctx)
if err != nil {
return ErrRecoveryDispositionUnavailable
}
for _, record := range records {
sameServer := record.ServerID == server.ID
if record.Evidence != nil {
sameServer = sameServer || (record.Evidence.Address == server.Address && record.Evidence.Port == server.Port)
}
if sameServer {
if err := s.checkRecoveryClaim(ctx, record.ID); err != nil {
return err
}
}
}
return ctx.Err()
}
@@ -0,0 +1,243 @@
package deployment
import (
"context"
"errors"
"maps"
"reflect"
"sync"
"sync/atomic"
"testing"
"time"
)
type operationWitnessContext struct {
context.Context
once sync.Once
waiting chan struct{}
}
func (c *operationWitnessContext) Done() <-chan struct{} {
c.once.Do(func() { close(c.waiting) })
return c.Context.Done()
}
type operationFactory struct {
client Client
calls atomic.Int32
}
func (f *operationFactory) Connect(context.Context, Server) (Client, error) {
f.calls.Add(1)
return f.client, nil
}
type operationCloseClient struct {
*memoryClient
closeHook func()
}
func (c *operationCloseClient) Close() error {
if c.closeHook != nil {
c.closeHook()
}
return nil
}
type operationJournal struct {
memoryTransactionJournal
commitHook func()
}
func (j *operationJournal) UpdateDeploymentTransaction(ctx context.Context, item TransactionRecord) error {
if item.Phase == TransactionPhaseCommitting && j.commitHook != nil {
j.commitHook()
}
return j.memoryTransactionJournal.UpdateDeploymentTransaction(ctx, item)
}
func awaitOperationSignal(t *testing.T, signal <-chan struct{}) {
t.Helper()
select {
case <-signal:
case <-time.After(5 * time.Second):
t.Fatal("operation did not reach expected stage")
}
}
func TestOperationGateCoversVerifiedLifecycle(t *testing.T) {
for _, stage := range []string{"verify", "commit", "restore", "close"} {
t.Run(stage, func(t *testing.T) {
entered, resume, done := make(chan struct{}), make(chan struct{}), make(chan struct{})
var unblock sync.Once
release := func() { unblock.Do(func() { close(resume) }) }
defer release()
var hookOnce sync.Once
hook := func() { hookOnce.Do(func() { close(entered); <-resume }) }
server, memory := testServer("gate"), newMemoryClient()
seedVerifiedFiles(memory, server)
client := &operationCloseClient{memoryClient: memory}
journal := &operationJournal{}
if stage == "close" {
client.closeHook = hook
}
if stage == "commit" {
journal.commitHook = hook
}
factory := &operationFactory{client: client}
service := NewServiceWithJournal(factory, journal)
ctx := WithTransactionContext(t.Context(), TransactionContext{TaskID: "task", AttemptID: "attempt", AttemptNumber: 1})
artifact := testArtifact(t)
var report Report
var runErr error
go func() {
defer close(done)
report, runErr = service.DeployVerified(ctx, []Target{{Server: server}}, artifact, Verification{
Verify: func(context.Context) error {
if stage == "verify" {
hook()
}
if stage == "restore" {
return errors.New("injected verification failure")
}
return nil
},
VerifyRecovery: func(context.Context) error { hook(); return nil },
})
}()
awaitOperationSignal(t, entered)
calls := factory.calls.Load()
for _, operation := range []string{"deploy", "verified", "recover"} {
waitCtx, cancel := context.WithCancel(t.Context())
witness := &operationWitnessContext{Context: waitCtx, waiting: make(chan struct{})}
otherDone := make(chan struct{})
var otherErr error
var otherReport Report
go func() {
defer close(otherDone)
if operation == "recover" {
otherErr = service.RecoverPending(witness, func(context.Context, string) (Server, error) { return server, nil })
} else if operation == "verified" {
otherReport, otherErr = service.DeployVerified(witness, []Target{{Server: server}}, artifact, Verification{Verify: func(context.Context) error { return nil }})
} else {
otherReport = service.Deploy(witness, []Target{{Server: server}}, artifact)
}
}()
awaitOperationSignal(t, witness.waiting)
cancel()
awaitOperationSignal(t, otherDone)
if factory.calls.Load() != calls || (operation != "deploy" && !errors.Is(otherErr, context.Canceled)) || (operation != "recover" && otherReport.Status != "failed") {
t.Fatalf("concurrent %s entered during %s: %+v %v", operation, stage, otherReport, otherErr)
}
}
release()
awaitOperationSignal(t, done)
if stage == "restore" {
if runErr == nil || report.RecoveryStatus != "success" {
t.Fatalf("restore failed: %+v %v", report, runErr)
}
} else if runErr != nil || report.Status != "success" {
t.Fatalf("deployment failed: %+v %v", report, runErr)
}
unlock, err := service.acquireOperation(t.Context())
if err != nil {
t.Fatal(err)
}
unlock()
})
}
}
type claimedOperationJournal struct {
memoryTransactionJournal
claimed bool
claimErr error
listErr error
}
func (j *claimedOperationJournal) HasRecoveryClaim(context.Context, string) (bool, error) {
return j.claimed, j.claimErr
}
func (j *claimedOperationJournal) ListPendingDeploymentTransactions(ctx context.Context) ([]TransactionRecord, error) {
if j.listErr != nil {
return nil, j.listErr
}
return j.memoryTransactionJournal.ListPendingDeploymentTransactions(ctx)
}
func TestRecoveryClaimPreventsAutomaticMutation(t *testing.T) {
for _, failure := range []bool{false, true} {
t.Run(map[bool]string{false: "claimed", true: "read_failure"}[failure], func(t *testing.T) {
server, client, record := recoveryPreflightFixture()
original := maps.Clone(client.files)
journal := &claimedOperationJournal{memoryTransactionJournal: memoryTransactionJournal{pending: []TransactionRecord{record}}, claimed: true}
want := ErrRecoveryDispositionBusy
if failure {
journal.claimErr = errors.New("private database error")
want = ErrRecoveryDispositionUnavailable
}
factory := &operationFactory{client: client}
service := NewServiceWithJournal(factory, journal)
err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { t.Error("claimed record resolved"); return server, nil })
if !errors.Is(err, want) || factory.calls.Load() != 0 || len(journal.updated) != 0 || len(journal.finished) != 0 || !reflect.DeepEqual(original, client.files) {
t.Fatalf("claimed recovery changed state: %v", err)
}
})
}
}
func TestRecoveryClaimPreventsDeploymentBeforeConnection(t *testing.T) {
for _, scenario := range []string{"same_server", "alias", "claim_failure", "list_failure"} {
t.Run(scenario, func(t *testing.T) {
server, client, record := recoveryPreflightFixture()
journal := &claimedOperationJournal{memoryTransactionJournal: memoryTransactionJournal{pending: []TransactionRecord{record}}, claimed: true}
if scenario == "alias" {
server.ID = "alias"
}
if scenario == "claim_failure" {
journal.claimErr = errors.New("private claim error")
}
if scenario == "list_failure" {
journal.listErr = errors.New("private list error")
}
factory := &operationFactory{client: client}
service := NewServiceWithJournal(factory, journal)
report := service.Deploy(t.Context(), []Target{{Server: server}}, testArtifact(t))
if report.Status != "failed" || report.Failed != 1 || factory.calls.Load() != 0 || len(journal.pending) != 1 {
t.Fatalf("claimed deployment entered remote: %+v", report)
}
})
}
}
func TestUnclaimedRecoveryStillRuns(t *testing.T) {
server, client, record := recoveryPreflightFixture()
journal := &claimedOperationJournal{memoryTransactionJournal: memoryTransactionJournal{pending: []TransactionRecord{record}}}
factory := &operationFactory{client: client}
service := NewServiceWithJournal(factory, journal)
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { return nil }
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err != nil {
t.Fatal(err)
}
if factory.calls.Load() != 1 || len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRolledBack {
t.Fatal("unclaimed recovery was suppressed")
}
}
func TestCanceledOperationDoesNotLeakGate(t *testing.T) {
service := NewService(nil)
canceled, cancel := context.WithCancel(t.Context())
cancel()
if _, err := service.acquireOperation(canceled); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
for range 3 {
ctx, stop := context.WithTimeout(t.Context(), time.Second)
unlock, err := service.acquireOperation(ctx)
stop()
if err != nil {
t.Fatal(err)
}
unlock()
}
}
+22
View File
@@ -11,6 +11,7 @@ import (
"os"
"sort"
"strings"
"sync"
"time"
)
@@ -57,6 +58,8 @@ type Report struct {
}
type Service struct {
operationOnce sync.Once
operationGate chan struct{}
factory ClientFactory
journal TransactionJournal
now func() time.Time
@@ -72,6 +75,11 @@ func NewServiceWithJournal(factory ClientFactory, journal TransactionJournal) *S
}
func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifact) Report {
release, err := s.acquireOperation(ctx)
if err != nil {
return operationFailure(targets, err)
}
defer release()
return s.deploy(ctx, targets, artifact, nil)
}
@@ -150,6 +158,11 @@ func (s *Service) deployOne(ctx context.Context, server Server, domainName strin
result.Message = err.Error()
return result
}
if err := s.checkDeploymentClaims(ctx, server); err != nil {
result.FailureCode = FailureUnavailable
result.Message = err.Error()
return result
}
client, err := s.factory.Connect(ctx, server)
if err != nil {
result.FailureCode = FailureConnection
@@ -752,6 +765,11 @@ func (s *Service) RecoverPending(ctx context.Context, resolve func(context.Conte
if s == nil || s.journal == nil {
return nil
}
release, err := s.acquireOperation(ctx)
if err != nil {
return err
}
defer release()
if resolve == nil {
return errors.New("部署事务恢复缺少服务器解析器")
}
@@ -761,6 +779,10 @@ func (s *Service) RecoverPending(ctx context.Context, resolve func(context.Conte
}
var recoveryErrors []error
for _, record := range records {
if err := s.checkRecoveryClaim(ctx, record.ID); err != nil {
recoveryErrors = append(recoveryErrors, fmt.Errorf("事务 %s 未执行自动恢复:%w", record.ID, err))
continue
}
if err := s.recoverTransaction(ctx, &record, resolve); err != nil {
recoveryErrors = append(recoveryErrors, fmt.Errorf("事务 %s 恢复失败:%w", record.ID, err))
record.Status = TransactionStatusRecoveryRequired
+5
View File
@@ -31,6 +31,11 @@ func (s *Service) DeployVerified(ctx context.Context, targets []Target, artifact
if verification.Verify == nil {
return Report{Status: "failed"}, errors.New("缺少部署后站点校验,未执行推送")
}
release, err := s.acquireOperation(ctx)
if err != nil {
return operationFailure(targets, err), err
}
defer release()
pending := make([]*pendingDeployment, 0, len(targets))
// 异常退出时关闭连接但保留现场;不得以释放锁掩盖未确认的事务。
defer func() {
+3 -2
View File
@@ -12,6 +12,7 @@ import (
type lateCancellationContext struct {
context.Context
armed atomic.Bool
errCalls atomic.Int32
}
@@ -20,7 +21,7 @@ func (c *lateCancellationContext) Done() <-chan struct{} {
}
func (c *lateCancellationContext) Err() error {
if c.errCalls.Add(1) > 2 {
if c.armed.Load() && c.errCalls.Add(1) > 1 {
return context.Canceled
}
return nil
@@ -49,7 +50,7 @@ func TestVerifiedDeploymentCommitUsesIndependentContextAfterCancellationRace(t *
artifact := testArtifact(t)
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
report, err := service.DeployVerified(ctx, []Target{{Server: server}}, artifact, Verification{
Verify: func(context.Context) error { return nil },
Verify: func(context.Context) error { parent.armed.Store(true); return nil },
})
if err != nil || report.Status != "success" || !journal.commitContextOK {
t.Fatalf("unexpected deployment result: %+v, %v", report, err)