接入完整部署互斥与持久化人工占用防护
This commit is contained in:
1 parent
d801fb3833
commit
b3f27e88df
12 files changed
+430
-2
No files matched your search
@@ -0,0 +1,85 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// acquireOperation 覆盖整个部署、站点核验及收尾;人工执行器必须共用此入口。
|
||||
// 它不替代数据库持久化占用和远端文件锁,等待可以取消,不能提前释放。
|
||||
func (s *Service) acquireOperation(ctx context.Context) (func(), error) {
|
||||
if s == nil {
|
||||
return nil, ErrRecoveryDispositionUnavailable
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.operationOnce.Do(func() { s.operationGate = make(chan struct{}, 1) })
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case s.operationGate <- struct{}{}:
|
||||
if err := ctx.Err(); err != nil {
|
||||
<-s.operationGate
|
||||
return nil, err
|
||||
}
|
||||
return func() { <-s.operationGate }, nil
|
||||
}
|
||||
}
|
||||
|
||||
func operationFailure(targets []Target, err error) Report {
|
||||
report := Report{Total: len(targets), Failed: len(targets), Status: "failed", Results: make([]Result, 0, len(targets))}
|
||||
for _, target := range targets {
|
||||
report.Results = append(report.Results, Result{
|
||||
ServerID: target.Server.ID, Server: target.Server.Name, Status: "failed",
|
||||
FailureCode: FailureUnavailable, Message: fmt.Sprintf("未获得部署操作许可:%v", err),
|
||||
})
|
||||
}
|
||||
return report
|
||||
}
|
||||
|
||||
type recoveryClaimReader interface {
|
||||
HasRecoveryClaim(context.Context, string) (bool, error)
|
||||
}
|
||||
|
||||
func (s *Service) checkRecoveryClaim(ctx context.Context, id string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
reader, ok := s.journal.(recoveryClaimReader)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
claimed, err := reader.HasRecoveryClaim(ctx, id)
|
||||
if err != nil {
|
||||
return ErrRecoveryDispositionUnavailable
|
||||
}
|
||||
if claimed {
|
||||
return ErrRecoveryDispositionBusy
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// checkDeploymentClaims 在连接远端前拦截仍有人工占用的服务器。
|
||||
// 同地址同端口的别名也须检查;凭据或路径修改不能绕过原服务器占用。
|
||||
func (s *Service) checkDeploymentClaims(ctx context.Context, server Server) error {
|
||||
if _, ok := s.journal.(recoveryClaimReader); !ok {
|
||||
return nil
|
||||
}
|
||||
records, err := s.journal.ListPendingDeploymentTransactions(ctx)
|
||||
if err != nil {
|
||||
return ErrRecoveryDispositionUnavailable
|
||||
}
|
||||
for _, record := range records {
|
||||
sameServer := record.ServerID == server.ID
|
||||
if record.Evidence != nil {
|
||||
sameServer = sameServer || (record.Evidence.Address == server.Address && record.Evidence.Port == server.Port)
|
||||
}
|
||||
if sameServer {
|
||||
if err := s.checkRecoveryClaim(ctx, record.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"maps"
|
||||
"reflect"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type operationWitnessContext struct {
|
||||
context.Context
|
||||
once sync.Once
|
||||
waiting chan struct{}
|
||||
}
|
||||
|
||||
func (c *operationWitnessContext) Done() <-chan struct{} {
|
||||
c.once.Do(func() { close(c.waiting) })
|
||||
return c.Context.Done()
|
||||
}
|
||||
|
||||
type operationFactory struct {
|
||||
client Client
|
||||
calls atomic.Int32
|
||||
}
|
||||
|
||||
func (f *operationFactory) Connect(context.Context, Server) (Client, error) {
|
||||
f.calls.Add(1)
|
||||
return f.client, nil
|
||||
}
|
||||
|
||||
type operationCloseClient struct {
|
||||
*memoryClient
|
||||
closeHook func()
|
||||
}
|
||||
|
||||
func (c *operationCloseClient) Close() error {
|
||||
if c.closeHook != nil {
|
||||
c.closeHook()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type operationJournal struct {
|
||||
memoryTransactionJournal
|
||||
commitHook func()
|
||||
}
|
||||
|
||||
func (j *operationJournal) UpdateDeploymentTransaction(ctx context.Context, item TransactionRecord) error {
|
||||
if item.Phase == TransactionPhaseCommitting && j.commitHook != nil {
|
||||
j.commitHook()
|
||||
}
|
||||
return j.memoryTransactionJournal.UpdateDeploymentTransaction(ctx, item)
|
||||
}
|
||||
|
||||
func awaitOperationSignal(t *testing.T, signal <-chan struct{}) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-signal:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("operation did not reach expected stage")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOperationGateCoversVerifiedLifecycle(t *testing.T) {
|
||||
for _, stage := range []string{"verify", "commit", "restore", "close"} {
|
||||
t.Run(stage, func(t *testing.T) {
|
||||
entered, resume, done := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||
var unblock sync.Once
|
||||
release := func() { unblock.Do(func() { close(resume) }) }
|
||||
defer release()
|
||||
var hookOnce sync.Once
|
||||
hook := func() { hookOnce.Do(func() { close(entered); <-resume }) }
|
||||
server, memory := testServer("gate"), newMemoryClient()
|
||||
seedVerifiedFiles(memory, server)
|
||||
client := &operationCloseClient{memoryClient: memory}
|
||||
journal := &operationJournal{}
|
||||
if stage == "close" {
|
||||
client.closeHook = hook
|
||||
}
|
||||
if stage == "commit" {
|
||||
journal.commitHook = hook
|
||||
}
|
||||
factory := &operationFactory{client: client}
|
||||
service := NewServiceWithJournal(factory, journal)
|
||||
ctx := WithTransactionContext(t.Context(), TransactionContext{TaskID: "task", AttemptID: "attempt", AttemptNumber: 1})
|
||||
artifact := testArtifact(t)
|
||||
var report Report
|
||||
var runErr error
|
||||
go func() {
|
||||
defer close(done)
|
||||
report, runErr = service.DeployVerified(ctx, []Target{{Server: server}}, artifact, Verification{
|
||||
Verify: func(context.Context) error {
|
||||
if stage == "verify" {
|
||||
hook()
|
||||
}
|
||||
if stage == "restore" {
|
||||
return errors.New("injected verification failure")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
VerifyRecovery: func(context.Context) error { hook(); return nil },
|
||||
})
|
||||
}()
|
||||
awaitOperationSignal(t, entered)
|
||||
calls := factory.calls.Load()
|
||||
for _, operation := range []string{"deploy", "verified", "recover"} {
|
||||
waitCtx, cancel := context.WithCancel(t.Context())
|
||||
witness := &operationWitnessContext{Context: waitCtx, waiting: make(chan struct{})}
|
||||
otherDone := make(chan struct{})
|
||||
var otherErr error
|
||||
var otherReport Report
|
||||
go func() {
|
||||
defer close(otherDone)
|
||||
if operation == "recover" {
|
||||
otherErr = service.RecoverPending(witness, func(context.Context, string) (Server, error) { return server, nil })
|
||||
} else if operation == "verified" {
|
||||
otherReport, otherErr = service.DeployVerified(witness, []Target{{Server: server}}, artifact, Verification{Verify: func(context.Context) error { return nil }})
|
||||
} else {
|
||||
otherReport = service.Deploy(witness, []Target{{Server: server}}, artifact)
|
||||
}
|
||||
}()
|
||||
awaitOperationSignal(t, witness.waiting)
|
||||
cancel()
|
||||
awaitOperationSignal(t, otherDone)
|
||||
if factory.calls.Load() != calls || (operation != "deploy" && !errors.Is(otherErr, context.Canceled)) || (operation != "recover" && otherReport.Status != "failed") {
|
||||
t.Fatalf("concurrent %s entered during %s: %+v %v", operation, stage, otherReport, otherErr)
|
||||
}
|
||||
}
|
||||
release()
|
||||
awaitOperationSignal(t, done)
|
||||
if stage == "restore" {
|
||||
if runErr == nil || report.RecoveryStatus != "success" {
|
||||
t.Fatalf("restore failed: %+v %v", report, runErr)
|
||||
}
|
||||
} else if runErr != nil || report.Status != "success" {
|
||||
t.Fatalf("deployment failed: %+v %v", report, runErr)
|
||||
}
|
||||
unlock, err := service.acquireOperation(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unlock()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type claimedOperationJournal struct {
|
||||
memoryTransactionJournal
|
||||
claimed bool
|
||||
claimErr error
|
||||
listErr error
|
||||
}
|
||||
|
||||
func (j *claimedOperationJournal) HasRecoveryClaim(context.Context, string) (bool, error) {
|
||||
return j.claimed, j.claimErr
|
||||
}
|
||||
func (j *claimedOperationJournal) ListPendingDeploymentTransactions(ctx context.Context) ([]TransactionRecord, error) {
|
||||
if j.listErr != nil {
|
||||
return nil, j.listErr
|
||||
}
|
||||
return j.memoryTransactionJournal.ListPendingDeploymentTransactions(ctx)
|
||||
}
|
||||
|
||||
func TestRecoveryClaimPreventsAutomaticMutation(t *testing.T) {
|
||||
for _, failure := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "claimed", true: "read_failure"}[failure], func(t *testing.T) {
|
||||
server, client, record := recoveryPreflightFixture()
|
||||
original := maps.Clone(client.files)
|
||||
journal := &claimedOperationJournal{memoryTransactionJournal: memoryTransactionJournal{pending: []TransactionRecord{record}}, claimed: true}
|
||||
want := ErrRecoveryDispositionBusy
|
||||
if failure {
|
||||
journal.claimErr = errors.New("private database error")
|
||||
want = ErrRecoveryDispositionUnavailable
|
||||
}
|
||||
factory := &operationFactory{client: client}
|
||||
service := NewServiceWithJournal(factory, journal)
|
||||
err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { t.Error("claimed record resolved"); return server, nil })
|
||||
if !errors.Is(err, want) || factory.calls.Load() != 0 || len(journal.updated) != 0 || len(journal.finished) != 0 || !reflect.DeepEqual(original, client.files) {
|
||||
t.Fatalf("claimed recovery changed state: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryClaimPreventsDeploymentBeforeConnection(t *testing.T) {
|
||||
for _, scenario := range []string{"same_server", "alias", "claim_failure", "list_failure"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
server, client, record := recoveryPreflightFixture()
|
||||
journal := &claimedOperationJournal{memoryTransactionJournal: memoryTransactionJournal{pending: []TransactionRecord{record}}, claimed: true}
|
||||
if scenario == "alias" {
|
||||
server.ID = "alias"
|
||||
}
|
||||
if scenario == "claim_failure" {
|
||||
journal.claimErr = errors.New("private claim error")
|
||||
}
|
||||
if scenario == "list_failure" {
|
||||
journal.listErr = errors.New("private list error")
|
||||
}
|
||||
factory := &operationFactory{client: client}
|
||||
service := NewServiceWithJournal(factory, journal)
|
||||
report := service.Deploy(t.Context(), []Target{{Server: server}}, testArtifact(t))
|
||||
if report.Status != "failed" || report.Failed != 1 || factory.calls.Load() != 0 || len(journal.pending) != 1 {
|
||||
t.Fatalf("claimed deployment entered remote: %+v", report)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnclaimedRecoveryStillRuns(t *testing.T) {
|
||||
server, client, record := recoveryPreflightFixture()
|
||||
journal := &claimedOperationJournal{memoryTransactionJournal: memoryTransactionJournal{pending: []TransactionRecord{record}}}
|
||||
factory := &operationFactory{client: client}
|
||||
service := NewServiceWithJournal(factory, journal)
|
||||
service.VerifyRecoveryTarget = func(context.Context, Server, RecoveryBaseline, bool) error { return nil }
|
||||
if err := service.RecoverPending(t.Context(), func(context.Context, string) (Server, error) { return server, nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if factory.calls.Load() != 1 || len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRolledBack {
|
||||
t.Fatal("unclaimed recovery was suppressed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanceledOperationDoesNotLeakGate(t *testing.T) {
|
||||
service := NewService(nil)
|
||||
canceled, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
if _, err := service.acquireOperation(canceled); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 3 {
|
||||
ctx, stop := context.WithTimeout(t.Context(), time.Second)
|
||||
unlock, err := service.acquireOperation(ctx)
|
||||
stop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unlock()
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -57,6 +58,8 @@ type Report struct {
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
operationOnce sync.Once
|
||||
operationGate chan struct{}
|
||||
factory ClientFactory
|
||||
journal TransactionJournal
|
||||
now func() time.Time
|
||||
@@ -72,6 +75,11 @@ func NewServiceWithJournal(factory ClientFactory, journal TransactionJournal) *S
|
||||
}
|
||||
|
||||
func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifact) Report {
|
||||
release, err := s.acquireOperation(ctx)
|
||||
if err != nil {
|
||||
return operationFailure(targets, err)
|
||||
}
|
||||
defer release()
|
||||
return s.deploy(ctx, targets, artifact, nil)
|
||||
}
|
||||
|
||||
@@ -150,6 +158,11 @@ func (s *Service) deployOne(ctx context.Context, server Server, domainName strin
|
||||
result.Message = err.Error()
|
||||
return result
|
||||
}
|
||||
if err := s.checkDeploymentClaims(ctx, server); err != nil {
|
||||
result.FailureCode = FailureUnavailable
|
||||
result.Message = err.Error()
|
||||
return result
|
||||
}
|
||||
client, err := s.factory.Connect(ctx, server)
|
||||
if err != nil {
|
||||
result.FailureCode = FailureConnection
|
||||
@@ -752,6 +765,11 @@ func (s *Service) RecoverPending(ctx context.Context, resolve func(context.Conte
|
||||
if s == nil || s.journal == nil {
|
||||
return nil
|
||||
}
|
||||
release, err := s.acquireOperation(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
if resolve == nil {
|
||||
return errors.New("部署事务恢复缺少服务器解析器")
|
||||
}
|
||||
@@ -761,6 +779,10 @@ func (s *Service) RecoverPending(ctx context.Context, resolve func(context.Conte
|
||||
}
|
||||
var recoveryErrors []error
|
||||
for _, record := range records {
|
||||
if err := s.checkRecoveryClaim(ctx, record.ID); err != nil {
|
||||
recoveryErrors = append(recoveryErrors, fmt.Errorf("事务 %s 未执行自动恢复:%w", record.ID, err))
|
||||
continue
|
||||
}
|
||||
if err := s.recoverTransaction(ctx, &record, resolve); err != nil {
|
||||
recoveryErrors = append(recoveryErrors, fmt.Errorf("事务 %s 恢复失败:%w", record.ID, err))
|
||||
record.Status = TransactionStatusRecoveryRequired
|
||||
|
||||
@@ -31,6 +31,11 @@ func (s *Service) DeployVerified(ctx context.Context, targets []Target, artifact
|
||||
if verification.Verify == nil {
|
||||
return Report{Status: "failed"}, errors.New("缺少部署后站点校验,未执行推送")
|
||||
}
|
||||
release, err := s.acquireOperation(ctx)
|
||||
if err != nil {
|
||||
return operationFailure(targets, err), err
|
||||
}
|
||||
defer release()
|
||||
pending := make([]*pendingDeployment, 0, len(targets))
|
||||
// 异常退出时关闭连接但保留现场;不得以释放锁掩盖未确认的事务。
|
||||
defer func() {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
type lateCancellationContext struct {
|
||||
context.Context
|
||||
armed atomic.Bool
|
||||
errCalls atomic.Int32
|
||||
}
|
||||
|
||||
@@ -20,7 +21,7 @@ func (c *lateCancellationContext) Done() <-chan struct{} {
|
||||
}
|
||||
|
||||
func (c *lateCancellationContext) Err() error {
|
||||
if c.errCalls.Add(1) > 2 {
|
||||
if c.armed.Load() && c.errCalls.Add(1) > 1 {
|
||||
return context.Canceled
|
||||
}
|
||||
return nil
|
||||
@@ -49,7 +50,7 @@ func TestVerifiedDeploymentCommitUsesIndependentContextAfterCancellationRace(t *
|
||||
artifact := testArtifact(t)
|
||||
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
|
||||
report, err := service.DeployVerified(ctx, []Target{{Server: server}}, artifact, Verification{
|
||||
Verify: func(context.Context) error { return nil },
|
||||
Verify: func(context.Context) error { parent.armed.Store(true); return nil },
|
||||
})
|
||||
if err != nil || report.Status != "success" || !journal.commitContextOK {
|
||||
t.Fatalf("unexpected deployment result: %+v, %v", report, err)
|
||||
|
||||
Reference in new issue
Block a user