完成部署后校验失败恢复整改并更新实机进度
延后事务提交,校验失败时恢复目标文件、配置和服务状态;新增恢复边界回归、前端提示及生产测试记录。
This commit is contained in:
1 parent
c623e2d324
commit
3cc8fb6470
13 files changed
+588
-58
No files matched your search
@@ -3,36 +3,41 @@ package deployment
|
||||
type FailureCode string
|
||||
|
||||
const (
|
||||
FailureUnavailable FailureCode = "unavailable"
|
||||
FailureArtifact FailureCode = "artifact_invalid"
|
||||
FailureConfiguration FailureCode = "configuration_invalid"
|
||||
FailureConnection FailureCode = "connection_failed"
|
||||
FailureDirectory FailureCode = "directory_failed"
|
||||
FailureLock FailureCode = "lock_unavailable"
|
||||
FailureUpload FailureCode = "upload_failed"
|
||||
FailureReplace FailureCode = "replace_failed"
|
||||
FailureValidation FailureCode = "validation_failed"
|
||||
FailureRollback FailureCode = "rollback_failed"
|
||||
FailureRuntimeUnknown FailureCode = "runtime_unconfirmed"
|
||||
FailureCleanup FailureCode = "cleanup_failed"
|
||||
FailureLockRelease FailureCode = "lock_release_failed"
|
||||
FailureUnavailable FailureCode = "unavailable"
|
||||
FailureArtifact FailureCode = "artifact_invalid"
|
||||
FailureConfiguration FailureCode = "configuration_invalid"
|
||||
FailureConnection FailureCode = "connection_failed"
|
||||
FailureDirectory FailureCode = "directory_failed"
|
||||
FailureLock FailureCode = "lock_unavailable"
|
||||
FailureUpload FailureCode = "upload_failed"
|
||||
FailureReplace FailureCode = "replace_failed"
|
||||
FailureValidation FailureCode = "validation_failed"
|
||||
FailureRollback FailureCode = "rollback_failed"
|
||||
FailureRuntimeUnknown FailureCode = "runtime_unconfirmed"
|
||||
FailureCleanup FailureCode = "cleanup_failed"
|
||||
FailureLockRelease FailureCode = "lock_release_failed"
|
||||
FailureRecoveryPending FailureCode = "recovery_pending"
|
||||
FailureRecoveryUnknown FailureCode = "recovery_unconfirmed"
|
||||
FailureRecoveryRestored FailureCode = "recovery_restored"
|
||||
)
|
||||
|
||||
// 只公开固定诊断,不从远端输出、错误文本或文件路径推断可展示的消息。
|
||||
var publicFailureMessages = map[FailureCode]string{
|
||||
FailureUnavailable: "部署服务不可用,请联系管理员检查服务配置",
|
||||
FailureArtifact: "待部署的证书或私钥校验失败,尚未推送到服务器",
|
||||
FailureConfiguration: "服务器部署配置无效,请检查认证、证书路径和命令配置",
|
||||
FailureConnection: "连接部署服务器失败,请检查网络、SSH 认证及主机身份配置",
|
||||
FailureDirectory: "无法创建证书目录,请检查部署账户的目录权限",
|
||||
FailureLock: "无法取得部署锁,可能有任务正在执行或存在中断遗留;请管理员核查远端文件、部署锁及服务状态后再重试,勿直接删除锁",
|
||||
FailureUpload: "证书上传失败,已尝试回滚;请检查受保护的服务日志后再重试",
|
||||
FailureReplace: "证书文件替换失败,已尝试回滚;请检查受保护的服务日志后再重试",
|
||||
FailureValidation: "服务配置校验失败,文件已回滚,未执行重载;请修正配置后再重试",
|
||||
FailureRollback: "部署失败且文件回滚未完成,已保留部署锁;请管理员核查远端文件及服务状态,勿直接重试或删除锁",
|
||||
FailureRuntimeUnknown: "服务重载失败,文件已回滚,但服务运行状态未确认;请管理员核对配置并恢复服务后再重试",
|
||||
FailureCleanup: "证书已推送并重载,但旧文件清理未完成;请管理员核查现场,不能据此确认站点新证书已生效",
|
||||
FailureLockRelease: "部署锁释放未完成;请管理员核查远端文件、部署锁及服务状态后再重试",
|
||||
FailureRecoveryUnknown: "本次部署未完成,已恢复旧文件,但服务或站点恢复状态未确认,已保留部署锁;请管理员核查,勿直接重试或删除锁",
|
||||
FailureRecoveryRestored: "本次部署未完成,旧文件已恢复并通过配置校验及重载,站点已恢复部署前证书;请核查失败原因后再重试",
|
||||
FailureUnavailable: "部署服务不可用,请联系管理员检查服务配置",
|
||||
FailureArtifact: "待部署的证书或私钥校验失败,尚未推送到服务器",
|
||||
FailureConfiguration: "服务器部署配置无效,请检查认证、证书路径和命令配置",
|
||||
FailureConnection: "连接部署服务器失败,请检查网络、SSH 认证及主机身份配置",
|
||||
FailureDirectory: "无法创建证书目录,请检查部署账户的目录权限",
|
||||
FailureLock: "无法取得部署锁,可能有任务正在执行或存在中断遗留;请管理员核查远端文件、部署锁及服务状态后再重试,勿直接删除锁",
|
||||
FailureUpload: "证书上传失败,已尝试回滚;请检查受保护的服务日志后再重试",
|
||||
FailureReplace: "证书文件替换失败,已尝试回滚;请检查受保护的服务日志后再重试",
|
||||
FailureValidation: "服务配置校验失败,文件已回滚,未执行重载;请修正配置后再重试",
|
||||
FailureRollback: "部署失败且文件回滚未完成,已保留部署锁;请管理员核查远端文件及服务状态,勿直接重试或删除锁",
|
||||
FailureRuntimeUnknown: "服务重载失败,文件已回滚,但服务运行状态未确认;请管理员核对配置并恢复服务后再重试",
|
||||
FailureCleanup: "证书已推送并重载,但旧文件清理未完成;请管理员核查现场,不能据此确认站点新证书已生效",
|
||||
FailureLockRelease: "部署锁释放未完成;请管理员核查远端文件、部署锁及服务状态后再重试",
|
||||
}
|
||||
|
||||
// PublicResult 保留结果事实,同时只允许固定错误码及中文诊断进入公开响应。
|
||||
|
||||
@@ -47,6 +47,7 @@ type Report struct {
|
||||
Results []Result `json:"results"`
|
||||
// 站点校验与各服务器文件推送分开记录,不表示所有服务器均完成独立 HTTPS 验证。
|
||||
VerificationStatus string `json:"verification_status,omitempty"`
|
||||
RecoveryStatus string `json:"recovery_status,omitempty"`
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
@@ -59,6 +60,10 @@ func NewService(factory ClientFactory) *Service {
|
||||
}
|
||||
|
||||
func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifact) Report {
|
||||
return s.deploy(ctx, targets, artifact, nil)
|
||||
}
|
||||
|
||||
func (s *Service) deploy(ctx context.Context, targets []Target, artifact Artifact, pending *[]*pendingDeployment) Report {
|
||||
report := Report{Total: len(targets), Results: make([]Result, 0, len(targets))}
|
||||
if len(targets) == 0 {
|
||||
report.Status = "failed"
|
||||
@@ -80,8 +85,8 @@ func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifac
|
||||
}
|
||||
return report
|
||||
}
|
||||
for _, target := range targets {
|
||||
result := s.deployOne(ctx, target.Server, artifact)
|
||||
for index, target := range targets {
|
||||
result := s.deployOne(ctx, target.Server, artifact, pending, index)
|
||||
report.Results = append(report.Results, result)
|
||||
if result.Status == "success" {
|
||||
report.Success++
|
||||
@@ -101,22 +106,30 @@ func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifac
|
||||
}
|
||||
|
||||
func (s *Service) DeploySaved(ctx context.Context, targets []Target, saved SavedFiles) (Report, error) {
|
||||
artifact, err := loadArtifact(saved)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
return s.Deploy(ctx, targets, artifact), nil
|
||||
}
|
||||
|
||||
func loadArtifact(saved SavedFiles) (Artifact, error) {
|
||||
certificateData, err := os.ReadFile(saved.CertificatePath)
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("读取证书文件失败:%w", err)
|
||||
return Artifact{}, fmt.Errorf("读取证书文件失败:%w", err)
|
||||
}
|
||||
fullChainData, err := os.ReadFile(saved.FullChainPath)
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("读取完整证书链失败:%w", err)
|
||||
return Artifact{}, fmt.Errorf("读取完整证书链失败:%w", err)
|
||||
}
|
||||
privateKeyData, err := os.ReadFile(saved.PrivateKeyPath)
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("读取私钥文件失败:%w", err)
|
||||
return Artifact{}, fmt.Errorf("读取私钥文件失败:%w", err)
|
||||
}
|
||||
return s.Deploy(ctx, targets, Artifact{Certificate: certificateData, FullChain: fullChainData, PrivateKey: privateKeyData}), nil
|
||||
return Artifact{Certificate: certificateData, FullChain: fullChainData, PrivateKey: privateKeyData}, nil
|
||||
}
|
||||
|
||||
func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifact) (result Result) {
|
||||
func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifact, pending *[]*pendingDeployment, index int) (result Result) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
server = server.Normalize()
|
||||
@@ -131,7 +144,12 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
result.Message = fmt.Sprintf("连接服务器失败:%v", err)
|
||||
return result
|
||||
}
|
||||
defer func() { _ = client.Close() }()
|
||||
handedOff := false
|
||||
defer func() {
|
||||
if !handedOff {
|
||||
_ = client.Close()
|
||||
}
|
||||
}()
|
||||
paths := []remoteFile{
|
||||
{path: RemotePath(server.CertificatePath), data: artifact.Certificate, mode: 0o644},
|
||||
{path: RemotePath(server.CertificateChainPath), data: artifact.FullChain, mode: 0o644},
|
||||
@@ -160,7 +178,7 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
sort.Strings(lockPaths)
|
||||
heldLocks := make([]string, 0, len(lockPaths))
|
||||
retainLocks := false
|
||||
defer func() {
|
||||
releaseLocks := func() {
|
||||
if len(heldLocks) == 0 || retainLocks {
|
||||
return
|
||||
}
|
||||
@@ -179,6 +197,11 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
result.Message += fmt.Sprintf(";释放部署锁 %s 失败,需人工检查:%v", heldLocks[index], err)
|
||||
}
|
||||
}
|
||||
}
|
||||
defer func() {
|
||||
if !handedOff {
|
||||
releaseLocks()
|
||||
}
|
||||
}()
|
||||
for _, lockPath := range lockPaths {
|
||||
if err := client.Lock(ctx, lockPath); err != nil {
|
||||
@@ -300,6 +323,59 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
}
|
||||
return failed(fmt.Sprintf("reload 失败:%v", err))
|
||||
}
|
||||
if pending != nil {
|
||||
// 旧文件与锁在所有目标及站点校验结束前始终保留,禁止提前提交。
|
||||
result.Status = "success"
|
||||
result.FailureCode = ""
|
||||
result.Message = "证书已推送并重载,等待站点校验"
|
||||
closed := false
|
||||
transaction := &pendingDeployment{index: index}
|
||||
transaction.restore = func(recoveryContext context.Context) Result {
|
||||
result.Status = "failed"
|
||||
if err := rollback(); err != nil {
|
||||
result.FailureCode = FailureRollback
|
||||
result.Message = fmt.Sprintf("部署未提交,自动回滚失败:%v", err)
|
||||
return result
|
||||
}
|
||||
result.FailureCode = FailureRecoveryUnknown
|
||||
if err := reconnectDeploymentClient(recoveryContext, s.factory, server, &client); err != nil {
|
||||
result.Message = fmt.Sprintf("旧文件已恢复,重新连接以恢复服务失败:%v", err)
|
||||
return result
|
||||
}
|
||||
for _, command := range [][]string{server.ValidationCommand, server.ReloadCommand} {
|
||||
outcome, err := client.Run(recoveryContext, command)
|
||||
if err != nil || outcome.ExitCode != 0 {
|
||||
result.Message = fmt.Sprintf("旧文件已恢复,但恢复配置校验或重载失败,退出码 %d:%v", outcome.ExitCode, err)
|
||||
return result
|
||||
}
|
||||
}
|
||||
result.FailureCode = FailureRecoveryPending
|
||||
result.Message = "本次部署未提交,旧文件已恢复并通过配置校验及重载,等待站点恢复核验"
|
||||
return result
|
||||
}
|
||||
transaction.commit = func(commitContext context.Context) Result {
|
||||
if err := cleanupBackups(commitContext, client, backups, paths); err != nil {
|
||||
result.Status = "failed"
|
||||
result.FailureCode = FailureCleanup
|
||||
result.Message = fmt.Sprintf("站点校验通过,但事务旧文件清理失败:%v", err)
|
||||
return result
|
||||
}
|
||||
result.Message = "证书部署、配置校验并 reload 成功"
|
||||
return result
|
||||
}
|
||||
transaction.finish = func(retain bool) Result {
|
||||
if !closed {
|
||||
retainLocks = retain
|
||||
releaseLocks()
|
||||
_ = client.Close()
|
||||
closed = true
|
||||
}
|
||||
return result
|
||||
}
|
||||
handedOff = true
|
||||
*pending = append(*pending, transaction)
|
||||
return result
|
||||
}
|
||||
if err := cleanupBackups(ctx, client, backups, paths); err != nil {
|
||||
result.Status = "failed"
|
||||
result.FailureCode = FailureCleanup
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Verification 将站点生效校验与恢复核验置于远端文件事务之内。
|
||||
type Verification struct {
|
||||
Verify func(context.Context) error
|
||||
VerifyRecovery func(context.Context) error
|
||||
}
|
||||
|
||||
type pendingDeployment struct {
|
||||
index int
|
||||
commit func(context.Context) Result
|
||||
restore func(context.Context) Result
|
||||
finish func(bool) Result
|
||||
}
|
||||
|
||||
func (s *Service) DeploySavedVerified(ctx context.Context, targets []Target, saved SavedFiles, verification Verification) (Report, error) {
|
||||
artifact, err := loadArtifact(saved)
|
||||
if err != nil {
|
||||
return Report{Status: "failed"}, err
|
||||
}
|
||||
return s.DeployVerified(ctx, targets, artifact, verification)
|
||||
}
|
||||
|
||||
func (s *Service) DeployVerified(ctx context.Context, targets []Target, artifact Artifact, verification Verification) (Report, error) {
|
||||
if verification.Verify == nil {
|
||||
return Report{Status: "failed"}, errors.New("缺少部署后站点校验,未执行推送")
|
||||
}
|
||||
pending := make([]*pendingDeployment, 0, len(targets))
|
||||
// 异常退出时关闭连接但保留现场;不得以释放锁掩盖未确认的事务。
|
||||
defer func() {
|
||||
for _, transaction := range pending {
|
||||
transaction.finish(true)
|
||||
}
|
||||
}()
|
||||
report := s.deploy(ctx, targets, artifact, &pending)
|
||||
var deployErr error
|
||||
if report.Status != "success" {
|
||||
deployErr = fmt.Errorf("证书部署未全部成功:成功 %d 台,失败 %d 台", report.Success, report.Failed)
|
||||
} else {
|
||||
report.VerificationStatus = "failed"
|
||||
deployErr = ctx.Err()
|
||||
if deployErr == nil {
|
||||
deployErr = verification.Verify(ctx)
|
||||
}
|
||||
if deployErr == nil {
|
||||
deployErr = ctx.Err()
|
||||
}
|
||||
if deployErr != nil {
|
||||
deployErr = fmt.Errorf("部署后 HTTPS 校验失败:%w", deployErr)
|
||||
} else {
|
||||
report.VerificationStatus = "success"
|
||||
}
|
||||
}
|
||||
if deployErr != nil {
|
||||
if len(pending) > 0 {
|
||||
s.restorePending(ctx, &report, pending, verification)
|
||||
}
|
||||
summarizeReport(&report)
|
||||
report.Status = "failed"
|
||||
return report, deployErr
|
||||
}
|
||||
for _, transaction := range pending {
|
||||
result := transaction.commit(ctx)
|
||||
report.Results[transaction.index] = transaction.finish(result.Status != "success")
|
||||
}
|
||||
summarizeReport(&report)
|
||||
if report.Status != "success" {
|
||||
return report, errors.New("站点校验通过,但事务清理或释放部署锁未完成")
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (s *Service) restorePending(ctx context.Context, report *Report, pending []*pendingDeployment, verification Verification) {
|
||||
report.RecoveryStatus = "unconfirmed"
|
||||
for index := len(pending) - 1; index >= 0; index-- {
|
||||
transaction := pending[index]
|
||||
// 请求取消后仍使用独立有界上下文恢复,不重放签发或文件推送。
|
||||
recoveryContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
|
||||
report.Results[transaction.index] = transaction.restore(recoveryContext)
|
||||
cancel()
|
||||
}
|
||||
restored := true
|
||||
for _, result := range report.Results {
|
||||
switch result.FailureCode {
|
||||
case FailureRecoveryPending, FailureConfiguration, FailureConnection, FailureDirectory, FailureUpload, FailureReplace, FailureValidation:
|
||||
// 这些失败要么未触碰目标,要么已完成文件回滚且未执行重载。
|
||||
default:
|
||||
restored = false
|
||||
}
|
||||
}
|
||||
var recoveryErr error
|
||||
if !restored {
|
||||
report.RecoveryStatus = "failed"
|
||||
} else if verification.VerifyRecovery != nil {
|
||||
recoveryContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
|
||||
err := verification.VerifyRecovery(recoveryContext)
|
||||
if err == nil {
|
||||
err = recoveryContext.Err()
|
||||
}
|
||||
cancel()
|
||||
if err == nil {
|
||||
report.RecoveryStatus = "success"
|
||||
} else {
|
||||
recoveryErr = err
|
||||
}
|
||||
}
|
||||
for _, transaction := range pending {
|
||||
result := transaction.finish(report.RecoveryStatus != "success")
|
||||
if result.FailureCode == FailureLockRelease {
|
||||
report.RecoveryStatus = "failed"
|
||||
}
|
||||
if result.FailureCode == FailureRecoveryPending {
|
||||
if report.RecoveryStatus == "success" {
|
||||
result.FailureCode = FailureRecoveryRestored
|
||||
} else {
|
||||
result.FailureCode = FailureRecoveryUnknown
|
||||
}
|
||||
result.Message = publicFailureMessages[result.FailureCode]
|
||||
if recoveryErr != nil {
|
||||
result.Message += fmt.Sprintf(";站点恢复核验失败:%v", recoveryErr)
|
||||
}
|
||||
}
|
||||
report.Results[transaction.index] = result
|
||||
}
|
||||
}
|
||||
|
||||
func summarizeReport(report *Report) {
|
||||
report.Success, report.Failed = 0, 0
|
||||
for _, result := range report.Results {
|
||||
if result.Status == "success" {
|
||||
report.Success++
|
||||
} else {
|
||||
report.Failed++
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case report.Total == 0 || report.Success == 0:
|
||||
report.Status = "failed"
|
||||
case report.Failed == 0:
|
||||
report.Status = "success"
|
||||
default:
|
||||
report.Status = "partial"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func seedVerifiedFiles(client *memoryClient, server Server) map[string]memoryFile {
|
||||
files := map[string]memoryFile{
|
||||
server.CertificatePath: {data: []byte("原证书"), mode: 0o644},
|
||||
server.CertificateChainPath: {data: []byte("原证书链"), mode: 0o644},
|
||||
server.PrivateKeyPath: {data: []byte("原私钥"), mode: 0o600},
|
||||
}
|
||||
for path, file := range files {
|
||||
client.files[path] = file
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func assertVerifiedFiles(t *testing.T, client *memoryClient, expected map[string]memoryFile) {
|
||||
t.Helper()
|
||||
for path, file := range expected {
|
||||
actual, ok := client.files[path]
|
||||
if !ok || !bytes.Equal(actual.data, file.data) || actual.mode != file.mode {
|
||||
t.Fatalf("未恢复原内容或权限:%s", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertVerifiedLocks(t *testing.T, client *memoryClient, server Server, held bool) {
|
||||
t.Helper()
|
||||
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
|
||||
_, exists := client.files[path+".scert-lock"]
|
||||
if exists != held {
|
||||
t.Fatalf("部署锁状态不符合预期:%s,锁定 %t", path, exists)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentDefersCommitUntilAllTargetsAndSitePass(t *testing.T) {
|
||||
clients := map[string]*memoryClient{"first": newMemoryClient(), "second": newMemoryClient()}
|
||||
servers := []Server{testServer("first"), testServer("second")}
|
||||
for _, server := range servers {
|
||||
seedVerifiedFiles(clients[server.ID], server)
|
||||
}
|
||||
artifact := testArtifact(t)
|
||||
called := false
|
||||
service := NewService(memoryFactory{clients: clients})
|
||||
report, err := service.DeployVerified(t.Context(), []Target{{Server: servers[0]}, {Server: servers[1]}}, artifact, Verification{
|
||||
Verify: func(context.Context) error {
|
||||
called = true
|
||||
for _, server := range servers {
|
||||
client := clients[server.ID]
|
||||
assertVerifiedLocks(t, client, server, true)
|
||||
if !bytes.Equal(client.files[server.CertificatePath].data, artifact.Certificate) || len(client.commands) != 2 {
|
||||
t.Fatal("必须在所有服务器推送和重载后才核验站点")
|
||||
}
|
||||
backups := 0
|
||||
for path := range client.files {
|
||||
if strings.Contains(path, ".scert-backup-") {
|
||||
backups++
|
||||
}
|
||||
}
|
||||
if backups != 3 {
|
||||
t.Fatal("站点确认前必须保留三份事务旧文件")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
VerifyRecovery: func(context.Context) error { t.Fatal("成功部署不得回滚"); return nil },
|
||||
})
|
||||
if err != nil || !called || report.Status != "success" || report.Success != 2 || report.VerificationStatus != "success" || report.RecoveryStatus != "" {
|
||||
t.Fatalf("事务提交结果错误:%+v,%v", report, err)
|
||||
}
|
||||
for _, server := range servers {
|
||||
client := clients[server.ID]
|
||||
assertVerifiedLocks(t, client, server, false)
|
||||
if len(client.files) != 3 {
|
||||
t.Fatal("提交后不应残留事务文件")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentFailureRestoresAllTargetsBeforeRecoveryCheck(t *testing.T) {
|
||||
clients := map[string]*memoryClient{"first": newMemoryClient(), "second": newMemoryClient()}
|
||||
servers := []Server{testServer("first"), testServer("second")}
|
||||
originals := map[string]map[string]memoryFile{}
|
||||
for _, server := range servers {
|
||||
originals[server.ID] = seedVerifiedFiles(clients[server.ID], server)
|
||||
}
|
||||
recovered := false
|
||||
verificationErr := errors.New("公网仍返回错误证书")
|
||||
report, err := NewService(memoryFactory{clients: clients}).DeployVerified(t.Context(), []Target{{Server: servers[0]}, {Server: servers[1]}}, testArtifact(t), Verification{
|
||||
Verify: func(context.Context) error { return verificationErr },
|
||||
VerifyRecovery: func(ctx context.Context) error {
|
||||
recovered = true
|
||||
if _, ok := ctx.Deadline(); !ok || ctx.Err() != nil {
|
||||
t.Fatal("恢复核验必须有独立有效期限")
|
||||
}
|
||||
for _, server := range servers {
|
||||
client := clients[server.ID]
|
||||
assertVerifiedFiles(t, client, originals[server.ID])
|
||||
assertVerifiedLocks(t, client, server, true)
|
||||
expectedCommands := [][]string{{"nginx", "-t"}, {"nginx", "-s", "reload"}, {"nginx", "-t"}, {"nginx", "-s", "reload"}}
|
||||
if !reflect.DeepEqual(client.commands, expectedCommands) {
|
||||
t.Fatalf("必须恢复配置并重载后核验站点:实际=%#v,期望=%#v", client.commands, expectedCommands)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
if !errors.Is(err, verificationErr) || !recovered || report.Status != "failed" || report.Success != 0 || report.Failed != 2 || report.VerificationStatus != "failed" || report.RecoveryStatus != "success" {
|
||||
t.Fatalf("失败与恢复状态错误:%+v,%v", report, err)
|
||||
}
|
||||
for index, server := range servers {
|
||||
assertVerifiedFiles(t, clients[server.ID], originals[server.ID])
|
||||
assertVerifiedLocks(t, clients[server.ID], server, false)
|
||||
if report.Results[index].FailureCode != FailureRecoveryRestored {
|
||||
t.Fatalf("应保留部署失败并提供恢复诊断:%+v", report.Results[index])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentCannotClaimRecoveryOnIncompleteEvidence(t *testing.T) {
|
||||
for _, scenario := range []struct {
|
||||
name string
|
||||
configure func(*memoryClient)
|
||||
recovery func(context.Context) error
|
||||
status string
|
||||
code FailureCode
|
||||
}{
|
||||
{"missing_baseline", func(*memoryClient) {}, nil, "unconfirmed", FailureRecoveryUnknown},
|
||||
{"wrong_runtime", func(*memoryClient) {}, func(context.Context) error { return errors.New("secret-path 旧站点身份不符") }, "unconfirmed", FailureRecoveryUnknown},
|
||||
{"restore_failed", func(c *memoryClient) { c.restoreError = true }, nil, "failed", FailureRollback},
|
||||
{"validation_failed", func(c *memoryClient) { c.runResults = []CommandResult{{}, {}, {ExitCode: 1}} }, nil, "failed", FailureRecoveryUnknown},
|
||||
{"reload_failed", func(c *memoryClient) { c.runResults = []CommandResult{{}, {}, {}, {ExitCode: 1}} }, nil, "failed", FailureRecoveryUnknown},
|
||||
} {
|
||||
t.Run(scenario.name, func(t *testing.T) {
|
||||
client, server := newMemoryClient(), testServer("recovery")
|
||||
seedVerifiedFiles(client, server)
|
||||
scenario.configure(client)
|
||||
report, err := NewService(fixedClientFactory{client}).DeployVerified(t.Context(), []Target{{Server: server}}, testArtifact(t), Verification{
|
||||
Verify: func(context.Context) error { return errors.New("部署后校验失败") }, VerifyRecovery: scenario.recovery,
|
||||
})
|
||||
if err == nil || report.RecoveryStatus != scenario.status || report.Results[0].FailureCode != scenario.code {
|
||||
t.Fatalf("不得夸大恢复能力:%+v,%v", report, err)
|
||||
}
|
||||
assertVerifiedLocks(t, client, server, true)
|
||||
public := PublicResult(report.Results[0])
|
||||
if strings.Contains(public.Message, "secret-path") || public.Message == "证书部署失败" {
|
||||
t.Fatalf("恢复诊断应安全且明确:%+v", public)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentCancellationStillRestoresWithBoundedContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
client, server := newMemoryClient(), testServer("cancel")
|
||||
original := seedVerifiedFiles(client, server)
|
||||
report, err := NewService(fixedClientFactory{client}).DeployVerified(ctx, []Target{{Server: server}}, testArtifact(t), Verification{
|
||||
Verify: func(context.Context) error { cancel(); return nil },
|
||||
VerifyRecovery: func(ctx context.Context) error {
|
||||
if ctx.Err() != nil {
|
||||
t.Fatal("请求取消不得取消恢复核验")
|
||||
}
|
||||
assertVerifiedFiles(t, client, original)
|
||||
return nil
|
||||
},
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) || report.Status != "failed" || report.RecoveryStatus != "success" || len(client.commands) != 4 {
|
||||
t.Fatalf("取消后应恢复但不得转为部署成功:%+v,%v", report, err)
|
||||
}
|
||||
assertVerifiedLocks(t, client, server, false)
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentPartialPreparationDoesNotCommitEarlierTarget(t *testing.T) {
|
||||
client, first := newMemoryClient(), testServer("first")
|
||||
second := testServer("second")
|
||||
original := seedVerifiedFiles(client, first)
|
||||
service := NewService(memoryFactory{clients: map[string]*memoryClient{"first": client}, errors: map[string]error{"second": errors.New("连接失败")}})
|
||||
report, err := service.DeployVerified(t.Context(), []Target{{Server: first}, {Server: second}}, testArtifact(t), Verification{
|
||||
Verify: func(context.Context) error { t.Fatal("部分准备失败不得进行提交校验"); return nil },
|
||||
VerifyRecovery: func(context.Context) error { assertVerifiedFiles(t, client, original); return nil },
|
||||
})
|
||||
if err == nil || report.Status != "failed" || report.Success != 0 || report.RecoveryStatus != "success" || report.Results[1].FailureCode != FailureConnection {
|
||||
t.Fatalf("部分目标失败必须恢复已推送目标:%+v,%v", report, err)
|
||||
}
|
||||
assertVerifiedLocks(t, client, first, false)
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentRequiresVerifierAndDoesNotHideCleanupFailure(t *testing.T) {
|
||||
client, server := newMemoryClient(), testServer("cleanup")
|
||||
original := seedVerifiedFiles(client, server)
|
||||
service := NewService(fixedClientFactory{client})
|
||||
report, err := service.DeployVerified(t.Context(), []Target{{Server: server}}, testArtifact(t), Verification{})
|
||||
if err == nil || report.Status != "failed" || len(client.commands) != 0 {
|
||||
t.Fatal("缺少校验器不得推送")
|
||||
}
|
||||
assertVerifiedFiles(t, client, original)
|
||||
client.removeError = ".scert-backup-"
|
||||
report, err = service.DeployVerified(t.Context(), []Target{{Server: server}}, testArtifact(t), Verification{Verify: func(context.Context) error { return nil }})
|
||||
if err == nil || report.Status != "failed" || report.VerificationStatus != "success" || report.Results[0].FailureCode != FailureCleanup {
|
||||
t.Fatalf("校验成功不能掩盖清理失败:%+v,%v", report, err)
|
||||
}
|
||||
assertVerifiedLocks(t, client, server, true)
|
||||
}
|
||||
Reference in new issue
Block a user