完成部署后校验失败恢复整改并更新实机进度
延后事务提交,校验失败时恢复目标文件、配置和服务状态;新增恢复边界回归、前端提示及生产测试记录。
This commit is contained in:
1 parent
c623e2d324
commit
3cc8fb6470
13 files changed
+588
-58
No files matched your search
@@ -26,6 +26,11 @@ func (d forbiddenDeployer) DeploySaved(context.Context, []deployment.Target, dep
|
||||
return deployment.Report{}, nil
|
||||
}
|
||||
|
||||
func (d forbiddenDeployer) DeploySavedVerified(context.Context, []deployment.Target, deployment.SavedFiles, deployment.Verification) (deployment.Report, error) {
|
||||
d.t.Fatal("权限检查失败后不得发起事务部署")
|
||||
return deployment.Report{}, nil
|
||||
}
|
||||
|
||||
func TestWorkflowDoesNotDeployAfterDNSPermissionDenied(t *testing.T) {
|
||||
workflow := &CertificateWorkflow{
|
||||
Certificates: &deniedWorkflowCertificateService{},
|
||||
|
||||
@@ -43,6 +43,7 @@ type serverLister interface {
|
||||
|
||||
type savedDeployer interface {
|
||||
DeploySaved(context.Context, []deployment.Target, deployment.SavedFiles) (deployment.Report, error)
|
||||
DeploySavedVerified(context.Context, []deployment.Target, deployment.SavedFiles, deployment.Verification) (deployment.Report, error)
|
||||
}
|
||||
|
||||
type domainDeploymentRepository interface {
|
||||
@@ -253,31 +254,41 @@ func (w *CertificateWorkflow) deploy(ctx context.Context, result certificate.Iss
|
||||
}
|
||||
expectedFingerprint = validation.FingerprintSHA256
|
||||
}
|
||||
report, err := w.Deployment.DeploySaved(ctx, targets, deployment.SavedFiles{
|
||||
saved := deployment.SavedFiles{
|
||||
CertificatePath: result.Saved.CertificatePath,
|
||||
FullChainPath: result.Saved.FullChainPath,
|
||||
PrivateKeyPath: result.Saved.PrivateKeyPath,
|
||||
})
|
||||
}
|
||||
if w.RemoteVerifier != nil {
|
||||
verificationDomain := verificationName(domainItem, result.Domains)
|
||||
if verificationDomain == "" {
|
||||
return deployment.Report{Status: "failed"}, errors.New("无法确定部署后的 HTTPS 校验域名,未执行推送")
|
||||
}
|
||||
endpoint := "https://" + verificationDomain
|
||||
// 首次部署允许不存在旧站点,但不得在缺少可信基线时宣称运行态已恢复。
|
||||
baseline, baselineErr := w.RemoteVerifier(ctx, endpoint, result.Domains)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return deployment.Report{Status: "failed"}, err
|
||||
}
|
||||
verification := deployment.Verification{
|
||||
Verify: func(verificationContext context.Context) error {
|
||||
return w.verifyRemoteDeployment(verificationContext, endpoint, result.Domains, expectedFingerprint)
|
||||
},
|
||||
}
|
||||
if baselineErr == nil && baseline.FingerprintSHA256 != "" {
|
||||
verification.VerifyRecovery = func(recoveryContext context.Context) error {
|
||||
return w.verifyRemoteDeployment(recoveryContext, endpoint, result.Domains, baseline.FingerprintSHA256)
|
||||
}
|
||||
}
|
||||
return w.Deployment.DeploySavedVerified(ctx, targets, saved, verification)
|
||||
}
|
||||
report, err := w.Deployment.DeploySaved(ctx, targets, saved)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if report.Status != "success" {
|
||||
return report, fmt.Errorf("证书部署未全部成功:成功 %d 台,失败 %d 台", report.Success, report.Failed)
|
||||
}
|
||||
if w.RemoteVerifier != nil {
|
||||
verificationDomain := verificationName(domainItem, result.Domains)
|
||||
// 单台结果保留真实文件推送结果,站点生效校验单独记录,不能宣称已回滚。
|
||||
report.VerificationStatus = "failed"
|
||||
report.Status = "failed"
|
||||
if verificationDomain == "" {
|
||||
return report, errors.New("无法确定部署后的 HTTPS 校验域名")
|
||||
}
|
||||
if err := w.verifyRemoteDeployment(ctx, "https://"+verificationDomain, result.Domains, expectedFingerprint); err != nil {
|
||||
return report, fmt.Errorf("部署后 HTTPS 校验失败:%w", err)
|
||||
}
|
||||
report.VerificationStatus = "success"
|
||||
report.Status = "success"
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -97,6 +97,21 @@ func (d *workflowDeployer) DeploySaved(_ context.Context, targets []deployment.T
|
||||
return d.report, nil
|
||||
}
|
||||
|
||||
// 工作流替身只模拟推送和校验,不伪造远端事务恢复;恢复边界由部署包实测。
|
||||
func (d *workflowDeployer) DeploySavedVerified(ctx context.Context, targets []deployment.Target, files deployment.SavedFiles, verification deployment.Verification) (deployment.Report, error) {
|
||||
report, err := d.DeploySaved(ctx, targets, files)
|
||||
if err != nil || report.Status != "success" {
|
||||
return report, err
|
||||
}
|
||||
report.VerificationStatus = "failed"
|
||||
if err := verification.Verify(ctx); err != nil {
|
||||
report.Status = "failed"
|
||||
return report, err
|
||||
}
|
||||
report.VerificationStatus = "success"
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func TestCertificateWorkflowDeploysIssuedCertificateToAssociatedServers(t *testing.T) {
|
||||
certificateService := &workflowCertificateService{result: certificate.IssueResult{
|
||||
DomainID: "domain-1",
|
||||
@@ -223,7 +238,7 @@ func TestCertificateWorkflowRetriesRemoteVerificationUntilSuccess(t *testing.T)
|
||||
if endpoint != "https://example.com" || len(expectedDomains) != 1 || expectedDomains[0] != "example.com" {
|
||||
t.Fatalf("HTTPS 校验参数错误:%s %+v", endpoint, expectedDomains)
|
||||
}
|
||||
if verificationCalls < 3 {
|
||||
if verificationCalls < 4 {
|
||||
return certificate.ValidationResult{}, errors.New("仍返回旧证书")
|
||||
}
|
||||
return certificate.ValidationResult{FingerprintSHA256: saved.FingerprintSHA256}, nil
|
||||
@@ -233,7 +248,7 @@ func TestCertificateWorkflowRetriesRemoteVerificationUntilSuccess(t *testing.T)
|
||||
if _, err := workflow.IssueWithReport(context.Background(), "domain-1"); err != nil {
|
||||
t.Fatalf("HTTPS 校验重试后应成功:%v", err)
|
||||
}
|
||||
if verificationCalls != 3 || waitCalls != 2 {
|
||||
if verificationCalls != 4 || waitCalls != 2 {
|
||||
t.Fatalf("HTTPS 校验重试次数错误:校验 %d 次,等待 %d 次", verificationCalls, waitCalls)
|
||||
}
|
||||
}
|
||||
@@ -264,7 +279,7 @@ func TestCertificateWorkflowRemoteVerificationStopsAfterConfiguredAttempts(t *te
|
||||
if err == nil || !strings.Contains(err.Error(), "重试 3 次后仍未通过") {
|
||||
t.Fatalf("永久校验错误应在有限重试后返回:%v", err)
|
||||
}
|
||||
if verificationCalls != 3 || waitCalls != 2 {
|
||||
if verificationCalls != 4 || waitCalls != 2 {
|
||||
t.Fatalf("永久错误的重试次数错误:校验 %d 次,等待 %d 次", verificationCalls, waitCalls)
|
||||
}
|
||||
}
|
||||
@@ -284,6 +299,9 @@ func TestCertificateWorkflowRemoteVerificationWaitHonorsContextCancellation(t *t
|
||||
RemoteVerificationInterval: time.Minute,
|
||||
RemoteVerifier: func(context.Context, string, []string) (certificate.ValidationResult, error) {
|
||||
verificationCalls++
|
||||
if verificationCalls == 1 {
|
||||
return certificate.ValidationResult{FingerprintSHA256: saved.FingerprintSHA256}, nil
|
||||
}
|
||||
cancel()
|
||||
return certificate.ValidationResult{}, errors.New("等待新证书")
|
||||
},
|
||||
@@ -293,7 +311,7 @@ func TestCertificateWorkflowRemoteVerificationWaitHonorsContextCancellation(t *t
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("取消上下文后应立即终止 HTTPS 校验重试:%v", err)
|
||||
}
|
||||
if verificationCalls != 1 {
|
||||
if verificationCalls != 2 {
|
||||
t.Fatalf("取消后不应继续校验,实际调用 %d 次", verificationCalls)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -199,14 +199,14 @@ func TestDeploymentVerificationRetriesRealValidOldCertificate(t *testing.T) {
|
||||
RemoteVerifier: func(ctx context.Context, _ string, domains []string) (certificate.ValidationResult, error) {
|
||||
calls++
|
||||
endpoint := oldServer.URL
|
||||
if calls > 1 {
|
||||
if calls > 2 {
|
||||
endpoint = newServer.URL
|
||||
}
|
||||
return validator.ValidateHTTPS(ctx, endpoint, domains)
|
||||
},
|
||||
}
|
||||
report, err := workflow.deploy(t.Context(), certificate.IssueResult{DomainID: "domain-1", Domains: []string{"example.com"}, Saved: newSaved})
|
||||
if err != nil || report.VerificationStatus != "success" || calls != 2 || waits != 1 {
|
||||
if err != nil || report.VerificationStatus != "success" || calls != 3 || waits != 1 {
|
||||
t.Fatalf("有效旧证书必须重试,只有实际新证书才能通过:%+v,调用 %d,等待 %d,%v", report, calls, waits, err)
|
||||
}
|
||||
}
|
||||
@@ -215,11 +215,15 @@ func TestDeploymentVerificationCancellationCannotBecomeSuccess(t *testing.T) {
|
||||
saved := workflowSavedCertificate(t)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
calls := 0
|
||||
workflow := &CertificateWorkflow{
|
||||
Domains: &workflowDomainState{}, Servers: workflowServerLister{servers: []deployment.Server{{ID: "server-1"}}},
|
||||
Deployment: &workflowDeployer{report: deployment.Report{Total: 1, Success: 1, Status: "success"}},
|
||||
RemoteVerifier: func(context.Context, string, []string) (certificate.ValidationResult, error) {
|
||||
cancel()
|
||||
calls++
|
||||
if calls > 1 {
|
||||
cancel()
|
||||
}
|
||||
return certificate.ValidationResult{FingerprintSHA256: saved.FingerprintSHA256}, nil
|
||||
},
|
||||
}
|
||||
|
||||
@@ -3,36 +3,41 @@ package deployment
|
||||
type FailureCode string
|
||||
|
||||
const (
|
||||
FailureUnavailable FailureCode = "unavailable"
|
||||
FailureArtifact FailureCode = "artifact_invalid"
|
||||
FailureConfiguration FailureCode = "configuration_invalid"
|
||||
FailureConnection FailureCode = "connection_failed"
|
||||
FailureDirectory FailureCode = "directory_failed"
|
||||
FailureLock FailureCode = "lock_unavailable"
|
||||
FailureUpload FailureCode = "upload_failed"
|
||||
FailureReplace FailureCode = "replace_failed"
|
||||
FailureValidation FailureCode = "validation_failed"
|
||||
FailureRollback FailureCode = "rollback_failed"
|
||||
FailureRuntimeUnknown FailureCode = "runtime_unconfirmed"
|
||||
FailureCleanup FailureCode = "cleanup_failed"
|
||||
FailureLockRelease FailureCode = "lock_release_failed"
|
||||
FailureUnavailable FailureCode = "unavailable"
|
||||
FailureArtifact FailureCode = "artifact_invalid"
|
||||
FailureConfiguration FailureCode = "configuration_invalid"
|
||||
FailureConnection FailureCode = "connection_failed"
|
||||
FailureDirectory FailureCode = "directory_failed"
|
||||
FailureLock FailureCode = "lock_unavailable"
|
||||
FailureUpload FailureCode = "upload_failed"
|
||||
FailureReplace FailureCode = "replace_failed"
|
||||
FailureValidation FailureCode = "validation_failed"
|
||||
FailureRollback FailureCode = "rollback_failed"
|
||||
FailureRuntimeUnknown FailureCode = "runtime_unconfirmed"
|
||||
FailureCleanup FailureCode = "cleanup_failed"
|
||||
FailureLockRelease FailureCode = "lock_release_failed"
|
||||
FailureRecoveryPending FailureCode = "recovery_pending"
|
||||
FailureRecoveryUnknown FailureCode = "recovery_unconfirmed"
|
||||
FailureRecoveryRestored FailureCode = "recovery_restored"
|
||||
)
|
||||
|
||||
// 只公开固定诊断,不从远端输出、错误文本或文件路径推断可展示的消息。
|
||||
var publicFailureMessages = map[FailureCode]string{
|
||||
FailureUnavailable: "部署服务不可用,请联系管理员检查服务配置",
|
||||
FailureArtifact: "待部署的证书或私钥校验失败,尚未推送到服务器",
|
||||
FailureConfiguration: "服务器部署配置无效,请检查认证、证书路径和命令配置",
|
||||
FailureConnection: "连接部署服务器失败,请检查网络、SSH 认证及主机身份配置",
|
||||
FailureDirectory: "无法创建证书目录,请检查部署账户的目录权限",
|
||||
FailureLock: "无法取得部署锁,可能有任务正在执行或存在中断遗留;请管理员核查远端文件、部署锁及服务状态后再重试,勿直接删除锁",
|
||||
FailureUpload: "证书上传失败,已尝试回滚;请检查受保护的服务日志后再重试",
|
||||
FailureReplace: "证书文件替换失败,已尝试回滚;请检查受保护的服务日志后再重试",
|
||||
FailureValidation: "服务配置校验失败,文件已回滚,未执行重载;请修正配置后再重试",
|
||||
FailureRollback: "部署失败且文件回滚未完成,已保留部署锁;请管理员核查远端文件及服务状态,勿直接重试或删除锁",
|
||||
FailureRuntimeUnknown: "服务重载失败,文件已回滚,但服务运行状态未确认;请管理员核对配置并恢复服务后再重试",
|
||||
FailureCleanup: "证书已推送并重载,但旧文件清理未完成;请管理员核查现场,不能据此确认站点新证书已生效",
|
||||
FailureLockRelease: "部署锁释放未完成;请管理员核查远端文件、部署锁及服务状态后再重试",
|
||||
FailureRecoveryUnknown: "本次部署未完成,已恢复旧文件,但服务或站点恢复状态未确认,已保留部署锁;请管理员核查,勿直接重试或删除锁",
|
||||
FailureRecoveryRestored: "本次部署未完成,旧文件已恢复并通过配置校验及重载,站点已恢复部署前证书;请核查失败原因后再重试",
|
||||
FailureUnavailable: "部署服务不可用,请联系管理员检查服务配置",
|
||||
FailureArtifact: "待部署的证书或私钥校验失败,尚未推送到服务器",
|
||||
FailureConfiguration: "服务器部署配置无效,请检查认证、证书路径和命令配置",
|
||||
FailureConnection: "连接部署服务器失败,请检查网络、SSH 认证及主机身份配置",
|
||||
FailureDirectory: "无法创建证书目录,请检查部署账户的目录权限",
|
||||
FailureLock: "无法取得部署锁,可能有任务正在执行或存在中断遗留;请管理员核查远端文件、部署锁及服务状态后再重试,勿直接删除锁",
|
||||
FailureUpload: "证书上传失败,已尝试回滚;请检查受保护的服务日志后再重试",
|
||||
FailureReplace: "证书文件替换失败,已尝试回滚;请检查受保护的服务日志后再重试",
|
||||
FailureValidation: "服务配置校验失败,文件已回滚,未执行重载;请修正配置后再重试",
|
||||
FailureRollback: "部署失败且文件回滚未完成,已保留部署锁;请管理员核查远端文件及服务状态,勿直接重试或删除锁",
|
||||
FailureRuntimeUnknown: "服务重载失败,文件已回滚,但服务运行状态未确认;请管理员核对配置并恢复服务后再重试",
|
||||
FailureCleanup: "证书已推送并重载,但旧文件清理未完成;请管理员核查现场,不能据此确认站点新证书已生效",
|
||||
FailureLockRelease: "部署锁释放未完成;请管理员核查远端文件、部署锁及服务状态后再重试",
|
||||
}
|
||||
|
||||
// PublicResult 保留结果事实,同时只允许固定错误码及中文诊断进入公开响应。
|
||||
|
||||
@@ -47,6 +47,7 @@ type Report struct {
|
||||
Results []Result `json:"results"`
|
||||
// 站点校验与各服务器文件推送分开记录,不表示所有服务器均完成独立 HTTPS 验证。
|
||||
VerificationStatus string `json:"verification_status,omitempty"`
|
||||
RecoveryStatus string `json:"recovery_status,omitempty"`
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
@@ -59,6 +60,10 @@ func NewService(factory ClientFactory) *Service {
|
||||
}
|
||||
|
||||
func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifact) Report {
|
||||
return s.deploy(ctx, targets, artifact, nil)
|
||||
}
|
||||
|
||||
func (s *Service) deploy(ctx context.Context, targets []Target, artifact Artifact, pending *[]*pendingDeployment) Report {
|
||||
report := Report{Total: len(targets), Results: make([]Result, 0, len(targets))}
|
||||
if len(targets) == 0 {
|
||||
report.Status = "failed"
|
||||
@@ -80,8 +85,8 @@ func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifac
|
||||
}
|
||||
return report
|
||||
}
|
||||
for _, target := range targets {
|
||||
result := s.deployOne(ctx, target.Server, artifact)
|
||||
for index, target := range targets {
|
||||
result := s.deployOne(ctx, target.Server, artifact, pending, index)
|
||||
report.Results = append(report.Results, result)
|
||||
if result.Status == "success" {
|
||||
report.Success++
|
||||
@@ -101,22 +106,30 @@ func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifac
|
||||
}
|
||||
|
||||
func (s *Service) DeploySaved(ctx context.Context, targets []Target, saved SavedFiles) (Report, error) {
|
||||
artifact, err := loadArtifact(saved)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
return s.Deploy(ctx, targets, artifact), nil
|
||||
}
|
||||
|
||||
func loadArtifact(saved SavedFiles) (Artifact, error) {
|
||||
certificateData, err := os.ReadFile(saved.CertificatePath)
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("读取证书文件失败:%w", err)
|
||||
return Artifact{}, fmt.Errorf("读取证书文件失败:%w", err)
|
||||
}
|
||||
fullChainData, err := os.ReadFile(saved.FullChainPath)
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("读取完整证书链失败:%w", err)
|
||||
return Artifact{}, fmt.Errorf("读取完整证书链失败:%w", err)
|
||||
}
|
||||
privateKeyData, err := os.ReadFile(saved.PrivateKeyPath)
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("读取私钥文件失败:%w", err)
|
||||
return Artifact{}, fmt.Errorf("读取私钥文件失败:%w", err)
|
||||
}
|
||||
return s.Deploy(ctx, targets, Artifact{Certificate: certificateData, FullChain: fullChainData, PrivateKey: privateKeyData}), nil
|
||||
return Artifact{Certificate: certificateData, FullChain: fullChainData, PrivateKey: privateKeyData}, nil
|
||||
}
|
||||
|
||||
func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifact) (result Result) {
|
||||
func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifact, pending *[]*pendingDeployment, index int) (result Result) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
server = server.Normalize()
|
||||
@@ -131,7 +144,12 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
result.Message = fmt.Sprintf("连接服务器失败:%v", err)
|
||||
return result
|
||||
}
|
||||
defer func() { _ = client.Close() }()
|
||||
handedOff := false
|
||||
defer func() {
|
||||
if !handedOff {
|
||||
_ = client.Close()
|
||||
}
|
||||
}()
|
||||
paths := []remoteFile{
|
||||
{path: RemotePath(server.CertificatePath), data: artifact.Certificate, mode: 0o644},
|
||||
{path: RemotePath(server.CertificateChainPath), data: artifact.FullChain, mode: 0o644},
|
||||
@@ -160,7 +178,7 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
sort.Strings(lockPaths)
|
||||
heldLocks := make([]string, 0, len(lockPaths))
|
||||
retainLocks := false
|
||||
defer func() {
|
||||
releaseLocks := func() {
|
||||
if len(heldLocks) == 0 || retainLocks {
|
||||
return
|
||||
}
|
||||
@@ -179,6 +197,11 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
result.Message += fmt.Sprintf(";释放部署锁 %s 失败,需人工检查:%v", heldLocks[index], err)
|
||||
}
|
||||
}
|
||||
}
|
||||
defer func() {
|
||||
if !handedOff {
|
||||
releaseLocks()
|
||||
}
|
||||
}()
|
||||
for _, lockPath := range lockPaths {
|
||||
if err := client.Lock(ctx, lockPath); err != nil {
|
||||
@@ -300,6 +323,59 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
}
|
||||
return failed(fmt.Sprintf("reload 失败:%v", err))
|
||||
}
|
||||
if pending != nil {
|
||||
// 旧文件与锁在所有目标及站点校验结束前始终保留,禁止提前提交。
|
||||
result.Status = "success"
|
||||
result.FailureCode = ""
|
||||
result.Message = "证书已推送并重载,等待站点校验"
|
||||
closed := false
|
||||
transaction := &pendingDeployment{index: index}
|
||||
transaction.restore = func(recoveryContext context.Context) Result {
|
||||
result.Status = "failed"
|
||||
if err := rollback(); err != nil {
|
||||
result.FailureCode = FailureRollback
|
||||
result.Message = fmt.Sprintf("部署未提交,自动回滚失败:%v", err)
|
||||
return result
|
||||
}
|
||||
result.FailureCode = FailureRecoveryUnknown
|
||||
if err := reconnectDeploymentClient(recoveryContext, s.factory, server, &client); err != nil {
|
||||
result.Message = fmt.Sprintf("旧文件已恢复,重新连接以恢复服务失败:%v", err)
|
||||
return result
|
||||
}
|
||||
for _, command := range [][]string{server.ValidationCommand, server.ReloadCommand} {
|
||||
outcome, err := client.Run(recoveryContext, command)
|
||||
if err != nil || outcome.ExitCode != 0 {
|
||||
result.Message = fmt.Sprintf("旧文件已恢复,但恢复配置校验或重载失败,退出码 %d:%v", outcome.ExitCode, err)
|
||||
return result
|
||||
}
|
||||
}
|
||||
result.FailureCode = FailureRecoveryPending
|
||||
result.Message = "本次部署未提交,旧文件已恢复并通过配置校验及重载,等待站点恢复核验"
|
||||
return result
|
||||
}
|
||||
transaction.commit = func(commitContext context.Context) Result {
|
||||
if err := cleanupBackups(commitContext, client, backups, paths); err != nil {
|
||||
result.Status = "failed"
|
||||
result.FailureCode = FailureCleanup
|
||||
result.Message = fmt.Sprintf("站点校验通过,但事务旧文件清理失败:%v", err)
|
||||
return result
|
||||
}
|
||||
result.Message = "证书部署、配置校验并 reload 成功"
|
||||
return result
|
||||
}
|
||||
transaction.finish = func(retain bool) Result {
|
||||
if !closed {
|
||||
retainLocks = retain
|
||||
releaseLocks()
|
||||
_ = client.Close()
|
||||
closed = true
|
||||
}
|
||||
return result
|
||||
}
|
||||
handedOff = true
|
||||
*pending = append(*pending, transaction)
|
||||
return result
|
||||
}
|
||||
if err := cleanupBackups(ctx, client, backups, paths); err != nil {
|
||||
result.Status = "failed"
|
||||
result.FailureCode = FailureCleanup
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Verification 将站点生效校验与恢复核验置于远端文件事务之内。
|
||||
type Verification struct {
|
||||
Verify func(context.Context) error
|
||||
VerifyRecovery func(context.Context) error
|
||||
}
|
||||
|
||||
type pendingDeployment struct {
|
||||
index int
|
||||
commit func(context.Context) Result
|
||||
restore func(context.Context) Result
|
||||
finish func(bool) Result
|
||||
}
|
||||
|
||||
func (s *Service) DeploySavedVerified(ctx context.Context, targets []Target, saved SavedFiles, verification Verification) (Report, error) {
|
||||
artifact, err := loadArtifact(saved)
|
||||
if err != nil {
|
||||
return Report{Status: "failed"}, err
|
||||
}
|
||||
return s.DeployVerified(ctx, targets, artifact, verification)
|
||||
}
|
||||
|
||||
func (s *Service) DeployVerified(ctx context.Context, targets []Target, artifact Artifact, verification Verification) (Report, error) {
|
||||
if verification.Verify == nil {
|
||||
return Report{Status: "failed"}, errors.New("缺少部署后站点校验,未执行推送")
|
||||
}
|
||||
pending := make([]*pendingDeployment, 0, len(targets))
|
||||
// 异常退出时关闭连接但保留现场;不得以释放锁掩盖未确认的事务。
|
||||
defer func() {
|
||||
for _, transaction := range pending {
|
||||
transaction.finish(true)
|
||||
}
|
||||
}()
|
||||
report := s.deploy(ctx, targets, artifact, &pending)
|
||||
var deployErr error
|
||||
if report.Status != "success" {
|
||||
deployErr = fmt.Errorf("证书部署未全部成功:成功 %d 台,失败 %d 台", report.Success, report.Failed)
|
||||
} else {
|
||||
report.VerificationStatus = "failed"
|
||||
deployErr = ctx.Err()
|
||||
if deployErr == nil {
|
||||
deployErr = verification.Verify(ctx)
|
||||
}
|
||||
if deployErr == nil {
|
||||
deployErr = ctx.Err()
|
||||
}
|
||||
if deployErr != nil {
|
||||
deployErr = fmt.Errorf("部署后 HTTPS 校验失败:%w", deployErr)
|
||||
} else {
|
||||
report.VerificationStatus = "success"
|
||||
}
|
||||
}
|
||||
if deployErr != nil {
|
||||
if len(pending) > 0 {
|
||||
s.restorePending(ctx, &report, pending, verification)
|
||||
}
|
||||
summarizeReport(&report)
|
||||
report.Status = "failed"
|
||||
return report, deployErr
|
||||
}
|
||||
for _, transaction := range pending {
|
||||
result := transaction.commit(ctx)
|
||||
report.Results[transaction.index] = transaction.finish(result.Status != "success")
|
||||
}
|
||||
summarizeReport(&report)
|
||||
if report.Status != "success" {
|
||||
return report, errors.New("站点校验通过,但事务清理或释放部署锁未完成")
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (s *Service) restorePending(ctx context.Context, report *Report, pending []*pendingDeployment, verification Verification) {
|
||||
report.RecoveryStatus = "unconfirmed"
|
||||
for index := len(pending) - 1; index >= 0; index-- {
|
||||
transaction := pending[index]
|
||||
// 请求取消后仍使用独立有界上下文恢复,不重放签发或文件推送。
|
||||
recoveryContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
|
||||
report.Results[transaction.index] = transaction.restore(recoveryContext)
|
||||
cancel()
|
||||
}
|
||||
restored := true
|
||||
for _, result := range report.Results {
|
||||
switch result.FailureCode {
|
||||
case FailureRecoveryPending, FailureConfiguration, FailureConnection, FailureDirectory, FailureUpload, FailureReplace, FailureValidation:
|
||||
// 这些失败要么未触碰目标,要么已完成文件回滚且未执行重载。
|
||||
default:
|
||||
restored = false
|
||||
}
|
||||
}
|
||||
var recoveryErr error
|
||||
if !restored {
|
||||
report.RecoveryStatus = "failed"
|
||||
} else if verification.VerifyRecovery != nil {
|
||||
recoveryContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
|
||||
err := verification.VerifyRecovery(recoveryContext)
|
||||
if err == nil {
|
||||
err = recoveryContext.Err()
|
||||
}
|
||||
cancel()
|
||||
if err == nil {
|
||||
report.RecoveryStatus = "success"
|
||||
} else {
|
||||
recoveryErr = err
|
||||
}
|
||||
}
|
||||
for _, transaction := range pending {
|
||||
result := transaction.finish(report.RecoveryStatus != "success")
|
||||
if result.FailureCode == FailureLockRelease {
|
||||
report.RecoveryStatus = "failed"
|
||||
}
|
||||
if result.FailureCode == FailureRecoveryPending {
|
||||
if report.RecoveryStatus == "success" {
|
||||
result.FailureCode = FailureRecoveryRestored
|
||||
} else {
|
||||
result.FailureCode = FailureRecoveryUnknown
|
||||
}
|
||||
result.Message = publicFailureMessages[result.FailureCode]
|
||||
if recoveryErr != nil {
|
||||
result.Message += fmt.Sprintf(";站点恢复核验失败:%v", recoveryErr)
|
||||
}
|
||||
}
|
||||
report.Results[transaction.index] = result
|
||||
}
|
||||
}
|
||||
|
||||
func summarizeReport(report *Report) {
|
||||
report.Success, report.Failed = 0, 0
|
||||
for _, result := range report.Results {
|
||||
if result.Status == "success" {
|
||||
report.Success++
|
||||
} else {
|
||||
report.Failed++
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case report.Total == 0 || report.Success == 0:
|
||||
report.Status = "failed"
|
||||
case report.Failed == 0:
|
||||
report.Status = "success"
|
||||
default:
|
||||
report.Status = "partial"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func seedVerifiedFiles(client *memoryClient, server Server) map[string]memoryFile {
|
||||
files := map[string]memoryFile{
|
||||
server.CertificatePath: {data: []byte("原证书"), mode: 0o644},
|
||||
server.CertificateChainPath: {data: []byte("原证书链"), mode: 0o644},
|
||||
server.PrivateKeyPath: {data: []byte("原私钥"), mode: 0o600},
|
||||
}
|
||||
for path, file := range files {
|
||||
client.files[path] = file
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func assertVerifiedFiles(t *testing.T, client *memoryClient, expected map[string]memoryFile) {
|
||||
t.Helper()
|
||||
for path, file := range expected {
|
||||
actual, ok := client.files[path]
|
||||
if !ok || !bytes.Equal(actual.data, file.data) || actual.mode != file.mode {
|
||||
t.Fatalf("未恢复原内容或权限:%s", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertVerifiedLocks(t *testing.T, client *memoryClient, server Server, held bool) {
|
||||
t.Helper()
|
||||
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
|
||||
_, exists := client.files[path+".scert-lock"]
|
||||
if exists != held {
|
||||
t.Fatalf("部署锁状态不符合预期:%s,锁定 %t", path, exists)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentDefersCommitUntilAllTargetsAndSitePass(t *testing.T) {
|
||||
clients := map[string]*memoryClient{"first": newMemoryClient(), "second": newMemoryClient()}
|
||||
servers := []Server{testServer("first"), testServer("second")}
|
||||
for _, server := range servers {
|
||||
seedVerifiedFiles(clients[server.ID], server)
|
||||
}
|
||||
artifact := testArtifact(t)
|
||||
called := false
|
||||
service := NewService(memoryFactory{clients: clients})
|
||||
report, err := service.DeployVerified(t.Context(), []Target{{Server: servers[0]}, {Server: servers[1]}}, artifact, Verification{
|
||||
Verify: func(context.Context) error {
|
||||
called = true
|
||||
for _, server := range servers {
|
||||
client := clients[server.ID]
|
||||
assertVerifiedLocks(t, client, server, true)
|
||||
if !bytes.Equal(client.files[server.CertificatePath].data, artifact.Certificate) || len(client.commands) != 2 {
|
||||
t.Fatal("必须在所有服务器推送和重载后才核验站点")
|
||||
}
|
||||
backups := 0
|
||||
for path := range client.files {
|
||||
if strings.Contains(path, ".scert-backup-") {
|
||||
backups++
|
||||
}
|
||||
}
|
||||
if backups != 3 {
|
||||
t.Fatal("站点确认前必须保留三份事务旧文件")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
VerifyRecovery: func(context.Context) error { t.Fatal("成功部署不得回滚"); return nil },
|
||||
})
|
||||
if err != nil || !called || report.Status != "success" || report.Success != 2 || report.VerificationStatus != "success" || report.RecoveryStatus != "" {
|
||||
t.Fatalf("事务提交结果错误:%+v,%v", report, err)
|
||||
}
|
||||
for _, server := range servers {
|
||||
client := clients[server.ID]
|
||||
assertVerifiedLocks(t, client, server, false)
|
||||
if len(client.files) != 3 {
|
||||
t.Fatal("提交后不应残留事务文件")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentFailureRestoresAllTargetsBeforeRecoveryCheck(t *testing.T) {
|
||||
clients := map[string]*memoryClient{"first": newMemoryClient(), "second": newMemoryClient()}
|
||||
servers := []Server{testServer("first"), testServer("second")}
|
||||
originals := map[string]map[string]memoryFile{}
|
||||
for _, server := range servers {
|
||||
originals[server.ID] = seedVerifiedFiles(clients[server.ID], server)
|
||||
}
|
||||
recovered := false
|
||||
verificationErr := errors.New("公网仍返回错误证书")
|
||||
report, err := NewService(memoryFactory{clients: clients}).DeployVerified(t.Context(), []Target{{Server: servers[0]}, {Server: servers[1]}}, testArtifact(t), Verification{
|
||||
Verify: func(context.Context) error { return verificationErr },
|
||||
VerifyRecovery: func(ctx context.Context) error {
|
||||
recovered = true
|
||||
if _, ok := ctx.Deadline(); !ok || ctx.Err() != nil {
|
||||
t.Fatal("恢复核验必须有独立有效期限")
|
||||
}
|
||||
for _, server := range servers {
|
||||
client := clients[server.ID]
|
||||
assertVerifiedFiles(t, client, originals[server.ID])
|
||||
assertVerifiedLocks(t, client, server, true)
|
||||
expectedCommands := [][]string{{"nginx", "-t"}, {"nginx", "-s", "reload"}, {"nginx", "-t"}, {"nginx", "-s", "reload"}}
|
||||
if !reflect.DeepEqual(client.commands, expectedCommands) {
|
||||
t.Fatalf("必须恢复配置并重载后核验站点:实际=%#v,期望=%#v", client.commands, expectedCommands)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
if !errors.Is(err, verificationErr) || !recovered || report.Status != "failed" || report.Success != 0 || report.Failed != 2 || report.VerificationStatus != "failed" || report.RecoveryStatus != "success" {
|
||||
t.Fatalf("失败与恢复状态错误:%+v,%v", report, err)
|
||||
}
|
||||
for index, server := range servers {
|
||||
assertVerifiedFiles(t, clients[server.ID], originals[server.ID])
|
||||
assertVerifiedLocks(t, clients[server.ID], server, false)
|
||||
if report.Results[index].FailureCode != FailureRecoveryRestored {
|
||||
t.Fatalf("应保留部署失败并提供恢复诊断:%+v", report.Results[index])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentCannotClaimRecoveryOnIncompleteEvidence(t *testing.T) {
|
||||
for _, scenario := range []struct {
|
||||
name string
|
||||
configure func(*memoryClient)
|
||||
recovery func(context.Context) error
|
||||
status string
|
||||
code FailureCode
|
||||
}{
|
||||
{"missing_baseline", func(*memoryClient) {}, nil, "unconfirmed", FailureRecoveryUnknown},
|
||||
{"wrong_runtime", func(*memoryClient) {}, func(context.Context) error { return errors.New("secret-path 旧站点身份不符") }, "unconfirmed", FailureRecoveryUnknown},
|
||||
{"restore_failed", func(c *memoryClient) { c.restoreError = true }, nil, "failed", FailureRollback},
|
||||
{"validation_failed", func(c *memoryClient) { c.runResults = []CommandResult{{}, {}, {ExitCode: 1}} }, nil, "failed", FailureRecoveryUnknown},
|
||||
{"reload_failed", func(c *memoryClient) { c.runResults = []CommandResult{{}, {}, {}, {ExitCode: 1}} }, nil, "failed", FailureRecoveryUnknown},
|
||||
} {
|
||||
t.Run(scenario.name, func(t *testing.T) {
|
||||
client, server := newMemoryClient(), testServer("recovery")
|
||||
seedVerifiedFiles(client, server)
|
||||
scenario.configure(client)
|
||||
report, err := NewService(fixedClientFactory{client}).DeployVerified(t.Context(), []Target{{Server: server}}, testArtifact(t), Verification{
|
||||
Verify: func(context.Context) error { return errors.New("部署后校验失败") }, VerifyRecovery: scenario.recovery,
|
||||
})
|
||||
if err == nil || report.RecoveryStatus != scenario.status || report.Results[0].FailureCode != scenario.code {
|
||||
t.Fatalf("不得夸大恢复能力:%+v,%v", report, err)
|
||||
}
|
||||
assertVerifiedLocks(t, client, server, true)
|
||||
public := PublicResult(report.Results[0])
|
||||
if strings.Contains(public.Message, "secret-path") || public.Message == "证书部署失败" {
|
||||
t.Fatalf("恢复诊断应安全且明确:%+v", public)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentCancellationStillRestoresWithBoundedContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
client, server := newMemoryClient(), testServer("cancel")
|
||||
original := seedVerifiedFiles(client, server)
|
||||
report, err := NewService(fixedClientFactory{client}).DeployVerified(ctx, []Target{{Server: server}}, testArtifact(t), Verification{
|
||||
Verify: func(context.Context) error { cancel(); return nil },
|
||||
VerifyRecovery: func(ctx context.Context) error {
|
||||
if ctx.Err() != nil {
|
||||
t.Fatal("请求取消不得取消恢复核验")
|
||||
}
|
||||
assertVerifiedFiles(t, client, original)
|
||||
return nil
|
||||
},
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) || report.Status != "failed" || report.RecoveryStatus != "success" || len(client.commands) != 4 {
|
||||
t.Fatalf("取消后应恢复但不得转为部署成功:%+v,%v", report, err)
|
||||
}
|
||||
assertVerifiedLocks(t, client, server, false)
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentPartialPreparationDoesNotCommitEarlierTarget(t *testing.T) {
|
||||
client, first := newMemoryClient(), testServer("first")
|
||||
second := testServer("second")
|
||||
original := seedVerifiedFiles(client, first)
|
||||
service := NewService(memoryFactory{clients: map[string]*memoryClient{"first": client}, errors: map[string]error{"second": errors.New("连接失败")}})
|
||||
report, err := service.DeployVerified(t.Context(), []Target{{Server: first}, {Server: second}}, testArtifact(t), Verification{
|
||||
Verify: func(context.Context) error { t.Fatal("部分准备失败不得进行提交校验"); return nil },
|
||||
VerifyRecovery: func(context.Context) error { assertVerifiedFiles(t, client, original); return nil },
|
||||
})
|
||||
if err == nil || report.Status != "failed" || report.Success != 0 || report.RecoveryStatus != "success" || report.Results[1].FailureCode != FailureConnection {
|
||||
t.Fatalf("部分目标失败必须恢复已推送目标:%+v,%v", report, err)
|
||||
}
|
||||
assertVerifiedLocks(t, client, first, false)
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentRequiresVerifierAndDoesNotHideCleanupFailure(t *testing.T) {
|
||||
client, server := newMemoryClient(), testServer("cleanup")
|
||||
original := seedVerifiedFiles(client, server)
|
||||
service := NewService(fixedClientFactory{client})
|
||||
report, err := service.DeployVerified(t.Context(), []Target{{Server: server}}, testArtifact(t), Verification{})
|
||||
if err == nil || report.Status != "failed" || len(client.commands) != 0 {
|
||||
t.Fatal("缺少校验器不得推送")
|
||||
}
|
||||
assertVerifiedFiles(t, client, original)
|
||||
client.removeError = ".scert-backup-"
|
||||
report, err = service.DeployVerified(t.Context(), []Target{{Server: server}}, testArtifact(t), Verification{Verify: func(context.Context) error { return nil }})
|
||||
if err == nil || report.Status != "failed" || report.VerificationStatus != "success" || report.Results[0].FailureCode != FailureCleanup {
|
||||
t.Fatalf("校验成功不能掩盖清理失败:%+v,%v", report, err)
|
||||
}
|
||||
assertVerifiedLocks(t, client, server, true)
|
||||
}
|
||||
Reference in new issue
Block a user