完成部署后校验失败恢复整改并更新实机进度

延后事务提交,校验失败时恢复目标文件、配置和服务状态;新增恢复边界回归、前端提示及生产测试记录。
This commit is contained in:
xuans committed 2026-09-30 23:14:42 +08:00
1 parent c623e2d324
commit 3cc8fb6470
13 files changed
+588 -58

No files matched your search

@@ -26,6 +26,11 @@ func (d forbiddenDeployer) DeploySaved(context.Context, []deployment.Target, dep
return deployment.Report{}, nil
}
func (d forbiddenDeployer) DeploySavedVerified(context.Context, []deployment.Target, deployment.SavedFiles, deployment.Verification) (deployment.Report, error) {
d.t.Fatal("权限检查失败后不得发起事务部署")
return deployment.Report{}, nil
}
func TestWorkflowDoesNotDeployAfterDNSPermissionDenied(t *testing.T) {
workflow := &CertificateWorkflow{
Certificates: &deniedWorkflowCertificateService{},
+27 -16
View File
@@ -43,6 +43,7 @@ type serverLister interface {
type savedDeployer interface {
DeploySaved(context.Context, []deployment.Target, deployment.SavedFiles) (deployment.Report, error)
DeploySavedVerified(context.Context, []deployment.Target, deployment.SavedFiles, deployment.Verification) (deployment.Report, error)
}
type domainDeploymentRepository interface {
@@ -253,31 +254,41 @@ func (w *CertificateWorkflow) deploy(ctx context.Context, result certificate.Iss
}
expectedFingerprint = validation.FingerprintSHA256
}
report, err := w.Deployment.DeploySaved(ctx, targets, deployment.SavedFiles{
saved := deployment.SavedFiles{
CertificatePath: result.Saved.CertificatePath,
FullChainPath: result.Saved.FullChainPath,
PrivateKeyPath: result.Saved.PrivateKeyPath,
})
}
if w.RemoteVerifier != nil {
verificationDomain := verificationName(domainItem, result.Domains)
if verificationDomain == "" {
return deployment.Report{Status: "failed"}, errors.New("无法确定部署后的 HTTPS 校验域名,未执行推送")
}
endpoint := "https://" + verificationDomain
// 首次部署允许不存在旧站点,但不得在缺少可信基线时宣称运行态已恢复。
baseline, baselineErr := w.RemoteVerifier(ctx, endpoint, result.Domains)
if err := ctx.Err(); err != nil {
return deployment.Report{Status: "failed"}, err
}
verification := deployment.Verification{
Verify: func(verificationContext context.Context) error {
return w.verifyRemoteDeployment(verificationContext, endpoint, result.Domains, expectedFingerprint)
},
}
if baselineErr == nil && baseline.FingerprintSHA256 != "" {
verification.VerifyRecovery = func(recoveryContext context.Context) error {
return w.verifyRemoteDeployment(recoveryContext, endpoint, result.Domains, baseline.FingerprintSHA256)
}
}
return w.Deployment.DeploySavedVerified(ctx, targets, saved, verification)
}
report, err := w.Deployment.DeploySaved(ctx, targets, saved)
if err != nil {
return report, err
}
if report.Status != "success" {
return report, fmt.Errorf("证书部署未全部成功:成功 %d 台,失败 %d 台", report.Success, report.Failed)
}
if w.RemoteVerifier != nil {
verificationDomain := verificationName(domainItem, result.Domains)
// 单台结果保留真实文件推送结果,站点生效校验单独记录,不能宣称已回滚。
report.VerificationStatus = "failed"
report.Status = "failed"
if verificationDomain == "" {
return report, errors.New("无法确定部署后的 HTTPS 校验域名")
}
if err := w.verifyRemoteDeployment(ctx, "https://"+verificationDomain, result.Domains, expectedFingerprint); err != nil {
return report, fmt.Errorf("部署后 HTTPS 校验失败:%w", err)
}
report.VerificationStatus = "success"
report.Status = "success"
}
return report, nil
}
@@ -97,6 +97,21 @@ func (d *workflowDeployer) DeploySaved(_ context.Context, targets []deployment.T
return d.report, nil
}
// 工作流替身只模拟推送和校验,不伪造远端事务恢复;恢复边界由部署包实测。
func (d *workflowDeployer) DeploySavedVerified(ctx context.Context, targets []deployment.Target, files deployment.SavedFiles, verification deployment.Verification) (deployment.Report, error) {
report, err := d.DeploySaved(ctx, targets, files)
if err != nil || report.Status != "success" {
return report, err
}
report.VerificationStatus = "failed"
if err := verification.Verify(ctx); err != nil {
report.Status = "failed"
return report, err
}
report.VerificationStatus = "success"
return report, nil
}
func TestCertificateWorkflowDeploysIssuedCertificateToAssociatedServers(t *testing.T) {
certificateService := &workflowCertificateService{result: certificate.IssueResult{
DomainID: "domain-1",
@@ -223,7 +238,7 @@ func TestCertificateWorkflowRetriesRemoteVerificationUntilSuccess(t *testing.T)
if endpoint != "https://example.com" || len(expectedDomains) != 1 || expectedDomains[0] != "example.com" {
t.Fatalf("HTTPS 校验参数错误:%s %+v", endpoint, expectedDomains)
}
if verificationCalls < 3 {
if verificationCalls < 4 {
return certificate.ValidationResult{}, errors.New("仍返回旧证书")
}
return certificate.ValidationResult{FingerprintSHA256: saved.FingerprintSHA256}, nil
@@ -233,7 +248,7 @@ func TestCertificateWorkflowRetriesRemoteVerificationUntilSuccess(t *testing.T)
if _, err := workflow.IssueWithReport(context.Background(), "domain-1"); err != nil {
t.Fatalf("HTTPS 校验重试后应成功:%v", err)
}
if verificationCalls != 3 || waitCalls != 2 {
if verificationCalls != 4 || waitCalls != 2 {
t.Fatalf("HTTPS 校验重试次数错误:校验 %d 次,等待 %d 次", verificationCalls, waitCalls)
}
}
@@ -264,7 +279,7 @@ func TestCertificateWorkflowRemoteVerificationStopsAfterConfiguredAttempts(t *te
if err == nil || !strings.Contains(err.Error(), "重试 3 次后仍未通过") {
t.Fatalf("永久校验错误应在有限重试后返回:%v", err)
}
if verificationCalls != 3 || waitCalls != 2 {
if verificationCalls != 4 || waitCalls != 2 {
t.Fatalf("永久错误的重试次数错误:校验 %d 次,等待 %d 次", verificationCalls, waitCalls)
}
}
@@ -284,6 +299,9 @@ func TestCertificateWorkflowRemoteVerificationWaitHonorsContextCancellation(t *t
RemoteVerificationInterval: time.Minute,
RemoteVerifier: func(context.Context, string, []string) (certificate.ValidationResult, error) {
verificationCalls++
if verificationCalls == 1 {
return certificate.ValidationResult{FingerprintSHA256: saved.FingerprintSHA256}, nil
}
cancel()
return certificate.ValidationResult{}, errors.New("等待新证书")
},
@@ -293,7 +311,7 @@ func TestCertificateWorkflowRemoteVerificationWaitHonorsContextCancellation(t *t
if !errors.Is(err, context.Canceled) {
t.Fatalf("取消上下文后应立即终止 HTTPS 校验重试:%v", err)
}
if verificationCalls != 1 {
if verificationCalls != 2 {
t.Fatalf("取消后不应继续校验,实际调用 %d 次", verificationCalls)
}
}
@@ -199,14 +199,14 @@ func TestDeploymentVerificationRetriesRealValidOldCertificate(t *testing.T) {
RemoteVerifier: func(ctx context.Context, _ string, domains []string) (certificate.ValidationResult, error) {
calls++
endpoint := oldServer.URL
if calls > 1 {
if calls > 2 {
endpoint = newServer.URL
}
return validator.ValidateHTTPS(ctx, endpoint, domains)
},
}
report, err := workflow.deploy(t.Context(), certificate.IssueResult{DomainID: "domain-1", Domains: []string{"example.com"}, Saved: newSaved})
if err != nil || report.VerificationStatus != "success" || calls != 2 || waits != 1 {
if err != nil || report.VerificationStatus != "success" || calls != 3 || waits != 1 {
t.Fatalf("有效旧证书必须重试,只有实际新证书才能通过:%+v,调用 %d,等待 %d,%v", report, calls, waits, err)
}
}
@@ -215,11 +215,15 @@ func TestDeploymentVerificationCancellationCannotBecomeSuccess(t *testing.T) {
saved := workflowSavedCertificate(t)
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
calls := 0
workflow := &CertificateWorkflow{
Domains: &workflowDomainState{}, Servers: workflowServerLister{servers: []deployment.Server{{ID: "server-1"}}},
Deployment: &workflowDeployer{report: deployment.Report{Total: 1, Success: 1, Status: "success"}},
RemoteVerifier: func(context.Context, string, []string) (certificate.ValidationResult, error) {
cancel()
calls++
if calls > 1 {
cancel()
}
return certificate.ValidationResult{FingerprintSHA256: saved.FingerprintSHA256}, nil
},
}