完成部署后校验失败恢复整改并更新实机进度
延后事务提交,校验失败时恢复目标文件、配置和服务状态;新增恢复边界回归、前端提示及生产测试记录。
This commit is contained in:
1 parent
c623e2d324
commit
3cc8fb6470
13 files changed
+588
-58
No files matched your search
@@ -26,6 +26,11 @@ func (d forbiddenDeployer) DeploySaved(context.Context, []deployment.Target, dep
|
||||
return deployment.Report{}, nil
|
||||
}
|
||||
|
||||
func (d forbiddenDeployer) DeploySavedVerified(context.Context, []deployment.Target, deployment.SavedFiles, deployment.Verification) (deployment.Report, error) {
|
||||
d.t.Fatal("权限检查失败后不得发起事务部署")
|
||||
return deployment.Report{}, nil
|
||||
}
|
||||
|
||||
func TestWorkflowDoesNotDeployAfterDNSPermissionDenied(t *testing.T) {
|
||||
workflow := &CertificateWorkflow{
|
||||
Certificates: &deniedWorkflowCertificateService{},
|
||||
|
||||
@@ -43,6 +43,7 @@ type serverLister interface {
|
||||
|
||||
type savedDeployer interface {
|
||||
DeploySaved(context.Context, []deployment.Target, deployment.SavedFiles) (deployment.Report, error)
|
||||
DeploySavedVerified(context.Context, []deployment.Target, deployment.SavedFiles, deployment.Verification) (deployment.Report, error)
|
||||
}
|
||||
|
||||
type domainDeploymentRepository interface {
|
||||
@@ -253,31 +254,41 @@ func (w *CertificateWorkflow) deploy(ctx context.Context, result certificate.Iss
|
||||
}
|
||||
expectedFingerprint = validation.FingerprintSHA256
|
||||
}
|
||||
report, err := w.Deployment.DeploySaved(ctx, targets, deployment.SavedFiles{
|
||||
saved := deployment.SavedFiles{
|
||||
CertificatePath: result.Saved.CertificatePath,
|
||||
FullChainPath: result.Saved.FullChainPath,
|
||||
PrivateKeyPath: result.Saved.PrivateKeyPath,
|
||||
})
|
||||
}
|
||||
if w.RemoteVerifier != nil {
|
||||
verificationDomain := verificationName(domainItem, result.Domains)
|
||||
if verificationDomain == "" {
|
||||
return deployment.Report{Status: "failed"}, errors.New("无法确定部署后的 HTTPS 校验域名,未执行推送")
|
||||
}
|
||||
endpoint := "https://" + verificationDomain
|
||||
// 首次部署允许不存在旧站点,但不得在缺少可信基线时宣称运行态已恢复。
|
||||
baseline, baselineErr := w.RemoteVerifier(ctx, endpoint, result.Domains)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return deployment.Report{Status: "failed"}, err
|
||||
}
|
||||
verification := deployment.Verification{
|
||||
Verify: func(verificationContext context.Context) error {
|
||||
return w.verifyRemoteDeployment(verificationContext, endpoint, result.Domains, expectedFingerprint)
|
||||
},
|
||||
}
|
||||
if baselineErr == nil && baseline.FingerprintSHA256 != "" {
|
||||
verification.VerifyRecovery = func(recoveryContext context.Context) error {
|
||||
return w.verifyRemoteDeployment(recoveryContext, endpoint, result.Domains, baseline.FingerprintSHA256)
|
||||
}
|
||||
}
|
||||
return w.Deployment.DeploySavedVerified(ctx, targets, saved, verification)
|
||||
}
|
||||
report, err := w.Deployment.DeploySaved(ctx, targets, saved)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if report.Status != "success" {
|
||||
return report, fmt.Errorf("证书部署未全部成功:成功 %d 台,失败 %d 台", report.Success, report.Failed)
|
||||
}
|
||||
if w.RemoteVerifier != nil {
|
||||
verificationDomain := verificationName(domainItem, result.Domains)
|
||||
// 单台结果保留真实文件推送结果,站点生效校验单独记录,不能宣称已回滚。
|
||||
report.VerificationStatus = "failed"
|
||||
report.Status = "failed"
|
||||
if verificationDomain == "" {
|
||||
return report, errors.New("无法确定部署后的 HTTPS 校验域名")
|
||||
}
|
||||
if err := w.verifyRemoteDeployment(ctx, "https://"+verificationDomain, result.Domains, expectedFingerprint); err != nil {
|
||||
return report, fmt.Errorf("部署后 HTTPS 校验失败:%w", err)
|
||||
}
|
||||
report.VerificationStatus = "success"
|
||||
report.Status = "success"
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -97,6 +97,21 @@ func (d *workflowDeployer) DeploySaved(_ context.Context, targets []deployment.T
|
||||
return d.report, nil
|
||||
}
|
||||
|
||||
// 工作流替身只模拟推送和校验,不伪造远端事务恢复;恢复边界由部署包实测。
|
||||
func (d *workflowDeployer) DeploySavedVerified(ctx context.Context, targets []deployment.Target, files deployment.SavedFiles, verification deployment.Verification) (deployment.Report, error) {
|
||||
report, err := d.DeploySaved(ctx, targets, files)
|
||||
if err != nil || report.Status != "success" {
|
||||
return report, err
|
||||
}
|
||||
report.VerificationStatus = "failed"
|
||||
if err := verification.Verify(ctx); err != nil {
|
||||
report.Status = "failed"
|
||||
return report, err
|
||||
}
|
||||
report.VerificationStatus = "success"
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func TestCertificateWorkflowDeploysIssuedCertificateToAssociatedServers(t *testing.T) {
|
||||
certificateService := &workflowCertificateService{result: certificate.IssueResult{
|
||||
DomainID: "domain-1",
|
||||
@@ -223,7 +238,7 @@ func TestCertificateWorkflowRetriesRemoteVerificationUntilSuccess(t *testing.T)
|
||||
if endpoint != "https://example.com" || len(expectedDomains) != 1 || expectedDomains[0] != "example.com" {
|
||||
t.Fatalf("HTTPS 校验参数错误:%s %+v", endpoint, expectedDomains)
|
||||
}
|
||||
if verificationCalls < 3 {
|
||||
if verificationCalls < 4 {
|
||||
return certificate.ValidationResult{}, errors.New("仍返回旧证书")
|
||||
}
|
||||
return certificate.ValidationResult{FingerprintSHA256: saved.FingerprintSHA256}, nil
|
||||
@@ -233,7 +248,7 @@ func TestCertificateWorkflowRetriesRemoteVerificationUntilSuccess(t *testing.T)
|
||||
if _, err := workflow.IssueWithReport(context.Background(), "domain-1"); err != nil {
|
||||
t.Fatalf("HTTPS 校验重试后应成功:%v", err)
|
||||
}
|
||||
if verificationCalls != 3 || waitCalls != 2 {
|
||||
if verificationCalls != 4 || waitCalls != 2 {
|
||||
t.Fatalf("HTTPS 校验重试次数错误:校验 %d 次,等待 %d 次", verificationCalls, waitCalls)
|
||||
}
|
||||
}
|
||||
@@ -264,7 +279,7 @@ func TestCertificateWorkflowRemoteVerificationStopsAfterConfiguredAttempts(t *te
|
||||
if err == nil || !strings.Contains(err.Error(), "重试 3 次后仍未通过") {
|
||||
t.Fatalf("永久校验错误应在有限重试后返回:%v", err)
|
||||
}
|
||||
if verificationCalls != 3 || waitCalls != 2 {
|
||||
if verificationCalls != 4 || waitCalls != 2 {
|
||||
t.Fatalf("永久错误的重试次数错误:校验 %d 次,等待 %d 次", verificationCalls, waitCalls)
|
||||
}
|
||||
}
|
||||
@@ -284,6 +299,9 @@ func TestCertificateWorkflowRemoteVerificationWaitHonorsContextCancellation(t *t
|
||||
RemoteVerificationInterval: time.Minute,
|
||||
RemoteVerifier: func(context.Context, string, []string) (certificate.ValidationResult, error) {
|
||||
verificationCalls++
|
||||
if verificationCalls == 1 {
|
||||
return certificate.ValidationResult{FingerprintSHA256: saved.FingerprintSHA256}, nil
|
||||
}
|
||||
cancel()
|
||||
return certificate.ValidationResult{}, errors.New("等待新证书")
|
||||
},
|
||||
@@ -293,7 +311,7 @@ func TestCertificateWorkflowRemoteVerificationWaitHonorsContextCancellation(t *t
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("取消上下文后应立即终止 HTTPS 校验重试:%v", err)
|
||||
}
|
||||
if verificationCalls != 1 {
|
||||
if verificationCalls != 2 {
|
||||
t.Fatalf("取消后不应继续校验,实际调用 %d 次", verificationCalls)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -199,14 +199,14 @@ func TestDeploymentVerificationRetriesRealValidOldCertificate(t *testing.T) {
|
||||
RemoteVerifier: func(ctx context.Context, _ string, domains []string) (certificate.ValidationResult, error) {
|
||||
calls++
|
||||
endpoint := oldServer.URL
|
||||
if calls > 1 {
|
||||
if calls > 2 {
|
||||
endpoint = newServer.URL
|
||||
}
|
||||
return validator.ValidateHTTPS(ctx, endpoint, domains)
|
||||
},
|
||||
}
|
||||
report, err := workflow.deploy(t.Context(), certificate.IssueResult{DomainID: "domain-1", Domains: []string{"example.com"}, Saved: newSaved})
|
||||
if err != nil || report.VerificationStatus != "success" || calls != 2 || waits != 1 {
|
||||
if err != nil || report.VerificationStatus != "success" || calls != 3 || waits != 1 {
|
||||
t.Fatalf("有效旧证书必须重试,只有实际新证书才能通过:%+v,调用 %d,等待 %d,%v", report, calls, waits, err)
|
||||
}
|
||||
}
|
||||
@@ -215,11 +215,15 @@ func TestDeploymentVerificationCancellationCannotBecomeSuccess(t *testing.T) {
|
||||
saved := workflowSavedCertificate(t)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
calls := 0
|
||||
workflow := &CertificateWorkflow{
|
||||
Domains: &workflowDomainState{}, Servers: workflowServerLister{servers: []deployment.Server{{ID: "server-1"}}},
|
||||
Deployment: &workflowDeployer{report: deployment.Report{Total: 1, Success: 1, Status: "success"}},
|
||||
RemoteVerifier: func(context.Context, string, []string) (certificate.ValidationResult, error) {
|
||||
cancel()
|
||||
calls++
|
||||
if calls > 1 {
|
||||
cancel()
|
||||
}
|
||||
return certificate.ValidationResult{FingerprintSHA256: saved.FingerprintSHA256}, nil
|
||||
},
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user