完成部署后校验失败恢复整改并更新实机进度
延后事务提交，校验失败时恢复目标文件、配置和服务状态；新增恢复边界回归、前端提示及生产测试记录。
This commit is contained in:
parent
c623e2d324
commit
3cc8fb6470
@ -26,6 +26,11 @@ func (d forbiddenDeployer) DeploySaved(context.Context, []deployment.Target, dep
|
||||
return deployment.Report{}, nil
|
||||
}
|
||||
|
||||
func (d forbiddenDeployer) DeploySavedVerified(context.Context, []deployment.Target, deployment.SavedFiles, deployment.Verification) (deployment.Report, error) {
|
||||
d.t.Fatal("权限检查失败后不得发起事务部署")
|
||||
return deployment.Report{}, nil
|
||||
}
|
||||
|
||||
func TestWorkflowDoesNotDeployAfterDNSPermissionDenied(t *testing.T) {
|
||||
workflow := &CertificateWorkflow{
|
||||
Certificates: &deniedWorkflowCertificateService{},
|
||||
|
||||
@ -43,6 +43,7 @@ type serverLister interface {
|
||||
|
||||
type savedDeployer interface {
|
||||
DeploySaved(context.Context, []deployment.Target, deployment.SavedFiles) (deployment.Report, error)
|
||||
DeploySavedVerified(context.Context, []deployment.Target, deployment.SavedFiles, deployment.Verification) (deployment.Report, error)
|
||||
}
|
||||
|
||||
type domainDeploymentRepository interface {
|
||||
@ -253,31 +254,41 @@ func (w *CertificateWorkflow) deploy(ctx context.Context, result certificate.Iss
|
||||
}
|
||||
expectedFingerprint = validation.FingerprintSHA256
|
||||
}
|
||||
report, err := w.Deployment.DeploySaved(ctx, targets, deployment.SavedFiles{
|
||||
saved := deployment.SavedFiles{
|
||||
CertificatePath: result.Saved.CertificatePath,
|
||||
FullChainPath: result.Saved.FullChainPath,
|
||||
PrivateKeyPath: result.Saved.PrivateKeyPath,
|
||||
})
|
||||
}
|
||||
if w.RemoteVerifier != nil {
|
||||
verificationDomain := verificationName(domainItem, result.Domains)
|
||||
if verificationDomain == "" {
|
||||
return deployment.Report{Status: "failed"}, errors.New("无法确定部署后的 HTTPS 校验域名，未执行推送")
|
||||
}
|
||||
endpoint := "https://" + verificationDomain
|
||||
// 首次部署允许不存在旧站点，但不得在缺少可信基线时宣称运行态已恢复。
|
||||
baseline, baselineErr := w.RemoteVerifier(ctx, endpoint, result.Domains)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return deployment.Report{Status: "failed"}, err
|
||||
}
|
||||
verification := deployment.Verification{
|
||||
Verify: func(verificationContext context.Context) error {
|
||||
return w.verifyRemoteDeployment(verificationContext, endpoint, result.Domains, expectedFingerprint)
|
||||
},
|
||||
}
|
||||
if baselineErr == nil && baseline.FingerprintSHA256 != "" {
|
||||
verification.VerifyRecovery = func(recoveryContext context.Context) error {
|
||||
return w.verifyRemoteDeployment(recoveryContext, endpoint, result.Domains, baseline.FingerprintSHA256)
|
||||
}
|
||||
}
|
||||
return w.Deployment.DeploySavedVerified(ctx, targets, saved, verification)
|
||||
}
|
||||
report, err := w.Deployment.DeploySaved(ctx, targets, saved)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if report.Status != "success" {
|
||||
return report, fmt.Errorf("证书部署未全部成功：成功 %d 台，失败 %d 台", report.Success, report.Failed)
|
||||
}
|
||||
if w.RemoteVerifier != nil {
|
||||
verificationDomain := verificationName(domainItem, result.Domains)
|
||||
// 单台结果保留真实文件推送结果，站点生效校验单独记录，不能宣称已回滚。
|
||||
report.VerificationStatus = "failed"
|
||||
report.Status = "failed"
|
||||
if verificationDomain == "" {
|
||||
return report, errors.New("无法确定部署后的 HTTPS 校验域名")
|
||||
}
|
||||
if err := w.verifyRemoteDeployment(ctx, "https://"+verificationDomain, result.Domains, expectedFingerprint); err != nil {
|
||||
return report, fmt.Errorf("部署后 HTTPS 校验失败：%w", err)
|
||||
}
|
||||
report.VerificationStatus = "success"
|
||||
report.Status = "success"
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
|
||||
@ -97,6 +97,21 @@ func (d *workflowDeployer) DeploySaved(_ context.Context, targets []deployment.T
|
||||
return d.report, nil
|
||||
}
|
||||
|
||||
// 工作流替身只模拟推送和校验，不伪造远端事务恢复；恢复边界由部署包实测。
|
||||
func (d *workflowDeployer) DeploySavedVerified(ctx context.Context, targets []deployment.Target, files deployment.SavedFiles, verification deployment.Verification) (deployment.Report, error) {
|
||||
report, err := d.DeploySaved(ctx, targets, files)
|
||||
if err != nil || report.Status != "success" {
|
||||
return report, err
|
||||
}
|
||||
report.VerificationStatus = "failed"
|
||||
if err := verification.Verify(ctx); err != nil {
|
||||
report.Status = "failed"
|
||||
return report, err
|
||||
}
|
||||
report.VerificationStatus = "success"
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func TestCertificateWorkflowDeploysIssuedCertificateToAssociatedServers(t *testing.T) {
|
||||
certificateService := &workflowCertificateService{result: certificate.IssueResult{
|
||||
DomainID: "domain-1",
|
||||
@ -223,7 +238,7 @@ func TestCertificateWorkflowRetriesRemoteVerificationUntilSuccess(t *testing.T)
|
||||
if endpoint != "https://example.com" || len(expectedDomains) != 1 || expectedDomains[0] != "example.com" {
|
||||
t.Fatalf("HTTPS 校验参数错误：%s %+v", endpoint, expectedDomains)
|
||||
}
|
||||
if verificationCalls < 3 {
|
||||
if verificationCalls < 4 {
|
||||
return certificate.ValidationResult{}, errors.New("仍返回旧证书")
|
||||
}
|
||||
return certificate.ValidationResult{FingerprintSHA256: saved.FingerprintSHA256}, nil
|
||||
@ -233,7 +248,7 @@ func TestCertificateWorkflowRetriesRemoteVerificationUntilSuccess(t *testing.T)
|
||||
if _, err := workflow.IssueWithReport(context.Background(), "domain-1"); err != nil {
|
||||
t.Fatalf("HTTPS 校验重试后应成功：%v", err)
|
||||
}
|
||||
if verificationCalls != 3 || waitCalls != 2 {
|
||||
if verificationCalls != 4 || waitCalls != 2 {
|
||||
t.Fatalf("HTTPS 校验重试次数错误：校验 %d 次，等待 %d 次", verificationCalls, waitCalls)
|
||||
}
|
||||
}
|
||||
@ -264,7 +279,7 @@ func TestCertificateWorkflowRemoteVerificationStopsAfterConfiguredAttempts(t *te
|
||||
if err == nil || !strings.Contains(err.Error(), "重试 3 次后仍未通过") {
|
||||
t.Fatalf("永久校验错误应在有限重试后返回：%v", err)
|
||||
}
|
||||
if verificationCalls != 3 || waitCalls != 2 {
|
||||
if verificationCalls != 4 || waitCalls != 2 {
|
||||
t.Fatalf("永久错误的重试次数错误：校验 %d 次，等待 %d 次", verificationCalls, waitCalls)
|
||||
}
|
||||
}
|
||||
@ -284,6 +299,9 @@ func TestCertificateWorkflowRemoteVerificationWaitHonorsContextCancellation(t *t
|
||||
RemoteVerificationInterval: time.Minute,
|
||||
RemoteVerifier: func(context.Context, string, []string) (certificate.ValidationResult, error) {
|
||||
verificationCalls++
|
||||
if verificationCalls == 1 {
|
||||
return certificate.ValidationResult{FingerprintSHA256: saved.FingerprintSHA256}, nil
|
||||
}
|
||||
cancel()
|
||||
return certificate.ValidationResult{}, errors.New("等待新证书")
|
||||
},
|
||||
@ -293,7 +311,7 @@ func TestCertificateWorkflowRemoteVerificationWaitHonorsContextCancellation(t *t
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("取消上下文后应立即终止 HTTPS 校验重试：%v", err)
|
||||
}
|
||||
if verificationCalls != 1 {
|
||||
if verificationCalls != 2 {
|
||||
t.Fatalf("取消后不应继续校验，实际调用 %d 次", verificationCalls)
|
||||
}
|
||||
}
|
||||
|
||||
@ -199,14 +199,14 @@ func TestDeploymentVerificationRetriesRealValidOldCertificate(t *testing.T) {
|
||||
RemoteVerifier: func(ctx context.Context, _ string, domains []string) (certificate.ValidationResult, error) {
|
||||
calls++
|
||||
endpoint := oldServer.URL
|
||||
if calls > 1 {
|
||||
if calls > 2 {
|
||||
endpoint = newServer.URL
|
||||
}
|
||||
return validator.ValidateHTTPS(ctx, endpoint, domains)
|
||||
},
|
||||
}
|
||||
report, err := workflow.deploy(t.Context(), certificate.IssueResult{DomainID: "domain-1", Domains: []string{"example.com"}, Saved: newSaved})
|
||||
if err != nil || report.VerificationStatus != "success" || calls != 2 || waits != 1 {
|
||||
if err != nil || report.VerificationStatus != "success" || calls != 3 || waits != 1 {
|
||||
t.Fatalf("有效旧证书必须重试，只有实际新证书才能通过：%+v，调用 %d，等待 %d，%v", report, calls, waits, err)
|
||||
}
|
||||
}
|
||||
@ -215,11 +215,15 @@ func TestDeploymentVerificationCancellationCannotBecomeSuccess(t *testing.T) {
|
||||
saved := workflowSavedCertificate(t)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
calls := 0
|
||||
workflow := &CertificateWorkflow{
|
||||
Domains: &workflowDomainState{}, Servers: workflowServerLister{servers: []deployment.Server{{ID: "server-1"}}},
|
||||
Deployment: &workflowDeployer{report: deployment.Report{Total: 1, Success: 1, Status: "success"}},
|
||||
RemoteVerifier: func(context.Context, string, []string) (certificate.ValidationResult, error) {
|
||||
cancel()
|
||||
calls++
|
||||
if calls > 1 {
|
||||
cancel()
|
||||
}
|
||||
return certificate.ValidationResult{FingerprintSHA256: saved.FingerprintSHA256}, nil
|
||||
},
|
||||
}
|
||||
|
||||
@ -3,36 +3,41 @@ package deployment
|
||||
type FailureCode string
|
||||
|
||||
const (
|
||||
FailureUnavailable FailureCode = "unavailable"
|
||||
FailureArtifact FailureCode = "artifact_invalid"
|
||||
FailureConfiguration FailureCode = "configuration_invalid"
|
||||
FailureConnection FailureCode = "connection_failed"
|
||||
FailureDirectory FailureCode = "directory_failed"
|
||||
FailureLock FailureCode = "lock_unavailable"
|
||||
FailureUpload FailureCode = "upload_failed"
|
||||
FailureReplace FailureCode = "replace_failed"
|
||||
FailureValidation FailureCode = "validation_failed"
|
||||
FailureRollback FailureCode = "rollback_failed"
|
||||
FailureRuntimeUnknown FailureCode = "runtime_unconfirmed"
|
||||
FailureCleanup FailureCode = "cleanup_failed"
|
||||
FailureLockRelease FailureCode = "lock_release_failed"
|
||||
FailureUnavailable FailureCode = "unavailable"
|
||||
FailureArtifact FailureCode = "artifact_invalid"
|
||||
FailureConfiguration FailureCode = "configuration_invalid"
|
||||
FailureConnection FailureCode = "connection_failed"
|
||||
FailureDirectory FailureCode = "directory_failed"
|
||||
FailureLock FailureCode = "lock_unavailable"
|
||||
FailureUpload FailureCode = "upload_failed"
|
||||
FailureReplace FailureCode = "replace_failed"
|
||||
FailureValidation FailureCode = "validation_failed"
|
||||
FailureRollback FailureCode = "rollback_failed"
|
||||
FailureRuntimeUnknown FailureCode = "runtime_unconfirmed"
|
||||
FailureCleanup FailureCode = "cleanup_failed"
|
||||
FailureLockRelease FailureCode = "lock_release_failed"
|
||||
FailureRecoveryPending FailureCode = "recovery_pending"
|
||||
FailureRecoveryUnknown FailureCode = "recovery_unconfirmed"
|
||||
FailureRecoveryRestored FailureCode = "recovery_restored"
|
||||
)
|
||||
|
||||
// 只公开固定诊断，不从远端输出、错误文本或文件路径推断可展示的消息。
|
||||
var publicFailureMessages = map[FailureCode]string{
|
||||
FailureUnavailable: "部署服务不可用，请联系管理员检查服务配置",
|
||||
FailureArtifact: "待部署的证书或私钥校验失败，尚未推送到服务器",
|
||||
FailureConfiguration: "服务器部署配置无效，请检查认证、证书路径和命令配置",
|
||||
FailureConnection: "连接部署服务器失败，请检查网络、SSH 认证及主机身份配置",
|
||||
FailureDirectory: "无法创建证书目录，请检查部署账户的目录权限",
|
||||
FailureLock: "无法取得部署锁，可能有任务正在执行或存在中断遗留；请管理员核查远端文件、部署锁及服务状态后再重试，勿直接删除锁",
|
||||
FailureUpload: "证书上传失败，已尝试回滚；请检查受保护的服务日志后再重试",
|
||||
FailureReplace: "证书文件替换失败，已尝试回滚；请检查受保护的服务日志后再重试",
|
||||
FailureValidation: "服务配置校验失败，文件已回滚，未执行重载；请修正配置后再重试",
|
||||
FailureRollback: "部署失败且文件回滚未完成，已保留部署锁；请管理员核查远端文件及服务状态，勿直接重试或删除锁",
|
||||
FailureRuntimeUnknown: "服务重载失败，文件已回滚，但服务运行状态未确认；请管理员核对配置并恢复服务后再重试",
|
||||
FailureCleanup: "证书已推送并重载，但旧文件清理未完成；请管理员核查现场，不能据此确认站点新证书已生效",
|
||||
FailureLockRelease: "部署锁释放未完成；请管理员核查远端文件、部署锁及服务状态后再重试",
|
||||
FailureRecoveryUnknown: "本次部署未完成，已恢复旧文件，但服务或站点恢复状态未确认，已保留部署锁；请管理员核查，勿直接重试或删除锁",
|
||||
FailureRecoveryRestored: "本次部署未完成，旧文件已恢复并通过配置校验及重载，站点已恢复部署前证书；请核查失败原因后再重试",
|
||||
FailureUnavailable: "部署服务不可用，请联系管理员检查服务配置",
|
||||
FailureArtifact: "待部署的证书或私钥校验失败，尚未推送到服务器",
|
||||
FailureConfiguration: "服务器部署配置无效，请检查认证、证书路径和命令配置",
|
||||
FailureConnection: "连接部署服务器失败，请检查网络、SSH 认证及主机身份配置",
|
||||
FailureDirectory: "无法创建证书目录，请检查部署账户的目录权限",
|
||||
FailureLock: "无法取得部署锁，可能有任务正在执行或存在中断遗留；请管理员核查远端文件、部署锁及服务状态后再重试，勿直接删除锁",
|
||||
FailureUpload: "证书上传失败，已尝试回滚；请检查受保护的服务日志后再重试",
|
||||
FailureReplace: "证书文件替换失败，已尝试回滚；请检查受保护的服务日志后再重试",
|
||||
FailureValidation: "服务配置校验失败，文件已回滚，未执行重载；请修正配置后再重试",
|
||||
FailureRollback: "部署失败且文件回滚未完成，已保留部署锁；请管理员核查远端文件及服务状态，勿直接重试或删除锁",
|
||||
FailureRuntimeUnknown: "服务重载失败，文件已回滚，但服务运行状态未确认；请管理员核对配置并恢复服务后再重试",
|
||||
FailureCleanup: "证书已推送并重载，但旧文件清理未完成；请管理员核查现场，不能据此确认站点新证书已生效",
|
||||
FailureLockRelease: "部署锁释放未完成；请管理员核查远端文件、部署锁及服务状态后再重试",
|
||||
}
|
||||
|
||||
// PublicResult 保留结果事实，同时只允许固定错误码及中文诊断进入公开响应。
|
||||
|
||||
@ -47,6 +47,7 @@ type Report struct {
|
||||
Results []Result `json:"results"`
|
||||
// 站点校验与各服务器文件推送分开记录，不表示所有服务器均完成独立 HTTPS 验证。
|
||||
VerificationStatus string `json:"verification_status,omitempty"`
|
||||
RecoveryStatus string `json:"recovery_status,omitempty"`
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
@ -59,6 +60,10 @@ func NewService(factory ClientFactory) *Service {
|
||||
}
|
||||
|
||||
func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifact) Report {
|
||||
return s.deploy(ctx, targets, artifact, nil)
|
||||
}
|
||||
|
||||
func (s *Service) deploy(ctx context.Context, targets []Target, artifact Artifact, pending *[]*pendingDeployment) Report {
|
||||
report := Report{Total: len(targets), Results: make([]Result, 0, len(targets))}
|
||||
if len(targets) == 0 {
|
||||
report.Status = "failed"
|
||||
@ -80,8 +85,8 @@ func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifac
|
||||
}
|
||||
return report
|
||||
}
|
||||
for _, target := range targets {
|
||||
result := s.deployOne(ctx, target.Server, artifact)
|
||||
for index, target := range targets {
|
||||
result := s.deployOne(ctx, target.Server, artifact, pending, index)
|
||||
report.Results = append(report.Results, result)
|
||||
if result.Status == "success" {
|
||||
report.Success++
|
||||
@ -101,22 +106,30 @@ func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifac
|
||||
}
|
||||
|
||||
func (s *Service) DeploySaved(ctx context.Context, targets []Target, saved SavedFiles) (Report, error) {
|
||||
artifact, err := loadArtifact(saved)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
return s.Deploy(ctx, targets, artifact), nil
|
||||
}
|
||||
|
||||
func loadArtifact(saved SavedFiles) (Artifact, error) {
|
||||
certificateData, err := os.ReadFile(saved.CertificatePath)
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("读取证书文件失败：%w", err)
|
||||
return Artifact{}, fmt.Errorf("读取证书文件失败：%w", err)
|
||||
}
|
||||
fullChainData, err := os.ReadFile(saved.FullChainPath)
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("读取完整证书链失败：%w", err)
|
||||
return Artifact{}, fmt.Errorf("读取完整证书链失败：%w", err)
|
||||
}
|
||||
privateKeyData, err := os.ReadFile(saved.PrivateKeyPath)
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("读取私钥文件失败：%w", err)
|
||||
return Artifact{}, fmt.Errorf("读取私钥文件失败：%w", err)
|
||||
}
|
||||
return s.Deploy(ctx, targets, Artifact{Certificate: certificateData, FullChain: fullChainData, PrivateKey: privateKeyData}), nil
|
||||
return Artifact{Certificate: certificateData, FullChain: fullChainData, PrivateKey: privateKeyData}, nil
|
||||
}
|
||||
|
||||
func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifact) (result Result) {
|
||||
func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifact, pending *[]*pendingDeployment, index int) (result Result) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
server = server.Normalize()
|
||||
@ -131,7 +144,12 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
result.Message = fmt.Sprintf("连接服务器失败：%v", err)
|
||||
return result
|
||||
}
|
||||
defer func() { _ = client.Close() }()
|
||||
handedOff := false
|
||||
defer func() {
|
||||
if !handedOff {
|
||||
_ = client.Close()
|
||||
}
|
||||
}()
|
||||
paths := []remoteFile{
|
||||
{path: RemotePath(server.CertificatePath), data: artifact.Certificate, mode: 0o644},
|
||||
{path: RemotePath(server.CertificateChainPath), data: artifact.FullChain, mode: 0o644},
|
||||
@ -160,7 +178,7 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
sort.Strings(lockPaths)
|
||||
heldLocks := make([]string, 0, len(lockPaths))
|
||||
retainLocks := false
|
||||
defer func() {
|
||||
releaseLocks := func() {
|
||||
if len(heldLocks) == 0 || retainLocks {
|
||||
return
|
||||
}
|
||||
@ -179,6 +197,11 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
result.Message += fmt.Sprintf("；释放部署锁 %s 失败，需人工检查：%v", heldLocks[index], err)
|
||||
}
|
||||
}
|
||||
}
|
||||
defer func() {
|
||||
if !handedOff {
|
||||
releaseLocks()
|
||||
}
|
||||
}()
|
||||
for _, lockPath := range lockPaths {
|
||||
if err := client.Lock(ctx, lockPath); err != nil {
|
||||
@ -300,6 +323,59 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
}
|
||||
return failed(fmt.Sprintf("reload 失败：%v", err))
|
||||
}
|
||||
if pending != nil {
|
||||
// 旧文件与锁在所有目标及站点校验结束前始终保留，禁止提前提交。
|
||||
result.Status = "success"
|
||||
result.FailureCode = ""
|
||||
result.Message = "证书已推送并重载，等待站点校验"
|
||||
closed := false
|
||||
transaction := &pendingDeployment{index: index}
|
||||
transaction.restore = func(recoveryContext context.Context) Result {
|
||||
result.Status = "failed"
|
||||
if err := rollback(); err != nil {
|
||||
result.FailureCode = FailureRollback
|
||||
result.Message = fmt.Sprintf("部署未提交，自动回滚失败：%v", err)
|
||||
return result
|
||||
}
|
||||
result.FailureCode = FailureRecoveryUnknown
|
||||
if err := reconnectDeploymentClient(recoveryContext, s.factory, server, &client); err != nil {
|
||||
result.Message = fmt.Sprintf("旧文件已恢复，重新连接以恢复服务失败：%v", err)
|
||||
return result
|
||||
}
|
||||
for _, command := range [][]string{server.ValidationCommand, server.ReloadCommand} {
|
||||
outcome, err := client.Run(recoveryContext, command)
|
||||
if err != nil || outcome.ExitCode != 0 {
|
||||
result.Message = fmt.Sprintf("旧文件已恢复，但恢复配置校验或重载失败，退出码 %d：%v", outcome.ExitCode, err)
|
||||
return result
|
||||
}
|
||||
}
|
||||
result.FailureCode = FailureRecoveryPending
|
||||
result.Message = "本次部署未提交，旧文件已恢复并通过配置校验及重载，等待站点恢复核验"
|
||||
return result
|
||||
}
|
||||
transaction.commit = func(commitContext context.Context) Result {
|
||||
if err := cleanupBackups(commitContext, client, backups, paths); err != nil {
|
||||
result.Status = "failed"
|
||||
result.FailureCode = FailureCleanup
|
||||
result.Message = fmt.Sprintf("站点校验通过，但事务旧文件清理失败：%v", err)
|
||||
return result
|
||||
}
|
||||
result.Message = "证书部署、配置校验并 reload 成功"
|
||||
return result
|
||||
}
|
||||
transaction.finish = func(retain bool) Result {
|
||||
if !closed {
|
||||
retainLocks = retain
|
||||
releaseLocks()
|
||||
_ = client.Close()
|
||||
closed = true
|
||||
}
|
||||
return result
|
||||
}
|
||||
handedOff = true
|
||||
*pending = append(*pending, transaction)
|
||||
return result
|
||||
}
|
||||
if err := cleanupBackups(ctx, client, backups, paths); err != nil {
|
||||
result.Status = "failed"
|
||||
result.FailureCode = FailureCleanup
|
||||
|
||||
150
backend/internal/deployment/verified.go
Normal file
150
backend/internal/deployment/verified.go
Normal file
@ -0,0 +1,150 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Verification 将站点生效校验与恢复核验置于远端文件事务之内。
|
||||
type Verification struct {
|
||||
Verify func(context.Context) error
|
||||
VerifyRecovery func(context.Context) error
|
||||
}
|
||||
|
||||
type pendingDeployment struct {
|
||||
index int
|
||||
commit func(context.Context) Result
|
||||
restore func(context.Context) Result
|
||||
finish func(bool) Result
|
||||
}
|
||||
|
||||
func (s *Service) DeploySavedVerified(ctx context.Context, targets []Target, saved SavedFiles, verification Verification) (Report, error) {
|
||||
artifact, err := loadArtifact(saved)
|
||||
if err != nil {
|
||||
return Report{Status: "failed"}, err
|
||||
}
|
||||
return s.DeployVerified(ctx, targets, artifact, verification)
|
||||
}
|
||||
|
||||
func (s *Service) DeployVerified(ctx context.Context, targets []Target, artifact Artifact, verification Verification) (Report, error) {
|
||||
if verification.Verify == nil {
|
||||
return Report{Status: "failed"}, errors.New("缺少部署后站点校验，未执行推送")
|
||||
}
|
||||
pending := make([]*pendingDeployment, 0, len(targets))
|
||||
// 异常退出时关闭连接但保留现场；不得以释放锁掩盖未确认的事务。
|
||||
defer func() {
|
||||
for _, transaction := range pending {
|
||||
transaction.finish(true)
|
||||
}
|
||||
}()
|
||||
report := s.deploy(ctx, targets, artifact, &pending)
|
||||
var deployErr error
|
||||
if report.Status != "success" {
|
||||
deployErr = fmt.Errorf("证书部署未全部成功：成功 %d 台，失败 %d 台", report.Success, report.Failed)
|
||||
} else {
|
||||
report.VerificationStatus = "failed"
|
||||
deployErr = ctx.Err()
|
||||
if deployErr == nil {
|
||||
deployErr = verification.Verify(ctx)
|
||||
}
|
||||
if deployErr == nil {
|
||||
deployErr = ctx.Err()
|
||||
}
|
||||
if deployErr != nil {
|
||||
deployErr = fmt.Errorf("部署后 HTTPS 校验失败：%w", deployErr)
|
||||
} else {
|
||||
report.VerificationStatus = "success"
|
||||
}
|
||||
}
|
||||
if deployErr != nil {
|
||||
if len(pending) > 0 {
|
||||
s.restorePending(ctx, &report, pending, verification)
|
||||
}
|
||||
summarizeReport(&report)
|
||||
report.Status = "failed"
|
||||
return report, deployErr
|
||||
}
|
||||
for _, transaction := range pending {
|
||||
result := transaction.commit(ctx)
|
||||
report.Results[transaction.index] = transaction.finish(result.Status != "success")
|
||||
}
|
||||
summarizeReport(&report)
|
||||
if report.Status != "success" {
|
||||
return report, errors.New("站点校验通过，但事务清理或释放部署锁未完成")
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (s *Service) restorePending(ctx context.Context, report *Report, pending []*pendingDeployment, verification Verification) {
|
||||
report.RecoveryStatus = "unconfirmed"
|
||||
for index := len(pending) - 1; index >= 0; index-- {
|
||||
transaction := pending[index]
|
||||
// 请求取消后仍使用独立有界上下文恢复，不重放签发或文件推送。
|
||||
recoveryContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
|
||||
report.Results[transaction.index] = transaction.restore(recoveryContext)
|
||||
cancel()
|
||||
}
|
||||
restored := true
|
||||
for _, result := range report.Results {
|
||||
switch result.FailureCode {
|
||||
case FailureRecoveryPending, FailureConfiguration, FailureConnection, FailureDirectory, FailureUpload, FailureReplace, FailureValidation:
|
||||
// 这些失败要么未触碰目标，要么已完成文件回滚且未执行重载。
|
||||
default:
|
||||
restored = false
|
||||
}
|
||||
}
|
||||
var recoveryErr error
|
||||
if !restored {
|
||||
report.RecoveryStatus = "failed"
|
||||
} else if verification.VerifyRecovery != nil {
|
||||
recoveryContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
|
||||
err := verification.VerifyRecovery(recoveryContext)
|
||||
if err == nil {
|
||||
err = recoveryContext.Err()
|
||||
}
|
||||
cancel()
|
||||
if err == nil {
|
||||
report.RecoveryStatus = "success"
|
||||
} else {
|
||||
recoveryErr = err
|
||||
}
|
||||
}
|
||||
for _, transaction := range pending {
|
||||
result := transaction.finish(report.RecoveryStatus != "success")
|
||||
if result.FailureCode == FailureLockRelease {
|
||||
report.RecoveryStatus = "failed"
|
||||
}
|
||||
if result.FailureCode == FailureRecoveryPending {
|
||||
if report.RecoveryStatus == "success" {
|
||||
result.FailureCode = FailureRecoveryRestored
|
||||
} else {
|
||||
result.FailureCode = FailureRecoveryUnknown
|
||||
}
|
||||
result.Message = publicFailureMessages[result.FailureCode]
|
||||
if recoveryErr != nil {
|
||||
result.Message += fmt.Sprintf("；站点恢复核验失败：%v", recoveryErr)
|
||||
}
|
||||
}
|
||||
report.Results[transaction.index] = result
|
||||
}
|
||||
}
|
||||
|
||||
func summarizeReport(report *Report) {
|
||||
report.Success, report.Failed = 0, 0
|
||||
for _, result := range report.Results {
|
||||
if result.Status == "success" {
|
||||
report.Success++
|
||||
} else {
|
||||
report.Failed++
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case report.Total == 0 || report.Success == 0:
|
||||
report.Status = "failed"
|
||||
case report.Failed == 0:
|
||||
report.Status = "success"
|
||||
default:
|
||||
report.Status = "partial"
|
||||
}
|
||||
}
|
||||
212
backend/internal/deployment/verified_test.go
Normal file
212
backend/internal/deployment/verified_test.go
Normal file
@ -0,0 +1,212 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func seedVerifiedFiles(client *memoryClient, server Server) map[string]memoryFile {
|
||||
files := map[string]memoryFile{
|
||||
server.CertificatePath: {data: []byte("原证书"), mode: 0o644},
|
||||
server.CertificateChainPath: {data: []byte("原证书链"), mode: 0o644},
|
||||
server.PrivateKeyPath: {data: []byte("原私钥"), mode: 0o600},
|
||||
}
|
||||
for path, file := range files {
|
||||
client.files[path] = file
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func assertVerifiedFiles(t *testing.T, client *memoryClient, expected map[string]memoryFile) {
|
||||
t.Helper()
|
||||
for path, file := range expected {
|
||||
actual, ok := client.files[path]
|
||||
if !ok || !bytes.Equal(actual.data, file.data) || actual.mode != file.mode {
|
||||
t.Fatalf("未恢复原内容或权限：%s", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertVerifiedLocks(t *testing.T, client *memoryClient, server Server, held bool) {
|
||||
t.Helper()
|
||||
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
|
||||
_, exists := client.files[path+".scert-lock"]
|
||||
if exists != held {
|
||||
t.Fatalf("部署锁状态不符合预期：%s，锁定 %t", path, exists)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentDefersCommitUntilAllTargetsAndSitePass(t *testing.T) {
|
||||
clients := map[string]*memoryClient{"first": newMemoryClient(), "second": newMemoryClient()}
|
||||
servers := []Server{testServer("first"), testServer("second")}
|
||||
for _, server := range servers {
|
||||
seedVerifiedFiles(clients[server.ID], server)
|
||||
}
|
||||
artifact := testArtifact(t)
|
||||
called := false
|
||||
service := NewService(memoryFactory{clients: clients})
|
||||
report, err := service.DeployVerified(t.Context(), []Target{{Server: servers[0]}, {Server: servers[1]}}, artifact, Verification{
|
||||
Verify: func(context.Context) error {
|
||||
called = true
|
||||
for _, server := range servers {
|
||||
client := clients[server.ID]
|
||||
assertVerifiedLocks(t, client, server, true)
|
||||
if !bytes.Equal(client.files[server.CertificatePath].data, artifact.Certificate) || len(client.commands) != 2 {
|
||||
t.Fatal("必须在所有服务器推送和重载后才核验站点")
|
||||
}
|
||||
backups := 0
|
||||
for path := range client.files {
|
||||
if strings.Contains(path, ".scert-backup-") {
|
||||
backups++
|
||||
}
|
||||
}
|
||||
if backups != 3 {
|
||||
t.Fatal("站点确认前必须保留三份事务旧文件")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
VerifyRecovery: func(context.Context) error { t.Fatal("成功部署不得回滚"); return nil },
|
||||
})
|
||||
if err != nil || !called || report.Status != "success" || report.Success != 2 || report.VerificationStatus != "success" || report.RecoveryStatus != "" {
|
||||
t.Fatalf("事务提交结果错误：%+v，%v", report, err)
|
||||
}
|
||||
for _, server := range servers {
|
||||
client := clients[server.ID]
|
||||
assertVerifiedLocks(t, client, server, false)
|
||||
if len(client.files) != 3 {
|
||||
t.Fatal("提交后不应残留事务文件")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentFailureRestoresAllTargetsBeforeRecoveryCheck(t *testing.T) {
|
||||
clients := map[string]*memoryClient{"first": newMemoryClient(), "second": newMemoryClient()}
|
||||
servers := []Server{testServer("first"), testServer("second")}
|
||||
originals := map[string]map[string]memoryFile{}
|
||||
for _, server := range servers {
|
||||
originals[server.ID] = seedVerifiedFiles(clients[server.ID], server)
|
||||
}
|
||||
recovered := false
|
||||
verificationErr := errors.New("公网仍返回错误证书")
|
||||
report, err := NewService(memoryFactory{clients: clients}).DeployVerified(t.Context(), []Target{{Server: servers[0]}, {Server: servers[1]}}, testArtifact(t), Verification{
|
||||
Verify: func(context.Context) error { return verificationErr },
|
||||
VerifyRecovery: func(ctx context.Context) error {
|
||||
recovered = true
|
||||
if _, ok := ctx.Deadline(); !ok || ctx.Err() != nil {
|
||||
t.Fatal("恢复核验必须有独立有效期限")
|
||||
}
|
||||
for _, server := range servers {
|
||||
client := clients[server.ID]
|
||||
assertVerifiedFiles(t, client, originals[server.ID])
|
||||
assertVerifiedLocks(t, client, server, true)
|
||||
expectedCommands := [][]string{{"nginx", "-t"}, {"nginx", "-s", "reload"}, {"nginx", "-t"}, {"nginx", "-s", "reload"}}
|
||||
if !reflect.DeepEqual(client.commands, expectedCommands) {
|
||||
t.Fatalf("必须恢复配置并重载后核验站点：实际=%#v，期望=%#v", client.commands, expectedCommands)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
if !errors.Is(err, verificationErr) || !recovered || report.Status != "failed" || report.Success != 0 || report.Failed != 2 || report.VerificationStatus != "failed" || report.RecoveryStatus != "success" {
|
||||
t.Fatalf("失败与恢复状态错误：%+v，%v", report, err)
|
||||
}
|
||||
for index, server := range servers {
|
||||
assertVerifiedFiles(t, clients[server.ID], originals[server.ID])
|
||||
assertVerifiedLocks(t, clients[server.ID], server, false)
|
||||
if report.Results[index].FailureCode != FailureRecoveryRestored {
|
||||
t.Fatalf("应保留部署失败并提供恢复诊断：%+v", report.Results[index])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentCannotClaimRecoveryOnIncompleteEvidence(t *testing.T) {
|
||||
for _, scenario := range []struct {
|
||||
name string
|
||||
configure func(*memoryClient)
|
||||
recovery func(context.Context) error
|
||||
status string
|
||||
code FailureCode
|
||||
}{
|
||||
{"missing_baseline", func(*memoryClient) {}, nil, "unconfirmed", FailureRecoveryUnknown},
|
||||
{"wrong_runtime", func(*memoryClient) {}, func(context.Context) error { return errors.New("secret-path 旧站点身份不符") }, "unconfirmed", FailureRecoveryUnknown},
|
||||
{"restore_failed", func(c *memoryClient) { c.restoreError = true }, nil, "failed", FailureRollback},
|
||||
{"validation_failed", func(c *memoryClient) { c.runResults = []CommandResult{{}, {}, {ExitCode: 1}} }, nil, "failed", FailureRecoveryUnknown},
|
||||
{"reload_failed", func(c *memoryClient) { c.runResults = []CommandResult{{}, {}, {}, {ExitCode: 1}} }, nil, "failed", FailureRecoveryUnknown},
|
||||
} {
|
||||
t.Run(scenario.name, func(t *testing.T) {
|
||||
client, server := newMemoryClient(), testServer("recovery")
|
||||
seedVerifiedFiles(client, server)
|
||||
scenario.configure(client)
|
||||
report, err := NewService(fixedClientFactory{client}).DeployVerified(t.Context(), []Target{{Server: server}}, testArtifact(t), Verification{
|
||||
Verify: func(context.Context) error { return errors.New("部署后校验失败") }, VerifyRecovery: scenario.recovery,
|
||||
})
|
||||
if err == nil || report.RecoveryStatus != scenario.status || report.Results[0].FailureCode != scenario.code {
|
||||
t.Fatalf("不得夸大恢复能力：%+v，%v", report, err)
|
||||
}
|
||||
assertVerifiedLocks(t, client, server, true)
|
||||
public := PublicResult(report.Results[0])
|
||||
if strings.Contains(public.Message, "secret-path") || public.Message == "证书部署失败" {
|
||||
t.Fatalf("恢复诊断应安全且明确：%+v", public)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentCancellationStillRestoresWithBoundedContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
client, server := newMemoryClient(), testServer("cancel")
|
||||
original := seedVerifiedFiles(client, server)
|
||||
report, err := NewService(fixedClientFactory{client}).DeployVerified(ctx, []Target{{Server: server}}, testArtifact(t), Verification{
|
||||
Verify: func(context.Context) error { cancel(); return nil },
|
||||
VerifyRecovery: func(ctx context.Context) error {
|
||||
if ctx.Err() != nil {
|
||||
t.Fatal("请求取消不得取消恢复核验")
|
||||
}
|
||||
assertVerifiedFiles(t, client, original)
|
||||
return nil
|
||||
},
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) || report.Status != "failed" || report.RecoveryStatus != "success" || len(client.commands) != 4 {
|
||||
t.Fatalf("取消后应恢复但不得转为部署成功：%+v，%v", report, err)
|
||||
}
|
||||
assertVerifiedLocks(t, client, server, false)
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentPartialPreparationDoesNotCommitEarlierTarget(t *testing.T) {
|
||||
client, first := newMemoryClient(), testServer("first")
|
||||
second := testServer("second")
|
||||
original := seedVerifiedFiles(client, first)
|
||||
service := NewService(memoryFactory{clients: map[string]*memoryClient{"first": client}, errors: map[string]error{"second": errors.New("连接失败")}})
|
||||
report, err := service.DeployVerified(t.Context(), []Target{{Server: first}, {Server: second}}, testArtifact(t), Verification{
|
||||
Verify: func(context.Context) error { t.Fatal("部分准备失败不得进行提交校验"); return nil },
|
||||
VerifyRecovery: func(context.Context) error { assertVerifiedFiles(t, client, original); return nil },
|
||||
})
|
||||
if err == nil || report.Status != "failed" || report.Success != 0 || report.RecoveryStatus != "success" || report.Results[1].FailureCode != FailureConnection {
|
||||
t.Fatalf("部分目标失败必须恢复已推送目标：%+v，%v", report, err)
|
||||
}
|
||||
assertVerifiedLocks(t, client, first, false)
|
||||
}
|
||||
|
||||
func TestVerifiedDeploymentRequiresVerifierAndDoesNotHideCleanupFailure(t *testing.T) {
|
||||
client, server := newMemoryClient(), testServer("cleanup")
|
||||
original := seedVerifiedFiles(client, server)
|
||||
service := NewService(fixedClientFactory{client})
|
||||
report, err := service.DeployVerified(t.Context(), []Target{{Server: server}}, testArtifact(t), Verification{})
|
||||
if err == nil || report.Status != "failed" || len(client.commands) != 0 {
|
||||
t.Fatal("缺少校验器不得推送")
|
||||
}
|
||||
assertVerifiedFiles(t, client, original)
|
||||
client.removeError = ".scert-backup-"
|
||||
report, err = service.DeployVerified(t.Context(), []Target{{Server: server}}, testArtifact(t), Verification{Verify: func(context.Context) error { return nil }})
|
||||
if err == nil || report.Status != "failed" || report.VerificationStatus != "success" || report.Results[0].FailureCode != FailureCleanup {
|
||||
t.Fatalf("校验成功不能掩盖清理失败：%+v，%v", report, err)
|
||||
}
|
||||
assertVerifiedLocks(t, client, server, true)
|
||||
}
|
||||
@ -1,4 +1,13 @@
|
||||
# Linux 生产流程测试记录
|
||||
## 固定诊断版本部署与锁测试收尾（2026-09-30 22:55）
|
||||
|
||||
- 6efc89061bc87fdcdc41182203749b82e598865a 干净源码正式构建部署；归档 SHA256 为 443cd7db2a98832c5b1d2eb5d27b0ee704192c5be97ddd0f7585531008c13b5c，主程序 SHA256 为 ee0a5a98c993eb8f512cb53936dd05e0fde079505d2ea78af8f2cde24cbe220f。
|
||||
- 保留已有数据与证书，仅升级及负向复测，不作为最终清库首次流程。复用隔离证据中的空锁进行锁冲突测试，没有新增辅助脚本。
|
||||
- 失败任务 d6b3be69-c4e1-4a10-b86d-70b8809fcfd8：页面与部署结果接口均显示固定中文核查提示；接口返回 200，不泄露底层错误或凭据。
|
||||
- 空锁已移回原证据目录；22:52 确认活动任务为零，原六项证据仍保留。正式页面重新部署任务 179bb071-486d-4356-8ba2-e15ce1164eb5 成功，页面两域名均正常。
|
||||
- 22:55 核对目标目录无锁；两个站点真实 DNS、可信 TLS 请求均返回 200，公网指纹分别为 5DE99CD681FB8A84745CE4ED6091D76A7B86616421B3048D3C42844B8A4460ED、43063C0726D34362B254CE28B66D7EB637E00B165F68EC0D07FCED0DF2C64AD2。
|
||||
- 本节只证明诊断与正常重新部署，不代表崩溃事务自动恢复或后置 HTTPS 失败自动恢复已完成。
|
||||
|
||||
## 真实部署中断与人工恢复（2026-09-30 22:35）
|
||||
|
||||
- 候选仍为 91e1123。沿用已有证书和数据执行破坏性中断验证，没有清库，不能替代最终首次生产流程；未新增辅助验收脚本。
|
||||
|
||||
@ -1,4 +1,16 @@
|
||||
# Scert 整改任务执行清单
|
||||
## 当前任务看板（2026-09-30 22:56，锁冲突测试已收尾）
|
||||
|
||||
需求确认：继续完整整改与生产验收；本批先补齐部署后 HTTPS 校验失败的事务恢复，不以提示改善代替恢复能力。修改前检查点 c623e2d 已推送。
|
||||
|
||||
- [x] 新诊断版本 6efc890 已正式构建部署；锁冲突页面提示与任务结果查询接口均已核实。
|
||||
- [x] 测试锁移回受限证据目录后，正式页面重新部署成功，任务 179bb071-486d-4356-8ba2-e15ce1164eb5 为 success；两站点 HTTPS 200、证书指纹正确，目标目录无残留锁。
|
||||
- [~] 将事务旧文件与部署锁保留至站点校验完成；失败时恢复各服务器文件、校验和重载，并核对部署前可信站点证书。
|
||||
- [ ] 增加多目标、取消、恢复失败及诊断回归，正式构建部署复测。
|
||||
- [!] 进程崩溃事务自动恢复、第二台真实服务器、三数据库、双平台服务、备份恢复和最终清库首次流程仍未完成。
|
||||
|
||||
以下时间章节保留历史记录，最新状态以本节为准。
|
||||
|
||||
## 当前任务看板（2026-09-30 22:35，中断恢复实测与诊断整改）
|
||||
|
||||
需求确认：沿用完整 P0-10 与生产验收范围。修改前检查点 b27eedc 已推送；本轮验证部署中断、遗留锁、人工核查后的恢复，并修正公开诊断，不把诊断改善当作远端事务自动恢复。
|
||||
|
||||
@ -1,4 +1,13 @@
|
||||
# Scert 整改、合并与优化清单
|
||||
## 当前验收总览（2026-09-30 22:56）
|
||||
|
||||
- [✓] P0-03 源码与产物边界局部复审通过，不扩大为其他门禁通过。
|
||||
- [x] 6efc890 已正式部署，锁冲突诊断页面与任务结果接口实测通过；测试现场已恢复，两域名正常 HTTPS，正常重新部署任务 179bb071-486d-4356-8ba2-e15ce1164eb5 成功。
|
||||
- [~] P0-10：开始整改站点 HTTPS 校验前过早清理事务旧文件与释放锁的问题；目标为失败自动恢复文件、配置重载及部署前可信站点身份核验。
|
||||
- [!] 崩溃事务自动恢复、多服务器、数据库矩阵、双平台、备份恢复和最终首次流程仍未通过；整体生产验收未完成。
|
||||
|
||||
下方带时间章节均为历史记录，涉及候选版本和阻塞位置时以本节为准。
|
||||
|
||||
## 当前验收总览（2026-09-30 22:35，中断恢复边界已实测）
|
||||
|
||||
- [✓] P0-03 局部复审结论保持，不把后续事务门禁重新挂到源码目录任务下。
|
||||
|
||||
@ -10,6 +10,12 @@ export function serverAuthLabel(server) {
|
||||
|
||||
export function deploymentSummary(report) {
|
||||
if (!report || !Array.isArray(report.results)) return "未收到有效部署结果";
|
||||
if (report.status !== "success" && report.recovery_status === "success") {
|
||||
return "本次部署失败，已恢复旧文件并完成配置校验与重载，站点已恢复部署前证书";
|
||||
}
|
||||
if (report.status !== "success" && ["failed", "unconfirmed"].includes(report.recovery_status)) {
|
||||
return "本次部署失败，自动恢复未完成或站点状态未确认；请检查各服务器结果及部署锁，勿直接重试";
|
||||
}
|
||||
if (report.verification_status === "failed") {
|
||||
return "证书已推送，但站点 HTTPS 校验失败，尚未确认新证书生效；未自动回滚";
|
||||
}
|
||||
|
||||
@ -63,6 +63,19 @@ test("手动部署展示真实返回结果并阻止执行期间重复提交", as
|
||||
assert.equal(state.notice.value, "证书部署完成：1 台成功");
|
||||
});
|
||||
|
||||
test("失败部署区分已恢复旧站点与恢复未确认，不再统一显示未自动回滚", () => {
|
||||
const failed = { ...report, status: "failed", success: 0, failed: 1, verification_status: "failed" };
|
||||
const restored = deploymentSummary({ ...failed, recovery_status: "success" });
|
||||
assert.match(restored, /本次部署失败/);
|
||||
assert.match(restored, /站点已恢复部署前证书/);
|
||||
assert.doesNotMatch(restored, /未自动回滚|部署完成/);
|
||||
for (const recovery_status of ["failed", "unconfirmed"]) {
|
||||
const message = deploymentSummary({ ...failed, recovery_status });
|
||||
assert.match(message, /勿直接重试/);
|
||||
assert.doesNotMatch(message, /站点已恢复部署前证书|部署完成/);
|
||||
}
|
||||
});
|
||||
|
||||
test("部署失败仍保留各服务器结果，网络失败不会残留执行中状态", async (t) => {
|
||||
let failNetwork = false;
|
||||
const failed = { ...report, status: "failed", success: 0, failed: 1, results: [{ ...report.results[0], status: "failed" }] };
|
||||
|
||||
Loading…
Reference in New Issue
Block a user