From 3cc8fb6470ac8c695b59ce0ac3edc5c7334b457c Mon Sep 17 00:00:00 2001 From: xuans Date: Wed, 30 Sep 2026 23:14:42 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90=E9=83=A8=E7=BD=B2=E5=90=8E?= =?UTF-8?q?=E6=A0=A1=E9=AA=8C=E5=A4=B1=E8=B4=A5=E6=81=A2=E5=A4=8D=E6=95=B4?= =?UTF-8?q?=E6=94=B9=E5=B9=B6=E6=9B=B4=E6=96=B0=E5=AE=9E=E6=9C=BA=E8=BF=9B?= =?UTF-8?q?=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 延后事务提交，校验失败时恢复目标文件、配置和服务状态；新增恢复边界回归、前端提示及生产测试记录。 --- .../app/certificate_authorization_test.go | 5 + backend/internal/app/certificate_workflow.go | 43 ++-- .../internal/app/certificate_workflow_test.go | 26 ++- .../app/deployment_verification_test.go | 10 +- backend/internal/deployment/diagnostics.go | 57 ++--- backend/internal/deployment/service.go | 94 +++++++- backend/internal/deployment/verified.go | 150 +++++++++++++ backend/internal/deployment/verified_test.go | 212 ++++++++++++++++++ doc/Linux生产流程测试记录.md | 9 + doc/Scert整改任务执行清单.md | 12 + doc/Scert整改合并优化清单.md | 9 + .../src/views/Dashboard/DeploymentFeedback.js | 6 + .../views/Dashboard/DeploymentResult.test.js | 13 ++ 13 files changed, 588 insertions(+), 58 deletions(-) create mode 100644 backend/internal/deployment/verified.go create mode 100644 backend/internal/deployment/verified_test.go diff --git a/backend/internal/app/certificate_authorization_test.go b/backend/internal/app/certificate_authorization_test.go index 0a7c703..2853166 100644 --- a/backend/internal/app/certificate_authorization_test.go +++ b/backend/internal/app/certificate_authorization_test.go @@ -26,6 +26,11 @@ func (d forbiddenDeployer) DeploySaved(context.Context, []deployment.Target, dep return deployment.Report{}, nil } +func (d forbiddenDeployer) DeploySavedVerified(context.Context, []deployment.Target, deployment.SavedFiles, deployment.Verification) (deployment.Report, error) { + d.t.Fatal("权限检查失败后不得发起事务部署") + return deployment.Report{}, nil +} + func TestWorkflowDoesNotDeployAfterDNSPermissionDenied(t *testing.T) { workflow := &CertificateWorkflow{ Certificates: &deniedWorkflowCertificateService{}, diff --git a/backend/internal/app/certificate_workflow.go b/backend/internal/app/certificate_workflow.go index 87225e0..a1c232c 100644 --- a/backend/internal/app/certificate_workflow.go +++ b/backend/internal/app/certificate_workflow.go @@ -43,6 +43,7 @@ type serverLister interface { type savedDeployer interface { DeploySaved(context.Context, []deployment.Target, deployment.SavedFiles) (deployment.Report, error) + DeploySavedVerified(context.Context, []deployment.Target, deployment.SavedFiles, deployment.Verification) (deployment.Report, error) } type domainDeploymentRepository interface { @@ -253,31 +254,41 @@ func (w *CertificateWorkflow) deploy(ctx context.Context, result certificate.Iss } expectedFingerprint = validation.FingerprintSHA256 } - report, err := w.Deployment.DeploySaved(ctx, targets, deployment.SavedFiles{ + saved := deployment.SavedFiles{ CertificatePath: result.Saved.CertificatePath, FullChainPath: result.Saved.FullChainPath, PrivateKeyPath: result.Saved.PrivateKeyPath, - }) + } + if w.RemoteVerifier != nil { + verificationDomain := verificationName(domainItem, result.Domains) + if verificationDomain == "" { + return deployment.Report{Status: "failed"}, errors.New("无法确定部署后的 HTTPS 校验域名，未执行推送") + } + endpoint := "https://" + verificationDomain + // 首次部署允许不存在旧站点，但不得在缺少可信基线时宣称运行态已恢复。 + baseline, baselineErr := w.RemoteVerifier(ctx, endpoint, result.Domains) + if err := ctx.Err(); err != nil { + return deployment.Report{Status: "failed"}, err + } + verification := deployment.Verification{ + Verify: func(verificationContext context.Context) error { + return w.verifyRemoteDeployment(verificationContext, endpoint, result.Domains, expectedFingerprint) + }, + } + if baselineErr == nil && baseline.FingerprintSHA256 != "" { + verification.VerifyRecovery = func(recoveryContext context.Context) error { + return w.verifyRemoteDeployment(recoveryContext, endpoint, result.Domains, baseline.FingerprintSHA256) + } + } + return w.Deployment.DeploySavedVerified(ctx, targets, saved, verification) + } + report, err := w.Deployment.DeploySaved(ctx, targets, saved) if err != nil { return report, err } if report.Status != "success" { return report, fmt.Errorf("证书部署未全部成功：成功 %d 台，失败 %d 台", report.Success, report.Failed) } - if w.RemoteVerifier != nil { - verificationDomain := verificationName(domainItem, result.Domains) - // 单台结果保留真实文件推送结果，站点生效校验单独记录，不能宣称已回滚。 - report.VerificationStatus = "failed" - report.Status = "failed" - if verificationDomain == "" { - return report, errors.New("无法确定部署后的 HTTPS 校验域名") - } - if err := w.verifyRemoteDeployment(ctx, "https://"+verificationDomain, result.Domains, expectedFingerprint); err != nil { - return report, fmt.Errorf("部署后 HTTPS 校验失败：%w", err) - } - report.VerificationStatus = "success" - report.Status = "success" - } return report, nil } diff --git a/backend/internal/app/certificate_workflow_test.go b/backend/internal/app/certificate_workflow_test.go index aa15ec8..94a7306 100644 --- a/backend/internal/app/certificate_workflow_test.go +++ b/backend/internal/app/certificate_workflow_test.go @@ -97,6 +97,21 @@ func (d *workflowDeployer) DeploySaved(_ context.Context, targets []deployment.T return d.report, nil } +// 工作流替身只模拟推送和校验，不伪造远端事务恢复；恢复边界由部署包实测。 +func (d *workflowDeployer) DeploySavedVerified(ctx context.Context, targets []deployment.Target, files deployment.SavedFiles, verification deployment.Verification) (deployment.Report, error) { + report, err := d.DeploySaved(ctx, targets, files) + if err != nil || report.Status != "success" { + return report, err + } + report.VerificationStatus = "failed" + if err := verification.Verify(ctx); err != nil { + report.Status = "failed" + return report, err + } + report.VerificationStatus = "success" + return report, nil +} + func TestCertificateWorkflowDeploysIssuedCertificateToAssociatedServers(t *testing.T) { certificateService := &workflowCertificateService{result: certificate.IssueResult{ DomainID: "domain-1", @@ -223,7 +238,7 @@ func TestCertificateWorkflowRetriesRemoteVerificationUntilSuccess(t *testing.T) if endpoint != "https://example.com" || len(expectedDomains) != 1 || expectedDomains[0] != "example.com" { t.Fatalf("HTTPS 校验参数错误：%s %+v", endpoint, expectedDomains) } - if verificationCalls < 3 { + if verificationCalls < 4 { return certificate.ValidationResult{}, errors.New("仍返回旧证书") } return certificate.ValidationResult{FingerprintSHA256: saved.FingerprintSHA256}, nil @@ -233,7 +248,7 @@ func TestCertificateWorkflowRetriesRemoteVerificationUntilSuccess(t *testing.T) if _, err := workflow.IssueWithReport(context.Background(), "domain-1"); err != nil { t.Fatalf("HTTPS 校验重试后应成功：%v", err) } - if verificationCalls != 3 || waitCalls != 2 { + if verificationCalls != 4 || waitCalls != 2 { t.Fatalf("HTTPS 校验重试次数错误：校验 %d 次，等待 %d 次", verificationCalls, waitCalls) } } @@ -264,7 +279,7 @@ func TestCertificateWorkflowRemoteVerificationStopsAfterConfiguredAttempts(t *te if err == nil || !strings.Contains(err.Error(), "重试 3 次后仍未通过") { t.Fatalf("永久校验错误应在有限重试后返回：%v", err) } - if verificationCalls != 3 || waitCalls != 2 { + if verificationCalls != 4 || waitCalls != 2 { t.Fatalf("永久错误的重试次数错误：校验 %d 次，等待 %d 次", verificationCalls, waitCalls) } } @@ -284,6 +299,9 @@ func TestCertificateWorkflowRemoteVerificationWaitHonorsContextCancellation(t *t RemoteVerificationInterval: time.Minute, RemoteVerifier: func(context.Context, string, []string) (certificate.ValidationResult, error) { verificationCalls++ + if verificationCalls == 1 { + return certificate.ValidationResult{FingerprintSHA256: saved.FingerprintSHA256}, nil + } cancel() return certificate.ValidationResult{}, errors.New("等待新证书") }, @@ -293,7 +311,7 @@ func TestCertificateWorkflowRemoteVerificationWaitHonorsContextCancellation(t *t if !errors.Is(err, context.Canceled) { t.Fatalf("取消上下文后应立即终止 HTTPS 校验重试：%v", err) } - if verificationCalls != 1 { + if verificationCalls != 2 { t.Fatalf("取消后不应继续校验，实际调用 %d 次", verificationCalls) } } diff --git a/backend/internal/app/deployment_verification_test.go b/backend/internal/app/deployment_verification_test.go index 34d5256..1adcd90 100644 --- a/backend/internal/app/deployment_verification_test.go +++ b/backend/internal/app/deployment_verification_test.go @@ -199,14 +199,14 @@ func TestDeploymentVerificationRetriesRealValidOldCertificate(t *testing.T) { RemoteVerifier: func(ctx context.Context, _ string, domains []string) (certificate.ValidationResult, error) { calls++ endpoint := oldServer.URL - if calls > 1 { + if calls > 2 { endpoint = newServer.URL } return validator.ValidateHTTPS(ctx, endpoint, domains) }, } report, err := workflow.deploy(t.Context(), certificate.IssueResult{DomainID: "domain-1", Domains: []string{"example.com"}, Saved: newSaved}) - if err != nil || report.VerificationStatus != "success" || calls != 2 || waits != 1 { + if err != nil || report.VerificationStatus != "success" || calls != 3 || waits != 1 { t.Fatalf("有效旧证书必须重试，只有实际新证书才能通过：%+v，调用 %d，等待 %d，%v", report, calls, waits, err) } } @@ -215,11 +215,15 @@ func TestDeploymentVerificationCancellationCannotBecomeSuccess(t *testing.T) { saved := workflowSavedCertificate(t) ctx, cancel := context.WithCancel(t.Context()) defer cancel() + calls := 0 workflow := &CertificateWorkflow{ Domains: &workflowDomainState{}, Servers: workflowServerLister{servers: []deployment.Server{{ID: "server-1"}}}, Deployment: &workflowDeployer{report: deployment.Report{Total: 1, Success: 1, Status: "success"}}, RemoteVerifier: func(context.Context, string, []string) (certificate.ValidationResult, error) { - cancel() + calls++ + if calls > 1 { + cancel() + } return certificate.ValidationResult{FingerprintSHA256: saved.FingerprintSHA256}, nil }, } diff --git a/backend/internal/deployment/diagnostics.go b/backend/internal/deployment/diagnostics.go index 247aee6..18d0cf9 100644 --- a/backend/internal/deployment/diagnostics.go +++ b/backend/internal/deployment/diagnostics.go @@ -3,36 +3,41 @@ package deployment type FailureCode string const ( - FailureUnavailable FailureCode = "unavailable" - FailureArtifact FailureCode = "artifact_invalid" - FailureConfiguration FailureCode = "configuration_invalid" - FailureConnection FailureCode = "connection_failed" - FailureDirectory FailureCode = "directory_failed" - FailureLock FailureCode = "lock_unavailable" - FailureUpload FailureCode = "upload_failed" - FailureReplace FailureCode = "replace_failed" - FailureValidation FailureCode = "validation_failed" - FailureRollback FailureCode = "rollback_failed" - FailureRuntimeUnknown FailureCode = "runtime_unconfirmed" - FailureCleanup FailureCode = "cleanup_failed" - FailureLockRelease FailureCode = "lock_release_failed" + FailureUnavailable FailureCode = "unavailable" + FailureArtifact FailureCode = "artifact_invalid" + FailureConfiguration FailureCode = "configuration_invalid" + FailureConnection FailureCode = "connection_failed" + FailureDirectory FailureCode = "directory_failed" + FailureLock FailureCode = "lock_unavailable" + FailureUpload FailureCode = "upload_failed" + FailureReplace FailureCode = "replace_failed" + FailureValidation FailureCode = "validation_failed" + FailureRollback FailureCode = "rollback_failed" + FailureRuntimeUnknown FailureCode = "runtime_unconfirmed" + FailureCleanup FailureCode = "cleanup_failed" + FailureLockRelease FailureCode = "lock_release_failed" + FailureRecoveryPending FailureCode = "recovery_pending" + FailureRecoveryUnknown FailureCode = "recovery_unconfirmed" + FailureRecoveryRestored FailureCode = "recovery_restored" ) // 只公开固定诊断，不从远端输出、错误文本或文件路径推断可展示的消息。 var publicFailureMessages = map[FailureCode]string{ - FailureUnavailable: "部署服务不可用，请联系管理员检查服务配置", - FailureArtifact: "待部署的证书或私钥校验失败，尚未推送到服务器", - FailureConfiguration: "服务器部署配置无效，请检查认证、证书路径和命令配置", - FailureConnection: "连接部署服务器失败，请检查网络、SSH 认证及主机身份配置", - FailureDirectory: "无法创建证书目录，请检查部署账户的目录权限", - FailureLock: "无法取得部署锁，可能有任务正在执行或存在中断遗留；请管理员核查远端文件、部署锁及服务状态后再重试，勿直接删除锁", - FailureUpload: "证书上传失败，已尝试回滚；请检查受保护的服务日志后再重试", - FailureReplace: "证书文件替换失败，已尝试回滚；请检查受保护的服务日志后再重试", - FailureValidation: "服务配置校验失败，文件已回滚，未执行重载；请修正配置后再重试", - FailureRollback: "部署失败且文件回滚未完成，已保留部署锁；请管理员核查远端文件及服务状态，勿直接重试或删除锁", - FailureRuntimeUnknown: "服务重载失败，文件已回滚，但服务运行状态未确认；请管理员核对配置并恢复服务后再重试", - FailureCleanup: "证书已推送并重载，但旧文件清理未完成；请管理员核查现场，不能据此确认站点新证书已生效", - FailureLockRelease: "部署锁释放未完成；请管理员核查远端文件、部署锁及服务状态后再重试", + FailureRecoveryUnknown: "本次部署未完成，已恢复旧文件，但服务或站点恢复状态未确认，已保留部署锁；请管理员核查，勿直接重试或删除锁", + FailureRecoveryRestored: "本次部署未完成，旧文件已恢复并通过配置校验及重载，站点已恢复部署前证书；请核查失败原因后再重试", + FailureUnavailable: "部署服务不可用，请联系管理员检查服务配置", + FailureArtifact: "待部署的证书或私钥校验失败，尚未推送到服务器", + FailureConfiguration: "服务器部署配置无效，请检查认证、证书路径和命令配置", + FailureConnection: "连接部署服务器失败，请检查网络、SSH 认证及主机身份配置", + FailureDirectory: "无法创建证书目录，请检查部署账户的目录权限", + FailureLock: "无法取得部署锁，可能有任务正在执行或存在中断遗留；请管理员核查远端文件、部署锁及服务状态后再重试，勿直接删除锁", + FailureUpload: "证书上传失败，已尝试回滚；请检查受保护的服务日志后再重试", + FailureReplace: "证书文件替换失败，已尝试回滚；请检查受保护的服务日志后再重试", + FailureValidation: "服务配置校验失败，文件已回滚，未执行重载；请修正配置后再重试", + FailureRollback: "部署失败且文件回滚未完成，已保留部署锁；请管理员核查远端文件及服务状态，勿直接重试或删除锁", + FailureRuntimeUnknown: "服务重载失败，文件已回滚，但服务运行状态未确认；请管理员核对配置并恢复服务后再重试", + FailureCleanup: "证书已推送并重载，但旧文件清理未完成；请管理员核查现场，不能据此确认站点新证书已生效", + FailureLockRelease: "部署锁释放未完成；请管理员核查远端文件、部署锁及服务状态后再重试", } // PublicResult 保留结果事实，同时只允许固定错误码及中文诊断进入公开响应。 diff --git a/backend/internal/deployment/service.go b/backend/internal/deployment/service.go index c41cff8..7685b0f 100644 --- a/backend/internal/deployment/service.go +++ b/backend/internal/deployment/service.go @@ -47,6 +47,7 @@ type Report struct { Results []Result `json:"results"` // 站点校验与各服务器文件推送分开记录，不表示所有服务器均完成独立 HTTPS 验证。 VerificationStatus string `json:"verification_status,omitempty"` + RecoveryStatus string `json:"recovery_status,omitempty"` } type Service struct { @@ -59,6 +60,10 @@ func NewService(factory ClientFactory) *Service { } func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifact) Report { + return s.deploy(ctx, targets, artifact, nil) +} + +func (s *Service) deploy(ctx context.Context, targets []Target, artifact Artifact, pending *[]*pendingDeployment) Report { report := Report{Total: len(targets), Results: make([]Result, 0, len(targets))} if len(targets) == 0 { report.Status = "failed" @@ -80,8 +85,8 @@ func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifac } return report } - for _, target := range targets { - result := s.deployOne(ctx, target.Server, artifact) + for index, target := range targets { + result := s.deployOne(ctx, target.Server, artifact, pending, index) report.Results = append(report.Results, result) if result.Status == "success" { report.Success++ @@ -101,22 +106,30 @@ func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifac } func (s *Service) DeploySaved(ctx context.Context, targets []Target, saved SavedFiles) (Report, error) { + artifact, err := loadArtifact(saved) + if err != nil { + return Report{}, err + } + return s.Deploy(ctx, targets, artifact), nil +} + +func loadArtifact(saved SavedFiles) (Artifact, error) { certificateData, err := os.ReadFile(saved.CertificatePath) if err != nil { - return Report{}, fmt.Errorf("读取证书文件失败：%w", err) + return Artifact{}, fmt.Errorf("读取证书文件失败：%w", err) } fullChainData, err := os.ReadFile(saved.FullChainPath) if err != nil { - return Report{}, fmt.Errorf("读取完整证书链失败：%w", err) + return Artifact{}, fmt.Errorf("读取完整证书链失败：%w", err) } privateKeyData, err := os.ReadFile(saved.PrivateKeyPath) if err != nil { - return Report{}, fmt.Errorf("读取私钥文件失败：%w", err) + return Artifact{}, fmt.Errorf("读取私钥文件失败：%w", err) } - return s.Deploy(ctx, targets, Artifact{Certificate: certificateData, FullChain: fullChainData, PrivateKey: privateKeyData}), nil + return Artifact{Certificate: certificateData, FullChain: fullChainData, PrivateKey: privateKeyData}, nil } -func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifact) (result Result) { +func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifact, pending *[]*pendingDeployment, index int) (result Result) { ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() server = server.Normalize() @@ -131,7 +144,12 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac result.Message = fmt.Sprintf("连接服务器失败：%v", err) return result } - defer func() { _ = client.Close() }() + handedOff := false + defer func() { + if !handedOff { + _ = client.Close() + } + }() paths := []remoteFile{ {path: RemotePath(server.CertificatePath), data: artifact.Certificate, mode: 0o644}, {path: RemotePath(server.CertificateChainPath), data: artifact.FullChain, mode: 0o644}, @@ -160,7 +178,7 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac sort.Strings(lockPaths) heldLocks := make([]string, 0, len(lockPaths)) retainLocks := false - defer func() { + releaseLocks := func() { if len(heldLocks) == 0 || retainLocks { return } @@ -179,6 +197,11 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac result.Message += fmt.Sprintf("；释放部署锁 %s 失败，需人工检查：%v", heldLocks[index], err) } } + } + defer func() { + if !handedOff { + releaseLocks() + } }() for _, lockPath := range lockPaths { if err := client.Lock(ctx, lockPath); err != nil { @@ -300,6 +323,59 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac } return failed(fmt.Sprintf("reload 失败：%v", err)) } + if pending != nil { + // 旧文件与锁在所有目标及站点校验结束前始终保留，禁止提前提交。 + result.Status = "success" + result.FailureCode = "" + result.Message = "证书已推送并重载，等待站点校验" + closed := false + transaction := &pendingDeployment{index: index} + transaction.restore = func(recoveryContext context.Context) Result { + result.Status = "failed" + if err := rollback(); err != nil { + result.FailureCode = FailureRollback + result.Message = fmt.Sprintf("部署未提交，自动回滚失败：%v", err) + return result + } + result.FailureCode = FailureRecoveryUnknown + if err := reconnectDeploymentClient(recoveryContext, s.factory, server, &client); err != nil { + result.Message = fmt.Sprintf("旧文件已恢复，重新连接以恢复服务失败：%v", err) + return result + } + for _, command := range [][]string{server.ValidationCommand, server.ReloadCommand} { + outcome, err := client.Run(recoveryContext, command) + if err != nil || outcome.ExitCode != 0 { + result.Message = fmt.Sprintf("旧文件已恢复，但恢复配置校验或重载失败，退出码 %d：%v", outcome.ExitCode, err) + return result + } + } + result.FailureCode = FailureRecoveryPending + result.Message = "本次部署未提交，旧文件已恢复并通过配置校验及重载，等待站点恢复核验" + return result + } + transaction.commit = func(commitContext context.Context) Result { + if err := cleanupBackups(commitContext, client, backups, paths); err != nil { + result.Status = "failed" + result.FailureCode = FailureCleanup + result.Message = fmt.Sprintf("站点校验通过，但事务旧文件清理失败：%v", err) + return result + } + result.Message = "证书部署、配置校验并 reload 成功" + return result + } + transaction.finish = func(retain bool) Result { + if !closed { + retainLocks = retain + releaseLocks() + _ = client.Close() + closed = true + } + return result + } + handedOff = true + *pending = append(*pending, transaction) + return result + } if err := cleanupBackups(ctx, client, backups, paths); err != nil { result.Status = "failed" result.FailureCode = FailureCleanup diff --git a/backend/internal/deployment/verified.go b/backend/internal/deployment/verified.go new file mode 100644 index 0000000..ebdc102 --- /dev/null +++ b/backend/internal/deployment/verified.go @@ -0,0 +1,150 @@ +package deployment + +import ( + "context" + "errors" + "fmt" +) + +// Verification 将站点生效校验与恢复核验置于远端文件事务之内。 +type Verification struct { + Verify func(context.Context) error + VerifyRecovery func(context.Context) error +} + +type pendingDeployment struct { + index int + commit func(context.Context) Result + restore func(context.Context) Result + finish func(bool) Result +} + +func (s *Service) DeploySavedVerified(ctx context.Context, targets []Target, saved SavedFiles, verification Verification) (Report, error) { + artifact, err := loadArtifact(saved) + if err != nil { + return Report{Status: "failed"}, err + } + return s.DeployVerified(ctx, targets, artifact, verification) +} + +func (s *Service) DeployVerified(ctx context.Context, targets []Target, artifact Artifact, verification Verification) (Report, error) { + if verification.Verify == nil { + return Report{Status: "failed"}, errors.New("缺少部署后站点校验，未执行推送") + } + pending := make([]*pendingDeployment, 0, len(targets)) + // 异常退出时关闭连接但保留现场；不得以释放锁掩盖未确认的事务。 + defer func() { + for _, transaction := range pending { + transaction.finish(true) + } + }() + report := s.deploy(ctx, targets, artifact, &pending) + var deployErr error + if report.Status != "success" { + deployErr = fmt.Errorf("证书部署未全部成功：成功 %d 台，失败 %d 台", report.Success, report.Failed) + } else { + report.VerificationStatus = "failed" + deployErr = ctx.Err() + if deployErr == nil { + deployErr = verification.Verify(ctx) + } + if deployErr == nil { + deployErr = ctx.Err() + } + if deployErr != nil { + deployErr = fmt.Errorf("部署后 HTTPS 校验失败：%w", deployErr) + } else { + report.VerificationStatus = "success" + } + } + if deployErr != nil { + if len(pending) > 0 { + s.restorePending(ctx, &report, pending, verification) + } + summarizeReport(&report) + report.Status = "failed" + return report, deployErr + } + for _, transaction := range pending { + result := transaction.commit(ctx) + report.Results[transaction.index] = transaction.finish(result.Status != "success") + } + summarizeReport(&report) + if report.Status != "success" { + return report, errors.New("站点校验通过，但事务清理或释放部署锁未完成") + } + return report, nil +} + +func (s *Service) restorePending(ctx context.Context, report *Report, pending []*pendingDeployment, verification Verification) { + report.RecoveryStatus = "unconfirmed" + for index := len(pending) - 1; index >= 0; index-- { + transaction := pending[index] + // 请求取消后仍使用独立有界上下文恢复，不重放签发或文件推送。 + recoveryContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout) + report.Results[transaction.index] = transaction.restore(recoveryContext) + cancel() + } + restored := true + for _, result := range report.Results { + switch result.FailureCode { + case FailureRecoveryPending, FailureConfiguration, FailureConnection, FailureDirectory, FailureUpload, FailureReplace, FailureValidation: + // 这些失败要么未触碰目标，要么已完成文件回滚且未执行重载。 + default: + restored = false + } + } + var recoveryErr error + if !restored { + report.RecoveryStatus = "failed" + } else if verification.VerifyRecovery != nil { + recoveryContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout) + err := verification.VerifyRecovery(recoveryContext) + if err == nil { + err = recoveryContext.Err() + } + cancel() + if err == nil { + report.RecoveryStatus = "success" + } else { + recoveryErr = err + } + } + for _, transaction := range pending { + result := transaction.finish(report.RecoveryStatus != "success") + if result.FailureCode == FailureLockRelease { + report.RecoveryStatus = "failed" + } + if result.FailureCode == FailureRecoveryPending { + if report.RecoveryStatus == "success" { + result.FailureCode = FailureRecoveryRestored + } else { + result.FailureCode = FailureRecoveryUnknown + } + result.Message = publicFailureMessages[result.FailureCode] + if recoveryErr != nil { + result.Message += fmt.Sprintf("；站点恢复核验失败：%v", recoveryErr) + } + } + report.Results[transaction.index] = result + } +} + +func summarizeReport(report *Report) { + report.Success, report.Failed = 0, 0 + for _, result := range report.Results { + if result.Status == "success" { + report.Success++ + } else { + report.Failed++ + } + } + switch { + case report.Total == 0 || report.Success == 0: + report.Status = "failed" + case report.Failed == 0: + report.Status = "success" + default: + report.Status = "partial" + } +} diff --git a/backend/internal/deployment/verified_test.go b/backend/internal/deployment/verified_test.go new file mode 100644 index 0000000..a72c784 --- /dev/null +++ b/backend/internal/deployment/verified_test.go @@ -0,0 +1,212 @@ +package deployment + +import ( + "bytes" + "context" + "errors" + "reflect" + "strings" + "testing" +) + +func seedVerifiedFiles(client *memoryClient, server Server) map[string]memoryFile { + files := map[string]memoryFile{ + server.CertificatePath: {data: []byte("原证书"), mode: 0o644}, + server.CertificateChainPath: {data: []byte("原证书链"), mode: 0o644}, + server.PrivateKeyPath: {data: []byte("原私钥"), mode: 0o600}, + } + for path, file := range files { + client.files[path] = file + } + return files +} + +func assertVerifiedFiles(t *testing.T, client *memoryClient, expected map[string]memoryFile) { + t.Helper() + for path, file := range expected { + actual, ok := client.files[path] + if !ok || !bytes.Equal(actual.data, file.data) || actual.mode != file.mode { + t.Fatalf("未恢复原内容或权限：%s", path) + } + } +} + +func assertVerifiedLocks(t *testing.T, client *memoryClient, server Server, held bool) { + t.Helper() + for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} { + _, exists := client.files[path+".scert-lock"] + if exists != held { + t.Fatalf("部署锁状态不符合预期：%s，锁定 %t", path, exists) + } + } +} + +func TestVerifiedDeploymentDefersCommitUntilAllTargetsAndSitePass(t *testing.T) { + clients := map[string]*memoryClient{"first": newMemoryClient(), "second": newMemoryClient()} + servers := []Server{testServer("first"), testServer("second")} + for _, server := range servers { + seedVerifiedFiles(clients[server.ID], server) + } + artifact := testArtifact(t) + called := false + service := NewService(memoryFactory{clients: clients}) + report, err := service.DeployVerified(t.Context(), []Target{{Server: servers[0]}, {Server: servers[1]}}, artifact, Verification{ + Verify: func(context.Context) error { + called = true + for _, server := range servers { + client := clients[server.ID] + assertVerifiedLocks(t, client, server, true) + if !bytes.Equal(client.files[server.CertificatePath].data, artifact.Certificate) || len(client.commands) != 2 { + t.Fatal("必须在所有服务器推送和重载后才核验站点") + } + backups := 0 + for path := range client.files { + if strings.Contains(path, ".scert-backup-") { + backups++ + } + } + if backups != 3 { + t.Fatal("站点确认前必须保留三份事务旧文件") + } + } + return nil + }, + VerifyRecovery: func(context.Context) error { t.Fatal("成功部署不得回滚"); return nil }, + }) + if err != nil || !called || report.Status != "success" || report.Success != 2 || report.VerificationStatus != "success" || report.RecoveryStatus != "" { + t.Fatalf("事务提交结果错误：%+v，%v", report, err) + } + for _, server := range servers { + client := clients[server.ID] + assertVerifiedLocks(t, client, server, false) + if len(client.files) != 3 { + t.Fatal("提交后不应残留事务文件") + } + } +} + +func TestVerifiedDeploymentFailureRestoresAllTargetsBeforeRecoveryCheck(t *testing.T) { + clients := map[string]*memoryClient{"first": newMemoryClient(), "second": newMemoryClient()} + servers := []Server{testServer("first"), testServer("second")} + originals := map[string]map[string]memoryFile{} + for _, server := range servers { + originals[server.ID] = seedVerifiedFiles(clients[server.ID], server) + } + recovered := false + verificationErr := errors.New("公网仍返回错误证书") + report, err := NewService(memoryFactory{clients: clients}).DeployVerified(t.Context(), []Target{{Server: servers[0]}, {Server: servers[1]}}, testArtifact(t), Verification{ + Verify: func(context.Context) error { return verificationErr }, + VerifyRecovery: func(ctx context.Context) error { + recovered = true + if _, ok := ctx.Deadline(); !ok || ctx.Err() != nil { + t.Fatal("恢复核验必须有独立有效期限") + } + for _, server := range servers { + client := clients[server.ID] + assertVerifiedFiles(t, client, originals[server.ID]) + assertVerifiedLocks(t, client, server, true) + expectedCommands := [][]string{{"nginx", "-t"}, {"nginx", "-s", "reload"}, {"nginx", "-t"}, {"nginx", "-s", "reload"}} + if !reflect.DeepEqual(client.commands, expectedCommands) { + t.Fatalf("必须恢复配置并重载后核验站点：实际=%#v，期望=%#v", client.commands, expectedCommands) + } + } + return nil + }, + }) + if !errors.Is(err, verificationErr) || !recovered || report.Status != "failed" || report.Success != 0 || report.Failed != 2 || report.VerificationStatus != "failed" || report.RecoveryStatus != "success" { + t.Fatalf("失败与恢复状态错误：%+v，%v", report, err) + } + for index, server := range servers { + assertVerifiedFiles(t, clients[server.ID], originals[server.ID]) + assertVerifiedLocks(t, clients[server.ID], server, false) + if report.Results[index].FailureCode != FailureRecoveryRestored { + t.Fatalf("应保留部署失败并提供恢复诊断：%+v", report.Results[index]) + } + } +} + +func TestVerifiedDeploymentCannotClaimRecoveryOnIncompleteEvidence(t *testing.T) { + for _, scenario := range []struct { + name string + configure func(*memoryClient) + recovery func(context.Context) error + status string + code FailureCode + }{ + {"missing_baseline", func(*memoryClient) {}, nil, "unconfirmed", FailureRecoveryUnknown}, + {"wrong_runtime", func(*memoryClient) {}, func(context.Context) error { return errors.New("secret-path 旧站点身份不符") }, "unconfirmed", FailureRecoveryUnknown}, + {"restore_failed", func(c *memoryClient) { c.restoreError = true }, nil, "failed", FailureRollback}, + {"validation_failed", func(c *memoryClient) { c.runResults = []CommandResult{{}, {}, {ExitCode: 1}} }, nil, "failed", FailureRecoveryUnknown}, + {"reload_failed", func(c *memoryClient) { c.runResults = []CommandResult{{}, {}, {}, {ExitCode: 1}} }, nil, "failed", FailureRecoveryUnknown}, + } { + t.Run(scenario.name, func(t *testing.T) { + client, server := newMemoryClient(), testServer("recovery") + seedVerifiedFiles(client, server) + scenario.configure(client) + report, err := NewService(fixedClientFactory{client}).DeployVerified(t.Context(), []Target{{Server: server}}, testArtifact(t), Verification{ + Verify: func(context.Context) error { return errors.New("部署后校验失败") }, VerifyRecovery: scenario.recovery, + }) + if err == nil || report.RecoveryStatus != scenario.status || report.Results[0].FailureCode != scenario.code { + t.Fatalf("不得夸大恢复能力：%+v，%v", report, err) + } + assertVerifiedLocks(t, client, server, true) + public := PublicResult(report.Results[0]) + if strings.Contains(public.Message, "secret-path") || public.Message == "证书部署失败" { + t.Fatalf("恢复诊断应安全且明确：%+v", public) + } + }) + } +} + +func TestVerifiedDeploymentCancellationStillRestoresWithBoundedContext(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + client, server := newMemoryClient(), testServer("cancel") + original := seedVerifiedFiles(client, server) + report, err := NewService(fixedClientFactory{client}).DeployVerified(ctx, []Target{{Server: server}}, testArtifact(t), Verification{ + Verify: func(context.Context) error { cancel(); return nil }, + VerifyRecovery: func(ctx context.Context) error { + if ctx.Err() != nil { + t.Fatal("请求取消不得取消恢复核验") + } + assertVerifiedFiles(t, client, original) + return nil + }, + }) + if !errors.Is(err, context.Canceled) || report.Status != "failed" || report.RecoveryStatus != "success" || len(client.commands) != 4 { + t.Fatalf("取消后应恢复但不得转为部署成功：%+v，%v", report, err) + } + assertVerifiedLocks(t, client, server, false) +} + +func TestVerifiedDeploymentPartialPreparationDoesNotCommitEarlierTarget(t *testing.T) { + client, first := newMemoryClient(), testServer("first") + second := testServer("second") + original := seedVerifiedFiles(client, first) + service := NewService(memoryFactory{clients: map[string]*memoryClient{"first": client}, errors: map[string]error{"second": errors.New("连接失败")}}) + report, err := service.DeployVerified(t.Context(), []Target{{Server: first}, {Server: second}}, testArtifact(t), Verification{ + Verify: func(context.Context) error { t.Fatal("部分准备失败不得进行提交校验"); return nil }, + VerifyRecovery: func(context.Context) error { assertVerifiedFiles(t, client, original); return nil }, + }) + if err == nil || report.Status != "failed" || report.Success != 0 || report.RecoveryStatus != "success" || report.Results[1].FailureCode != FailureConnection { + t.Fatalf("部分目标失败必须恢复已推送目标：%+v，%v", report, err) + } + assertVerifiedLocks(t, client, first, false) +} + +func TestVerifiedDeploymentRequiresVerifierAndDoesNotHideCleanupFailure(t *testing.T) { + client, server := newMemoryClient(), testServer("cleanup") + original := seedVerifiedFiles(client, server) + service := NewService(fixedClientFactory{client}) + report, err := service.DeployVerified(t.Context(), []Target{{Server: server}}, testArtifact(t), Verification{}) + if err == nil || report.Status != "failed" || len(client.commands) != 0 { + t.Fatal("缺少校验器不得推送") + } + assertVerifiedFiles(t, client, original) + client.removeError = ".scert-backup-" + report, err = service.DeployVerified(t.Context(), []Target{{Server: server}}, testArtifact(t), Verification{Verify: func(context.Context) error { return nil }}) + if err == nil || report.Status != "failed" || report.VerificationStatus != "success" || report.Results[0].FailureCode != FailureCleanup { + t.Fatalf("校验成功不能掩盖清理失败：%+v，%v", report, err) + } + assertVerifiedLocks(t, client, server, true) +} diff --git a/doc/Linux生产流程测试记录.md b/doc/Linux生产流程测试记录.md index 47dc1ee..cdc7737 100644 --- a/doc/Linux生产流程测试记录.md +++ b/doc/Linux生产流程测试记录.md @@ -1,4 +1,13 @@ # Linux 生产流程测试记录 +## 固定诊断版本部署与锁测试收尾（2026-09-30 22:55） + +- 6efc89061bc87fdcdc41182203749b82e598865a 干净源码正式构建部署；归档 SHA256 为 443cd7db2a98832c5b1d2eb5d27b0ee704192c5be97ddd0f7585531008c13b5c，主程序 SHA256 为 ee0a5a98c993eb8f512cb53936dd05e0fde079505d2ea78af8f2cde24cbe220f。 +- 保留已有数据与证书，仅升级及负向复测，不作为最终清库首次流程。复用隔离证据中的空锁进行锁冲突测试，没有新增辅助脚本。 +- 失败任务 d6b3be69-c4e1-4a10-b86d-70b8809fcfd8：页面与部署结果接口均显示固定中文核查提示；接口返回 200，不泄露底层错误或凭据。 +- 空锁已移回原证据目录；22:52 确认活动任务为零，原六项证据仍保留。正式页面重新部署任务 179bb071-486d-4356-8ba2-e15ce1164eb5 成功，页面两域名均正常。 +- 22:55 核对目标目录无锁；两个站点真实 DNS、可信 TLS 请求均返回 200，公网指纹分别为 5DE99CD681FB8A84745CE4ED6091D76A7B86616421B3048D3C42844B8A4460ED、43063C0726D34362B254CE28B66D7EB637E00B165F68EC0D07FCED0DF2C64AD2。 +- 本节只证明诊断与正常重新部署，不代表崩溃事务自动恢复或后置 HTTPS 失败自动恢复已完成。 + ## 真实部署中断与人工恢复（2026-09-30 22:35） - 候选仍为 91e1123。沿用已有证书和数据执行破坏性中断验证，没有清库，不能替代最终首次生产流程；未新增辅助验收脚本。 diff --git a/doc/Scert整改任务执行清单.md b/doc/Scert整改任务执行清单.md index e6dad1e..f35b7a7 100644 --- a/doc/Scert整改任务执行清单.md +++ b/doc/Scert整改任务执行清单.md @@ -1,4 +1,16 @@ # Scert 整改任务执行清单 +## 当前任务看板（2026-09-30 22:56，锁冲突测试已收尾） + +需求确认：继续完整整改与生产验收；本批先补齐部署后 HTTPS 校验失败的事务恢复，不以提示改善代替恢复能力。修改前检查点 c623e2d 已推送。 + +- [x] 新诊断版本 6efc890 已正式构建部署；锁冲突页面提示与任务结果查询接口均已核实。 +- [x] 测试锁移回受限证据目录后，正式页面重新部署成功，任务 179bb071-486d-4356-8ba2-e15ce1164eb5 为 success；两站点 HTTPS 200、证书指纹正确，目标目录无残留锁。 +- [~] 将事务旧文件与部署锁保留至站点校验完成；失败时恢复各服务器文件、校验和重载，并核对部署前可信站点证书。 +- [ ] 增加多目标、取消、恢复失败及诊断回归，正式构建部署复测。 +- [!] 进程崩溃事务自动恢复、第二台真实服务器、三数据库、双平台服务、备份恢复和最终清库首次流程仍未完成。 + +以下时间章节保留历史记录，最新状态以本节为准。 + ## 当前任务看板（2026-09-30 22:35，中断恢复实测与诊断整改） 需求确认：沿用完整 P0-10 与生产验收范围。修改前检查点 b27eedc 已推送；本轮验证部署中断、遗留锁、人工核查后的恢复，并修正公开诊断，不把诊断改善当作远端事务自动恢复。 diff --git a/doc/Scert整改合并优化清单.md b/doc/Scert整改合并优化清单.md index 4a919b5..5ef3291 100644 --- a/doc/Scert整改合并优化清单.md +++ b/doc/Scert整改合并优化清单.md @@ -1,4 +1,13 @@ # Scert 整改、合并与优化清单 +## 当前验收总览（2026-09-30 22:56） + +- [✓] P0-03 源码与产物边界局部复审通过，不扩大为其他门禁通过。 +- [x] 6efc890 已正式部署，锁冲突诊断页面与任务结果接口实测通过；测试现场已恢复，两域名正常 HTTPS，正常重新部署任务 179bb071-486d-4356-8ba2-e15ce1164eb5 成功。 +- [~] P0-10：开始整改站点 HTTPS 校验前过早清理事务旧文件与释放锁的问题；目标为失败自动恢复文件、配置重载及部署前可信站点身份核验。 +- [!] 崩溃事务自动恢复、多服务器、数据库矩阵、双平台、备份恢复和最终首次流程仍未通过；整体生产验收未完成。 + +下方带时间章节均为历史记录，涉及候选版本和阻塞位置时以本节为准。 + ## 当前验收总览（2026-09-30 22:35，中断恢复边界已实测） - [✓] P0-03 局部复审结论保持，不把后续事务门禁重新挂到源码目录任务下。 diff --git a/frontend/src/views/Dashboard/DeploymentFeedback.js b/frontend/src/views/Dashboard/DeploymentFeedback.js index 4da1940..b4d96f6 100644 --- a/frontend/src/views/Dashboard/DeploymentFeedback.js +++ b/frontend/src/views/Dashboard/DeploymentFeedback.js @@ -10,6 +10,12 @@ export function serverAuthLabel(server) { export function deploymentSummary(report) { if (!report || !Array.isArray(report.results)) return "未收到有效部署结果"; + if (report.status !== "success" && report.recovery_status === "success") { + return "本次部署失败，已恢复旧文件并完成配置校验与重载，站点已恢复部署前证书"; + } + if (report.status !== "success" && ["failed", "unconfirmed"].includes(report.recovery_status)) { + return "本次部署失败，自动恢复未完成或站点状态未确认；请检查各服务器结果及部署锁，勿直接重试"; + } if (report.verification_status === "failed") { return "证书已推送，但站点 HTTPS 校验失败，尚未确认新证书生效；未自动回滚"; } diff --git a/frontend/src/views/Dashboard/DeploymentResult.test.js b/frontend/src/views/Dashboard/DeploymentResult.test.js index 07f095e..726bfc4 100644 --- a/frontend/src/views/Dashboard/DeploymentResult.test.js +++ b/frontend/src/views/Dashboard/DeploymentResult.test.js @@ -63,6 +63,19 @@ test("手动部署展示真实返回结果并阻止执行期间重复提交", as assert.equal(state.notice.value, "证书部署完成：1 台成功"); }); +test("失败部署区分已恢复旧站点与恢复未确认，不再统一显示未自动回滚", () => { + const failed = { ...report, status: "failed", success: 0, failed: 1, verification_status: "failed" }; + const restored = deploymentSummary({ ...failed, recovery_status: "success" }); + assert.match(restored, /本次部署失败/); + assert.match(restored, /站点已恢复部署前证书/); + assert.doesNotMatch(restored, /未自动回滚|部署完成/); + for (const recovery_status of ["failed", "unconfirmed"]) { + const message = deploymentSummary({ ...failed, recovery_status }); + assert.match(message, /勿直接重试/); + assert.doesNotMatch(message, /站点已恢复部署前证书|部署完成/); + } +}); + test("部署失败仍保留各服务器结果，网络失败不会残留执行中状态", async (t) => { let failNetwork = false; const failed = { ...report, status: "failed", success: 0, failed: 1, results: [{ ...report.results[0], status: "failed" }] };