完善部署中断恢复提示和安全阶段诊断并记录实机边界

This commit is contained in:
xuans committed 2026-09-30 22:41:28 +08:00
1 parent b27eedc1d0
commit 6efc89061b
17 files changed
+386 -14

No files matched your search

+1 -1
View File
@@ -87,7 +87,7 @@ func TestNewWithConfigPathRecoversInterruptedTasks(t *testing.T) {
if task.Status != "failed" || task.Stage != "service_restart" || task.FinishedAt == nil {
t.Fatalf("任务 %s 未按服务重启恢复:%+v", expectedID, task)
}
if task.Message != "服务重启导致任务中断,请重新执行" {
if task.Message != "服务重启导致任务中断,执行结果未确认;请先核查关联资源,证书部署需检查远端文件、部署锁及服务状态,确认安全后再重试" {
t.Fatalf("任务 %s 中断说明不正确:%s", expectedID, task.Message)
}
}
@@ -0,0 +1,22 @@
package app
import (
"testing"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
)
func TestTaskDeploymentResultsPersistSafeFailureDiagnostics(t *testing.T) {
report := deployment.Report{Results: []deployment.Result{
{Status: "failed", FailureCode: deployment.FailureLock, Message: "secret-path"},
{Status: "failed", FailureCode: "secret-code", Message: "secret-output"},
{Status: "success", Message: "部署完成"},
}}
results := taskDeploymentResults(report)
if len(results) != 3 || results[0].Message != deployment.PublicResult(report.Results[0]).Message || results[1].Message != "证书部署失败" || results[2].Message != "部署完成" {
t.Fatalf("持久化部署诊断不正确:%+v", results)
}
if report.Results[0].Message != "secret-path" {
t.Fatal("不得覆盖受保护的原始错误")
}
}
@@ -179,6 +179,7 @@ func taskDeploymentResults(report deployment.Report) []repository.TaskDeployment
}
items := make([]repository.TaskDeploymentResult, 0, len(report.Results))
for index, item := range report.Results {
item = deployment.PublicResult(item)
items = append(items, repository.TaskDeploymentResult{
ResultIndex: index,
ServerID: item.ServerID,
@@ -0,0 +1,61 @@
package deployment
type FailureCode string
const (
FailureUnavailable FailureCode = "unavailable"
FailureArtifact FailureCode = "artifact_invalid"
FailureConfiguration FailureCode = "configuration_invalid"
FailureConnection FailureCode = "connection_failed"
FailureDirectory FailureCode = "directory_failed"
FailureLock FailureCode = "lock_unavailable"
FailureUpload FailureCode = "upload_failed"
FailureReplace FailureCode = "replace_failed"
FailureValidation FailureCode = "validation_failed"
FailureRollback FailureCode = "rollback_failed"
FailureRuntimeUnknown FailureCode = "runtime_unconfirmed"
FailureCleanup FailureCode = "cleanup_failed"
FailureLockRelease FailureCode = "lock_release_failed"
)
// 只公开固定诊断,不从远端输出、错误文本或文件路径推断可展示的消息。
var publicFailureMessages = map[FailureCode]string{
FailureUnavailable: "部署服务不可用,请联系管理员检查服务配置",
FailureArtifact: "待部署的证书或私钥校验失败,尚未推送到服务器",
FailureConfiguration: "服务器部署配置无效,请检查认证、证书路径和命令配置",
FailureConnection: "连接部署服务器失败,请检查网络、SSH 认证及主机身份配置",
FailureDirectory: "无法创建证书目录,请检查部署账户的目录权限",
FailureLock: "无法取得部署锁,可能有任务正在执行或存在中断遗留;请管理员核查远端文件、部署锁及服务状态后再重试,勿直接删除锁",
FailureUpload: "证书上传失败,已尝试回滚;请检查受保护的服务日志后再重试",
FailureReplace: "证书文件替换失败,已尝试回滚;请检查受保护的服务日志后再重试",
FailureValidation: "服务配置校验失败,文件已回滚,未执行重载;请修正配置后再重试",
FailureRollback: "部署失败且文件回滚未完成,已保留部署锁;请管理员核查远端文件及服务状态,勿直接重试或删除锁",
FailureRuntimeUnknown: "服务重载失败,文件已回滚,但服务运行状态未确认;请管理员核对配置并恢复服务后再重试",
FailureCleanup: "证书已推送并重载,但旧文件清理未完成;请管理员核查现场,不能据此确认站点新证书已生效",
FailureLockRelease: "部署锁释放未完成;请管理员核查远端文件、部署锁及服务状态后再重试",
}
// PublicResult 保留结果事实,同时只允许固定错误码及中文诊断进入公开响应。
func PublicResult(result Result) Result {
if result.Status != "failed" {
result.FailureCode = ""
return result
}
message, ok := publicFailureMessages[result.FailureCode]
if !ok {
result.FailureCode = ""
message = "证书部署失败"
}
result.Message = message
return result
}
// PublicStoredFailureMessage 兼容历史任务;仅原样放行完整匹配的固定诊断。
func PublicStoredFailureMessage(message string) string {
for _, safe := range publicFailureMessages {
if message == safe {
return safe
}
}
return "证书部署失败"
}
@@ -0,0 +1,97 @@
package deployment
import (
"context"
"errors"
"strings"
"testing"
)
type lockReleaseFailureClient struct {
*memoryClient
}
func (c *lockReleaseFailureClient) Unlock(context.Context, string) error {
return errors.New("secret-path lock-release-error")
}
func TestDeploymentLockReleaseFailureOverridesSuccess(t *testing.T) {
client := &lockReleaseFailureClient{memoryClient: newMemoryClient()}
server := testServer("unlock")
report := NewService(fixedClientFactory{client}).Deploy(t.Context(), []Target{{Server: server}}, testArtifact(t))
if report.Status != "failed" || report.Success != 0 || report.Failed != 1 || report.Results[0].FailureCode != FailureLockRelease {
t.Fatalf("锁释放失败不得保留成功状态:%+v", report)
}
public := PublicResult(report.Results[0])
if strings.Contains(public.Message, "secret-path") || !strings.Contains(public.Message, "部署锁释放未完成") {
t.Fatalf("锁释放诊断错误:%+v", public)
}
}
func TestPublicDeploymentDiagnosticsAreAllowlisted(t *testing.T) {
for code, expected := range publicFailureMessages {
t.Run(string(code), func(t *testing.T) {
raw := Result{ServerID: "server-1", Status: "failed", FailureCode: code, Message: "secret-path private-key remote-output"}
public := PublicResult(raw)
if public.Message != expected || public.FailureCode != code || public.ServerID != raw.ServerID {
t.Fatalf("固定诊断或结果身份丢失:%+v", public)
}
if PublicStoredFailureMessage(public.Message) != expected {
t.Fatal("持久化诊断无法读取")
}
if PublicStoredFailureMessage(public.Message+" secret-path") != "证书部署失败" {
t.Fatal("包含额外敏感内容的诊断不应放行")
}
})
}
for _, code := range []FailureCode{"", "secret-path", "lock_unavailable\nprivate-key"} {
public := PublicResult(Result{Status: "failed", FailureCode: code, Message: "无法取得部署锁 /secret-path"})
if public.FailureCode != "" || public.Message != "证书部署失败" {
t.Fatalf("未知错误码或原始错误泄露:%+v", public)
}
}
public := PublicResult(Result{Status: "success", FailureCode: "secret-path", Message: "部署完成"})
if public.FailureCode != "" || public.Message != "部署完成" {
t.Fatalf("成功结果错误:%+v", public)
}
}
func TestDeploymentFailureCodesDescribeActualRecoveryBoundary(t *testing.T) {
for _, scenario := range []struct {
name string
code FailureCode
configure func(*memoryClient, Server)
}{
{"lock", FailureLock, func(c *memoryClient, s Server) { c.files[s.PrivateKeyPath+".scert-lock"] = memoryFile{} }},
{"upload", FailureUpload, func(c *memoryClient, _ Server) { c.uploadError = ".scert-tmp-" }},
{"replace", FailureReplace, func(c *memoryClient, _ Server) { c.renameError = ".scert-tmp-" }},
{"validation", FailureValidation, func(c *memoryClient, _ Server) { c.runErrors = []error{errors.New("secret-path")} }},
{"reload", FailureRuntimeUnknown, func(c *memoryClient, _ Server) { c.runErrors = []error{nil, errors.New("secret-path")} }},
{"validation_rollback", FailureRollback, func(c *memoryClient, _ Server) {
c.runErrors = []error{errors.New("secret-path")}
c.restoreError = true
}},
{"reload_rollback", FailureRollback, func(c *memoryClient, _ Server) {
c.runErrors = []error{nil, errors.New("secret-path")}
c.restoreError = true
}},
{"cleanup", FailureCleanup, func(c *memoryClient, _ Server) { c.removeError = ".scert-backup-" }},
} {
t.Run(scenario.name, func(t *testing.T) {
client := newMemoryClient()
server := testServer("diagnostics")
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
client.files[path] = memoryFile{data: []byte("original"), mode: 0o600}
}
scenario.configure(client, server)
report := NewService(fixedClientFactory{client}).Deploy(t.Context(), []Target{{Server: server}}, testArtifact(t))
if report.Status != "failed" || len(report.Results) != 1 || report.Results[0].FailureCode != scenario.code {
t.Fatalf("失败阶段或恢复边界错误:%+v", report)
}
public := PublicResult(report.Results[0])
if strings.Contains(public.Message, "secret-path") || public.Message == "证书部署失败" {
t.Fatalf("阶段诊断缺失或泄露底层信息:%+v", public)
}
})
}
}
+23 -8
View File
@@ -29,11 +29,12 @@ type SavedFiles struct {
}
type Result struct {
ServerID string `json:"server_id"`
Server string `json:"server"`
Status string `json:"status"`
Message string `json:"message"`
BackupID string `json:"backup_id,omitempty"`
ServerID string `json:"server_id"`
Server string `json:"server"`
Status string `json:"status"`
Message string `json:"message"`
BackupID string `json:"backup_id,omitempty"`
FailureCode FailureCode `json:"failure_code,omitempty"`
}
const deploymentRollbackTimeout = 30 * time.Second
@@ -67,7 +68,7 @@ func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifac
report.Status = "failed"
report.Failed = len(targets)
for _, target := range targets {
report.Results = append(report.Results, Result{ServerID: target.Server.ID, Server: target.Server.Name, Status: "failed", Message: "部署服务未初始化"})
report.Results = append(report.Results, Result{ServerID: target.Server.ID, Server: target.Server.Name, Status: "failed", Message: "部署服务未初始化", FailureCode: FailureUnavailable})
}
return report
}
@@ -75,7 +76,7 @@ func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifac
report.Status = "failed"
report.Failed = len(targets)
for _, target := range targets {
report.Results = append(report.Results, Result{ServerID: target.Server.ID, Server: target.Server.Name, Status: "failed", Message: err.Error()})
report.Results = append(report.Results, Result{ServerID: target.Server.ID, Server: target.Server.Name, Status: "failed", Message: err.Error(), FailureCode: FailureArtifact})
}
return report
}
@@ -119,13 +120,14 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
server = server.Normalize()
result = Result{ServerID: server.ID, Server: server.Name, Status: "failed"}
result = Result{ServerID: server.ID, Server: server.Name, Status: "failed", FailureCode: FailureConfiguration}
if err := server.Validate(); err != nil {
result.Message = err.Error()
return result
}
client, err := s.factory.Connect(ctx, server)
if err != nil {
result.FailureCode = FailureConnection
result.Message = fmt.Sprintf("连接服务器失败:%v", err)
return result
}
@@ -149,6 +151,7 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
lockPaths := make([]string, 0, len(paths))
for _, file := range paths {
if err := client.MkdirAll(ctx, parentPath(file.path)); err != nil {
result.FailureCode = FailureDirectory
result.Message = fmt.Sprintf("创建远程目录失败:%v", err)
return result
}
@@ -165,18 +168,21 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
defer cancel()
if err := reconnectDeploymentClient(releaseCtx, s.factory, server, &client); err != nil {
result.Status = "failed"
result.FailureCode = FailureLockRelease
result.Message += fmt.Sprintf(";释放部署锁失败,需人工检查:%v", err)
return
}
for index := len(heldLocks) - 1; index >= 0; index-- {
if err := client.Unlock(releaseCtx, heldLocks[index]); err != nil {
result.Status = "failed"
result.FailureCode = FailureLockRelease
result.Message += fmt.Sprintf(";释放部署锁 %s 失败,需人工检查:%v", heldLocks[index], err)
}
}
}()
for _, lockPath := range lockPaths {
if err := client.Lock(ctx, lockPath); err != nil {
result.FailureCode = FailureLock
result.Message = fmt.Sprintf("无法取得部署锁 %s,可能存在并发部署或遗留锁;未修改当前证书,请人工核查:%v", lockPath, err)
return result
}
@@ -224,6 +230,7 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
reloadAttempted := false
failed := func(message string) Result {
if rollbackErr := rollback(); rollbackErr != nil {
result.FailureCode = FailureRollback
message += fmt.Sprintf(";自动回滚失败:%v", rollbackErr)
retainLocks = true
message += ";已保留本次部署锁,修复并确认文件状态后再由管理员释放"
@@ -231,12 +238,16 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
message += ";文件已回滚"
}
if reloadAttempted {
if !retainLocks {
result.FailureCode = FailureRuntimeUnknown
}
// 命令可能已部分执行;文件回滚并不证明运行中的服务已经恢复。
message += ";服务运行状态未确认,请检查服务并在确认旧配置后重新加载"
}
result.Message = message
return result
}
result.FailureCode = FailureUpload
for _, file := range paths {
temporaryPath := file.path + ".scert-tmp-" + backupID
// 上传失败也可能留下半份私钥,必须在开始上传前登记清理路径。
@@ -251,6 +262,7 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
}
}
// 所有文件上传完成后才触碰当前版本,避免上传失败造成不必要的证书替换。
result.FailureCode = FailureReplace
for _, file := range paths {
exists, err := client.Exists(ctx, file.path)
if err != nil {
@@ -272,6 +284,7 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
return failed(fmt.Sprintf("替换远程证书失败:%v", err))
}
}
result.FailureCode = FailureValidation
validationResult, err := client.Run(ctx, server.ValidationCommand)
if err != nil || validationResult.ExitCode != 0 {
if err == nil {
@@ -289,10 +302,12 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
}
if err := cleanupBackups(ctx, client, backups, paths); err != nil {
result.Status = "failed"
result.FailureCode = FailureCleanup
result.Message = fmt.Sprintf("\u8bc1\u4e66\u5df2\u90e8\u7f72\u5e76 reload \u6210\u529f\uff0c\u4f46\u65e7\u8bc1\u4e66\u5907\u4efd\u6e05\u7406\u5931\u8d25\uff1a%v", err)
return result
}
result.Status = "success"
result.FailureCode = ""
result.Message = "证书部署、配置校验并 reload 成功"
result.BackupID = backupID
return result
@@ -26,9 +26,7 @@ func sanitizeDeploymentReport(report deployment.Report) deployment.Report {
result := report
result.Results = append([]deployment.Result(nil), report.Results...)
for index := range result.Results {
if result.Results[index].Status == "failed" {
result.Results[index].Message = "证书部署失败"
}
result.Results[index] = deployment.PublicResult(result.Results[index])
}
return result
}
@@ -0,0 +1,28 @@
package httpapi
import (
"testing"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
)
func TestSanitizeDeploymentReportKeepsSafeDiagnosisWithoutMutatingEvidence(t *testing.T) {
report := deployment.Report{Status: "failed", Results: []deployment.Result{
{Status: "failed", FailureCode: deployment.FailureLock, Message: "secret-path"},
{Status: "failed", FailureCode: "secret-code", Message: "secret-output"},
}}
public := sanitizeDeploymentReport(report)
if public.Results[0].FailureCode != deployment.FailureLock || public.Results[0].Message != deployment.PublicResult(report.Results[0]).Message {
t.Fatalf("固定锁诊断丢失:%+v", public)
}
if public.Results[1].FailureCode != "" || public.Results[1].Message != "证书部署失败" {
t.Fatalf("未知诊断未脱敏:%+v", public)
}
if report.Results[0].Message != "secret-path" || report.Results[1].FailureCode != "secret-code" {
t.Fatal("公开响应脱敏修改了受保护的原始证据")
}
stored := taskDeploymentResults(report)
if stored[0].Message != public.Results[0].Message || stored[1].Message != "证书部署失败" {
t.Fatalf("接口工作流未持久化安全诊断:%+v", stored)
}
}
+2 -1
View File
@@ -553,7 +553,7 @@ func (s *Server) listTaskDeploymentResults(c *gin.Context) {
}
for index := range items {
if items[index].Status == "failed" {
items[index].Message = "证书部署失败"
items[index].Message = deployment.PublicStoredFailureMessage(items[index].Message)
}
}
c.JSON(http.StatusOK, gin.H{"items": items})
@@ -565,6 +565,7 @@ func taskDeploymentResults(report deployment.Report) []repository.TaskDeployment
}
items := make([]repository.TaskDeploymentResult, 0, len(report.Results))
for index, item := range report.Results {
item = deployment.PublicResult(item)
items = append(items, repository.TaskDeploymentResult{
ResultIndex: index,
ServerID: item.ServerID,
@@ -78,6 +78,34 @@ func TestTaskDeploymentResultsIssueAndRead(t *testing.T) {
}
}
func TestTaskDeploymentResultsKeepAllowlistedLockDiagnosis(t *testing.T) {
raw := deployment.Result{ServerID: "server-a", Status: "failed", FailureCode: deployment.FailureLock, Message: "secret-path remote-command-output"}
server, tasks, _, closeDB := newTaskAPITestServer(t, func(_ context.Context, id string) (certificate.IssueResult, deployment.Report, error) {
return certificate.IssueResult{DomainID: id}, deployment.Report{Status: "failed", Total: 1, Failed: 1, Results: []deployment.Result{raw}}, errors.New("部署失败")
})
defer closeDB()
response := httptest.NewRecorder()
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
if response.Code != http.StatusBadGateway || strings.Contains(response.Body.String(), "secret-path") || !strings.Contains(response.Body.String(), "lock_unavailable") {
t.Fatalf("部署接口缺少安全的锁诊断:%d %s", response.Code, response.Body.String())
}
items, err := tasks.ListTasks(t.Context(), "", 10)
if err != nil || len(items) != 1 {
t.Fatalf("任务未保存:%+v %v", items, err)
}
response = httptest.NewRecorder()
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodGet, "/api/v1/tasks/"+items[0].ID+"/deployment-results", nil))
var payload struct {
Items []repository.TaskDeploymentResult `json:"items"`
}
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if response.Code != http.StatusOK || len(payload.Items) != 1 || payload.Items[0].Message != deployment.PublicResult(raw).Message {
t.Fatalf("任务查询丢失固定锁诊断:%d %s", response.Code, response.Body.String())
}
}
func TestTaskDeploymentResultsHTTPSFailurePreservesTransferEvidence(t *testing.T) {
server, tasks, _, closeDB := newTaskAPITestServer(t, func(_ context.Context, id string) (certificate.IssueResult, deployment.Report, error) {
return certificate.IssueResult{DomainID: id}, deployment.Report{
@@ -0,0 +1,39 @@
package repository
import (
"strings"
"testing"
"time"
)
func TestTaskRecoveryRequiresResourceInspectionBeforeRetry(t *testing.T) {
repo, db := newTaskRepositoryTest(t)
defer db.Close()
ctx := t.Context()
if err := repo.CreateTask(ctx, Task{ID: "interrupted-deploy", DomainID: "domain-1", TaskType: "certificate_deploy", Status: "running", Stage: "certificate_deploy"}); err != nil {
t.Fatal(err)
}
if err := repo.CreateAttempt(ctx, TaskAttempt{ID: "interrupted-attempt", TaskID: "interrupted-deploy", AttemptNumber: 1, Stage: "certificate_deploy", Status: "running"}); err != nil {
t.Fatal(err)
}
if count, err := repo.RecoverInterruptedTasks(ctx, time.Now()); err != nil || count != 1 {
t.Fatalf("恢复任务失败:%d %v", count, err)
}
task, err := repo.GetTask(ctx, "interrupted-deploy")
if err != nil {
t.Fatal(err)
}
for _, expected := range []string{"执行结果未确认", "远端文件", "部署锁", "服务状态", "确认安全后再重试"} {
if !strings.Contains(task.Message, expected) {
t.Errorf("中断任务缺少恢复边界提示 %q:%s", expected, task.Message)
}
}
attempts, err := repo.ListAttempts(ctx, task.ID)
if err != nil || len(attempts) != 1 || attempts[0].Message != task.Message {
t.Fatalf("执行尝试与任务恢复说明不一致:%+v %v", attempts, err)
}
var auditMessage string
if err := db.QueryRowContext(ctx, `SELECT message FROM audit_logs WHERE action = 'task_recovery' AND task_id = ?`, task.ID).Scan(&auditMessage); err != nil || auditMessage != task.Message {
t.Fatalf("恢复审计说明不一致:%s %v", auditMessage, err)
}
}
@@ -346,7 +346,7 @@ func (r *SQLTaskRepository) RecoverInterruptedTasks(ctx context.Context, finishe
rollback := func() {
_ = tx.Rollback()
}
recoveryMessage := "\u670d\u52a1\u91cd\u542f\u5bfc\u81f4\u4efb\u52a1\u4e2d\u65ad\uff0c\u8bf7\u91cd\u65b0\u6267\u884c"
recoveryMessage := "服务重启导致任务中断,执行结果未确认;请先核查关联资源,证书部署需检查远端文件、部署锁及服务状态,确认安全后再重试"
auditResult, err := tx.ExecContext(ctx, r.query(`INSERT INTO audit_logs
(action, resource_type, resource_id, status, message, created_at, scope_type, project_id, task_id, operator_id)
SELECT 'task_recovery', 'task', id, 'failed', ?, ?, scope_type, project_id, id, operator_id