补充证书部署任务重试流程

支持证书部署失败、取消和部分成功任务重试,补充任务执行阶段、权限判断、前端操作入口及回归测试。
This commit is contained in:
xuans committed 2026-10-01 15:42:24 +08:00
1 parent cfe8f35e42
commit 9edb7c11be
5 files changed
+157 -34

No files matched your search

+37 -16
View File
@@ -137,6 +137,14 @@ func (s *Server) issueDomainAsTask(c *gin.Context) {
}
func (s *Server) executeCertificateIssueTask(c *gin.Context, task repository.Task, attempt repository.TaskAttempt, auditAction string) {
s.executeCertificateTask(c, task, attempt, auditAction, s.issueCertificate, "certificate_issue", "正在申请证书并部署", "证书申请和部署完成")
}
func (s *Server) executeCertificateDeployTask(c *gin.Context, task repository.Task, attempt repository.TaskAttempt, auditAction string) {
s.executeCertificateTask(c, task, attempt, auditAction, s.deployCertificate, "certificate_deploy", "正在部署证书", "证书部署完成")
}
func (s *Server) executeCertificateTask(c *gin.Context, task repository.Task, attempt repository.TaskAttempt, auditAction string, operation func(context.Context, string) (certificate.IssueResult, deployment.Report, error), stage, runningMessage, successMessage string) {
executionCtx, cancel := context.WithTimeout(c.Request.Context(), certificateTaskTimeout)
executionCtx = deployment.WithTransactionContext(executionCtx, deployment.TransactionContext{
TaskID: task.ID, AttemptID: attempt.ID, AttemptNumber: attempt.AttemptNumber,
@@ -148,13 +156,13 @@ func (s *Server) executeCertificateIssueTask(c *gin.Context, task repository.Tas
}()
now := time.Now().UTC()
if err := s.tasks.UpdateTask(executionCtx, task.ID, "running", "certificate_issue", "正在申请证书并部署", &now, nil); err != nil {
if err := s.tasks.UpdateTask(executionCtx, task.ID, "running", stage, runningMessage, &now, nil); err != nil {
finished := time.Now().UTC()
failureMessage := "任务启动失败"
if s.writeCancelledTaskResponse(c, task.ID) {
return
}
_ = s.tasks.UpdateTask(context.Background(), task.ID, "failed", "certificate_issue", failureMessage, &now, &finished)
_ = s.tasks.UpdateTask(context.Background(), task.ID, "failed", stage, failureMessage, &now, &finished)
_ = s.tasks.FinishAttempt(context.Background(), attempt.ID, "failed", failureMessage, &finished)
if s.audit != nil {
_ = s.audit.CreateAuditLog(context.Background(), repository.AuditLog{
@@ -186,14 +194,14 @@ func (s *Server) executeCertificateIssueTask(c *gin.Context, task repository.Tas
executionCtx = certificate.WithProgressReporter(executionCtx, func(progressCtx context.Context, progressStage, progressMessage string) error {
return s.tasks.UpdateTask(progressCtx, task.ID, "running", progressStage, progressMessage, &now, nil)
})
certificateResult, deploymentReport, issueErr := s.issueCertificate(executionCtx, task.DomainID)
certificateResult, deploymentReport, issueErr := operation(executionCtx, task.DomainID)
deploymentReport = sanitizeDeploymentReport(deploymentReport)
if recordErr := s.tasks.RecordTaskDeploymentResults(context.Background(), task.ID, attempt.ID, attempt.AttemptNumber, taskDeploymentResults(deploymentReport)); recordErr != nil {
issueErr = errors.Join(issueErr, recordErr)
}
finished := time.Now().UTC()
status := "success"
message := "证书申请和部署完成"
message := successMessage
cancelled := false
partial := deploymentReport.Status == "partial"
if issueErr != nil {
@@ -221,11 +229,11 @@ func (s *Server) executeCertificateIssueTask(c *gin.Context, task repository.Tas
}
}
if !cancelled {
_ = s.tasks.UpdateTask(context.Background(), task.ID, status, "certificate_issue", message, &now, &finished)
_ = s.tasks.UpdateTask(context.Background(), task.ID, status, stage, message, &now, &finished)
}
_ = s.tasks.FinishAttempt(context.Background(), attempt.ID, status, message, &finished)
task.Status = status
task.Stage = "certificate_issue"
task.Stage = stage
task.Message = message
task.StartedAt = &now
task.FinishedAt = &finished
@@ -386,10 +394,6 @@ func (s *Server) cancelTask(c *gin.Context) {
}
func (s *Server) retryTask(c *gin.Context) {
if s.issueCertificate == nil {
s.writeError(c, http.StatusNotImplemented, "证书申请接口尚未配置", errors.New("证书申请工作流未配置"))
return
}
reader, ok := s.projectTaskReader(c)
if !ok {
return
@@ -405,14 +409,32 @@ func (s *Server) retryTask(c *gin.Context) {
s.writeError(c, http.StatusInternalServerError, "读取任务失败", err)
return
}
if item.TaskType != "certificate_issue" {
if item.TaskType != "certificate_issue" && item.TaskType != "certificate_deploy" {
c.JSON(http.StatusConflict, gin.H{"error": "当前任务类型不支持手动重试", "task": item})
return
}
if item.Status != "failed" && item.Status != "cancelled" {
c.JSON(http.StatusConflict, gin.H{"error": "仅失败或已取消的证书申请任务可以重试", "task": item})
retryable := item.Status == "failed" || item.Status == "cancelled"
if item.TaskType == "certificate_deploy" && item.Status == "partial_success" {
retryable = true
}
if !retryable {
c.JSON(http.StatusConflict, gin.H{"error": "仅失败、已取消或部分成功的证书任务可以重试", "task": item})
return
}
var runTask func(*gin.Context, repository.Task, repository.TaskAttempt, string)
if item.TaskType == "certificate_deploy" {
if s.deployCertificate == nil {
s.writeError(c, http.StatusNotImplemented, "证书部署接口尚未配置", errors.New("证书部署工作流未配置"))
return
}
runTask = s.executeCertificateDeployTask
} else {
if s.issueCertificate == nil {
s.writeError(c, http.StatusNotImplemented, "证书申请接口尚未配置", errors.New("证书申请工作流未配置"))
return
}
runTask = s.executeCertificateIssueTask
}
authorizer, ok := s.projectMembers.(repository.ProjectAuthorizer)
if !ok {
s.writeError(c, http.StatusServiceUnavailable, "任务重试权限服务尚未就绪", errors.New("项目授权服务未配置"))
@@ -433,7 +455,7 @@ func (s *Server) retryTask(c *gin.Context) {
return
}
}
if !s.authorizeDNSCredentialUse(c, projectID) {
if item.TaskType == "certificate_issue" && !s.authorizeDNSCredentialUse(c, projectID) {
return
}
startedAt := time.Now().UTC()
@@ -449,9 +471,8 @@ func (s *Server) retryTask(c *gin.Context) {
}
return
}
s.executeCertificateIssueTask(c, retryTask, attempt, "certificate_issue_retry")
runTask(c, retryTask, attempt, item.TaskType+"_retry")
}
func validTaskIdempotencyKey(value string) bool {
if len(value) < 8 || len(value) > 128 {
return false
+85
View File
@@ -742,3 +742,88 @@ func TestTaskAPIRecordsPartialDeploymentFailureAsPartialSuccess(t *testing.T) {
t.Fatalf("partial failure audit status: %+v %v", logs, err)
}
}
func TestTaskAPIRetriesFailedCertificateDeploymentWithoutDNSCredential(t *testing.T) {
calls := 0
server, taskRepo, auditRepo, closeDB := newTaskAPITestServer(t, func(context.Context, string) (certificate.IssueResult, deployment.Report, error) {
return certificate.IssueResult{}, deployment.Report{}, nil
})
defer closeDB()
server.deployCertificate = func(_ context.Context, id string) (certificate.IssueResult, deployment.Report, error) {
calls++
return certificate.IssueResult{DomainID: id}, deployment.Report{Status: "success"}, nil
}
finished := time.Now().UTC()
task := repository.Task{
HistoryScope: repository.HistoryScope{ScopeType: repository.ScopeProject, ProjectID: domain.DefaultProjectID},
ID: "deploy-retry-failed", DomainID: "domain-1", TaskType: "certificate_deploy",
Status: "failed", Stage: "certificate_deploy", Message: "deployment failed", FinishedAt: &finished,
}
if err := taskRepo.CreateTask(t.Context(), task); err != nil {
t.Fatalf("create deployment task: %v", err)
}
attempt := repository.TaskAttempt{
HistoryScope: repository.HistoryScope{ScopeType: repository.ScopeProject, ProjectID: domain.DefaultProjectID},
ID: "deploy-retry-failed-attempt-1", TaskID: task.ID, AttemptNumber: 1,
Stage: "certificate_deploy", Status: "failed", Message: "deployment failed", FinishedAt: &finished,
}
if err := taskRepo.CreateAttempt(t.Context(), attempt); err != nil {
t.Fatalf("create deployment attempt: %v", err)
}
response := httptest.NewRecorder()
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/tasks/"+task.ID+"/retry", nil))
if response.Code != http.StatusOK {
t.Fatalf("retry response: %d %s", response.Code, response.Body.String())
}
if calls != 1 {
t.Fatalf("deployment workflow calls: %d", calls)
}
updated, err := taskRepo.GetTask(t.Context(), task.ID)
if err != nil || updated.Status != "success" || updated.Stage != "certificate_deploy" {
t.Fatalf("deployment retry did not converge: %+v %v", updated, err)
}
attempts, err := taskRepo.ListAttempts(t.Context(), task.ID)
if err != nil || len(attempts) != 2 || attempts[1].Stage != "certificate_deploy" || attempts[1].Status != "success" {
t.Fatalf("deployment retry attempts: %+v %v", attempts, err)
}
logs, err := auditRepo.ListAuditLogs(t.Context(), 10)
if err != nil || len(logs) != 2 || logs[0].Action != "certificate_deploy_retry" || logs[0].Status != "success" {
t.Fatalf("deployment retry audit: %+v %v", logs, err)
}
}
func TestTaskAPIRetriesPartialCertificateDeployment(t *testing.T) {
server, taskRepo, _, closeDB := newTaskAPITestServer(t, func(context.Context, string) (certificate.IssueResult, deployment.Report, error) {
return certificate.IssueResult{}, deployment.Report{}, nil
})
defer closeDB()
server.deployCertificate = func(_ context.Context, id string) (certificate.IssueResult, deployment.Report, error) {
return certificate.IssueResult{DomainID: id}, deployment.Report{Status: "success"}, nil
}
finished := time.Now().UTC()
task := repository.Task{
HistoryScope: repository.HistoryScope{ScopeType: repository.ScopeProject, ProjectID: domain.DefaultProjectID},
ID: "deploy-retry-partial", DomainID: "domain-1", TaskType: "certificate_deploy",
Status: "partial_success", Stage: "certificate_deploy", Message: "partial deployment", FinishedAt: &finished,
}
if err := taskRepo.CreateTask(t.Context(), task); err != nil {
t.Fatalf("create partial deployment task: %v", err)
}
attempt := repository.TaskAttempt{
HistoryScope: repository.HistoryScope{ScopeType: repository.ScopeProject, ProjectID: domain.DefaultProjectID},
ID: "deploy-retry-partial-attempt-1", TaskID: task.ID, AttemptNumber: 1,
Stage: "certificate_deploy", Status: "partial_success", Message: "partial deployment", FinishedAt: &finished,
}
if err := taskRepo.CreateAttempt(t.Context(), attempt); err != nil {
t.Fatalf("create partial deployment attempt: %v", err)
}
response := httptest.NewRecorder()
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/tasks/"+task.ID+"/retry", nil))
if response.Code != http.StatusOK {
t.Fatalf("partial retry response: %d %s", response.Code, response.Body.String())
}
updated, err := taskRepo.GetTask(t.Context(), task.ID)
if err != nil || updated.Status != "success" {
t.Fatalf("partial deployment retry did not converge: %+v %v", updated, err)
}
}