diff --git a/backend/internal/httpapi/task_api.go b/backend/internal/httpapi/task_api.go index 69db7e3..914d05e 100644 --- a/backend/internal/httpapi/task_api.go +++ b/backend/internal/httpapi/task_api.go @@ -137,6 +137,14 @@ func (s *Server) issueDomainAsTask(c *gin.Context) { } func (s *Server) executeCertificateIssueTask(c *gin.Context, task repository.Task, attempt repository.TaskAttempt, auditAction string) { + s.executeCertificateTask(c, task, attempt, auditAction, s.issueCertificate, "certificate_issue", "正在申请证书并部署", "证书申请和部署完成") +} + +func (s *Server) executeCertificateDeployTask(c *gin.Context, task repository.Task, attempt repository.TaskAttempt, auditAction string) { + s.executeCertificateTask(c, task, attempt, auditAction, s.deployCertificate, "certificate_deploy", "正在部署证书", "证书部署完成") +} + +func (s *Server) executeCertificateTask(c *gin.Context, task repository.Task, attempt repository.TaskAttempt, auditAction string, operation func(context.Context, string) (certificate.IssueResult, deployment.Report, error), stage, runningMessage, successMessage string) { executionCtx, cancel := context.WithTimeout(c.Request.Context(), certificateTaskTimeout) executionCtx = deployment.WithTransactionContext(executionCtx, deployment.TransactionContext{ TaskID: task.ID, AttemptID: attempt.ID, AttemptNumber: attempt.AttemptNumber, @@ -148,13 +156,13 @@ func (s *Server) executeCertificateIssueTask(c *gin.Context, task repository.Tas }() now := time.Now().UTC() - if err := s.tasks.UpdateTask(executionCtx, task.ID, "running", "certificate_issue", "正在申请证书并部署", &now, nil); err != nil { + if err := s.tasks.UpdateTask(executionCtx, task.ID, "running", stage, runningMessage, &now, nil); err != nil { finished := time.Now().UTC() failureMessage := "任务启动失败" if s.writeCancelledTaskResponse(c, task.ID) { return } - _ = s.tasks.UpdateTask(context.Background(), task.ID, "failed", "certificate_issue", failureMessage, &now, &finished) + _ = s.tasks.UpdateTask(context.Background(), task.ID, "failed", stage, failureMessage, &now, &finished) _ = s.tasks.FinishAttempt(context.Background(), attempt.ID, "failed", failureMessage, &finished) if s.audit != nil { _ = s.audit.CreateAuditLog(context.Background(), repository.AuditLog{ @@ -186,14 +194,14 @@ func (s *Server) executeCertificateIssueTask(c *gin.Context, task repository.Tas executionCtx = certificate.WithProgressReporter(executionCtx, func(progressCtx context.Context, progressStage, progressMessage string) error { return s.tasks.UpdateTask(progressCtx, task.ID, "running", progressStage, progressMessage, &now, nil) }) - certificateResult, deploymentReport, issueErr := s.issueCertificate(executionCtx, task.DomainID) + certificateResult, deploymentReport, issueErr := operation(executionCtx, task.DomainID) deploymentReport = sanitizeDeploymentReport(deploymentReport) if recordErr := s.tasks.RecordTaskDeploymentResults(context.Background(), task.ID, attempt.ID, attempt.AttemptNumber, taskDeploymentResults(deploymentReport)); recordErr != nil { issueErr = errors.Join(issueErr, recordErr) } finished := time.Now().UTC() status := "success" - message := "证书申请和部署完成" + message := successMessage cancelled := false partial := deploymentReport.Status == "partial" if issueErr != nil { @@ -221,11 +229,11 @@ func (s *Server) executeCertificateIssueTask(c *gin.Context, task repository.Tas } } if !cancelled { - _ = s.tasks.UpdateTask(context.Background(), task.ID, status, "certificate_issue", message, &now, &finished) + _ = s.tasks.UpdateTask(context.Background(), task.ID, status, stage, message, &now, &finished) } _ = s.tasks.FinishAttempt(context.Background(), attempt.ID, status, message, &finished) task.Status = status - task.Stage = "certificate_issue" + task.Stage = stage task.Message = message task.StartedAt = &now task.FinishedAt = &finished @@ -386,10 +394,6 @@ func (s *Server) cancelTask(c *gin.Context) { } func (s *Server) retryTask(c *gin.Context) { - if s.issueCertificate == nil { - s.writeError(c, http.StatusNotImplemented, "证书申请接口尚未配置", errors.New("证书申请工作流未配置")) - return - } reader, ok := s.projectTaskReader(c) if !ok { return @@ -405,14 +409,32 @@ func (s *Server) retryTask(c *gin.Context) { s.writeError(c, http.StatusInternalServerError, "读取任务失败", err) return } - if item.TaskType != "certificate_issue" { + if item.TaskType != "certificate_issue" && item.TaskType != "certificate_deploy" { c.JSON(http.StatusConflict, gin.H{"error": "当前任务类型不支持手动重试", "task": item}) return } - if item.Status != "failed" && item.Status != "cancelled" { - c.JSON(http.StatusConflict, gin.H{"error": "仅失败或已取消的证书申请任务可以重试", "task": item}) + retryable := item.Status == "failed" || item.Status == "cancelled" + if item.TaskType == "certificate_deploy" && item.Status == "partial_success" { + retryable = true + } + if !retryable { + c.JSON(http.StatusConflict, gin.H{"error": "仅失败、已取消或部分成功的证书任务可以重试", "task": item}) return } + var runTask func(*gin.Context, repository.Task, repository.TaskAttempt, string) + if item.TaskType == "certificate_deploy" { + if s.deployCertificate == nil { + s.writeError(c, http.StatusNotImplemented, "证书部署接口尚未配置", errors.New("证书部署工作流未配置")) + return + } + runTask = s.executeCertificateDeployTask + } else { + if s.issueCertificate == nil { + s.writeError(c, http.StatusNotImplemented, "证书申请接口尚未配置", errors.New("证书申请工作流未配置")) + return + } + runTask = s.executeCertificateIssueTask + } authorizer, ok := s.projectMembers.(repository.ProjectAuthorizer) if !ok { s.writeError(c, http.StatusServiceUnavailable, "任务重试权限服务尚未就绪", errors.New("项目授权服务未配置")) @@ -433,7 +455,7 @@ func (s *Server) retryTask(c *gin.Context) { return } } - if !s.authorizeDNSCredentialUse(c, projectID) { + if item.TaskType == "certificate_issue" && !s.authorizeDNSCredentialUse(c, projectID) { return } startedAt := time.Now().UTC() @@ -449,9 +471,8 @@ func (s *Server) retryTask(c *gin.Context) { } return } - s.executeCertificateIssueTask(c, retryTask, attempt, "certificate_issue_retry") + runTask(c, retryTask, attempt, item.TaskType+"_retry") } - func validTaskIdempotencyKey(value string) bool { if len(value) < 8 || len(value) > 128 { return false diff --git a/backend/internal/httpapi/task_api_test.go b/backend/internal/httpapi/task_api_test.go index 3218d26..eb2881c 100644 --- a/backend/internal/httpapi/task_api_test.go +++ b/backend/internal/httpapi/task_api_test.go @@ -742,3 +742,88 @@ func TestTaskAPIRecordsPartialDeploymentFailureAsPartialSuccess(t *testing.T) { t.Fatalf("partial failure audit status: %+v %v", logs, err) } } + +func TestTaskAPIRetriesFailedCertificateDeploymentWithoutDNSCredential(t *testing.T) { + calls := 0 + server, taskRepo, auditRepo, closeDB := newTaskAPITestServer(t, func(context.Context, string) (certificate.IssueResult, deployment.Report, error) { + return certificate.IssueResult{}, deployment.Report{}, nil + }) + defer closeDB() + server.deployCertificate = func(_ context.Context, id string) (certificate.IssueResult, deployment.Report, error) { + calls++ + return certificate.IssueResult{DomainID: id}, deployment.Report{Status: "success"}, nil + } + finished := time.Now().UTC() + task := repository.Task{ + HistoryScope: repository.HistoryScope{ScopeType: repository.ScopeProject, ProjectID: domain.DefaultProjectID}, + ID: "deploy-retry-failed", DomainID: "domain-1", TaskType: "certificate_deploy", + Status: "failed", Stage: "certificate_deploy", Message: "deployment failed", FinishedAt: &finished, + } + if err := taskRepo.CreateTask(t.Context(), task); err != nil { + t.Fatalf("create deployment task: %v", err) + } + attempt := repository.TaskAttempt{ + HistoryScope: repository.HistoryScope{ScopeType: repository.ScopeProject, ProjectID: domain.DefaultProjectID}, + ID: "deploy-retry-failed-attempt-1", TaskID: task.ID, AttemptNumber: 1, + Stage: "certificate_deploy", Status: "failed", Message: "deployment failed", FinishedAt: &finished, + } + if err := taskRepo.CreateAttempt(t.Context(), attempt); err != nil { + t.Fatalf("create deployment attempt: %v", err) + } + response := httptest.NewRecorder() + serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/tasks/"+task.ID+"/retry", nil)) + if response.Code != http.StatusOK { + t.Fatalf("retry response: %d %s", response.Code, response.Body.String()) + } + if calls != 1 { + t.Fatalf("deployment workflow calls: %d", calls) + } + updated, err := taskRepo.GetTask(t.Context(), task.ID) + if err != nil || updated.Status != "success" || updated.Stage != "certificate_deploy" { + t.Fatalf("deployment retry did not converge: %+v %v", updated, err) + } + attempts, err := taskRepo.ListAttempts(t.Context(), task.ID) + if err != nil || len(attempts) != 2 || attempts[1].Stage != "certificate_deploy" || attempts[1].Status != "success" { + t.Fatalf("deployment retry attempts: %+v %v", attempts, err) + } + logs, err := auditRepo.ListAuditLogs(t.Context(), 10) + if err != nil || len(logs) != 2 || logs[0].Action != "certificate_deploy_retry" || logs[0].Status != "success" { + t.Fatalf("deployment retry audit: %+v %v", logs, err) + } +} + +func TestTaskAPIRetriesPartialCertificateDeployment(t *testing.T) { + server, taskRepo, _, closeDB := newTaskAPITestServer(t, func(context.Context, string) (certificate.IssueResult, deployment.Report, error) { + return certificate.IssueResult{}, deployment.Report{}, nil + }) + defer closeDB() + server.deployCertificate = func(_ context.Context, id string) (certificate.IssueResult, deployment.Report, error) { + return certificate.IssueResult{DomainID: id}, deployment.Report{Status: "success"}, nil + } + finished := time.Now().UTC() + task := repository.Task{ + HistoryScope: repository.HistoryScope{ScopeType: repository.ScopeProject, ProjectID: domain.DefaultProjectID}, + ID: "deploy-retry-partial", DomainID: "domain-1", TaskType: "certificate_deploy", + Status: "partial_success", Stage: "certificate_deploy", Message: "partial deployment", FinishedAt: &finished, + } + if err := taskRepo.CreateTask(t.Context(), task); err != nil { + t.Fatalf("create partial deployment task: %v", err) + } + attempt := repository.TaskAttempt{ + HistoryScope: repository.HistoryScope{ScopeType: repository.ScopeProject, ProjectID: domain.DefaultProjectID}, + ID: "deploy-retry-partial-attempt-1", TaskID: task.ID, AttemptNumber: 1, + Stage: "certificate_deploy", Status: "partial_success", Message: "partial deployment", FinishedAt: &finished, + } + if err := taskRepo.CreateAttempt(t.Context(), attempt); err != nil { + t.Fatalf("create partial deployment attempt: %v", err) + } + response := httptest.NewRecorder() + serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/tasks/"+task.ID+"/retry", nil)) + if response.Code != http.StatusOK { + t.Fatalf("partial retry response: %d %s", response.Code, response.Body.String()) + } + updated, err := taskRepo.GetTask(t.Context(), task.ID) + if err != nil || updated.Status != "success" { + t.Fatalf("partial deployment retry did not converge: %+v %v", updated, err) + } +} diff --git a/backend/internal/repository/project_task_reader.go b/backend/internal/repository/project_task_reader.go index 34d9446..fd92630 100644 --- a/backend/internal/repository/project_task_reader.go +++ b/backend/internal/repository/project_task_reader.go @@ -242,12 +242,16 @@ func (r *SQLTaskRepository) RetryVisibleTask(ctx context.Context, actorID, taskI predicate, predicateArgs := taskVisibilityPredicate("tasks", domain.PermissionTaskRetry) predicateArgs[0] = actorID updateArgs := []any{ - actorID, "certificate_issue", "正在重试证书申请", nullableTime(startedAt), time.Now().UTC(), taskID, + actorID, nullableTime(startedAt), time.Now().UTC(), taskID, } updateArgs = append(updateArgs, predicateArgs...) - result, err := tx.ExecContext(ctx, r.query(`UPDATE tasks SET operator_id = ?, status = 'running', stage = ?, message = ?, + result, err := tx.ExecContext(ctx, r.query(`UPDATE tasks SET operator_id = ?, status = 'running', + stage = CASE task_type WHEN 'certificate_deploy' THEN 'certificate_deploy' ELSE 'certificate_issue' END, + message = CASE task_type WHEN 'certificate_deploy' THEN '正在重试证书部署' ELSE '正在重试证书申请' END, started_at = ?, finished_at = NULL, updated_at = ? - WHERE id = ? AND task_type = 'certificate_issue' AND status IN ('failed', 'cancelled') AND `+predicate), updateArgs...) + WHERE id = ? AND task_type IN ('certificate_issue', 'certificate_deploy') + AND (status IN ('failed', 'cancelled') OR (task_type = 'certificate_deploy' AND status = 'partial_success')) + AND `+predicate), updateArgs...) if err != nil { rollback() return Task{}, TaskAttempt{}, fmt.Errorf("更新重试任务状态失败：%w", err) @@ -266,25 +270,37 @@ func (r *SQLTaskRepository) RetryVisibleTask(ctx context.Context, actorID, taskI if getErr != nil { return Task{}, TaskAttempt{}, getErr } - if item.Status != "failed" && item.Status != "cancelled" { + if !retryableTaskStatus(item) { return Task{}, TaskAttempt{}, ErrTaskNotRetryable } return Task{}, TaskAttempt{}, ErrProjectAccessDenied } + task, err := scanTask(tx.QueryRowContext(ctx, r.query(taskSelect+` WHERE id = ?`), taskID)) + if err != nil { + rollback() + return Task{}, TaskAttempt{}, err + } + stage := "certificate_issue" + message := "开始重试证书申请" + if task.TaskType == "certificate_deploy" { + stage = "certificate_deploy" + message = "开始重试证书部署" + } + var attemptNumber int if err := tx.QueryRowContext(ctx, r.query(`SELECT COALESCE(MAX(attempt_number), 0) + 1 FROM task_attempts WHERE task_id = ?`), taskID).Scan(&attemptNumber); err != nil { rollback() return Task{}, TaskAttempt{}, fmt.Errorf("读取任务重试次数失败：%w", err) } attempt := TaskAttempt{ - HistoryScope: HistoryScope{ScopeType: ScopeProject}, + HistoryScope: task.HistoryScope, ID: domain.NewID(), TaskID: taskID, AttemptNumber: attemptNumber, - Stage: "certificate_issue", + Stage: stage, Status: "running", - Message: "开始重试证书申请", + Message: message, StartedAt: startedAt, CreatedAt: startedAt.UTC(), } @@ -305,14 +321,15 @@ func (r *SQLTaskRepository) RetryVisibleTask(ctx context.Context, actorID, taskI return Task{}, TaskAttempt{}, ErrTaskNotRetryable } - task, err := scanTask(tx.QueryRowContext(ctx, r.query(taskSelect+` WHERE id = ?`), taskID)) - if err != nil { - rollback() - return Task{}, TaskAttempt{}, err - } - attempt.HistoryScope = task.HistoryScope if err := tx.Commit(); err != nil { return Task{}, TaskAttempt{}, fmt.Errorf("提交重试任务事务失败：%w", err) } return task, attempt, nil } + +func retryableTaskStatus(item Task) bool { + if item.Status == "failed" || item.Status == "cancelled" { + return item.TaskType == "certificate_issue" || item.TaskType == "certificate_deploy" + } + return item.TaskType == "certificate_deploy" && item.Status == "partial_success" +} diff --git a/frontend/src/views/Dashboard/Dashboard.test.js b/frontend/src/views/Dashboard/Dashboard.test.js index 553ecef..632c2e6 100644 --- a/frontend/src/views/Dashboard/Dashboard.test.js +++ b/frontend/src/views/Dashboard/Dashboard.test.js @@ -1096,7 +1096,7 @@ test("workflowTasks.renderOnlySupportedActions", async () => { assert.deepEqual(compiled.errors, []); const render = new Function("Vue", compiled.code)(Vue); for (const taskType of ["certificate_issue", "renewal_run", "certificate_deploy", "management_renewal", "unknown"]) { - for (const status of ["queued", "running", "success", "failed", "cancelled"]) { + for (const status of ["queued", "running", "success", "failed", "cancelled", "partial_success"]) { const html = await renderToString(Vue.createSSRApp({ render, setup: () => ({ @@ -1112,8 +1112,8 @@ test("workflowTasks.renderOnlySupportedActions", async () => { retryTask: () => {}, }), })); - const canCancel = taskType === "certificate_issue" && ["queued", "running"].includes(status); - const canRetry = taskType === "certificate_issue" && ["failed", "cancelled"].includes(status); + const canCancel = ["certificate_issue", "certificate_deploy"].includes(taskType) && ["queued", "running"].includes(status); + const canRetry = (taskType === "certificate_issue" && ["failed", "cancelled"].includes(status)) || (taskType === "certificate_deploy" && ["failed", "cancelled", "partial_success"].includes(status)); const buttonCount = (html.match(/