衔接异步任务前后端并记录自动回归进度,竞争复审待完成

This commit is contained in:
xuans committed 2026-10-02 16:47:30 +08:00
1 parent 13872ea3b1
commit 9e666be522
18 files changed
+767 -104

No files matched your search

+2 -2
View File
@@ -219,7 +219,7 @@ func (s *Server) registerRoutes() {
domainAPI.POST("/domains", s.dnsOperation, s.createDomain)
domainAPI.PUT("/domains/:id", s.dnsOperation, s.updateDomain)
domainAPI.DELETE("/domains/:id", s.deleteDomain)
domainAPI.POST("/domains/:id/issue", s.longOperation(certificateTaskTimeout), s.issueDomain)
domainAPI.POST("/domains/:id/issue", s.issueDomain)
api := s.router.Group("/api/v1", privateProjectResponse, s.requireAuthenticated)
api.GET("/tasks", s.listTasks)
api.GET("/tasks/:id", s.getTask)
@@ -237,7 +237,7 @@ func (s *Server) registerRoutes() {
adminAPI.POST("/renewal/run", s.requireSuperAdmin, s.longOperation(certificateTaskTimeout), s.runRenewalNow)
domainAPI.POST("/domains/:id/certificate/import", s.longOperation(certificateTaskTimeout), s.importCertificateFiles)
domainAPI.POST("/domains/:id/certificate/validate", s.longOperation(certificateTaskTimeout), s.validateCertificate)
domainAPI.POST("/domains/:id/deploy", s.longOperation(certificateTaskTimeout), s.deployCertificateCurrent)
domainAPI.POST("/domains/:id/deploy", s.deployCertificateCurrent)
domainAPI.POST("/domains/:id/certificate/versions/:version/restore", s.longOperation(certificateTaskTimeout), s.restoreCertificateVersion)
api.POST("/servers", s.createServer)
api.PUT("/servers/:id", s.updateServer)
+24 -20
View File
@@ -170,19 +170,20 @@ func (s *Server) runCertificateTask(ctx context.Context, task repository.Task, a
})
now := time.Now().UTC()
if err := executionCtx.Err(); err != nil {
s.finishTaskBeforeExecution(task, attempt, "cancelled", "任务在开始前已停止")
s.finishTaskContext(task, attempt, err)
return
}
if err := s.tasks.UpdateTask(executionCtx, task.ID, "running", stage, runningMessage, &now, nil); err != nil {
finished := time.Now().UTC()
failureMessage := "任务启动失败"
if s.finishCancelledTask(task, attempt, executionCtx) {
return
}
_ = s.tasks.UpdateTask(context.Background(), task.ID, "failed", stage, failureMessage, &now, &finished)
_ = s.tasks.FinishAttempt(context.Background(), attempt.ID, "failed", failureMessage, &finished)
attempt.Stage = stage
s.finishTaskBeforeExecution(task, attempt, "failed", failureMessage)
if s.audit != nil {
_ = s.audit.CreateAuditLog(context.Background(), repository.AuditLog{
auditCtx, auditCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer auditCancel()
_ = s.audit.CreateAuditLog(auditCtx, repository.AuditLog{
TaskID: task.ID, OperatorID: task.OperatorID,
Action: auditAction, ResourceType: "domain", ResourceID: task.DomainID,
Status: "error", Message: failureMessage,
@@ -196,13 +197,12 @@ func (s *Server) runCertificateTask(ctx context.Context, task repository.Task, a
Action: auditAction, ResourceType: "domain", ResourceID: task.DomainID,
Status: "running", Message: runningMessage,
}); err != nil {
finished := time.Now().UTC()
failureMessage := "任务启动审计记录失败"
if s.finishCancelledTask(task, attempt, executionCtx) {
return
}
_ = s.tasks.UpdateTask(context.Background(), task.ID, "failed", "record", failureMessage, &now, &finished)
_ = s.tasks.FinishAttempt(context.Background(), attempt.ID, "failed", failureMessage, &finished)
attempt.Stage = "record"
s.finishTaskBeforeExecution(task, attempt, "failed", failureMessage)
return
}
}
@@ -248,15 +248,6 @@ func (s *Server) runCertificateTask(ctx context.Context, task repository.Task, a
issueErr = errors.New(message)
}
}
if err := s.tasks.FinishAttempt(finishCtx, attempt.ID, status, message, &finished); err != nil {
status, message = "failed", "任务尝试结果保存失败,请检查任务恢复状态"
}
task.Status = status
task.Stage = stage
task.Message = message
task.StartedAt = &now
task.FinishedAt = &finished
task.UpdatedAt = finished
if s.audit != nil {
auditStatus := status
if issueErr != nil && status != "cancelled" && status != "partial_success" {
@@ -269,15 +260,28 @@ func (s *Server) runCertificateTask(ctx context.Context, task repository.Task, a
status, message = "failed", "任务完成审计保存失败,请检查任务恢复状态"
}
}
if err := s.tasks.FinishAttempt(finishCtx, attempt.ID, status, message, &finished); err != nil {
status, message = "failed", "任务尝试结果保存失败,请检查任务恢复状态"
}
if err := s.tasks.UpdateTask(finishCtx, task.ID, status, stage, message, &now, &finished); err != nil && s.logger != nil {
s.logger.Error("保存任务终态失败", "task_id", task.ID, "error", err)
}
}
func (s *Server) finishCancelledTask(task repository.Task, attempt repository.TaskAttempt, ctx context.Context) bool {
persisted, err := s.tasks.GetTask(context.Background(), task.ID)
if (err == nil && (persisted.Status == "cancelled" || persisted.Status == "cancelling")) || errors.Is(ctx.Err(), context.Canceled) {
s.finishTaskBeforeExecution(task, attempt, "cancelled", "任务已取消,未继续执行")
if ctx.Err() != nil {
s.finishTaskContext(task, attempt, ctx.Err())
return true
}
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
persisted, err := s.tasks.GetTask(checkCtx, task.ID)
if err != nil {
s.finishTaskBeforeExecution(task, attempt, "failed", "无法确认任务状态,未执行外部操作")
return true
}
if persisted.Status == "cancelled" || persisted.Status == "cancelling" {
s.finishTaskContext(task, attempt, context.Canceled)
return true
}
return false
@@ -28,9 +28,10 @@ func TestTaskDeploymentResultsIssueAndRead(t *testing.T) {
defer closeDB()
response := httptest.NewRecorder()
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
if response.Code != http.StatusMultiStatus {
t.Fatalf("部分部署失败响应错误:%d %s", response.Code, response.Body.String())
if response.Code != http.StatusAccepted {
t.Fatalf("任务提交响应错误:%d %s", response.Code, response.Body.String())
}
waitTaskWorkersForTest(t, server)
items, err := tasks.ListTasks(t.Context(), "", 10)
if err != nil || len(items) != 1 || items[0].Status != "partial_success" {
t.Fatalf("部分成功任务记录错误:%+v %v", items, err)
@@ -86,9 +87,10 @@ func TestTaskDeploymentResultsKeepAllowlistedLockDiagnosis(t *testing.T) {
defer closeDB()
response := httptest.NewRecorder()
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
if response.Code != http.StatusBadGateway || strings.Contains(response.Body.String(), "secret-path") || !strings.Contains(response.Body.String(), "lock_unavailable") {
t.Fatalf("部署接口缺少安全的锁诊断:%d %s", response.Code, response.Body.String())
if response.Code != http.StatusAccepted || strings.Contains(response.Body.String(), "secret-path") {
t.Fatalf("任务提交失败或泄露诊断原文:%d %s", response.Code, response.Body.String())
}
waitTaskWorkersForTest(t, server)
items, err := tasks.ListTasks(t.Context(), "", 10)
if err != nil || len(items) != 1 {
t.Fatalf("任务未保存:%+v %v", items, err)
@@ -116,21 +118,19 @@ func TestTaskDeploymentResultsHTTPSFailurePreservesTransferEvidence(t *testing.T
defer closeDB()
response := httptest.NewRecorder()
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
if response.Code != http.StatusBadGateway {
t.Fatalf("HTTPS 校验失败应返回失败响应:%d %s", response.Code, response.Body.String())
if response.Code != http.StatusAccepted {
t.Fatalf("任务提交失败:%d %s", response.Code, response.Body.String())
}
var payload struct {
Deployment deployment.Report `json:"deployment"`
TaskStatus string `json:"task_status"`
}
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
waitTaskWorkersForTest(t, server)
location := response.Header().Get("Location")
response = httptest.NewRecorder()
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodGet, location, nil))
var task repository.Task
if err := json.Unmarshal(response.Body.Bytes(), &task); err != nil {
t.Fatal(err)
}
if payload.TaskStatus != "failed" || payload.Deployment.Status != "failed" || payload.Deployment.VerificationStatus != "failed" {
t.Fatalf("接口未保留整体失败与站点校验失败:%+v", payload)
}
if payload.Deployment.Success != 1 || payload.Deployment.Results[0].Status != "success" || strings.Contains(response.Body.String(), "/secret-path") {
t.Fatalf("接口丢失真实推送结果或泄露底层错误:%s", response.Body.String())
if response.Code != http.StatusOK || task.Status != "failed" || strings.Contains(response.Body.String(), "/secret-path") {
t.Fatalf("状态查询未保留整体失败或泄露底层错误:%d %s", response.Code, response.Body.String())
}
items, err := tasks.ListTasks(t.Context(), "", 10)
if err != nil || len(items) != 1 || items[0].Status != "failed" {
+10 -1
View File
@@ -2,6 +2,7 @@ package httpapi
import (
"context"
"errors"
"net/http"
"time"
@@ -47,7 +48,7 @@ func (s *Server) enqueueCertificateTask(c *gin.Context, task repository.Task, at
case s.taskSlots <- struct{}{}:
defer func() { <-s.taskSlots }()
case <-ctx.Done():
s.finishTaskBeforeExecution(task, attempt, "cancelled", "任务在开始前已停止")
s.finishTaskContext(task, attempt, ctx.Err())
return
}
s.runCertificateTask(ctx, task, attempt, action, operation, stage, runningMessage, successMessage)
@@ -56,6 +57,14 @@ func (s *Server) enqueueCertificateTask(c *gin.Context, task repository.Task, at
c.JSON(http.StatusAccepted, gin.H{"task": task, "accepted": true})
}
func (s *Server) finishTaskContext(task repository.Task, attempt repository.TaskAttempt, err error) {
if errors.Is(err, context.DeadlineExceeded) {
s.finishTaskBeforeExecution(task, attempt, "failed", "任务执行超时,未继续执行")
return
}
s.finishTaskBeforeExecution(task, attempt, "cancelled", "任务已取消,未继续执行")
}
func (s *Server) finishTaskBeforeExecution(task repository.Task, attempt repository.TaskAttempt, status, message string) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
@@ -6,6 +6,8 @@ import (
"errors"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
@@ -15,6 +17,128 @@ import (
"git.pcux.cn/xuans/Scert/backend/internal/repository"
)
func TestAsyncQueueOverloadClosesPersistedTask(t *testing.T) {
server, tasks, _, cleanup := newTaskAPITestServer(t, func(context.Context, string) (certificate.IssueResult, deployment.Report, error) {
t.Error("队列已满时不能执行外部操作")
return certificate.IssueResult{}, deployment.Report{}, nil
})
defer cleanup()
server.asyncTasks = make(map[string]bool)
for i := 0; i < 32; i++ {
server.asyncTasks[strconv.Itoa(i)] = true
}
response := httptest.NewRecorder()
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
if response.Code != http.StatusServiceUnavailable {
t.Fatalf("队列过载未拒绝:%d %s", response.Code, response.Body.String())
}
items, err := tasks.ListTasks(t.Context(), "", 10)
if err != nil || len(items) != 1 || items[0].Status != "failed" {
t.Fatalf("过载后遗留活动任务:%+v %v", items, err)
}
attempts, err := tasks.ListAttempts(t.Context(), items[0].ID)
if err != nil || len(attempts) != 1 || attempts[0].Status != "failed" {
t.Fatalf("过载后尝试未关闭:%+v %v", attempts, err)
}
}
func TestAsyncQueuedCancellationNeverStartsOperation(t *testing.T) {
server, tasks, _, cleanup := newTaskAPITestServer(t, func(context.Context, string) (certificate.IssueResult, deployment.Report, error) {
t.Error("排队期间取消的任务不能执行外部操作")
return certificate.IssueResult{}, deployment.Report{}, nil
})
defer cleanup()
server.asyncTasks = make(map[string]bool)
server.taskSlots = make(chan struct{}, 4)
for i := 0; i < 4; i++ {
server.taskSlots <- struct{}{}
}
response := httptest.NewRecorder()
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
task := submittedTask(t, response)
response = httptest.NewRecorder()
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodDelete, "/api/v1/tasks/"+task.ID, nil))
if response.Code != http.StatusAccepted {
t.Fatalf("排队取消未接收:%d %s", response.Code, response.Body.String())
}
waitTaskWorkersForTest(t, server)
stored, err := tasks.GetTask(t.Context(), task.ID)
if err != nil || stored.Status != "cancelled" {
t.Fatalf("排队取消终态错误:%+v %v", stored, err)
}
}
func TestAsyncPanicClosesTaskWithoutExposingPanic(t *testing.T) {
server, tasks, _, cleanup := newTaskAPITestServer(t, func(context.Context, string) (certificate.IssueResult, deployment.Report, error) {
panic("private-panic-secret")
})
defer cleanup()
response := httptest.NewRecorder()
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
task := submittedTask(t, response)
waitTaskWorkersForTest(t, server)
stored, err := tasks.GetTask(t.Context(), task.ID)
if err != nil || stored.Status != "failed" || strings.Contains(stored.Message, "private-panic-secret") {
t.Fatalf("异常任务未安全关闭:%+v %v", stored, err)
}
attempts, err := tasks.ListAttempts(t.Context(), task.ID)
if err != nil || len(attempts) != 1 || attempts[0].Status != "failed" || strings.Contains(attempts[0].Message, "private-panic-secret") {
t.Fatalf("异常尝试未安全关闭:%+v %v", attempts, err)
}
}
func TestAsyncDeadlineBeforeExecutionIsFailureNotUserCancellation(t *testing.T) {
server, tasks, _, cleanup := newTaskAPITestServer(t, nil)
defer cleanup()
task := repository.Task{ID: "deadline-task", DomainID: "domain-1", TaskType: "certificate_issue", Status: "queued", Stage: "queued"}
attempt := repository.TaskAttempt{ID: "deadline-attempt", TaskID: task.ID, AttemptNumber: 1, Status: "queued", Stage: "queued"}
if err := tasks.CreateTask(t.Context(), task); err != nil {
t.Fatal(err)
}
if err := tasks.CreateAttempt(t.Context(), attempt); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithDeadline(t.Context(), time.Now().Add(-time.Second))
defer cancel()
server.runCertificateTask(ctx, task, attempt, "certificate_issue", func(context.Context, string) (certificate.IssueResult, deployment.Report, error) {
t.Error("超时任务不能执行外部操作")
return certificate.IssueResult{}, deployment.Report{}, nil
}, "issue", "执行中", "执行完成")
stored, err := tasks.GetTask(t.Context(), task.ID)
if err != nil || stored.Status != "failed" || !strings.Contains(stored.Message, "超时") {
t.Fatalf("超时被误记为用户取消:%+v %v", stored, err)
}
}
type failingTaskFinishAuditRepository struct{ repository.AuditRepository }
func (r failingTaskFinishAuditRepository) CreateAuditLog(ctx context.Context, item repository.AuditLog) error {
if item.Status == "success" {
return errors.New("模拟完成审计写入失败")
}
return r.AuditRepository.CreateAuditLog(ctx, item)
}
func TestAsyncFinishAuditFailureDoesNotLeaveSuccessfulAttempt(t *testing.T) {
server, tasks, audit, cleanup := newTaskAPITestServer(t, func(context.Context, string) (certificate.IssueResult, deployment.Report, error) {
return certificate.IssueResult{}, deployment.Report{Status: "success"}, nil
})
defer cleanup()
server.audit = failingTaskFinishAuditRepository{audit}
response := httptest.NewRecorder()
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
task := submittedTask(t, response)
waitTaskWorkersForTest(t, server)
stored, err := tasks.GetTask(t.Context(), task.ID)
if err != nil || stored.Status != "failed" {
t.Fatalf("完成审计失败却标记成功:%+v %v", stored, err)
}
attempts, err := tasks.ListAttempts(t.Context(), task.ID)
if err != nil || len(attempts) != 1 || attempts[0].Status != "failed" || attempts[0].Message != stored.Message {
t.Fatalf("任务与尝试终态不一致:%+v %v", attempts, err)
}
}
func submittedTask(t *testing.T, response *httptest.ResponseRecorder) repository.Task {
t.Helper()
if response.Code != http.StatusAccepted {