保存异步任务后端开发中基线,尚未完成回归
This commit is contained in:
parent
cd01aa3fc3
commit
13872ea3b1
@ -292,9 +292,11 @@ func (a *App) Close() error {
|
|||||||
<-a.managementDone
|
<-a.managementDone
|
||||||
}
|
}
|
||||||
if a.HTTP != nil {
|
if a.HTTP != nil {
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||||
if err := a.HTTP.Shutdown(ctx); err != nil && first == nil {
|
if err := a.HTTP.Shutdown(ctx); err != nil {
|
||||||
first = err
|
cancel()
|
||||||
|
// 后台任务可能仍在回滚;不得先关闭其数据库或日志。
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
cancel()
|
cancel()
|
||||||
}
|
}
|
||||||
|
|||||||
@ -136,6 +136,10 @@ func (s *Server) deployCertificateCurrent(c *gin.Context) {
|
|||||||
if !s.authorizeVisibleDomainPermission(c, domain.PermissionDeploymentExecute) {
|
if !s.authorizeVisibleDomainPermission(c, domain.PermissionDeploymentExecute) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if !taskRepositoryUnavailable(s.tasks) {
|
||||||
|
s.submitDomainTask(c, "certificate_deploy")
|
||||||
|
return
|
||||||
|
}
|
||||||
if s.deployCertificate == nil {
|
if s.deployCertificate == nil {
|
||||||
s.writeError(c, http.StatusNotImplemented, "证书部署接口尚未配置", errors.New("证书部署流程未配置"))
|
s.writeError(c, http.StatusNotImplemented, "证书部署接口尚未配置", errors.New("证书部署流程未配置"))
|
||||||
return
|
return
|
||||||
|
|||||||
@ -114,6 +114,10 @@ type Server struct {
|
|||||||
aliyunChecker aliyunScopeChecker
|
aliyunChecker aliyunScopeChecker
|
||||||
taskMu sync.Mutex
|
taskMu sync.Mutex
|
||||||
taskCancels map[string]context.CancelFunc
|
taskCancels map[string]context.CancelFunc
|
||||||
|
taskWorkers sync.WaitGroup
|
||||||
|
taskWorkersStopped bool
|
||||||
|
asyncTasks map[string]bool
|
||||||
|
taskSlots chan struct{}
|
||||||
issueTaskMu sync.Mutex
|
issueTaskMu sync.Mutex
|
||||||
setupMu sync.Mutex
|
setupMu sync.Mutex
|
||||||
resetMu sync.Mutex
|
resetMu sync.Mutex
|
||||||
@ -371,18 +375,20 @@ func isLoopbackHost(host string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) Shutdown(ctx context.Context) error {
|
func (s *Server) Shutdown(ctx context.Context) error {
|
||||||
|
s.stopTaskWorkers()
|
||||||
s.httpMu.Lock()
|
s.httpMu.Lock()
|
||||||
s.httpStopped = true
|
s.httpStopped = true
|
||||||
server := s.httpServer
|
server := s.httpServer
|
||||||
s.httpMu.Unlock()
|
s.httpMu.Unlock()
|
||||||
if server == nil {
|
if server == nil {
|
||||||
return nil
|
return s.waitTaskWorkers(ctx)
|
||||||
}
|
}
|
||||||
err := server.Shutdown(ctx)
|
err := server.Shutdown(ctx)
|
||||||
|
workerErr := s.waitTaskWorkers(ctx)
|
||||||
if errors.Is(err, http.ErrServerClosed) {
|
if errors.Is(err, http.ErrServerClosed) {
|
||||||
return nil
|
return workerErr
|
||||||
}
|
}
|
||||||
return err
|
return errors.Join(err, workerErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
type domainRequest struct {
|
type domainRequest struct {
|
||||||
|
|||||||
@ -19,7 +19,11 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func (s *Server) issueDomainAsTask(c *gin.Context) {
|
func (s *Server) issueDomainAsTask(c *gin.Context) {
|
||||||
if s.issueCertificate == nil {
|
s.submitDomainTask(c, "certificate_issue")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) submitDomainTask(c *gin.Context, taskType string) {
|
||||||
|
if (taskType == "certificate_issue" && s.issueCertificate == nil) || (taskType == "certificate_deploy" && s.deployCertificateTask == nil) {
|
||||||
s.writeError(c, http.StatusNotImplemented, "证书申请接口尚未配置", errors.New("证书申请工作流未配置"))
|
s.writeError(c, http.StatusNotImplemented, "证书申请接口尚未配置", errors.New("证书申请工作流未配置"))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -46,13 +50,15 @@ func (s *Server) issueDomainAsTask(c *gin.Context) {
|
|||||||
s.writeError(c, http.StatusServiceUnavailable, "证书任务权限服务尚未就绪", errors.New("项目授权服务未配置"))
|
s.writeError(c, http.StatusServiceUnavailable, "证书任务权限服务尚未就绪", errors.New("项目授权服务未配置"))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := authorizer.AuthorizeProject(c.Request.Context(), c.GetString(authenticatedActorKey), resource.ResourceScope.Normalize().ProjectID, domain.PermissionCertificateManage); err != nil {
|
if taskType == "certificate_issue" {
|
||||||
if errors.Is(err, repository.ErrProjectAccessDenied) {
|
if err := authorizer.AuthorizeProject(c.Request.Context(), c.GetString(authenticatedActorKey), resource.ResourceScope.Normalize().ProjectID, domain.PermissionCertificateManage); err != nil {
|
||||||
s.writeError(c, http.StatusForbidden, "无权管理此项目的证书任务", nil)
|
if errors.Is(err, repository.ErrProjectAccessDenied) {
|
||||||
|
s.writeError(c, http.StatusForbidden, "无权管理此项目的证书任务", nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.writeError(c, http.StatusInternalServerError, "检查证书任务权限失败", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.writeError(c, http.StatusInternalServerError, "检查证书任务权限失败", err)
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
if err := authorizer.AuthorizeProject(c.Request.Context(), c.GetString(authenticatedActorKey), resource.ResourceScope.Normalize().ProjectID, domain.PermissionDeploymentExecute); err != nil {
|
if err := authorizer.AuthorizeProject(c.Request.Context(), c.GetString(authenticatedActorKey), resource.ResourceScope.Normalize().ProjectID, domain.PermissionDeploymentExecute); err != nil {
|
||||||
if errors.Is(err, repository.ErrProjectAccessDenied) {
|
if errors.Is(err, repository.ErrProjectAccessDenied) {
|
||||||
@ -62,7 +68,7 @@ func (s *Server) issueDomainAsTask(c *gin.Context) {
|
|||||||
s.writeError(c, http.StatusInternalServerError, "检查证书部署权限失败", err)
|
s.writeError(c, http.StatusInternalServerError, "检查证书部署权限失败", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if !s.authorizeDNSCredentialUse(c, resource.ResourceScope.Normalize().ProjectID) {
|
if taskType == "certificate_issue" && !s.authorizeDNSCredentialUse(c, resource.ResourceScope.Normalize().ProjectID) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.issueTaskMu.Lock()
|
s.issueTaskMu.Lock()
|
||||||
@ -74,6 +80,11 @@ func (s *Server) issueDomainAsTask(c *gin.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if existing, err := s.tasks.FindTaskByIdempotencyKey(c.Request.Context(), domainID, idempotencyKey); err == nil {
|
if existing, err := s.tasks.FindTaskByIdempotencyKey(c.Request.Context(), domainID, idempotencyKey); err == nil {
|
||||||
|
if existing.TaskType != taskType {
|
||||||
|
s.issueTaskMu.Unlock()
|
||||||
|
s.writeError(c, http.StatusConflict, "幂等键已用于另一种操作", nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
status := http.StatusOK
|
status := http.StatusOK
|
||||||
if isActiveTaskStatus(existing.Status) {
|
if isActiveTaskStatus(existing.Status) {
|
||||||
status = http.StatusConflict
|
status = http.StatusConflict
|
||||||
@ -87,7 +98,7 @@ func (s *Server) issueDomainAsTask(c *gin.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if active, err := s.tasks.FindActiveTaskByDomain(c.Request.Context(), domainID, "certificate_issue"); err == nil {
|
if active, err := s.tasks.FindActiveTaskByDomain(c.Request.Context(), domainID, taskType); err == nil {
|
||||||
s.issueTaskMu.Unlock()
|
s.issueTaskMu.Unlock()
|
||||||
c.JSON(http.StatusConflict, gin.H{"error": "该域名已有正在执行的证书任务", "task": active})
|
c.JSON(http.StatusConflict, gin.H{"error": "该域名已有正在执行的证书任务", "task": active})
|
||||||
return
|
return
|
||||||
@ -100,12 +111,17 @@ func (s *Server) issueDomainAsTask(c *gin.Context) {
|
|||||||
task := repository.Task{
|
task := repository.Task{
|
||||||
HistoryScope: repository.HistoryScope{ScopeType: repository.ScopeProject, ProjectID: resource.ResourceScope.Normalize().ProjectID},
|
HistoryScope: repository.HistoryScope{ScopeType: repository.ScopeProject, ProjectID: resource.ResourceScope.Normalize().ProjectID},
|
||||||
OperatorID: s.currentOperatorID(c),
|
OperatorID: s.currentOperatorID(c),
|
||||||
ID: domain.NewID(), DomainID: domainID, TaskType: "certificate_issue", Status: "queued", Stage: "queued",
|
ID: domain.NewID(), DomainID: domainID, TaskType: taskType, Status: "queued", Stage: "queued",
|
||||||
Message: "等待执行证书申请", IdempotencyKey: idempotencyKey, CreatedAt: now, UpdatedAt: now,
|
Message: "等待执行证书任务", IdempotencyKey: idempotencyKey, CreatedAt: now, UpdatedAt: now,
|
||||||
}
|
}
|
||||||
if err := s.tasks.CreateTask(c.Request.Context(), task); err != nil {
|
if err := s.tasks.CreateTask(c.Request.Context(), task); err != nil {
|
||||||
if idempotencyKey != "" {
|
if idempotencyKey != "" {
|
||||||
if existing, lookupErr := s.tasks.FindTaskByIdempotencyKey(c.Request.Context(), domainID, idempotencyKey); lookupErr == nil {
|
if existing, lookupErr := s.tasks.FindTaskByIdempotencyKey(c.Request.Context(), domainID, idempotencyKey); lookupErr == nil {
|
||||||
|
if existing.TaskType != taskType {
|
||||||
|
s.issueTaskMu.Unlock()
|
||||||
|
s.writeError(c, http.StatusConflict, "幂等键已用于另一种操作", nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
status := http.StatusOK
|
status := http.StatusOK
|
||||||
if isActiveTaskStatus(existing.Status) {
|
if isActiveTaskStatus(existing.Status) {
|
||||||
status = http.StatusConflict
|
status = http.StatusConflict
|
||||||
@ -125,41 +141,42 @@ func (s *Server) issueDomainAsTask(c *gin.Context) {
|
|||||||
}
|
}
|
||||||
s.issueTaskMu.Unlock()
|
s.issueTaskMu.Unlock()
|
||||||
attempt := repository.TaskAttempt{
|
attempt := repository.TaskAttempt{
|
||||||
ID: domain.NewID(), TaskID: task.ID, AttemptNumber: 1, Stage: "certificate_issue", Status: "running",
|
ID: domain.NewID(), TaskID: task.ID, AttemptNumber: 1, Stage: taskType, Status: "running",
|
||||||
Message: "开始申请证书", StartedAt: &now, CreatedAt: now,
|
Message: "等待执行证书任务", StartedAt: &now, CreatedAt: now,
|
||||||
}
|
}
|
||||||
if err := s.tasks.CreateAttempt(c.Request.Context(), attempt); err != nil {
|
if err := s.tasks.CreateAttempt(c.Request.Context(), attempt); err != nil {
|
||||||
_ = s.tasks.UpdateTask(context.Background(), task.ID, "failed", "certificate_issue", "创建任务尝试失败", &now, &now)
|
_ = s.tasks.UpdateTask(context.Background(), task.ID, "failed", taskType, "创建任务尝试失败", &now, &now)
|
||||||
s.writeError(c, http.StatusInternalServerError, "创建任务尝试失败", err)
|
s.writeError(c, http.StatusInternalServerError, "创建任务尝试失败", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.executeCertificateIssueTask(c, task, attempt, "certificate_issue")
|
if taskType == "certificate_deploy" {
|
||||||
|
s.executeCertificateDeployTask(c, task, attempt, taskType)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.executeCertificateIssueTask(c, task, attempt, taskType)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) executeCertificateIssueTask(c *gin.Context, task repository.Task, attempt repository.TaskAttempt, auditAction string) {
|
func (s *Server) executeCertificateIssueTask(c *gin.Context, task repository.Task, attempt repository.TaskAttempt, auditAction string) {
|
||||||
s.executeCertificateTask(c, task, attempt, auditAction, s.issueCertificate, "certificate_issue", "正在申请证书并部署", "证书申请和部署完成")
|
s.enqueueCertificateTask(c, task, attempt, auditAction, s.issueCertificate, "certificate_issue", "正在申请证书并部署", "证书申请和部署完成")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) executeCertificateDeployTask(c *gin.Context, task repository.Task, attempt repository.TaskAttempt, auditAction string) {
|
func (s *Server) executeCertificateDeployTask(c *gin.Context, task repository.Task, attempt repository.TaskAttempt, auditAction string) {
|
||||||
s.executeCertificateTask(c, task, attempt, auditAction, s.deployCertificateTask, "certificate_deploy", "正在部署证书", "证书部署完成")
|
s.enqueueCertificateTask(c, task, attempt, auditAction, s.deployCertificateTask, "certificate_deploy", "正在部署证书", "证书部署完成")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) executeCertificateTask(c *gin.Context, task repository.Task, attempt repository.TaskAttempt, auditAction string, operation func(context.Context, string) (certificate.IssueResult, deployment.Report, error), stage, runningMessage, successMessage string) {
|
func (s *Server) runCertificateTask(ctx context.Context, task repository.Task, attempt repository.TaskAttempt, auditAction string, operation func(context.Context, string) (certificate.IssueResult, deployment.Report, error), stage, runningMessage, successMessage string) {
|
||||||
executionCtx, cancel := context.WithTimeout(c.Request.Context(), certificateTaskTimeout)
|
executionCtx := deployment.WithTransactionContext(ctx, deployment.TransactionContext{
|
||||||
executionCtx = deployment.WithTransactionContext(executionCtx, deployment.TransactionContext{
|
|
||||||
TaskID: task.ID, AttemptID: attempt.ID, AttemptNumber: attempt.AttemptNumber,
|
TaskID: task.ID, AttemptID: attempt.ID, AttemptNumber: attempt.AttemptNumber,
|
||||||
})
|
})
|
||||||
s.registerTaskCancel(task.ID, cancel)
|
|
||||||
defer func() {
|
|
||||||
cancel()
|
|
||||||
s.unregisterTaskCancel(task.ID)
|
|
||||||
}()
|
|
||||||
|
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
|
if err := executionCtx.Err(); err != nil {
|
||||||
|
s.finishTaskBeforeExecution(task, attempt, "cancelled", "任务在开始前已停止")
|
||||||
|
return
|
||||||
|
}
|
||||||
if err := s.tasks.UpdateTask(executionCtx, task.ID, "running", stage, runningMessage, &now, nil); err != nil {
|
if err := s.tasks.UpdateTask(executionCtx, task.ID, "running", stage, runningMessage, &now, nil); err != nil {
|
||||||
finished := time.Now().UTC()
|
finished := time.Now().UTC()
|
||||||
failureMessage := "任务启动失败"
|
failureMessage := "任务启动失败"
|
||||||
if s.writeCancelledTaskResponse(c, task.ID) {
|
if s.finishCancelledTask(task, attempt, executionCtx) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
_ = s.tasks.UpdateTask(context.Background(), task.ID, "failed", stage, failureMessage, &now, &finished)
|
_ = s.tasks.UpdateTask(context.Background(), task.ID, "failed", stage, failureMessage, &now, &finished)
|
||||||
@ -171,38 +188,40 @@ func (s *Server) executeCertificateTask(c *gin.Context, task repository.Task, at
|
|||||||
Status: "error", Message: failureMessage,
|
Status: "error", Message: failureMessage,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
s.writeError(c, http.StatusInternalServerError, "更新任务状态失败", err)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if s.audit != nil {
|
if s.audit != nil {
|
||||||
if err := s.audit.CreateAuditLog(executionCtx, repository.AuditLog{
|
if err := s.audit.CreateAuditLog(executionCtx, repository.AuditLog{
|
||||||
TaskID: task.ID, OperatorID: task.OperatorID,
|
TaskID: task.ID, OperatorID: task.OperatorID,
|
||||||
Action: auditAction, ResourceType: "domain", ResourceID: task.DomainID,
|
Action: auditAction, ResourceType: "domain", ResourceID: task.DomainID,
|
||||||
Status: "running", Message: "开始申请证书并部署",
|
Status: "running", Message: runningMessage,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
finished := time.Now().UTC()
|
finished := time.Now().UTC()
|
||||||
failureMessage := "任务启动审计记录失败"
|
failureMessage := "任务启动审计记录失败"
|
||||||
if s.writeCancelledTaskResponse(c, task.ID) {
|
if s.finishCancelledTask(task, attempt, executionCtx) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
_ = s.tasks.UpdateTask(context.Background(), task.ID, "failed", "record", failureMessage, &now, &finished)
|
_ = s.tasks.UpdateTask(context.Background(), task.ID, "failed", "record", failureMessage, &now, &finished)
|
||||||
_ = s.tasks.FinishAttempt(context.Background(), attempt.ID, "failed", failureMessage, &finished)
|
_ = s.tasks.FinishAttempt(context.Background(), attempt.ID, "failed", failureMessage, &finished)
|
||||||
s.writeError(c, http.StatusInternalServerError, failureMessage, err)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if s.finishCancelledTask(task, attempt, executionCtx) {
|
||||||
|
return
|
||||||
|
}
|
||||||
executionCtx = certificate.WithProgressReporter(executionCtx, func(progressCtx context.Context, progressStage, progressMessage string) error {
|
executionCtx = certificate.WithProgressReporter(executionCtx, func(progressCtx context.Context, progressStage, progressMessage string) error {
|
||||||
return s.tasks.UpdateTask(progressCtx, task.ID, "running", progressStage, progressMessage, &now, nil)
|
return s.tasks.UpdateTask(progressCtx, task.ID, "running", progressStage, progressMessage, &now, nil)
|
||||||
})
|
})
|
||||||
certificateResult, deploymentReport, issueErr := operation(executionCtx, task.DomainID)
|
_, deploymentReport, issueErr := operation(executionCtx, task.DomainID)
|
||||||
|
finishCtx, finishCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer finishCancel()
|
||||||
deploymentReport = sanitizeDeploymentReport(deploymentReport)
|
deploymentReport = sanitizeDeploymentReport(deploymentReport)
|
||||||
if recordErr := s.tasks.RecordTaskDeploymentResults(context.Background(), task.ID, attempt.ID, attempt.AttemptNumber, taskDeploymentResults(deploymentReport)); recordErr != nil {
|
if recordErr := s.tasks.RecordTaskDeploymentResults(finishCtx, task.ID, attempt.ID, attempt.AttemptNumber, taskDeploymentResults(deploymentReport)); recordErr != nil {
|
||||||
issueErr = errors.Join(issueErr, recordErr)
|
issueErr = errors.Join(issueErr, recordErr)
|
||||||
}
|
}
|
||||||
finished := time.Now().UTC()
|
finished := time.Now().UTC()
|
||||||
status := "success"
|
status := "success"
|
||||||
message := successMessage
|
message := successMessage
|
||||||
cancelled := false
|
|
||||||
partial := deploymentReport.Status == "partial"
|
partial := deploymentReport.Status == "partial"
|
||||||
if issueErr != nil {
|
if issueErr != nil {
|
||||||
status = "failed"
|
status = "failed"
|
||||||
@ -220,18 +239,18 @@ func (s *Server) executeCertificateTask(c *gin.Context, task repository.Task, at
|
|||||||
} else if partial {
|
} else if partial {
|
||||||
status = "partial_success"
|
status = "partial_success"
|
||||||
}
|
}
|
||||||
if persisted, err := s.tasks.GetTask(context.Background(), task.ID); err == nil && persisted.Status == "cancelled" {
|
s.taskMu.Lock()
|
||||||
|
defer s.taskMu.Unlock()
|
||||||
|
if persisted, err := s.tasks.GetTask(finishCtx, task.ID); err == nil && (persisted.Status == "cancelled" || persisted.Status == "cancelling") {
|
||||||
status = "cancelled"
|
status = "cancelled"
|
||||||
message = persisted.Message
|
message = persisted.Message
|
||||||
cancelled = true
|
|
||||||
if issueErr == nil {
|
if issueErr == nil {
|
||||||
issueErr = errors.New(message)
|
issueErr = errors.New(message)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !cancelled {
|
if err := s.tasks.FinishAttempt(finishCtx, attempt.ID, status, message, &finished); err != nil {
|
||||||
_ = s.tasks.UpdateTask(context.Background(), task.ID, status, stage, message, &now, &finished)
|
status, message = "failed", "任务尝试结果保存失败,请检查任务恢复状态"
|
||||||
}
|
}
|
||||||
_ = s.tasks.FinishAttempt(context.Background(), attempt.ID, status, message, &finished)
|
|
||||||
task.Status = status
|
task.Status = status
|
||||||
task.Stage = stage
|
task.Stage = stage
|
||||||
task.Message = message
|
task.Message = message
|
||||||
@ -243,46 +262,25 @@ func (s *Server) executeCertificateTask(c *gin.Context, task repository.Task, at
|
|||||||
if issueErr != nil && status != "cancelled" && status != "partial_success" {
|
if issueErr != nil && status != "cancelled" && status != "partial_success" {
|
||||||
auditStatus = "error"
|
auditStatus = "error"
|
||||||
}
|
}
|
||||||
_ = s.audit.CreateAuditLog(context.Background(), repository.AuditLog{
|
if err := s.audit.CreateAuditLog(finishCtx, repository.AuditLog{
|
||||||
TaskID: task.ID, OperatorID: task.OperatorID,
|
TaskID: task.ID, OperatorID: task.OperatorID,
|
||||||
Action: auditAction, ResourceType: "domain", ResourceID: task.DomainID, Status: auditStatus, Message: message,
|
Action: auditAction, ResourceType: "domain", ResourceID: task.DomainID, Status: auditStatus, Message: message,
|
||||||
})
|
}); err != nil {
|
||||||
}
|
status, message = "failed", "任务完成审计保存失败,请检查任务恢复状态"
|
||||||
response := gin.H{
|
|
||||||
"task": task, "certificate": certificateResult, "deployment": deploymentReport,
|
|
||||||
"task_status": status, "finished_at": finished,
|
|
||||||
}
|
|
||||||
if issueErr != nil {
|
|
||||||
response["error"] = message
|
|
||||||
responseStatus := http.StatusBadGateway
|
|
||||||
if status == "cancelled" {
|
|
||||||
responseStatus = http.StatusConflict
|
|
||||||
} else if status == "partial_success" {
|
|
||||||
responseStatus = http.StatusMultiStatus
|
|
||||||
} else if errors.Is(issueErr, context.DeadlineExceeded) {
|
|
||||||
responseStatus = http.StatusGatewayTimeout
|
|
||||||
}
|
}
|
||||||
c.JSON(responseStatus, response)
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
c.JSON(http.StatusOK, response)
|
if err := s.tasks.UpdateTask(finishCtx, task.ID, status, stage, message, &now, &finished); err != nil && s.logger != nil {
|
||||||
|
s.logger.Error("保存任务终态失败", "task_id", task.ID, "error", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) writeCancelledTaskResponse(c *gin.Context, taskID string) bool {
|
func (s *Server) finishCancelledTask(task repository.Task, attempt repository.TaskAttempt, ctx context.Context) bool {
|
||||||
persisted, err := s.tasks.GetTask(context.Background(), taskID)
|
persisted, err := s.tasks.GetTask(context.Background(), task.ID)
|
||||||
if err != nil || persisted.Status != "cancelled" {
|
if (err == nil && (persisted.Status == "cancelled" || persisted.Status == "cancelling")) || errors.Is(ctx.Err(), context.Canceled) {
|
||||||
return false
|
s.finishTaskBeforeExecution(task, attempt, "cancelled", "任务已取消,未继续执行")
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
message := persisted.Message
|
return false
|
||||||
if message == "" {
|
|
||||||
message = "任务已由用户取消"
|
|
||||||
}
|
|
||||||
c.JSON(http.StatusConflict, gin.H{
|
|
||||||
"task": persisted,
|
|
||||||
"task_status": persisted.Status,
|
|
||||||
"error": message,
|
|
||||||
})
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const certificateTaskTimeout = 30 * time.Minute
|
const certificateTaskTimeout = 30 * time.Minute
|
||||||
@ -300,6 +298,7 @@ func (s *Server) unregisterTaskCancel(taskID string) {
|
|||||||
s.taskMu.Lock()
|
s.taskMu.Lock()
|
||||||
defer s.taskMu.Unlock()
|
defer s.taskMu.Unlock()
|
||||||
delete(s.taskCancels, taskID)
|
delete(s.taskCancels, taskID)
|
||||||
|
delete(s.asyncTasks, taskID)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) taskWorkflowContext(c *gin.Context) context.Context {
|
func (s *Server) taskWorkflowContext(c *gin.Context) context.Context {
|
||||||
@ -365,6 +364,27 @@ func (s *Server) cancelTask(c *gin.Context) {
|
|||||||
if !s.authorizeProjectPermissions(c, item.ProjectID, domain.PermissionTaskCancel) {
|
if !s.authorizeProjectPermissions(c, item.ProjectID, domain.PermissionTaskCancel) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
s.taskMu.Lock()
|
||||||
|
if s.asyncTasks[taskID] {
|
||||||
|
if err := reader.RequestCancelVisibleTask(c.Request.Context(), c.GetString(authenticatedActorKey), taskID); err != nil {
|
||||||
|
s.taskMu.Unlock()
|
||||||
|
s.writeError(c, http.StatusConflict, "任务状态已变化,无法取消", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.taskCancels[taskID]()
|
||||||
|
s.taskMu.Unlock()
|
||||||
|
item.Status = "cancelling"
|
||||||
|
item.Message = "正在取消任务并等待部署收尾"
|
||||||
|
if s.audit != nil {
|
||||||
|
_ = s.audit.CreateAuditLog(c.Request.Context(), repository.AuditLog{
|
||||||
|
TaskID: taskID, OperatorID: s.currentOperatorID(c), Action: "task_cancel",
|
||||||
|
ResourceType: "task", ResourceID: taskID, Status: "success", Message: item.Message,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
c.JSON(http.StatusAccepted, gin.H{"task": item})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.taskMu.Unlock()
|
||||||
finished := time.Now().UTC()
|
finished := time.Now().UTC()
|
||||||
if err := reader.CancelVisibleTask(c.Request.Context(), c.GetString(authenticatedActorKey), taskID, "任务已由用户取消", &finished); err != nil {
|
if err := reader.CancelVisibleTask(c.Request.Context(), c.GetString(authenticatedActorKey), taskID, "任务已由用户取消", &finished); err != nil {
|
||||||
if errors.Is(err, repository.ErrProjectAccessDenied) || errors.Is(err, sql.ErrNoRows) {
|
if errors.Is(err, repository.ErrProjectAccessDenied) || errors.Is(err, sql.ErrNoRows) {
|
||||||
@ -409,6 +429,13 @@ func (s *Server) retryTask(c *gin.Context) {
|
|||||||
s.writeError(c, http.StatusInternalServerError, "读取任务失败", err)
|
s.writeError(c, http.StatusInternalServerError, "读取任务失败", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
s.taskMu.Lock()
|
||||||
|
workerActive := s.asyncTasks[taskID]
|
||||||
|
s.taskMu.Unlock()
|
||||||
|
if workerActive {
|
||||||
|
s.writeError(c, http.StatusConflict, "任务仍在执行或收尾,暂时不能重试", nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
if item.TaskType != "certificate_issue" && item.TaskType != "certificate_deploy" {
|
if item.TaskType != "certificate_issue" && item.TaskType != "certificate_deploy" {
|
||||||
c.JSON(http.StatusConflict, gin.H{"error": "当前任务类型不支持手动重试", "task": item})
|
c.JSON(http.StatusConflict, gin.H{"error": "当前任务类型不支持手动重试", "task": item})
|
||||||
return
|
return
|
||||||
|
|||||||
@ -61,7 +61,24 @@ func newTaskAPITestServer(t *testing.T, issue func(context.Context, string) (cer
|
|||||||
ProjectResources: repository.NewSQLProjectMemberRepository(db.DB, cfg.Database.Driver),
|
ProjectResources: repository.NewSQLProjectMemberRepository(db.DB, cfg.Database.Driver),
|
||||||
Users: repository.NewSQLUserRepository(db.DB, cfg.Database.Driver),
|
Users: repository.NewSQLUserRepository(db.DB, cfg.Database.Driver),
|
||||||
})
|
})
|
||||||
return server, taskRepo, auditRepo, func() { _ = db.Close() }
|
return server, taskRepo, auditRepo, func() {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := server.Shutdown(ctx); err != nil {
|
||||||
|
t.Errorf("测试任务未能停止:%v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = db.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitTaskWorkersForTest(t *testing.T, server *Server) {
|
||||||
|
t.Helper()
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := server.waitTaskWorkers(ctx); err != nil {
|
||||||
|
t.Fatalf("等待后台任务结束失败:%v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type blockingTaskAuditRepository struct {
|
type blockingTaskAuditRepository struct {
|
||||||
@ -94,9 +111,10 @@ func TestTaskAPIRecordsSuccessfulIssueAndAudit(t *testing.T) {
|
|||||||
|
|
||||||
response := httptest.NewRecorder()
|
response := httptest.NewRecorder()
|
||||||
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
||||||
if response.Code != http.StatusOK {
|
if response.Code != http.StatusAccepted {
|
||||||
t.Fatalf("成功任务响应错误:%d %s", response.Code, response.Body.String())
|
t.Fatalf("提交任务响应错误:%d %s", response.Code, response.Body.String())
|
||||||
}
|
}
|
||||||
|
waitTaskWorkersForTest(t, server)
|
||||||
items, err := taskRepo.ListTasks(context.Background(), "", 10)
|
items, err := taskRepo.ListTasks(context.Background(), "", 10)
|
||||||
if err != nil || len(items) != 1 || items[0].Status != "success" {
|
if err != nil || len(items) != 1 || items[0].Status != "success" {
|
||||||
t.Fatalf("任务没有进入成功状态:%+v %v", items, err)
|
t.Fatalf("任务没有进入成功状态:%+v %v", items, err)
|
||||||
@ -215,9 +233,10 @@ func TestTaskAPIClosesTaskWhenStartAuditFails(t *testing.T) {
|
|||||||
|
|
||||||
response := httptest.NewRecorder()
|
response := httptest.NewRecorder()
|
||||||
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
||||||
if response.Code != http.StatusInternalServerError || !strings.Contains(response.Body.String(), "任务启动审计记录失败") {
|
if response.Code != http.StatusAccepted {
|
||||||
t.Fatalf("任务启动审计失败响应错误:%d %s", response.Code, response.Body.String())
|
t.Fatalf("任务启动审计失败响应错误:%d %s", response.Code, response.Body.String())
|
||||||
}
|
}
|
||||||
|
waitTaskWorkersForTest(t, server)
|
||||||
items, err := taskRepo.ListTasks(context.Background(), "", 10)
|
items, err := taskRepo.ListTasks(context.Background(), "", 10)
|
||||||
if err != nil || len(items) != 1 || items[0].Status != "failed" || items[0].Stage != "record" {
|
if err != nil || len(items) != 1 || items[0].Status != "failed" || items[0].Stage != "record" {
|
||||||
t.Fatalf("任务启动审计失败后任务未收敛:%+v %v", items, err)
|
t.Fatalf("任务启动审计失败后任务未收敛:%+v %v", items, err)
|
||||||
@ -241,9 +260,10 @@ func TestTaskAPIClosesTaskWhenStartStateUpdateFails(t *testing.T) {
|
|||||||
|
|
||||||
response := httptest.NewRecorder()
|
response := httptest.NewRecorder()
|
||||||
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
||||||
if response.Code != http.StatusInternalServerError || !strings.Contains(response.Body.String(), "更新任务状态失败") {
|
if response.Code != http.StatusAccepted {
|
||||||
t.Fatalf("任务启动状态写入失败响应错误:%d %s", response.Code, response.Body.String())
|
t.Fatalf("任务启动状态写入失败响应错误:%d %s", response.Code, response.Body.String())
|
||||||
}
|
}
|
||||||
|
waitTaskWorkersForTest(t, server)
|
||||||
items, err := taskRepo.ListTasks(context.Background(), "", 10)
|
items, err := taskRepo.ListTasks(context.Background(), "", 10)
|
||||||
if err != nil || len(items) != 1 || items[0].Status != "failed" || items[0].Message != "任务启动失败" {
|
if err != nil || len(items) != 1 || items[0].Status != "failed" || items[0].Message != "任务启动失败" {
|
||||||
t.Fatalf("任务启动失败后未正确收敛任务状态:%+v %v", items, err)
|
t.Fatalf("任务启动失败后未正确收敛任务状态:%+v %v", items, err)
|
||||||
@ -269,30 +289,30 @@ func TestTaskAPIReturnsFailureAndKeepsTaskTrace(t *testing.T) {
|
|||||||
|
|
||||||
response := httptest.NewRecorder()
|
response := httptest.NewRecorder()
|
||||||
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
||||||
if response.Code != http.StatusBadGateway || !strings.Contains(response.Body.String(), "证书申请或部署失败") {
|
if response.Code != http.StatusAccepted {
|
||||||
t.Fatalf("失败任务响应错误:%d %s", response.Code, response.Body.String())
|
t.Fatalf("失败任务响应错误:%d %s", response.Code, response.Body.String())
|
||||||
}
|
}
|
||||||
|
waitTaskWorkersForTest(t, server)
|
||||||
|
location := response.Header().Get("Location")
|
||||||
|
if location == "" {
|
||||||
|
t.Fatal("任务提交未返回查询地址")
|
||||||
|
}
|
||||||
|
response = httptest.NewRecorder()
|
||||||
|
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodGet, location, nil))
|
||||||
|
if response.Code != http.StatusOK {
|
||||||
|
t.Fatalf("查询失败任务错误:%d %s", response.Code, response.Body.String())
|
||||||
|
}
|
||||||
if strings.Contains(response.Body.String(), "/etc/scert/secret/config.json") || strings.Contains(response.Body.String(), "/etc/nginx/private/key.pem") || strings.Contains(response.Body.String(), "permission denied") {
|
if strings.Contains(response.Body.String(), "/etc/scert/secret/config.json") || strings.Contains(response.Body.String(), "/etc/nginx/private/key.pem") || strings.Contains(response.Body.String(), "permission denied") {
|
||||||
t.Fatalf("失败任务响应泄露底层错误:%s", response.Body.String())
|
t.Fatalf("失败任务响应泄露底层错误:%s", response.Body.String())
|
||||||
}
|
}
|
||||||
var payload struct {
|
var payload repository.Task
|
||||||
Task struct {
|
|
||||||
Status string `json:"status"`
|
|
||||||
Stage string `json:"stage"`
|
|
||||||
Message string `json:"message"`
|
|
||||||
FinishedAt string `json:"finished_at"`
|
|
||||||
UpdatedAt string `json:"updated_at"`
|
|
||||||
} `json:"task"`
|
|
||||||
TaskStatus string `json:"task_status"`
|
|
||||||
FinishedAt string `json:"finished_at"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
||||||
t.Fatalf("失败任务响应不是有效 JSON:%v;响应:%s", err, response.Body.String())
|
t.Fatalf("失败任务响应不是有效 JSON:%v;响应:%s", err, response.Body.String())
|
||||||
}
|
}
|
||||||
if payload.Task.Status != "failed" || payload.Task.Stage != "certificate_issue" || payload.Task.Message != "证书申请或部署失败" {
|
if payload.Status != "failed" || payload.Stage != "certificate_issue" || payload.Message != "证书申请或部署失败" {
|
||||||
t.Fatalf("失败任务嵌套状态不一致:%+v", payload.Task)
|
t.Fatalf("失败任务状态不一致:%+v", payload)
|
||||||
}
|
}
|
||||||
if payload.TaskStatus != "failed" || payload.FinishedAt == "" || payload.Task.FinishedAt == "" || payload.Task.UpdatedAt == "" {
|
if payload.FinishedAt == nil || payload.UpdatedAt.IsZero() {
|
||||||
t.Fatalf("失败任务顶层状态或时间字段不完整:%+v", payload)
|
t.Fatalf("失败任务顶层状态或时间字段不完整:%+v", payload)
|
||||||
}
|
}
|
||||||
items, err := taskRepo.ListTasks(context.Background(), "", 10)
|
items, err := taskRepo.ListTasks(context.Background(), "", 10)
|
||||||
@ -359,7 +379,7 @@ func TestTaskAPICanCancelRunningIssue(t *testing.T) {
|
|||||||
}
|
}
|
||||||
cancelResponse := httptest.NewRecorder()
|
cancelResponse := httptest.NewRecorder()
|
||||||
serveAdminHTTP(server, cancelResponse, httptest.NewRequest(http.MethodDelete, "/api/v1/tasks/"+items[0].ID, nil))
|
serveAdminHTTP(server, cancelResponse, httptest.NewRequest(http.MethodDelete, "/api/v1/tasks/"+items[0].ID, nil))
|
||||||
if cancelResponse.Code != http.StatusOK || !strings.Contains(cancelResponse.Body.String(), "cancelled") {
|
if cancelResponse.Code != http.StatusAccepted || !strings.Contains(cancelResponse.Body.String(), "cancelling") {
|
||||||
t.Fatalf("取消任务响应错误:%d %s", cancelResponse.Code, cancelResponse.Body.String())
|
t.Fatalf("取消任务响应错误:%d %s", cancelResponse.Code, cancelResponse.Body.String())
|
||||||
}
|
}
|
||||||
select {
|
select {
|
||||||
@ -367,9 +387,10 @@ func TestTaskAPICanCancelRunningIssue(t *testing.T) {
|
|||||||
case <-time.After(2 * time.Second):
|
case <-time.After(2 * time.Second):
|
||||||
t.Fatal("取消后证书任务没有结束")
|
t.Fatal("取消后证书任务没有结束")
|
||||||
}
|
}
|
||||||
if issueResponse.Code != http.StatusConflict {
|
if issueResponse.Code != http.StatusAccepted {
|
||||||
t.Fatalf("取消任务的申请请求应返回 409:%d %s", issueResponse.Code, issueResponse.Body.String())
|
t.Fatalf("申请请求应先返回 202:%d %s", issueResponse.Code, issueResponse.Body.String())
|
||||||
}
|
}
|
||||||
|
waitTaskWorkersForTest(t, server)
|
||||||
item, err := taskRepo.GetTask(context.Background(), items[0].ID)
|
item, err := taskRepo.GetTask(context.Background(), items[0].ID)
|
||||||
if err != nil || item.Status != "cancelled" {
|
if err != nil || item.Status != "cancelled" {
|
||||||
t.Fatalf("任务取消状态没有持久化:%+v %v", item, err)
|
t.Fatalf("任务取消状态没有持久化:%+v %v", item, err)
|
||||||
@ -417,7 +438,7 @@ func TestTaskAPICancellationDuringStartupDoesNotExecuteCertificate(t *testing.T)
|
|||||||
}
|
}
|
||||||
cancelResponse := httptest.NewRecorder()
|
cancelResponse := httptest.NewRecorder()
|
||||||
serveAdminHTTP(server, cancelResponse, httptest.NewRequest(http.MethodDelete, "/api/v1/tasks/"+items[0].ID, nil))
|
serveAdminHTTP(server, cancelResponse, httptest.NewRequest(http.MethodDelete, "/api/v1/tasks/"+items[0].ID, nil))
|
||||||
if cancelResponse.Code != http.StatusOK {
|
if cancelResponse.Code != http.StatusAccepted {
|
||||||
t.Fatalf("启动阶段取消任务失败:%d %s", cancelResponse.Code, cancelResponse.Body.String())
|
t.Fatalf("启动阶段取消任务失败:%d %s", cancelResponse.Code, cancelResponse.Body.String())
|
||||||
}
|
}
|
||||||
close(blockingAudit.release)
|
close(blockingAudit.release)
|
||||||
@ -426,12 +447,13 @@ func TestTaskAPICancellationDuringStartupDoesNotExecuteCertificate(t *testing.T)
|
|||||||
case <-time.After(2 * time.Second):
|
case <-time.After(2 * time.Second):
|
||||||
t.Fatal("取消后的启动任务没有结束")
|
t.Fatal("取消后的启动任务没有结束")
|
||||||
}
|
}
|
||||||
|
waitTaskWorkersForTest(t, server)
|
||||||
select {
|
select {
|
||||||
case <-issueCalled:
|
case <-issueCalled:
|
||||||
t.Fatal("任务取消后不应再调用证书申请")
|
t.Fatal("任务取消后不应再调用证书申请")
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
if issueResponse.Code != http.StatusConflict {
|
if issueResponse.Code != http.StatusAccepted {
|
||||||
t.Fatalf("启动阶段取消响应错误:%d %s", issueResponse.Code, issueResponse.Body.String())
|
t.Fatalf("启动阶段取消响应错误:%d %s", issueResponse.Code, issueResponse.Body.String())
|
||||||
}
|
}
|
||||||
item, err := taskRepo.GetTask(context.Background(), items[0].ID)
|
item, err := taskRepo.GetTask(context.Background(), items[0].ID)
|
||||||
@ -456,9 +478,10 @@ func TestTaskAPIReplaysIdempotentIssue(t *testing.T) {
|
|||||||
firstRequest.Header.Set("Idempotency-Key", "issue-key-20260830")
|
firstRequest.Header.Set("Idempotency-Key", "issue-key-20260830")
|
||||||
firstResponse := httptest.NewRecorder()
|
firstResponse := httptest.NewRecorder()
|
||||||
serveAdminHTTP(server, firstResponse, firstRequest)
|
serveAdminHTTP(server, firstResponse, firstRequest)
|
||||||
if firstResponse.Code != http.StatusOK {
|
if firstResponse.Code != http.StatusAccepted {
|
||||||
t.Fatalf("首次幂等请求失败:%d %s", firstResponse.Code, firstResponse.Body.String())
|
t.Fatalf("首次幂等请求失败:%d %s", firstResponse.Code, firstResponse.Body.String())
|
||||||
}
|
}
|
||||||
|
waitTaskWorkersForTest(t, server)
|
||||||
secondRequest := httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil)
|
secondRequest := httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil)
|
||||||
secondRequest.Header.Set("Idempotency-Key", "issue-key-20260830")
|
secondRequest.Header.Set("Idempotency-Key", "issue-key-20260830")
|
||||||
secondResponse := httptest.NewRecorder()
|
secondResponse := httptest.NewRecorder()
|
||||||
@ -658,9 +681,10 @@ func TestTaskAPIRetriesFailedCertificateIssue(t *testing.T) {
|
|||||||
|
|
||||||
first := httptest.NewRecorder()
|
first := httptest.NewRecorder()
|
||||||
serveAdminHTTP(server, first, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
serveAdminHTTP(server, first, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
||||||
if first.Code != http.StatusBadGateway {
|
if first.Code != http.StatusAccepted {
|
||||||
t.Fatalf("initial failure response: %d %s", first.Code, first.Body.String())
|
t.Fatalf("initial failure response: %d %s", first.Code, first.Body.String())
|
||||||
}
|
}
|
||||||
|
waitTaskWorkersForTest(t, server)
|
||||||
tasks, err := taskRepo.ListTasks(context.Background(), "domain-1", 10)
|
tasks, err := taskRepo.ListTasks(context.Background(), "domain-1", 10)
|
||||||
if err != nil || len(tasks) != 1 || tasks[0].Status != "failed" {
|
if err != nil || len(tasks) != 1 || tasks[0].Status != "failed" {
|
||||||
t.Fatalf("failed task was not persisted: %+v %v", tasks, err)
|
t.Fatalf("failed task was not persisted: %+v %v", tasks, err)
|
||||||
@ -668,9 +692,10 @@ func TestTaskAPIRetriesFailedCertificateIssue(t *testing.T) {
|
|||||||
|
|
||||||
retry := httptest.NewRecorder()
|
retry := httptest.NewRecorder()
|
||||||
serveAdminHTTP(server, retry, httptest.NewRequest(http.MethodPost, "/api/v1/tasks/"+tasks[0].ID+"/retry", nil))
|
serveAdminHTTP(server, retry, httptest.NewRequest(http.MethodPost, "/api/v1/tasks/"+tasks[0].ID+"/retry", nil))
|
||||||
if retry.Code != http.StatusOK {
|
if retry.Code != http.StatusAccepted {
|
||||||
t.Fatalf("retry response: %d %s", retry.Code, retry.Body.String())
|
t.Fatalf("retry response: %d %s", retry.Code, retry.Body.String())
|
||||||
}
|
}
|
||||||
|
waitTaskWorkersForTest(t, server)
|
||||||
updated, err := taskRepo.GetTask(context.Background(), tasks[0].ID)
|
updated, err := taskRepo.GetTask(context.Background(), tasks[0].ID)
|
||||||
if err != nil || updated.Status != "success" || updated.OperatorID != "test-admin" {
|
if err != nil || updated.Status != "success" || updated.OperatorID != "test-admin" {
|
||||||
t.Fatalf("retried task did not converge: %+v %v", updated, err)
|
t.Fatalf("retried task did not converge: %+v %v", updated, err)
|
||||||
@ -699,9 +724,10 @@ func TestTaskAPIRecordsPartialDeploymentSuccess(t *testing.T) {
|
|||||||
|
|
||||||
response := httptest.NewRecorder()
|
response := httptest.NewRecorder()
|
||||||
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
||||||
if response.Code != http.StatusOK {
|
if response.Code != http.StatusAccepted {
|
||||||
t.Fatalf("partial response: %d %s", response.Code, response.Body.String())
|
t.Fatalf("部分成功任务提交响应错误:%d %s", response.Code, response.Body.String())
|
||||||
}
|
}
|
||||||
|
waitTaskWorkersForTest(t, server)
|
||||||
items, err := taskRepo.ListTasks(context.Background(), "domain-1", 10)
|
items, err := taskRepo.ListTasks(context.Background(), "domain-1", 10)
|
||||||
if err != nil || len(items) != 1 || items[0].Status != "partial_success" {
|
if err != nil || len(items) != 1 || items[0].Status != "partial_success" {
|
||||||
t.Fatalf("partial task status: %+v %v", items, err)
|
t.Fatalf("partial task status: %+v %v", items, err)
|
||||||
@ -726,9 +752,10 @@ func TestTaskAPIRecordsPartialDeploymentFailureAsPartialSuccess(t *testing.T) {
|
|||||||
|
|
||||||
response := httptest.NewRecorder()
|
response := httptest.NewRecorder()
|
||||||
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
||||||
if response.Code != http.StatusMultiStatus {
|
if response.Code != http.StatusAccepted {
|
||||||
t.Fatalf("partial failure response: %d %s", response.Code, response.Body.String())
|
t.Fatalf("partial failure response: %d %s", response.Code, response.Body.String())
|
||||||
}
|
}
|
||||||
|
waitTaskWorkersForTest(t, server)
|
||||||
items, err := taskRepo.ListTasks(context.Background(), "domain-1", 10)
|
items, err := taskRepo.ListTasks(context.Background(), "domain-1", 10)
|
||||||
if err != nil || len(items) != 1 || items[0].Status != "partial_success" {
|
if err != nil || len(items) != 1 || items[0].Status != "partial_success" {
|
||||||
t.Fatalf("partial failure task status: %+v %v", items, err)
|
t.Fatalf("partial failure task status: %+v %v", items, err)
|
||||||
@ -772,9 +799,10 @@ func TestTaskAPIRetriesFailedCertificateDeploymentWithoutDNSCredential(t *testin
|
|||||||
}
|
}
|
||||||
response := httptest.NewRecorder()
|
response := httptest.NewRecorder()
|
||||||
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/tasks/"+task.ID+"/retry", nil))
|
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/tasks/"+task.ID+"/retry", nil))
|
||||||
if response.Code != http.StatusOK {
|
if response.Code != http.StatusAccepted {
|
||||||
t.Fatalf("retry response: %d %s", response.Code, response.Body.String())
|
t.Fatalf("重试提交响应错误:%d %s", response.Code, response.Body.String())
|
||||||
}
|
}
|
||||||
|
waitTaskWorkersForTest(t, server)
|
||||||
if calls != 1 {
|
if calls != 1 {
|
||||||
t.Fatalf("deployment workflow calls: %d", calls)
|
t.Fatalf("deployment workflow calls: %d", calls)
|
||||||
}
|
}
|
||||||
@ -849,9 +877,10 @@ func TestTaskAPIRetriesPartialCertificateDeployment(t *testing.T) {
|
|||||||
}
|
}
|
||||||
response := httptest.NewRecorder()
|
response := httptest.NewRecorder()
|
||||||
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/tasks/"+task.ID+"/retry", nil))
|
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/tasks/"+task.ID+"/retry", nil))
|
||||||
if response.Code != http.StatusOK {
|
if response.Code != http.StatusAccepted {
|
||||||
t.Fatalf("partial retry response: %d %s", response.Code, response.Body.String())
|
t.Fatalf("部分成功重试提交响应错误:%d %s", response.Code, response.Body.String())
|
||||||
}
|
}
|
||||||
|
waitTaskWorkersForTest(t, server)
|
||||||
updated, err := taskRepo.GetTask(t.Context(), task.ID)
|
updated, err := taskRepo.GetTask(t.Context(), task.ID)
|
||||||
if err != nil || updated.Status != "success" {
|
if err != nil || updated.Status != "success" {
|
||||||
t.Fatalf("partial deployment retry did not converge: %+v %v", updated, err)
|
t.Fatalf("partial deployment retry did not converge: %+v %v", updated, err)
|
||||||
|
|||||||
92
backend/internal/httpapi/task_executor.go
Normal file
92
backend/internal/httpapi/task_executor.go
Normal file
@ -0,0 +1,92 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/gin-gonic/gin"
|
||||||
|
|
||||||
|
"git.pcux.cn/xuans/Scert/backend/internal/certificate"
|
||||||
|
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||||
|
"git.pcux.cn/xuans/Scert/backend/internal/repository"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 持久化任务与尝试后才调用此入口;后台执行不得持有请求或响应对象。
|
||||||
|
// 进程中断由启动恢复标记失败,不自动重放可能已经签发或部署的外部操作。
|
||||||
|
func (s *Server) enqueueCertificateTask(c *gin.Context, task repository.Task, attempt repository.TaskAttempt, action string, operation func(context.Context, string) (certificate.IssueResult, deployment.Report, error), stage, runningMessage, successMessage string) {
|
||||||
|
s.taskMu.Lock()
|
||||||
|
if s.taskWorkersStopped || len(s.asyncTasks) >= 32 {
|
||||||
|
s.taskMu.Unlock()
|
||||||
|
s.finishTaskBeforeExecution(task, attempt, "failed", "服务正在停止或任务队列已满,任务未执行")
|
||||||
|
s.writeError(c, http.StatusServiceUnavailable, "服务正在停止或任务队列已满,请稍后重试", nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), certificateTaskTimeout)
|
||||||
|
if s.taskCancels == nil {
|
||||||
|
s.taskCancels = make(map[string]context.CancelFunc)
|
||||||
|
}
|
||||||
|
if s.asyncTasks == nil {
|
||||||
|
s.asyncTasks = make(map[string]bool)
|
||||||
|
s.taskSlots = make(chan struct{}, 4)
|
||||||
|
}
|
||||||
|
s.taskCancels[task.ID] = cancel
|
||||||
|
s.asyncTasks[task.ID] = true
|
||||||
|
s.taskWorkers.Add(1)
|
||||||
|
s.taskMu.Unlock()
|
||||||
|
go func() {
|
||||||
|
defer s.taskWorkers.Done()
|
||||||
|
defer s.unregisterTaskCancel(task.ID)
|
||||||
|
defer cancel()
|
||||||
|
defer func() {
|
||||||
|
if recover() != nil {
|
||||||
|
s.finishTaskBeforeExecution(task, attempt, "failed", "任务执行异常中断,请检查部署恢复状态后重试")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case s.taskSlots <- struct{}{}:
|
||||||
|
defer func() { <-s.taskSlots }()
|
||||||
|
case <-ctx.Done():
|
||||||
|
s.finishTaskBeforeExecution(task, attempt, "cancelled", "任务在开始前已停止")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.runCertificateTask(ctx, task, attempt, action, operation, stage, runningMessage, successMessage)
|
||||||
|
}()
|
||||||
|
c.Header("Location", "/api/v1/tasks/"+task.ID)
|
||||||
|
c.JSON(http.StatusAccepted, gin.H{"task": task, "accepted": true})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) finishTaskBeforeExecution(task repository.Task, attempt repository.TaskAttempt, status, message string) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
finished := time.Now().UTC()
|
||||||
|
if err := s.tasks.FinishAttempt(ctx, attempt.ID, status, message, &finished); err != nil && s.logger != nil {
|
||||||
|
s.logger.Error("保存任务尝试失败", "task_id", task.ID, "error", err)
|
||||||
|
}
|
||||||
|
if err := s.tasks.UpdateTask(ctx, task.ID, status, attempt.Stage, message, nil, &finished); err != nil && s.logger != nil {
|
||||||
|
s.logger.Error("保存任务终态失败", "task_id", task.ID, "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) stopTaskWorkers() {
|
||||||
|
s.taskMu.Lock()
|
||||||
|
defer s.taskMu.Unlock()
|
||||||
|
s.taskWorkersStopped = true
|
||||||
|
for _, cancel := range s.taskCancels {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) waitTaskWorkers(ctx context.Context) error {
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
s.taskWorkers.Wait()
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
return nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
200
backend/internal/httpapi/task_executor_test.go
Normal file
200
backend/internal/httpapi/task_executor_test.go
Normal file
@ -0,0 +1,200 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.pcux.cn/xuans/Scert/backend/internal/certificate"
|
||||||
|
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||||
|
"git.pcux.cn/xuans/Scert/backend/internal/repository"
|
||||||
|
)
|
||||||
|
|
||||||
|
func submittedTask(t *testing.T, response *httptest.ResponseRecorder) repository.Task {
|
||||||
|
t.Helper()
|
||||||
|
if response.Code != http.StatusAccepted {
|
||||||
|
t.Fatalf("应快速返回已接收:%d %s", response.Code, response.Body.String())
|
||||||
|
}
|
||||||
|
var payload struct {
|
||||||
|
Task repository.Task `json:"task"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil || payload.Task.ID == "" {
|
||||||
|
t.Fatalf("任务编号缺失:%v %s", err, response.Body.String())
|
||||||
|
}
|
||||||
|
return payload.Task
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAsyncTaskSurvivesRequestDisconnect(t *testing.T) {
|
||||||
|
started, release := make(chan struct{}), make(chan struct{})
|
||||||
|
server, tasks, _, cleanup := newTaskAPITestServer(t, func(ctx context.Context, id string) (certificate.IssueResult, deployment.Report, error) {
|
||||||
|
close(started)
|
||||||
|
select {
|
||||||
|
case <-release:
|
||||||
|
return certificate.IssueResult{DomainID: id}, deployment.Report{Status: "success"}, nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return certificate.IssueResult{}, deployment.Report{}, ctx.Err()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
defer cleanup()
|
||||||
|
ctx, disconnect := context.WithCancel(t.Context())
|
||||||
|
defer disconnect()
|
||||||
|
response := httptest.NewRecorder()
|
||||||
|
request := httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil).WithContext(ctx)
|
||||||
|
request.Header.Set("Idempotency-Key", "disconnect-request")
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { serveAdminHTTP(server, response, request); close(done) }()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("请求仍等待工作流,未异步返回")
|
||||||
|
}
|
||||||
|
task := submittedTask(t, response)
|
||||||
|
if response.Header().Get("Location") != "/api/v1/tasks/"+task.ID {
|
||||||
|
t.Fatal("任务查询地址不正确")
|
||||||
|
}
|
||||||
|
disconnect()
|
||||||
|
select {
|
||||||
|
case <-started:
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("任务未开始")
|
||||||
|
}
|
||||||
|
stored, err := tasks.GetTask(t.Context(), task.ID)
|
||||||
|
if err != nil || stored.Status != "running" {
|
||||||
|
t.Fatalf("断线影响任务:%+v %v", stored, err)
|
||||||
|
}
|
||||||
|
duplicate := httptest.NewRecorder()
|
||||||
|
replay := httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil)
|
||||||
|
replay.Header.Set("Idempotency-Key", "disconnect-request")
|
||||||
|
serveAdminHTTP(server, duplicate, replay)
|
||||||
|
if duplicate.Code != http.StatusConflict {
|
||||||
|
t.Fatal("活动任务没有拒绝重复提交")
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
waitTaskWorkersForTest(t, server)
|
||||||
|
stored, err = tasks.GetTask(t.Context(), task.ID)
|
||||||
|
if err != nil || stored.Status != "success" {
|
||||||
|
t.Fatalf("后台任务未完成:%+v %v", stored, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAsyncCancellationBlocksRetryUntilCleanup(t *testing.T) {
|
||||||
|
started, cancelled, release := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||||
|
server, tasks, _, cleanup := newTaskAPITestServer(t, func(ctx context.Context, _ string) (certificate.IssueResult, deployment.Report, error) {
|
||||||
|
close(started)
|
||||||
|
<-ctx.Done()
|
||||||
|
close(cancelled)
|
||||||
|
<-release
|
||||||
|
return certificate.IssueResult{}, deployment.Report{}, ctx.Err()
|
||||||
|
})
|
||||||
|
defer cleanup()
|
||||||
|
defer close(release)
|
||||||
|
response := httptest.NewRecorder()
|
||||||
|
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
||||||
|
task := submittedTask(t, response)
|
||||||
|
select {
|
||||||
|
case <-started:
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("未开始执行")
|
||||||
|
}
|
||||||
|
response = httptest.NewRecorder()
|
||||||
|
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodDelete, "/api/v1/tasks/"+task.ID, nil))
|
||||||
|
if response.Code != http.StatusAccepted {
|
||||||
|
t.Fatal(response.Body.String())
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-cancelled:
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("取消信号未送达")
|
||||||
|
}
|
||||||
|
stored, err := tasks.GetTask(t.Context(), task.ID)
|
||||||
|
if err != nil || stored.Status != "cancelling" {
|
||||||
|
t.Fatalf("过早发布取消终态:%+v %v", stored, err)
|
||||||
|
}
|
||||||
|
if err := tasks.UpdateTask(t.Context(), task.ID, "running", "deployment", "迟到进度", nil, nil); err == nil {
|
||||||
|
t.Fatal("迟到进度覆盖取消状态")
|
||||||
|
}
|
||||||
|
response = httptest.NewRecorder()
|
||||||
|
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/tasks/"+task.ID+"/retry", nil))
|
||||||
|
if response.Code != http.StatusConflict {
|
||||||
|
t.Fatal("回滚期间允许了重试")
|
||||||
|
}
|
||||||
|
response = httptest.NewRecorder()
|
||||||
|
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
||||||
|
if response.Code != http.StatusConflict {
|
||||||
|
t.Fatal("回滚期间允许了新任务")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAsyncShutdownWaitsForCleanupAndRejectsNewWork(t *testing.T) {
|
||||||
|
started, release := make(chan struct{}), make(chan struct{})
|
||||||
|
server, tasks, _, cleanup := newTaskAPITestServer(t, func(ctx context.Context, _ string) (certificate.IssueResult, deployment.Report, error) {
|
||||||
|
close(started)
|
||||||
|
<-ctx.Done()
|
||||||
|
<-release
|
||||||
|
return certificate.IssueResult{}, deployment.Report{}, ctx.Err()
|
||||||
|
})
|
||||||
|
defer cleanup()
|
||||||
|
response := httptest.NewRecorder()
|
||||||
|
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
||||||
|
task := submittedTask(t, response)
|
||||||
|
select {
|
||||||
|
case <-started:
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
close(release)
|
||||||
|
t.Fatal("任务未开始")
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
if err := server.Shutdown(ctx); !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
close(release)
|
||||||
|
t.Fatalf("停服没有等待收尾:%v", err)
|
||||||
|
}
|
||||||
|
stored, err := tasks.GetTask(t.Context(), task.ID)
|
||||||
|
if err != nil || !isActiveTaskStatus(stored.Status) {
|
||||||
|
close(release)
|
||||||
|
t.Fatalf("收尾前终态错误:%+v %v", stored, err)
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
waitTaskWorkersForTest(t, server)
|
||||||
|
response = httptest.NewRecorder()
|
||||||
|
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
|
||||||
|
if response.Code != http.StatusServiceUnavailable {
|
||||||
|
t.Fatalf("停止后仍接收任务:%d", response.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAsyncDeploymentUsesExistingTaskAndRejectsCrossOperationKey(t *testing.T) {
|
||||||
|
var calls atomic.Int32
|
||||||
|
server, tasks, _, cleanup := newTaskAPITestServer(t, nil)
|
||||||
|
defer cleanup()
|
||||||
|
server.deployCertificateTask = func(ctx context.Context, id string) (certificate.IssueResult, deployment.Report, error) {
|
||||||
|
calls.Add(1)
|
||||||
|
return certificate.IssueResult{DomainID: id}, deployment.Report{Status: "success"}, nil
|
||||||
|
}
|
||||||
|
response := httptest.NewRecorder()
|
||||||
|
request := httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/deploy", nil)
|
||||||
|
request.Header.Set("Idempotency-Key", "shared-operation-key")
|
||||||
|
serveAdminHTTP(server, response, request)
|
||||||
|
task := submittedTask(t, response)
|
||||||
|
waitTaskWorkersForTest(t, server)
|
||||||
|
items, err := tasks.ListTasks(t.Context(), "domain-1", 10)
|
||||||
|
if err != nil || len(items) != 1 || items[0].ID != task.ID || items[0].Status != "success" || calls.Load() != 1 {
|
||||||
|
t.Fatalf("部署任务重复创建或未完成:%+v %v", items, err)
|
||||||
|
}
|
||||||
|
server.issueCertificate = func(context.Context, string) (certificate.IssueResult, deployment.Report, error) {
|
||||||
|
calls.Add(1)
|
||||||
|
return certificate.IssueResult{}, deployment.Report{}, nil
|
||||||
|
}
|
||||||
|
request = httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil)
|
||||||
|
request.Header.Set("Idempotency-Key", "shared-operation-key")
|
||||||
|
response = httptest.NewRecorder()
|
||||||
|
serveAdminHTTP(server, response, request)
|
||||||
|
if response.Code != http.StatusConflict || calls.Load() != 1 {
|
||||||
|
t.Fatal("跨操作复用了幂等键")
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -18,6 +18,7 @@ type ProjectTaskReader interface {
|
|||||||
ListVisibleTaskAttempts(context.Context, string, string) ([]TaskAttempt, error)
|
ListVisibleTaskAttempts(context.Context, string, string) ([]TaskAttempt, error)
|
||||||
ListVisibleTaskDeploymentResults(context.Context, string, string) ([]TaskDeploymentResult, error)
|
ListVisibleTaskDeploymentResults(context.Context, string, string) ([]TaskDeploymentResult, error)
|
||||||
CancelVisibleTask(context.Context, string, string, string, *time.Time) error
|
CancelVisibleTask(context.Context, string, string, string, *time.Time) error
|
||||||
|
RequestCancelVisibleTask(context.Context, string, string) error
|
||||||
RetryVisibleTask(context.Context, string, string, *time.Time) (Task, TaskAttempt, error)
|
RetryVisibleTask(context.Context, string, string, *time.Time) (Task, TaskAttempt, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -164,6 +165,27 @@ func (r *SQLTaskRepository) ListVisibleTaskDeploymentResults(ctx context.Context
|
|||||||
return items, nil
|
return items, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 请求取消仍保留活动状态,直到执行器完成回滚收尾,禁止提前重试。
|
||||||
|
func (r *SQLTaskRepository) RequestCancelVisibleTask(ctx context.Context, actorID, taskID string) error {
|
||||||
|
predicate, predicateArgs := taskVisibilityPredicate("tasks", domain.PermissionTaskCancel)
|
||||||
|
predicateArgs[0] = strings.TrimSpace(actorID)
|
||||||
|
args := []any{"正在取消任务并等待部署收尾", time.Now().UTC(), strings.TrimSpace(taskID)}
|
||||||
|
args = append(args, predicateArgs...)
|
||||||
|
result, err := r.db.ExecContext(ctx, r.query(`UPDATE tasks SET status = 'cancelling', message = ?, updated_at = ?
|
||||||
|
WHERE id = ? AND status IN ('queued', 'running', 'retry_wait', 'cancelling') AND `+predicate), args...)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("请求取消任务失败:%w", err)
|
||||||
|
}
|
||||||
|
count, err := result.RowsAffected()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if count != 1 {
|
||||||
|
return ErrProjectAccessDenied
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *SQLTaskRepository) CancelVisibleTask(ctx context.Context, actorID, taskID, message string, finishedAt *time.Time) error {
|
func (r *SQLTaskRepository) CancelVisibleTask(ctx context.Context, actorID, taskID, message string, finishedAt *time.Time) error {
|
||||||
if r == nil || r.db == nil {
|
if r == nil || r.db == nil {
|
||||||
return errors.New("数据库连接不能为空")
|
return errors.New("数据库连接不能为空")
|
||||||
|
|||||||
@ -264,8 +264,8 @@ func (r *SQLTaskRepository) UpdateTask(ctx context.Context, id, status, stage, m
|
|||||||
if !validTaskStatus(status) || stage == "" {
|
if !validTaskStatus(status) || stage == "" {
|
||||||
return errors.New("任务状态或阶段无效")
|
return errors.New("任务状态或阶段无效")
|
||||||
}
|
}
|
||||||
result, err := r.db.ExecContext(ctx, r.query(`UPDATE tasks SET status = ?, stage = ?, message = ?, started_at = ?, finished_at = ?, updated_at = ? WHERE id = ? AND status IN ('queued', 'running', 'retry_wait', 'cancelling')`),
|
result, err := r.db.ExecContext(ctx, r.query(`UPDATE tasks SET status = ?, stage = ?, message = ?, started_at = ?, finished_at = ?, updated_at = ? WHERE id = ? AND status IN ('queued', 'running', 'retry_wait', 'cancelling') AND NOT (status = 'cancelling' AND ? IN ('queued', 'running', 'retry_wait'))`),
|
||||||
status, stage, message, nullableTime(startedAt), nullableTime(finishedAt), time.Now().UTC(), id)
|
status, stage, message, nullableTime(startedAt), nullableTime(finishedAt), time.Now().UTC(), id, status)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("更新任务失败:%w", err)
|
return fmt.Errorf("更新任务失败:%w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user