统一后台证书任务报告及收尾事务并补齐故障回归

This commit is contained in:
xuans committed 2026-10-02 18:36:52 +08:00
1 parent cf3bc7aaa5
commit 657a9c0a36
9 files changed
+379 -67

No files matched your search

+58 -24
View File
@@ -13,15 +13,17 @@ import (
// TaskCompletion 同时关闭任务与对应尝试,避免只保存其中一方的终态。
type TaskCompletion struct {
TaskID string
AttemptID string
Status string
Stage string
Message string
StartedAt *time.Time
FinishedAt time.Time
Report *deployment.Report
AuditAction string
TaskID string
AttemptID string
Status string
Stage string
Message string
StartedAt *time.Time
FinishedAt time.Time
Report *deployment.Report
AuditAction string
Retry bool
WithoutAttempt bool
}
func (r *SQLTaskRepository) FinishTaskAttempt(ctx context.Context, completion TaskCompletion) error {
@@ -31,7 +33,7 @@ func (r *SQLTaskRepository) FinishTaskAttempt(ctx context.Context, completion Ta
completion.TaskID = strings.TrimSpace(completion.TaskID)
completion.AttemptID = strings.TrimSpace(completion.AttemptID)
completion.Stage = strings.TrimSpace(completion.Stage)
if completion.TaskID == "" || completion.AttemptID == "" || completion.Stage == "" || completion.FinishedAt.IsZero() {
if completion.TaskID == "" || (completion.AttemptID == "" && !completion.WithoutAttempt) || completion.Stage == "" || completion.FinishedAt.IsZero() {
return errors.New("任务终态记录不完整")
}
switch completion.Status {
@@ -39,17 +41,31 @@ func (r *SQLTaskRepository) FinishTaskAttempt(ctx context.Context, completion Ta
default:
return errors.New("任务终态无效")
}
if completion.WithoutAttempt && (completion.AttemptID != "" || completion.Report != nil || completion.Retry || (completion.Status != "failed" && completion.Status != "cancelled")) {
return errors.New("无活动尝试的收尾参数无效")
}
if completion.Retry && (completion.Status != "failed" || completion.AuditAction != "") {
return errors.New("重试只能保存失败尝试且不能发布完成审计")
}
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("开始任务收尾事务失败:%w", err)
}
defer tx.Rollback()
taskStatus, taskStage := completion.Status, completion.Stage
var taskFinished any = completion.FinishedAt.UTC()
activeStates := "('queued', 'running', 'retry_wait', 'cancelling')"
if completion.Retry {
taskStatus, taskStage, taskFinished = "retry_wait", "retry_wait", nil
// 已受理的取消不能被重试重新激活。
activeStates = "('running')"
}
// 与取消、重试保持任务在前、尝试在后的加锁顺序。旧尝试或跨任务尝试必须整体回滚。
result, err := tx.ExecContext(ctx, r.query(`UPDATE tasks
SET status = ?, stage = ?, message = ?, started_at = COALESCE(?, started_at), finished_at = ?, updated_at = ?
WHERE id = ? AND status IN ('queued', 'running', 'retry_wait', 'cancelling')`),
completion.Status, completion.Stage, completion.Message, nullableTime(completion.StartedAt),
completion.FinishedAt.UTC(), completion.FinishedAt.UTC(), completion.TaskID)
WHERE id = ? AND status IN `+activeStates),
taskStatus, taskStage, completion.Message, nullableTime(completion.StartedAt),
taskFinished, completion.FinishedAt.UTC(), completion.TaskID)
if err != nil {
return fmt.Errorf("保存任务终态失败:%w", err)
}
@@ -58,17 +74,8 @@ func (r *SQLTaskRepository) FinishTaskAttempt(ctx context.Context, completion Ta
} else if count != 1 {
return sql.ErrNoRows
}
result, err = tx.ExecContext(ctx, r.query(`UPDATE task_attempts
SET status = ?, stage = ?, message = ?, finished_at = ?
WHERE id = ? AND task_id = ? AND status IN ('queued', 'running', 'retry_wait', 'cancelling')`),
completion.Status, completion.Stage, completion.Message, completion.FinishedAt.UTC(), completion.AttemptID, completion.TaskID)
if err != nil {
return fmt.Errorf("保存任务尝试终态失败:%w", err)
}
if count, err := result.RowsAffected(); err != nil {
return fmt.Errorf("读取任务尝试收尾结果失败:%w", err)
} else if count != 1 {
return sql.ErrNoRows
if err := r.finishCompletionAttempt(ctx, tx, completion); err != nil {
return err
}
if completion.Report != nil {
if err := r.recordCompletionReport(ctx, tx, completion); err != nil {
@@ -85,3 +92,30 @@ func (r *SQLTaskRepository) FinishTaskAttempt(ctx context.Context, completion Ta
}
return nil
}
func (r *SQLTaskRepository) finishCompletionAttempt(ctx context.Context, tx *sql.Tx, completion TaskCompletion) error {
if completion.WithoutAttempt {
var active int
if err := tx.QueryRowContext(ctx, r.query(`SELECT COUNT(*) FROM task_attempts
WHERE task_id = ? AND status IN ('queued', 'running', 'retry_wait', 'cancelling')`), completion.TaskID).Scan(&active); err != nil {
return fmt.Errorf("检查活动尝试失败:%w", err)
}
if active != 0 {
return errors.New("任务仍有活动尝试,不能单独收尾")
}
return nil
}
result, err := tx.ExecContext(ctx, r.query(`UPDATE task_attempts
SET status = ?, stage = ?, message = ?, finished_at = ?
WHERE id = ? AND task_id = ? AND status IN ('queued', 'running', 'retry_wait', 'cancelling')`),
completion.Status, completion.Stage, completion.Message, completion.FinishedAt.UTC(), completion.AttemptID, completion.TaskID)
if err != nil {
return fmt.Errorf("保存任务尝试终态失败:%w", err)
}
if count, err := result.RowsAffected(); err != nil {
return fmt.Errorf("读取任务尝试收尾结果失败:%w", err)
} else if count != 1 {
return sql.ErrNoRows
}
return nil
}
@@ -0,0 +1,102 @@
package repository
import (
"testing"
"time"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
)
func TestTaskCompletionRetryKeepsTaskActiveAndPreservesReport(t *testing.T) {
repo, db := newTaskRepositoryTest(t)
defer db.Close()
completion := seedTaskCompletion(t, repo)
completion.Status, completion.Retry = "failed", true
completion.Report = &deployment.Report{Status: "failed", RecoveryStatus: "success"}
if err := repo.FinishTaskAttempt(t.Context(), completion); err != nil {
t.Fatal(err)
}
task, err := repo.GetTask(t.Context(), completion.TaskID)
if err != nil || task.Status != "retry_wait" || task.Stage != "retry_wait" || task.FinishedAt != nil {
t.Fatalf("重试不能提前关闭任务:%+v %v", task, err)
}
attempts, err := repo.ListAttempts(t.Context(), completion.TaskID)
if err != nil || len(attempts) != 1 || attempts[0].Status != "failed" || attempts[0].FinishedAt == nil {
t.Fatalf("失败尝试未收尾:%+v %v", attempts, err)
}
reports, err := repo.ListTaskDeploymentReports(t.Context(), completion.TaskID)
if err != nil || len(reports) != 1 || reports[0].Report.RecoveryStatus != "success" {
t.Fatalf("失败尝试报告未保存:%+v %v", reports, err)
}
// 重试等待期间取消:只关闭任务和审计,不覆盖已完成的尝试及报告。
completion.AttemptID, completion.Status = "", "cancelled"
completion.Retry, completion.WithoutAttempt = false, true
completion.Report, completion.AuditAction = nil, "certificate_deploy"
if err := repo.FinishTaskAttempt(t.Context(), completion); err != nil {
t.Fatal(err)
}
task, err = repo.GetTask(t.Context(), completion.TaskID)
if err != nil || task.Status != "cancelled" || task.FinishedAt == nil {
t.Fatalf("等待取消未收尾:%+v %v", task, err)
}
attempts, err = repo.ListAttempts(t.Context(), completion.TaskID)
if err != nil || len(attempts) != 1 || attempts[0].Status != "failed" {
t.Fatalf("等待取消覆盖历史尝试:%+v %v", attempts, err)
}
}
func TestTaskCompletionRejectsUnsafeRetryAndMissingAttempt(t *testing.T) {
for _, scenario := range []string{"active_attempt", "retry_cancelled", "retry_success", "retry_audit", "no_attempt_success", "no_attempt_report"} {
t.Run(scenario, func(t *testing.T) {
repo, db := newTaskRepositoryTest(t)
defer db.Close()
completion := seedTaskCompletion(t, repo)
completion.Status = "failed"
wantStatus := "running"
switch scenario {
case "active_attempt":
completion.AttemptID, completion.WithoutAttempt = "", true
case "retry_cancelled":
completion.Retry = true
if err := repo.UpdateTask(t.Context(), completion.TaskID, "cancelling", "deployment", "正在取消", nil, nil); err != nil {
t.Fatal(err)
}
wantStatus = "cancelling"
case "retry_success":
completion.Retry, completion.Status = true, "success"
case "retry_audit":
completion.Retry, completion.AuditAction = true, "certificate_deploy"
case "no_attempt_success":
completion.AttemptID, completion.WithoutAttempt, completion.Status = "", true, "success"
case "no_attempt_report":
completion.AttemptID, completion.WithoutAttempt = "", true
completion.Report = &deployment.Report{}
}
if err := repo.FinishTaskAttempt(t.Context(), completion); err == nil {
t.Fatal("不安全收尾未被拒绝")
}
task, err := repo.GetTask(t.Context(), completion.TaskID)
if err != nil || task.Status != wantStatus || task.FinishedAt != nil {
t.Fatalf("拒绝后改变任务状态:%+v %v", task, err)
}
})
}
}
func TestTaskCompletionWithoutAttemptRollsBackOnAuditFailure(t *testing.T) {
repo, db := newTaskRepositoryTest(t)
defer db.Close()
if err := repo.CreateTask(t.Context(), Task{ID: "before-attempt", DomainID: "domain-1", TaskType: "certificate_deploy", Status: "running", Stage: "deployment"}); err != nil {
t.Fatal(err)
}
if _, err := db.ExecContext(t.Context(), `CREATE TRIGGER reject_completion_audit BEFORE INSERT ON audit_logs BEGIN SELECT RAISE(ABORT, 'audit unavailable'); END`); err != nil {
t.Fatal(err)
}
if err := repo.FinishTaskAttempt(t.Context(), TaskCompletion{TaskID: "before-attempt", Status: "failed", Stage: "deployment", FinishedAt: time.Now().UTC(), WithoutAttempt: true, AuditAction: "certificate_deploy"}); err == nil {
t.Fatal("审计写入失败没有返回")
}
task, err := repo.GetTask(t.Context(), "before-attempt")
if err != nil || task.Status != "running" || task.FinishedAt != nil {
t.Fatalf("无尝试路径未整体回滚:%+v %v", task, err)
}
}