保存任务终态事务与取消一致性整改基线,生产复审待完成
This commit is contained in:
1 parent
9e666be522
commit
8a8b1a5e0e
12 files changed
+464
-18
No files matched your search
@@ -0,0 +1,73 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TaskCompletion 同时关闭任务与对应尝试,避免只保存其中一方的终态。
|
||||
type TaskCompletion struct {
|
||||
TaskID string
|
||||
AttemptID string
|
||||
Status string
|
||||
Stage string
|
||||
Message string
|
||||
StartedAt *time.Time
|
||||
FinishedAt time.Time
|
||||
}
|
||||
|
||||
func (r *SQLTaskRepository) FinishTaskAttempt(ctx context.Context, completion TaskCompletion) error {
|
||||
if r == nil || r.db == nil {
|
||||
return errors.New("数据库连接不能为空")
|
||||
}
|
||||
completion.TaskID = strings.TrimSpace(completion.TaskID)
|
||||
completion.AttemptID = strings.TrimSpace(completion.AttemptID)
|
||||
completion.Stage = strings.TrimSpace(completion.Stage)
|
||||
if completion.TaskID == "" || completion.AttemptID == "" || completion.Stage == "" || completion.FinishedAt.IsZero() {
|
||||
return errors.New("任务终态记录不完整")
|
||||
}
|
||||
switch completion.Status {
|
||||
case "success", "failed", "cancelled", "partial_success":
|
||||
default:
|
||||
return errors.New("任务终态无效")
|
||||
}
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("开始任务收尾事务失败:%w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
// 与取消、重试保持任务在前、尝试在后的加锁顺序。旧尝试或跨任务尝试必须整体回滚。
|
||||
result, err := tx.ExecContext(ctx, r.query(`UPDATE tasks
|
||||
SET status = ?, stage = ?, message = ?, started_at = COALESCE(?, started_at), finished_at = ?, updated_at = ?
|
||||
WHERE id = ? AND status IN ('queued', 'running', 'retry_wait', 'cancelling')`),
|
||||
completion.Status, completion.Stage, completion.Message, nullableTime(completion.StartedAt),
|
||||
completion.FinishedAt.UTC(), completion.FinishedAt.UTC(), completion.TaskID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("保存任务终态失败:%w", err)
|
||||
}
|
||||
if count, err := result.RowsAffected(); err != nil {
|
||||
return fmt.Errorf("读取任务收尾结果失败:%w", err)
|
||||
} else if count != 1 {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
result, err = tx.ExecContext(ctx, r.query(`UPDATE task_attempts
|
||||
SET status = ?, stage = ?, message = ?, finished_at = ?
|
||||
WHERE id = ? AND task_id = ? AND status IN ('queued', 'running', 'retry_wait', 'cancelling')`),
|
||||
completion.Status, completion.Stage, completion.Message, completion.FinishedAt.UTC(), completion.AttemptID, completion.TaskID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("保存任务尝试终态失败:%w", err)
|
||||
}
|
||||
if count, err := result.RowsAffected(); err != nil {
|
||||
return fmt.Errorf("读取任务尝试收尾结果失败:%w", err)
|
||||
} else if count != 1 {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("提交任务收尾事务失败:%w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func seedTaskCompletion(t *testing.T, repo *SQLTaskRepository) TaskCompletion {
|
||||
t.Helper()
|
||||
started := time.Now().UTC().Add(-time.Minute)
|
||||
if err := repo.CreateTask(t.Context(), Task{
|
||||
ID: "completion-task", DomainID: "domain-1", TaskType: "certificate_deploy",
|
||||
Status: "running", Stage: "deployment", StartedAt: &started,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repo.CreateAttempt(t.Context(), TaskAttempt{
|
||||
ID: "completion-attempt", TaskID: "completion-task", AttemptNumber: 1,
|
||||
Status: "running", Stage: "deployment", StartedAt: &started,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return TaskCompletion{
|
||||
TaskID: "completion-task", AttemptID: "completion-attempt", Status: "success",
|
||||
Stage: "deployment", Message: "部署完成", FinishedAt: time.Now().UTC(),
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskCompletionPersistsBothTerminalRecords(t *testing.T) {
|
||||
for _, status := range []string{"success", "failed", "cancelled", "partial_success"} {
|
||||
t.Run(status, func(t *testing.T) {
|
||||
repo, db := newTaskRepositoryTest(t)
|
||||
defer db.Close()
|
||||
completion := seedTaskCompletion(t, repo)
|
||||
completion.Status = status
|
||||
if err := repo.FinishTaskAttempt(t.Context(), completion); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task, err := repo.GetTask(t.Context(), completion.TaskID)
|
||||
if err != nil || task.Status != status || task.StartedAt == nil || task.FinishedAt == nil || task.Message != completion.Message {
|
||||
t.Fatalf("任务终态未完整保存:%+v %v", task, err)
|
||||
}
|
||||
attempts, err := repo.ListAttempts(t.Context(), completion.TaskID)
|
||||
if err != nil || len(attempts) != 1 || attempts[0].Status != status || attempts[0].FinishedAt == nil || !attempts[0].FinishedAt.Equal(*task.FinishedAt) || attempts[0].Message != task.Message {
|
||||
t.Fatalf("任务和尝试终态不一致:%+v %v", attempts, err)
|
||||
}
|
||||
if err := repo.FinishTaskAttempt(t.Context(), completion); !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatalf("终态不能被迟到的收尾重复覆盖:%v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskCompletionRollsBackWhenAttemptCannotFinish(t *testing.T) {
|
||||
for _, failure := range []string{"write_failure", "missing_attempt", "foreign_attempt", "stale_attempt", "invalid_status"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
repo, db := newTaskRepositoryTest(t)
|
||||
defer db.Close()
|
||||
completion := seedTaskCompletion(t, repo)
|
||||
wantAttemptStatus := "running"
|
||||
switch failure {
|
||||
case "write_failure":
|
||||
if _, err := db.ExecContext(t.Context(), `CREATE TRIGGER reject_attempt_completion BEFORE UPDATE ON task_attempts BEGIN SELECT RAISE(ABORT, 'injected completion failure'); END`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "missing_attempt":
|
||||
completion.AttemptID = "missing"
|
||||
case "foreign_attempt":
|
||||
if err := repo.CreateTask(t.Context(), Task{ID: "foreign-task", DomainID: "domain-2", TaskType: "certificate_deploy", Status: "running", Stage: "deployment"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repo.CreateAttempt(t.Context(), TaskAttempt{ID: "foreign-attempt", TaskID: "foreign-task", AttemptNumber: 1, Status: "running", Stage: "deployment"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
completion.AttemptID = "foreign-attempt"
|
||||
case "stale_attempt":
|
||||
if err := repo.FinishAttempt(t.Context(), completion.AttemptID, "failed", "此前失败", &completion.FinishedAt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantAttemptStatus = "failed"
|
||||
case "invalid_status":
|
||||
completion.Status = "running"
|
||||
}
|
||||
if err := repo.FinishTaskAttempt(t.Context(), completion); err == nil {
|
||||
t.Fatal("应拒绝无法同时关闭任务和尝试的操作")
|
||||
}
|
||||
task, err := repo.GetTask(t.Context(), completion.TaskID)
|
||||
if err != nil || task.Status != "running" || task.FinishedAt != nil {
|
||||
t.Fatalf("尝试保存失败后任务未回滚:%+v %v", task, err)
|
||||
}
|
||||
attempts, err := repo.ListAttempts(t.Context(), completion.TaskID)
|
||||
if err != nil || len(attempts) != 1 || attempts[0].Status != wantAttemptStatus {
|
||||
t.Fatalf("失败操作修改了尝试:%+v %v", attempts, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -68,6 +68,7 @@ type TaskRepository interface {
|
||||
FindTaskByIdempotencyKey(context.Context, string, string) (Task, error)
|
||||
CreateAttempt(context.Context, TaskAttempt) error
|
||||
FinishAttempt(context.Context, string, string, string, *time.Time) error
|
||||
FinishTaskAttempt(context.Context, TaskCompletion) error
|
||||
ListAttempts(context.Context, string) ([]TaskAttempt, error)
|
||||
RecordTaskDeploymentResults(context.Context, string, string, int, []TaskDeploymentResult) error
|
||||
ListTaskDeploymentResults(context.Context, string) ([]TaskDeploymentResult, error)
|
||||
|
||||
Reference in new issue
Block a user