开发失败任务重试权限闭环
新增失败证书任务重试接口、事务处理、权限校验、审计记录及前端重试入口,并补充后端测试。
This commit is contained in:
1 parent
35c9d7760c
commit
35260bcc4b
8 files changed
+350
-6
No files matched your search
@@ -17,8 +17,11 @@ type ProjectTaskReader interface {
|
||||
GetVisibleTask(context.Context, string, string) (Task, error)
|
||||
ListVisibleTaskAttempts(context.Context, string, string) ([]TaskAttempt, error)
|
||||
CancelVisibleTask(context.Context, string, string, string, *time.Time) error
|
||||
RetryVisibleTask(context.Context, string, string, *time.Time) (Task, TaskAttempt, error)
|
||||
}
|
||||
|
||||
var ErrTaskNotRetryable = errors.New("任务当前不可重试")
|
||||
|
||||
var _ ProjectTaskReader = (*SQLTaskRepository)(nil)
|
||||
|
||||
func taskVisibilityPredicate(alias string, permission domain.ProjectPermission) (string, []any) {
|
||||
@@ -30,6 +33,9 @@ func taskVisibilityPredicate(alias string, permission domain.ProjectPermission)
|
||||
domain.ProjectRoleReadOnly,
|
||||
}
|
||||
}
|
||||
if permission == domain.PermissionTaskRetry {
|
||||
roles = []domain.ProjectRole{domain.ProjectRoleAdmin}
|
||||
}
|
||||
placeholders := make([]string, 0, len(roles))
|
||||
args := []any{""}
|
||||
for _, role := range roles {
|
||||
@@ -178,3 +184,102 @@ func (r *SQLTaskRepository) CancelVisibleTask(ctx context.Context, actorID, task
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *SQLTaskRepository) RetryVisibleTask(ctx context.Context, actorID, taskID string, startedAt *time.Time) (Task, TaskAttempt, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return Task{}, TaskAttempt{}, errors.New("数据库连接不能为空")
|
||||
}
|
||||
actorID = strings.TrimSpace(actorID)
|
||||
taskID = strings.TrimSpace(taskID)
|
||||
if actorID == "" || taskID == "" {
|
||||
return Task{}, TaskAttempt{}, ErrProjectAccessDenied
|
||||
}
|
||||
if startedAt == nil {
|
||||
now := time.Now().UTC()
|
||||
startedAt = &now
|
||||
}
|
||||
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return Task{}, TaskAttempt{}, fmt.Errorf("开始重试任务事务失败:%w", err)
|
||||
}
|
||||
rollback := func() {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
predicate, predicateArgs := taskVisibilityPredicate("tasks", domain.PermissionTaskRetry)
|
||||
predicateArgs[0] = actorID
|
||||
updateArgs := []any{
|
||||
actorID, "certificate_issue", "正在重试证书申请", nullableTime(startedAt), time.Now().UTC(), taskID,
|
||||
}
|
||||
updateArgs = append(updateArgs, predicateArgs...)
|
||||
result, err := tx.ExecContext(ctx, r.query(`UPDATE tasks SET operator_id = ?, status = 'running', stage = ?, message = ?,
|
||||
started_at = ?, finished_at = NULL, updated_at = ?
|
||||
WHERE id = ? AND task_type = 'certificate_issue' AND status IN ('failed', 'cancelled') AND `+predicate), updateArgs...)
|
||||
if err != nil {
|
||||
rollback()
|
||||
return Task{}, TaskAttempt{}, fmt.Errorf("更新重试任务状态失败:%w", err)
|
||||
}
|
||||
count, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
rollback()
|
||||
return Task{}, TaskAttempt{}, fmt.Errorf("读取重试任务更新结果失败:%w", err)
|
||||
}
|
||||
if count != 1 {
|
||||
item, getErr := scanTask(tx.QueryRowContext(ctx, r.query(taskSelect+` WHERE id = ?`), taskID))
|
||||
rollback()
|
||||
if errors.Is(getErr, sql.ErrNoRows) {
|
||||
return Task{}, TaskAttempt{}, sql.ErrNoRows
|
||||
}
|
||||
if getErr != nil {
|
||||
return Task{}, TaskAttempt{}, getErr
|
||||
}
|
||||
if item.Status != "failed" && item.Status != "cancelled" {
|
||||
return Task{}, TaskAttempt{}, ErrTaskNotRetryable
|
||||
}
|
||||
return Task{}, TaskAttempt{}, ErrProjectAccessDenied
|
||||
}
|
||||
|
||||
var attemptNumber int
|
||||
if err := tx.QueryRowContext(ctx, r.query(`SELECT COALESCE(MAX(attempt_number), 0) + 1 FROM task_attempts WHERE task_id = ?`), taskID).Scan(&attemptNumber); err != nil {
|
||||
rollback()
|
||||
return Task{}, TaskAttempt{}, fmt.Errorf("读取任务重试次数失败:%w", err)
|
||||
}
|
||||
attempt := TaskAttempt{
|
||||
HistoryScope: HistoryScope{ScopeType: ScopeProject},
|
||||
ID: domain.NewID(),
|
||||
TaskID: taskID,
|
||||
AttemptNumber: attemptNumber,
|
||||
Stage: "certificate_issue",
|
||||
Status: "running",
|
||||
Message: "开始重试证书申请",
|
||||
StartedAt: startedAt,
|
||||
CreatedAt: startedAt.UTC(),
|
||||
}
|
||||
insertResult, err := tx.ExecContext(ctx, r.query(`INSERT INTO task_attempts
|
||||
(id, task_id, attempt_number, stage, status, message, started_at, finished_at, created_at)
|
||||
SELECT ?, id, ?, ?, ?, ?, ?, NULL, ? FROM tasks WHERE id = ?`),
|
||||
attempt.ID, attempt.AttemptNumber, attempt.Stage, attempt.Status, attempt.Message,
|
||||
nullableTime(attempt.StartedAt), attempt.CreatedAt, taskID)
|
||||
if err != nil {
|
||||
rollback()
|
||||
return Task{}, TaskAttempt{}, fmt.Errorf("创建重试任务尝试失败:%w", err)
|
||||
}
|
||||
if count, err := insertResult.RowsAffected(); err != nil {
|
||||
rollback()
|
||||
return Task{}, TaskAttempt{}, fmt.Errorf("读取重试任务尝试创建结果失败:%w", err)
|
||||
} else if count != 1 {
|
||||
rollback()
|
||||
return Task{}, TaskAttempt{}, ErrTaskNotRetryable
|
||||
}
|
||||
|
||||
task, err := scanTask(tx.QueryRowContext(ctx, r.query(taskSelect+` WHERE id = ?`), taskID))
|
||||
if err != nil {
|
||||
rollback()
|
||||
return Task{}, TaskAttempt{}, err
|
||||
}
|
||||
attempt.HistoryScope = task.HistoryScope
|
||||
if err := tx.Commit(); err != nil {
|
||||
return Task{}, TaskAttempt{}, fmt.Errorf("提交重试任务事务失败:%w", err)
|
||||
}
|
||||
return task, attempt, nil
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSQLTaskRepositoryRetryChecksPermissionAndCreatesAttempt(t *testing.T) {
|
||||
repository, db := newTaskRepositoryTest(t)
|
||||
defer db.Close()
|
||||
ctx := context.Background()
|
||||
for _, item := range []struct {
|
||||
id string
|
||||
role string
|
||||
}{
|
||||
{id: "retry-admin", role: "admin"},
|
||||
{id: "retry-reader", role: "user"},
|
||||
} {
|
||||
if _, err := db.ExecContext(ctx, `INSERT INTO users (id, username, password_hash, role, status) VALUES (?, ?, 'test', ?, 'active')`, item.id, item.id, item.role); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, `INSERT INTO project_members (project_id, user_id, role) VALUES ('default', 'retry-admin', 'project_admin'), ('default', 'retry-reader', 'read_only')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
created := time.Now().UTC().Add(-time.Minute)
|
||||
if err := repository.CreateTask(ctx, Task{
|
||||
ID: "task-retry", DomainID: "domain-1", TaskType: "certificate_issue", Status: "failed",
|
||||
Stage: "certificate_issue", Message: "申请失败", CreatedAt: created, UpdatedAt: created,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repository.CreateAttempt(ctx, TaskAttempt{
|
||||
ID: "attempt-retry-1", TaskID: "task-retry", AttemptNumber: 1, Stage: "certificate_issue",
|
||||
Status: "failed", Message: "申请失败", CreatedAt: created,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
started := time.Now().UTC()
|
||||
if _, _, err := repository.RetryVisibleTask(ctx, "retry-reader", "task-retry", &started); !errors.Is(err, ErrProjectAccessDenied) {
|
||||
t.Fatalf("read-only member should not retry: %v", err)
|
||||
}
|
||||
task, attempt, err := repository.RetryVisibleTask(ctx, "retry-admin", "task-retry", &started)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if task.Status != "running" || task.OperatorID != "retry-admin" || attempt.AttemptNumber != 2 || attempt.Status != "running" || attempt.ProjectID != task.ProjectID {
|
||||
t.Fatalf("retry state is incorrect: task=%+v attempt=%+v", task, attempt)
|
||||
}
|
||||
if _, _, err := repository.RetryVisibleTask(ctx, "retry-admin", "task-retry", &started); !errors.Is(err, ErrTaskNotRetryable) {
|
||||
t.Fatalf("running task should not be retried again: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLTaskRepositoryConcurrentRetryAllowsOnlyOneWinner(t *testing.T) {
|
||||
repository, db := newTaskRepositoryTest(t)
|
||||
defer db.Close()
|
||||
ctx := context.Background()
|
||||
if _, err := db.ExecContext(ctx, `INSERT INTO users (id, username, password_hash, role, status) VALUES ('retry-admin', 'retry-admin', 'test', 'admin', 'active')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, `INSERT INTO project_members (project_id, user_id, role) VALUES ('default', 'retry-admin', 'project_admin')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
created := time.Now().UTC().Add(-time.Minute)
|
||||
if err := repository.CreateTask(ctx, Task{
|
||||
ID: "task-concurrent-retry", DomainID: "domain-1", TaskType: "certificate_issue", Status: "failed",
|
||||
Stage: "certificate_issue", Message: "申请失败", CreatedAt: created, UpdatedAt: created,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repository.CreateAttempt(ctx, TaskAttempt{
|
||||
ID: "attempt-concurrent-retry-1", TaskID: "task-concurrent-retry", AttemptNumber: 1,
|
||||
Stage: "certificate_issue", Status: "failed", Message: "申请失败", CreatedAt: created,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
results := make(chan error, 2)
|
||||
for i := 0; i < 2; i++ {
|
||||
go func() {
|
||||
started := time.Now().UTC()
|
||||
_, _, err := repository.RetryVisibleTask(ctx, "retry-admin", "task-concurrent-retry", &started)
|
||||
results <- err
|
||||
}()
|
||||
}
|
||||
var successCount int
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := <-results; err == nil {
|
||||
successCount++
|
||||
} else if !errors.Is(err, ErrTaskNotRetryable) {
|
||||
t.Fatalf("unexpected concurrent retry error: %v", err)
|
||||
}
|
||||
}
|
||||
if successCount != 1 {
|
||||
t.Fatalf("expected exactly one retry winner, got %d", successCount)
|
||||
}
|
||||
attempts, err := repository.ListAttempts(ctx, "task-concurrent-retry")
|
||||
if err != nil || len(attempts) != 2 || attempts[1].AttemptNumber != 2 || attempts[1].Status != "running" {
|
||||
t.Fatalf("concurrent retry attempts are incorrect: %+v %v", attempts, err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user