保存任务终态事务与取消一致性整改基线,生产复审待完成

This commit is contained in:
xuans committed 2026-10-02 17:19:51 +08:00
1 parent 9e666be522
commit 8a8b1a5e0e
12 files changed
+464 -18

No files matched your search

+21 -12
View File
@@ -216,7 +216,8 @@ func (s *Server) runCertificateTask(ctx context.Context, task repository.Task, a
finishCtx, finishCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer finishCancel()
deploymentReport = sanitizeDeploymentReport(deploymentReport)
if recordErr := s.tasks.RecordTaskDeploymentResults(finishCtx, task.ID, attempt.ID, attempt.AttemptNumber, taskDeploymentResults(deploymentReport)); recordErr != nil {
recordErr := s.tasks.RecordTaskDeploymentResults(finishCtx, task.ID, attempt.ID, attempt.AttemptNumber, taskDeploymentResults(deploymentReport))
if recordErr != nil {
issueErr = errors.Join(issueErr, recordErr)
}
finished := time.Now().UTC()
@@ -233,19 +234,27 @@ func (s *Server) runCertificateTask(ctx context.Context, task repository.Task, a
if s.logger != nil {
s.logger.Error("证书任务执行失败", "task_id", task.ID, "domain_id", task.DomainID, "error", issueErr)
}
if errors.Is(issueErr, context.Canceled) {
if errors.Is(issueErr, context.Canceled) && !partial && deploymentReport.RecoveryStatus != "unconfirmed" && deploymentReport.RecoveryStatus != "failed" {
status = "cancelled"
}
} else if partial {
status = "partial_success"
}
if recordErr != nil {
status, message = "failed", "部署结果保存失败,请核查实际部署状态后再重试"
}
s.taskMu.Lock()
defer s.taskMu.Unlock()
if persisted, err := s.tasks.GetTask(finishCtx, task.ID); err == nil && (persisted.Status == "cancelled" || persisted.Status == "cancelling") {
status = "cancelled"
message = persisted.Message
if issueErr == nil {
issueErr = errors.New(message)
if persisted, err := s.tasks.GetTask(finishCtx, task.ID); err != nil {
status, message = "failed", "无法确认任务收尾状态,请核查部署结果后再重试"
issueErr = errors.Join(issueErr, err)
} else if persisted.Status == "cancelled" || persisted.Status == "cancelling" {
// 取消请求只是意图,不能覆盖已经完成的外部操作,也不能隐藏失败及部分成功。
if issueErr == nil && !partial {
message = "任务已完成,取消请求未撤销已完成的操作"
} else if status == "cancelled" {
status = "cancelled"
message = "任务已停止;已执行操作是否回滚请查看部署结果"
}
}
if s.audit != nil {
@@ -260,11 +269,11 @@ func (s *Server) runCertificateTask(ctx context.Context, task repository.Task, a
status, message = "failed", "任务完成审计保存失败,请检查任务恢复状态"
}
}
if err := s.tasks.FinishAttempt(finishCtx, attempt.ID, status, message, &finished); err != nil {
status, message = "failed", "任务尝试结果保存失败,请检查任务恢复状态"
}
if err := s.tasks.UpdateTask(finishCtx, task.ID, status, stage, message, &now, &finished); err != nil && s.logger != nil {
s.logger.Error("保存任务终态失败", "task_id", task.ID, "error", err)
if err := s.tasks.FinishTaskAttempt(finishCtx, repository.TaskCompletion{
TaskID: task.ID, AttemptID: attempt.ID, Status: status, Stage: stage,
Message: message, StartedAt: &now, FinishedAt: finished,
}); err != nil && s.logger != nil {
s.logger.Error("保存任务及尝试终态失败,保留中断恢复状态", "task_id", task.ID, "error", err)
}
}
@@ -0,0 +1,129 @@
package httpapi
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"git.pcux.cn/xuans/Scert/backend/internal/certificate"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
"git.pcux.cn/xuans/Scert/backend/internal/repository"
)
func TestAsyncCancellationPreservesActualOperationOutcome(t *testing.T) {
for _, test := range []struct {
name string
report deployment.Report
err error
want string
}{
{name: "already_completed", report: deployment.Report{Status: "success"}, want: "success"},
{name: "failed", report: deployment.Report{Status: "failed"}, err: errors.New("部署收尾失败"), want: "failed"},
{name: "partially_completed", report: deployment.Report{Status: "partial"}, err: context.Canceled, want: "partial_success"},
{name: "cancelled", err: context.Canceled, want: "cancelled"},
{name: "recovery_unconfirmed", report: deployment.Report{Status: "failed", RecoveryStatus: "unconfirmed"}, err: errors.Join(context.Canceled, errors.New("回滚未确认")), want: "failed"},
{name: "recovery_failed", report: deployment.Report{Status: "failed", RecoveryStatus: "failed"}, err: errors.Join(context.Canceled, errors.New("回滚失败")), want: "failed"},
} {
t.Run(test.name, func(t *testing.T) {
started := make(chan struct{})
server, tasks, _, cleanup := newTaskAPITestServer(t, func(ctx context.Context, _ string) (certificate.IssueResult, deployment.Report, error) {
close(started)
<-ctx.Done()
return certificate.IssueResult{}, test.report, test.err
})
defer cleanup()
response := httptest.NewRecorder()
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
task := submittedTask(t, response)
select {
case <-started:
case <-time.After(2 * time.Second):
t.Fatal("任务没有开始")
}
response = httptest.NewRecorder()
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodDelete, "/api/v1/tasks/"+task.ID, nil))
if response.Code != http.StatusAccepted {
t.Fatalf("取消未被接收:%s", response.Body.String())
}
waitTaskWorkersForTest(t, server)
stored, err := tasks.GetTask(t.Context(), task.ID)
if err != nil || stored.Status != test.want {
t.Fatalf("取消请求覆盖了实际结果:%+v %v", stored, err)
}
attempts, err := tasks.ListAttempts(t.Context(), task.ID)
if err != nil || len(attempts) != 1 || attempts[0].Status != stored.Status {
t.Fatalf("任务与尝试不一致:%+v %v", attempts, err)
}
if test.want == "success" && !strings.Contains(stored.Message, "未撤销") {
t.Fatal("必须提示取消未撤销已完成操作")
}
})
}
}
type failingCompletionTaskRepository struct {
repository.TaskRepository
operationFinished *atomic.Bool
failRead bool
}
func (r failingCompletionTaskRepository) GetTask(ctx context.Context, id string) (repository.Task, error) {
if r.failRead && r.operationFinished.Load() {
return repository.Task{}, errors.New("模拟收尾状态读取失败")
}
return r.TaskRepository.GetTask(ctx, id)
}
func (r failingCompletionTaskRepository) FinishTaskAttempt(ctx context.Context, item repository.TaskCompletion) error {
if !r.failRead {
return errors.New("模拟收尾事务保存失败")
}
return r.TaskRepository.FinishTaskAttempt(ctx, item)
}
func TestAsyncCompletionFailureDoesNotPublishFalseSuccess(t *testing.T) {
for _, failRead := range []bool{true, false} {
name := "atomic_write_failure"
if failRead {
name = "state_read_failure"
}
t.Run(name, func(t *testing.T) {
var finished atomic.Bool
server, tasks, _, cleanup := newTaskAPITestServer(t, func(context.Context, string) (certificate.IssueResult, deployment.Report, error) {
finished.Store(true)
return certificate.IssueResult{}, deployment.Report{Status: "success"}, nil
})
defer cleanup()
server.tasks = failingCompletionTaskRepository{TaskRepository: tasks, operationFinished: &finished, failRead: failRead}
response := httptest.NewRecorder()
serveAdminHTTP(server, response, httptest.NewRequest(http.MethodPost, "/api/v1/domains/domain-1/issue", nil))
task := submittedTask(t, response)
waitTaskWorkersForTest(t, server)
stored, err := tasks.GetTask(t.Context(), task.ID)
if err != nil || stored.Status == "success" {
t.Fatalf("无法可靠持久化却标记成功:%+v %v", stored, err)
}
attempts, err := tasks.ListAttempts(t.Context(), task.ID)
if err != nil || len(attempts) != 1 || attempts[0].Status == "success" {
t.Fatalf("尝试被单独保存为成功:%+v %v", attempts, err)
}
if failRead {
if stored.Status != "failed" || attempts[0].Status != "failed" {
t.Fatal("状态读取失败必须闭合为失败")
}
} else {
if !isActiveTaskStatus(stored.Status) || stored.FinishedAt != nil || attempts[0].FinishedAt != nil {
t.Fatal("事务失败必须保留可由启动恢复接管的中断状态")
}
if count, err := tasks.RecoverInterruptedTasks(t.Context(), time.Now().UTC()); err != nil || count != 1 {
t.Fatalf("中断记录无法恢复:%d %v", count, err)
}
}
})
}
}
+5 -5
View File
@@ -69,11 +69,11 @@ func (s *Server) finishTaskBeforeExecution(task repository.Task, attempt reposit
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
finished := time.Now().UTC()
if err := s.tasks.FinishAttempt(ctx, attempt.ID, status, message, &finished); err != nil && s.logger != nil {
s.logger.Error("保存任务尝试失败", "task_id", task.ID, "error", err)
}
if err := s.tasks.UpdateTask(ctx, task.ID, status, attempt.Stage, message, nil, &finished); err != nil && s.logger != nil {
s.logger.Error("保存任务终态失败", "task_id", task.ID, "error", err)
if err := s.tasks.FinishTaskAttempt(ctx, repository.TaskCompletion{
TaskID: task.ID, AttemptID: attempt.ID, Status: status, Stage: attempt.Stage,
Message: message, FinishedAt: finished,
}); err != nil && s.logger != nil {
s.logger.Error("保存任务及尝试终态失败,保留中断恢复状态", "task_id", task.ID, "error", err)
}
}
@@ -0,0 +1,73 @@
package repository
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"time"
)
// TaskCompletion 同时关闭任务与对应尝试,避免只保存其中一方的终态。
type TaskCompletion struct {
TaskID string
AttemptID string
Status string
Stage string
Message string
StartedAt *time.Time
FinishedAt time.Time
}
func (r *SQLTaskRepository) FinishTaskAttempt(ctx context.Context, completion TaskCompletion) error {
if r == nil || r.db == nil {
return errors.New("数据库连接不能为空")
}
completion.TaskID = strings.TrimSpace(completion.TaskID)
completion.AttemptID = strings.TrimSpace(completion.AttemptID)
completion.Stage = strings.TrimSpace(completion.Stage)
if completion.TaskID == "" || completion.AttemptID == "" || completion.Stage == "" || completion.FinishedAt.IsZero() {
return errors.New("任务终态记录不完整")
}
switch completion.Status {
case "success", "failed", "cancelled", "partial_success":
default:
return errors.New("任务终态无效")
}
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("开始任务收尾事务失败:%w", err)
}
defer tx.Rollback()
// 与取消、重试保持任务在前、尝试在后的加锁顺序。旧尝试或跨任务尝试必须整体回滚。
result, err := tx.ExecContext(ctx, r.query(`UPDATE tasks
SET status = ?, stage = ?, message = ?, started_at = COALESCE(?, started_at), finished_at = ?, updated_at = ?
WHERE id = ? AND status IN ('queued', 'running', 'retry_wait', 'cancelling')`),
completion.Status, completion.Stage, completion.Message, nullableTime(completion.StartedAt),
completion.FinishedAt.UTC(), completion.FinishedAt.UTC(), completion.TaskID)
if err != nil {
return fmt.Errorf("保存任务终态失败:%w", err)
}
if count, err := result.RowsAffected(); err != nil {
return fmt.Errorf("读取任务收尾结果失败:%w", err)
} else if count != 1 {
return sql.ErrNoRows
}
result, err = tx.ExecContext(ctx, r.query(`UPDATE task_attempts
SET status = ?, stage = ?, message = ?, finished_at = ?
WHERE id = ? AND task_id = ? AND status IN ('queued', 'running', 'retry_wait', 'cancelling')`),
completion.Status, completion.Stage, completion.Message, completion.FinishedAt.UTC(), completion.AttemptID, completion.TaskID)
if err != nil {
return fmt.Errorf("保存任务尝试终态失败:%w", err)
}
if count, err := result.RowsAffected(); err != nil {
return fmt.Errorf("读取任务尝试收尾结果失败:%w", err)
} else if count != 1 {
return sql.ErrNoRows
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("提交任务收尾事务失败:%w", err)
}
return nil
}
@@ -0,0 +1,99 @@
package repository
import (
"database/sql"
"errors"
"testing"
"time"
)
func seedTaskCompletion(t *testing.T, repo *SQLTaskRepository) TaskCompletion {
t.Helper()
started := time.Now().UTC().Add(-time.Minute)
if err := repo.CreateTask(t.Context(), Task{
ID: "completion-task", DomainID: "domain-1", TaskType: "certificate_deploy",
Status: "running", Stage: "deployment", StartedAt: &started,
}); err != nil {
t.Fatal(err)
}
if err := repo.CreateAttempt(t.Context(), TaskAttempt{
ID: "completion-attempt", TaskID: "completion-task", AttemptNumber: 1,
Status: "running", Stage: "deployment", StartedAt: &started,
}); err != nil {
t.Fatal(err)
}
return TaskCompletion{
TaskID: "completion-task", AttemptID: "completion-attempt", Status: "success",
Stage: "deployment", Message: "部署完成", FinishedAt: time.Now().UTC(),
}
}
func TestTaskCompletionPersistsBothTerminalRecords(t *testing.T) {
for _, status := range []string{"success", "failed", "cancelled", "partial_success"} {
t.Run(status, func(t *testing.T) {
repo, db := newTaskRepositoryTest(t)
defer db.Close()
completion := seedTaskCompletion(t, repo)
completion.Status = status
if err := repo.FinishTaskAttempt(t.Context(), completion); err != nil {
t.Fatal(err)
}
task, err := repo.GetTask(t.Context(), completion.TaskID)
if err != nil || task.Status != status || task.StartedAt == nil || task.FinishedAt == nil || task.Message != completion.Message {
t.Fatalf("任务终态未完整保存:%+v %v", task, err)
}
attempts, err := repo.ListAttempts(t.Context(), completion.TaskID)
if err != nil || len(attempts) != 1 || attempts[0].Status != status || attempts[0].FinishedAt == nil || !attempts[0].FinishedAt.Equal(*task.FinishedAt) || attempts[0].Message != task.Message {
t.Fatalf("任务和尝试终态不一致:%+v %v", attempts, err)
}
if err := repo.FinishTaskAttempt(t.Context(), completion); !errors.Is(err, sql.ErrNoRows) {
t.Fatalf("终态不能被迟到的收尾重复覆盖:%v", err)
}
})
}
}
func TestTaskCompletionRollsBackWhenAttemptCannotFinish(t *testing.T) {
for _, failure := range []string{"write_failure", "missing_attempt", "foreign_attempt", "stale_attempt", "invalid_status"} {
t.Run(failure, func(t *testing.T) {
repo, db := newTaskRepositoryTest(t)
defer db.Close()
completion := seedTaskCompletion(t, repo)
wantAttemptStatus := "running"
switch failure {
case "write_failure":
if _, err := db.ExecContext(t.Context(), `CREATE TRIGGER reject_attempt_completion BEFORE UPDATE ON task_attempts BEGIN SELECT RAISE(ABORT, 'injected completion failure'); END`); err != nil {
t.Fatal(err)
}
case "missing_attempt":
completion.AttemptID = "missing"
case "foreign_attempt":
if err := repo.CreateTask(t.Context(), Task{ID: "foreign-task", DomainID: "domain-2", TaskType: "certificate_deploy", Status: "running", Stage: "deployment"}); err != nil {
t.Fatal(err)
}
if err := repo.CreateAttempt(t.Context(), TaskAttempt{ID: "foreign-attempt", TaskID: "foreign-task", AttemptNumber: 1, Status: "running", Stage: "deployment"}); err != nil {
t.Fatal(err)
}
completion.AttemptID = "foreign-attempt"
case "stale_attempt":
if err := repo.FinishAttempt(t.Context(), completion.AttemptID, "failed", "此前失败", &completion.FinishedAt); err != nil {
t.Fatal(err)
}
wantAttemptStatus = "failed"
case "invalid_status":
completion.Status = "running"
}
if err := repo.FinishTaskAttempt(t.Context(), completion); err == nil {
t.Fatal("应拒绝无法同时关闭任务和尝试的操作")
}
task, err := repo.GetTask(t.Context(), completion.TaskID)
if err != nil || task.Status != "running" || task.FinishedAt != nil {
t.Fatalf("尝试保存失败后任务未回滚:%+v %v", task, err)
}
attempts, err := repo.ListAttempts(t.Context(), completion.TaskID)
if err != nil || len(attempts) != 1 || attempts[0].Status != wantAttemptStatus {
t.Fatalf("失败操作修改了尝试:%+v %v", attempts, err)
}
})
}
}
@@ -68,6 +68,7 @@ type TaskRepository interface {
FindTaskByIdempotencyKey(context.Context, string, string) (Task, error)
CreateAttempt(context.Context, TaskAttempt) error
FinishAttempt(context.Context, string, string, string, *time.Time) error
FinishTaskAttempt(context.Context, TaskCompletion) error
ListAttempts(context.Context, string) ([]TaskAttempt, error)
RecordTaskDeploymentResults(context.Context, string, string, int, []TaskDeploymentResult) error
ListTaskDeploymentResults(context.Context, string) ([]TaskDeploymentResult, error)
+1 -1
View File
@@ -180,7 +180,7 @@ func renderUnitFile(options Options) (string, error) {
"Environment=SCERT_BOOTSTRAP_FIREWALL_MODE=external",
"Environment=SCERT_SERVER_HOST=127.0.0.1",
"KillSignal=SIGTERM",
"TimeoutStopSec=30",
"TimeoutStopSec=180",
"Restart=on-failure",
"UMask=0077",
"RuntimeDirectory=scert",
@@ -35,6 +35,7 @@ func TestRenderUnitFileIncludesExecutableConfigurationAndWorkDir(t *testing.T) {
"Type=notify",
"NotifyAccess=main",
"TimeoutStartSec=90",
"TimeoutStopSec=180",
"User=Scert",
"Group=Scert",
"WorkingDirectory=\"/opt/Scert data\"",
@@ -78,6 +79,7 @@ func TestPackagedUnitAllowsInterfaceDiscoveryWithoutCapabilities(t *testing.T) {
"Type=notify",
"NotifyAccess=main",
"TimeoutStartSec=90",
"TimeoutStopSec=180",
"CapabilityBoundingSet=",
"AmbientCapabilities=",
"NoNewPrivileges=yes",