统一后台证书任务报告及收尾事务并补齐故障回归

This commit is contained in:
xuans committed 2026-10-02 18:36:52 +08:00
1 parent cf3bc7aaa5
commit 657a9c0a36
9 files changed
+379 -67

No files matched your search

@@ -0,0 +1,118 @@
package app
import (
"context"
"errors"
"testing"
"time"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
"git.pcux.cn/xuans/Scert/backend/internal/repository"
)
func TestCertificateCompletionPersistsFullReportsAcrossRetries(t *testing.T) {
w, a := newCertificateTaskFixture(t)
calls := 0
_, err := w.run(t.Context(), "task-domain", "renewal_run", "certificate_issue", 2, time.Nanosecond,
func(context.Context, string) (CertificateWorkflowResult, error) {
calls++
result := CertificateWorkflowResult{Deployment: deployment.Report{
Status: "success", RecoveryStatus: "success", Total: 1, Success: 1,
Results: []deployment.Result{{ServerID: "target", Status: "success"}},
}}
if calls == 1 {
result.Deployment.Status = "failed"
return result, errors.New("temporary failure")
}
return result, nil
})
if err != nil || calls != 2 {
t.Fatalf("重试执行失败:%d %v", calls, err)
}
task := assertCertificateTask(t, a, "success", "failed", "success")
reports, err := a.Tasks.ListTaskDeploymentReports(t.Context(), task.ID)
if err != nil || len(reports) != 2 {
t.Fatalf("每次尝试均应保存完整报告:%+v %v", reports, err)
}
if reports[0].Report.Status != "failed" || reports[1].Report.Status != "success" || reports[0].AttemptID == reports[1].AttemptID || reports[1].Report.RecoveryStatus != "success" {
t.Fatalf("重试覆盖报告或丢失恢复状态:%+v", reports)
}
}
type cancelAfterCertificateRetry struct {
repository.TaskRepository
cancel context.CancelFunc
}
func (r cancelAfterCertificateRetry) FinishTaskAttempt(ctx context.Context, completion repository.TaskCompletion) error {
err := r.TaskRepository.FinishTaskAttempt(ctx, completion)
if err == nil && completion.Retry {
r.cancel()
}
return err
}
func TestCertificateCompletionCancellationDuringRetryWait(t *testing.T) {
w, a := newCertificateTaskFixture(t)
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
w.tasks = cancelAfterCertificateRetry{TaskRepository: a.Tasks, cancel: cancel}
calls := 0
_, err := w.run(ctx, "task-domain", "renewal_run", "certificate_issue", 3, time.Hour,
func(context.Context, string) (CertificateWorkflowResult, error) {
calls++
return CertificateWorkflowResult{}, errors.New("temporary failure")
})
if !errors.Is(err, context.Canceled) || calls != 1 {
t.Fatalf("等待取消后继续重试或丢失取消原因:%d %v", calls, err)
}
task := assertCertificateTask(t, a, "cancelled", "failed")
reports, err := a.Tasks.ListTaskDeploymentReports(t.Context(), task.ID)
if err != nil || len(reports) != 1 {
t.Fatalf("等待取消丢失上次尝试报告:%+v %v", reports, err)
}
}
func TestCertificateCompletionFailureRollsBackAllRecords(t *testing.T) {
for _, table := range []string{"task_deployment_reports", "task_deployment_results", "audit_logs", "task_attempts", "tasks"} {
t.Run(table, func(t *testing.T) {
w, a := newCertificateTaskFixture(t)
calls := 0
_, err := w.run(t.Context(), "task-domain", "renewal_run", "certificate_issue", 3, time.Nanosecond,
func(context.Context, string) (CertificateWorkflowResult, error) {
calls++
operation := "INSERT"
if table == "tasks" || table == "task_attempts" {
operation = "UPDATE"
}
// 表名只来自本测试的固定枚举,不接受用户输入。
if _, err := a.Database.DB.ExecContext(t.Context(), "CREATE TRIGGER IF NOT EXISTS reject_completion BEFORE "+operation+" ON "+table+" BEGIN SELECT RAISE(ABORT, 'completion unavailable'); END"); err != nil {
t.Fatal(err)
}
return CertificateWorkflowResult{Deployment: deployment.Report{Status: "success", Total: 1, Success: 1,
Results: []deployment.Result{{ServerID: "target", Status: "success"}}}}, nil
})
if err == nil || calls != 1 {
t.Fatalf("保存失败必须返回错误且不重复外部操作:%d %v", calls, err)
}
tasks, err := a.Tasks.ListTasks(t.Context(), "task-domain", 10)
if err != nil || len(tasks) != 1 || tasks[0].Status != "running" || tasks[0].FinishedAt != nil {
t.Fatalf("事务失败必须保留活动任务供恢复:%+v %v", tasks, err)
}
attempts, err := a.Tasks.ListAttempts(t.Context(), tasks[0].ID)
if err != nil || len(attempts) != 1 || attempts[0].Status != "running" || attempts[0].FinishedAt != nil {
t.Fatalf("尝试终态未回滚:%+v %v", attempts, err)
}
for _, stored := range []string{"task_deployment_reports", "task_deployment_results", "audit_logs"} {
var count int
query := "SELECT COUNT(*) FROM " + stored + " WHERE task_id = ?"
if stored == "audit_logs" {
query += " AND status <> 'running'"
}
if err := a.Database.DB.QueryRowContext(t.Context(), query, tasks[0].ID).Scan(&count); err != nil || count != 0 {
t.Fatalf("失败后仍残留部分记录:%s %d %v", stored, count, err)
}
}
})
}
}
+35 -29
View File
@@ -101,26 +101,32 @@ func (w *certificateTaskWorkflow) run(ctx context.Context, id, taskType, stage s
taskCtx = certificate.WithProgressReporter(taskCtx, func(progressCtx context.Context, progressStage, progressMessage string) error {
return w.tasks.UpdateTask(progressCtx, task.ID, "running", progressStage, progressMessage, &started, nil)
})
var activeAttempt string
completionFailed := false
defer func() {
defer func() {
if resultErr != nil && w.logError != nil {
w.logError(resultErr)
}
}()
// 收尾事务失败后保留中断记录,不能再用分散写入覆盖现场。
if completionFailed {
return
}
finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
finished := time.Now().UTC()
status, message := certificateTaskOutcome(result, resultErr)
auditErr := w.audit.CreateAuditLog(finishCtx, repository.AuditLog{
TaskID: task.ID,
Action: taskType, ResourceType: "domain", ResourceID: id,
OperatorID: task.OperatorID,
Status: status, Message: message,
var report *deployment.Report
if activeAttempt != "" {
report = &result.Deployment
}
recordErr := w.tasks.FinishTaskAttempt(finishCtx, repository.TaskCompletion{
TaskID: task.ID, AttemptID: activeAttempt, Status: status, Stage: stage,
Message: message, StartedAt: &started, FinishedAt: finished,
Report: report, AuditAction: taskType, WithoutAttempt: activeAttempt == "",
})
if auditErr != nil {
status, message = certificateTaskOutcome(result, auditErr)
stage = "record"
}
taskErr := w.tasks.UpdateTask(finishCtx, task.ID, status, stage, message, &started, &finished)
resultErr = errors.Join(resultErr, taskErr, auditErr)
if resultErr != nil && w.logError != nil {
w.logError(resultErr)
}
resultErr = errors.Join(resultErr, recordErr)
}()
if err := w.audit.CreateAuditLog(taskCtx, repository.AuditLog{
TaskID: task.ID,
@@ -146,30 +152,30 @@ func (w *certificateTaskWorkflow) run(ctx context.Context, id, taskType, stage s
if err := w.tasks.CreateAttempt(taskCtx, attempt); err != nil {
return result, err
}
activeAttempt = attempt.ID
issueCtx, cancel := context.WithTimeout(taskCtx, 30*time.Minute)
issueCtx = deployment.WithTransactionContext(issueCtx, deployment.TransactionContext{
TaskID: task.ID, AttemptID: attempt.ID, AttemptNumber: attempt.AttemptNumber,
})
result, resultErr = execute(issueCtx, id)
cancel()
// 取消只停止外部操作,结果保存仍使用独立且有限时的收尾上下文。
finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
if recordErr := w.tasks.RecordTaskDeploymentResults(finishCtx, task.ID, attempt.ID, number, taskDeploymentResults(result.Deployment)); recordErr != nil {
resultErr = errors.Join(resultErr, recordErr)
}
finished := time.Now().UTC()
status, message := certificateTaskOutcome(result, resultErr)
recordErr := w.tasks.FinishAttempt(finishCtx, attempt.ID, status, message, &finished)
finishCancel()
if recordErr != nil {
return result, errors.Join(resultErr, recordErr)
}
if resultErr == nil || number == attempts || result.Certificate.Saved.CertificatePath != "" || errors.Is(resultErr, context.Canceled) || errors.Is(resultErr, context.DeadlineExceeded) {
if resultErr == nil || number == attempts || result.Certificate.Saved.CertificatePath != "" || result.Deployment.Status == "partial" || result.Deployment.RecoveryStatus == "unconfirmed" || result.Deployment.RecoveryStatus == "failed" || errors.Is(resultErr, context.Canceled) || errors.Is(resultErr, context.DeadlineExceeded) {
return result, resultErr
}
if err := w.tasks.UpdateTask(taskCtx, task.ID, "retry_wait", "retry_wait", "证书申请失败,等待重试", &started, nil); err != nil {
return result, errors.Join(resultErr, err)
// 取消只停止外部操作,结果保存仍使用独立且有限时的收尾上下文。
finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
finished := time.Now().UTC()
status, message := certificateTaskOutcome(result, resultErr)
recordErr := w.tasks.FinishTaskAttempt(finishCtx, repository.TaskCompletion{
TaskID: task.ID, AttemptID: attempt.ID, Status: status, Stage: stage,
Message: message, StartedAt: &started, FinishedAt: finished, Report: &result.Deployment, Retry: true,
})
finishCancel()
if recordErr != nil {
completionFailed = true
return result, errors.Join(resultErr, recordErr)
}
activeAttempt = ""
if err := waitForManagementRetry(taskCtx, interval); err != nil {
return result, errors.Join(resultErr, err)
}
+25 -13
View File
@@ -59,10 +59,14 @@ func assertCertificateTask(t *testing.T, a *App, status string, statuses ...stri
}
}
var count int
if err := a.Database.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM audit_logs WHERE action = ? AND resource_id = ? AND status = ?", item.TaskType, item.DomainID, status).Scan(&count); err != nil || count != 1 {
auditStatus := status
if auditStatus == "failed" {
auditStatus = "error"
}
if err := a.Database.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM audit_logs WHERE action = ? AND resource_id = ? AND status = ?", item.TaskType, item.DomainID, auditStatus).Scan(&count); err != nil || count != 1 {
t.Fatalf("审计记录缺失:%d %v", count, err)
}
if err := a.Database.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM audit_logs WHERE task_id = ? AND scope_type = 'project' AND project_id = ? AND operator_id = '' AND status IN ('running', ?)", item.ID, item.ProjectID, status).Scan(&count); err != nil || count != 2 {
if err := a.Database.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM audit_logs WHERE task_id = ? AND scope_type = 'project' AND project_id = ? AND operator_id = '' AND status IN ('running', ?)", item.ID, item.ProjectID, auditStatus).Scan(&count); err != nil || count != 2 {
t.Fatalf("后台任务开始与收尾审计关联缺失:%d,%v", count, err)
}
return item
@@ -168,11 +172,11 @@ func (r failingCertificateTasks) CreateTask(ctx context.Context, task repository
return r.TaskRepository.CreateTask(ctx, task)
}
func (r failingCertificateTasks) FinishAttempt(ctx context.Context, id, status, message string, at *time.Time) error {
func (r failingCertificateTasks) FinishTaskAttempt(ctx context.Context, completion repository.TaskCompletion) error {
if r.failFinish {
return errors.New("attempt-store-unavailable")
}
return r.TaskRepository.FinishAttempt(ctx, id, status, message, at)
return r.TaskRepository.FinishTaskAttempt(ctx, completion)
}
func TestCertificateTaskPersistenceFailuresStopExecutionOrRetries(t *testing.T) {
@@ -320,11 +324,8 @@ type failingCertificateTerminal struct {
failure error
}
func (r failingCertificateTerminal) UpdateTask(ctx context.Context, id, status, stage, message string, started, finished *time.Time) error {
if finished != nil {
return r.failure
}
return r.TaskRepository.UpdateTask(ctx, id, status, stage, message, started, finished)
func (r failingCertificateTerminal) FinishTaskAttempt(ctx context.Context, completion repository.TaskCompletion) error {
return r.failure
}
func TestCertificateAuditFailuresPersistFailedTaskWithoutReissue(t *testing.T) {
@@ -333,6 +334,13 @@ func TestCertificateAuditFailuresPersistFailedTaskWithoutReissue(t *testing.T) {
w, a := newCertificateTaskFixture(t)
failure := errors.New("audit-store-unavailable")
w.audit = failingCertificateAudit{AuditRepository: w.audit, failStatus: failStatus, failure: failure}
if failStatus == "success" {
// 完成审计已移入数据库事务,在真实写入点注入失败。
if _, err := a.Database.DB.ExecContext(t.Context(), `CREATE TRIGGER reject_success_audit BEFORE INSERT ON audit_logs
WHEN NEW.status = 'success' BEGIN SELECT RAISE(ABORT, 'audit-store-unavailable'); END`); err != nil {
t.Fatal(err)
}
}
calls := 0
var logged error
w.logError = func(err error) { logged = err }
@@ -344,15 +352,19 @@ func TestCertificateAuditFailuresPersistFailedTaskWithoutReissue(t *testing.T) {
if failStatus == "running" {
expectedCalls = 0
}
if !errors.Is(err, failure) || !errors.Is(logged, failure) || calls != expectedCalls {
if err == nil || logged == nil || !strings.Contains(err.Error(), failure.Error()) || !strings.Contains(logged.Error(), failure.Error()) || calls != expectedCalls {
t.Fatalf("calls=%d error=%v logged=%v", calls, err, logged)
}
tasks, listErr := a.Tasks.ListTasks(t.Context(), "task-domain", 10)
if listErr != nil || len(tasks) != 1 || tasks[0].Status != "failed" || tasks[0].FinishedAt == nil {
if listErr != nil || len(tasks) != 1 {
t.Fatalf("tasks=%+v error=%v", tasks, listErr)
}
if failStatus == "success" && tasks[0].Stage != "record" {
t.Fatalf("stage=%s", tasks[0].Stage)
if failStatus == "success" {
if tasks[0].Status != "running" || tasks[0].FinishedAt != nil {
t.Fatalf("完成审计失败必须回滚终态:%+v", tasks[0])
}
} else if tasks[0].Status != "failed" || tasks[0].FinishedAt == nil {
t.Fatalf("开始审计失败必须停止执行并收尾:%+v", tasks[0])
}
if strings.Contains(tasks[0].Message, failure.Error()) {
t.Fatal("audit-store-unavailable")