统一后台证书任务报告及收尾事务并补齐故障回归
This commit is contained in:
1 parent
cf3bc7aaa5
commit
657a9c0a36
9 files changed
+379
-67
No files matched your search
@@ -0,0 +1,118 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/repository"
|
||||
)
|
||||
|
||||
func TestCertificateCompletionPersistsFullReportsAcrossRetries(t *testing.T) {
|
||||
w, a := newCertificateTaskFixture(t)
|
||||
calls := 0
|
||||
_, err := w.run(t.Context(), "task-domain", "renewal_run", "certificate_issue", 2, time.Nanosecond,
|
||||
func(context.Context, string) (CertificateWorkflowResult, error) {
|
||||
calls++
|
||||
result := CertificateWorkflowResult{Deployment: deployment.Report{
|
||||
Status: "success", RecoveryStatus: "success", Total: 1, Success: 1,
|
||||
Results: []deployment.Result{{ServerID: "target", Status: "success"}},
|
||||
}}
|
||||
if calls == 1 {
|
||||
result.Deployment.Status = "failed"
|
||||
return result, errors.New("temporary failure")
|
||||
}
|
||||
return result, nil
|
||||
})
|
||||
if err != nil || calls != 2 {
|
||||
t.Fatalf("重试执行失败:%d %v", calls, err)
|
||||
}
|
||||
task := assertCertificateTask(t, a, "success", "failed", "success")
|
||||
reports, err := a.Tasks.ListTaskDeploymentReports(t.Context(), task.ID)
|
||||
if err != nil || len(reports) != 2 {
|
||||
t.Fatalf("每次尝试均应保存完整报告:%+v %v", reports, err)
|
||||
}
|
||||
if reports[0].Report.Status != "failed" || reports[1].Report.Status != "success" || reports[0].AttemptID == reports[1].AttemptID || reports[1].Report.RecoveryStatus != "success" {
|
||||
t.Fatalf("重试覆盖报告或丢失恢复状态:%+v", reports)
|
||||
}
|
||||
}
|
||||
|
||||
type cancelAfterCertificateRetry struct {
|
||||
repository.TaskRepository
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (r cancelAfterCertificateRetry) FinishTaskAttempt(ctx context.Context, completion repository.TaskCompletion) error {
|
||||
err := r.TaskRepository.FinishTaskAttempt(ctx, completion)
|
||||
if err == nil && completion.Retry {
|
||||
r.cancel()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func TestCertificateCompletionCancellationDuringRetryWait(t *testing.T) {
|
||||
w, a := newCertificateTaskFixture(t)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
w.tasks = cancelAfterCertificateRetry{TaskRepository: a.Tasks, cancel: cancel}
|
||||
calls := 0
|
||||
_, err := w.run(ctx, "task-domain", "renewal_run", "certificate_issue", 3, time.Hour,
|
||||
func(context.Context, string) (CertificateWorkflowResult, error) {
|
||||
calls++
|
||||
return CertificateWorkflowResult{}, errors.New("temporary failure")
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) || calls != 1 {
|
||||
t.Fatalf("等待取消后继续重试或丢失取消原因:%d %v", calls, err)
|
||||
}
|
||||
task := assertCertificateTask(t, a, "cancelled", "failed")
|
||||
reports, err := a.Tasks.ListTaskDeploymentReports(t.Context(), task.ID)
|
||||
if err != nil || len(reports) != 1 {
|
||||
t.Fatalf("等待取消丢失上次尝试报告:%+v %v", reports, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCertificateCompletionFailureRollsBackAllRecords(t *testing.T) {
|
||||
for _, table := range []string{"task_deployment_reports", "task_deployment_results", "audit_logs", "task_attempts", "tasks"} {
|
||||
t.Run(table, func(t *testing.T) {
|
||||
w, a := newCertificateTaskFixture(t)
|
||||
calls := 0
|
||||
_, err := w.run(t.Context(), "task-domain", "renewal_run", "certificate_issue", 3, time.Nanosecond,
|
||||
func(context.Context, string) (CertificateWorkflowResult, error) {
|
||||
calls++
|
||||
operation := "INSERT"
|
||||
if table == "tasks" || table == "task_attempts" {
|
||||
operation = "UPDATE"
|
||||
}
|
||||
// 表名只来自本测试的固定枚举,不接受用户输入。
|
||||
if _, err := a.Database.DB.ExecContext(t.Context(), "CREATE TRIGGER IF NOT EXISTS reject_completion BEFORE "+operation+" ON "+table+" BEGIN SELECT RAISE(ABORT, 'completion unavailable'); END"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return CertificateWorkflowResult{Deployment: deployment.Report{Status: "success", Total: 1, Success: 1,
|
||||
Results: []deployment.Result{{ServerID: "target", Status: "success"}}}}, nil
|
||||
})
|
||||
if err == nil || calls != 1 {
|
||||
t.Fatalf("保存失败必须返回错误且不重复外部操作:%d %v", calls, err)
|
||||
}
|
||||
tasks, err := a.Tasks.ListTasks(t.Context(), "task-domain", 10)
|
||||
if err != nil || len(tasks) != 1 || tasks[0].Status != "running" || tasks[0].FinishedAt != nil {
|
||||
t.Fatalf("事务失败必须保留活动任务供恢复:%+v %v", tasks, err)
|
||||
}
|
||||
attempts, err := a.Tasks.ListAttempts(t.Context(), tasks[0].ID)
|
||||
if err != nil || len(attempts) != 1 || attempts[0].Status != "running" || attempts[0].FinishedAt != nil {
|
||||
t.Fatalf("尝试终态未回滚:%+v %v", attempts, err)
|
||||
}
|
||||
for _, stored := range []string{"task_deployment_reports", "task_deployment_results", "audit_logs"} {
|
||||
var count int
|
||||
query := "SELECT COUNT(*) FROM " + stored + " WHERE task_id = ?"
|
||||
if stored == "audit_logs" {
|
||||
query += " AND status <> 'running'"
|
||||
}
|
||||
if err := a.Database.DB.QueryRowContext(t.Context(), query, tasks[0].ID).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatalf("失败后仍残留部分记录:%s %d %v", stored, count, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -101,26 +101,32 @@ func (w *certificateTaskWorkflow) run(ctx context.Context, id, taskType, stage s
|
||||
taskCtx = certificate.WithProgressReporter(taskCtx, func(progressCtx context.Context, progressStage, progressMessage string) error {
|
||||
return w.tasks.UpdateTask(progressCtx, task.ID, "running", progressStage, progressMessage, &started, nil)
|
||||
})
|
||||
var activeAttempt string
|
||||
completionFailed := false
|
||||
defer func() {
|
||||
defer func() {
|
||||
if resultErr != nil && w.logError != nil {
|
||||
w.logError(resultErr)
|
||||
}
|
||||
}()
|
||||
// 收尾事务失败后保留中断记录,不能再用分散写入覆盖现场。
|
||||
if completionFailed {
|
||||
return
|
||||
}
|
||||
finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
finished := time.Now().UTC()
|
||||
status, message := certificateTaskOutcome(result, resultErr)
|
||||
auditErr := w.audit.CreateAuditLog(finishCtx, repository.AuditLog{
|
||||
TaskID: task.ID,
|
||||
Action: taskType, ResourceType: "domain", ResourceID: id,
|
||||
OperatorID: task.OperatorID,
|
||||
Status: status, Message: message,
|
||||
var report *deployment.Report
|
||||
if activeAttempt != "" {
|
||||
report = &result.Deployment
|
||||
}
|
||||
recordErr := w.tasks.FinishTaskAttempt(finishCtx, repository.TaskCompletion{
|
||||
TaskID: task.ID, AttemptID: activeAttempt, Status: status, Stage: stage,
|
||||
Message: message, StartedAt: &started, FinishedAt: finished,
|
||||
Report: report, AuditAction: taskType, WithoutAttempt: activeAttempt == "",
|
||||
})
|
||||
if auditErr != nil {
|
||||
status, message = certificateTaskOutcome(result, auditErr)
|
||||
stage = "record"
|
||||
}
|
||||
taskErr := w.tasks.UpdateTask(finishCtx, task.ID, status, stage, message, &started, &finished)
|
||||
resultErr = errors.Join(resultErr, taskErr, auditErr)
|
||||
if resultErr != nil && w.logError != nil {
|
||||
w.logError(resultErr)
|
||||
}
|
||||
resultErr = errors.Join(resultErr, recordErr)
|
||||
}()
|
||||
if err := w.audit.CreateAuditLog(taskCtx, repository.AuditLog{
|
||||
TaskID: task.ID,
|
||||
@@ -146,30 +152,30 @@ func (w *certificateTaskWorkflow) run(ctx context.Context, id, taskType, stage s
|
||||
if err := w.tasks.CreateAttempt(taskCtx, attempt); err != nil {
|
||||
return result, err
|
||||
}
|
||||
activeAttempt = attempt.ID
|
||||
issueCtx, cancel := context.WithTimeout(taskCtx, 30*time.Minute)
|
||||
issueCtx = deployment.WithTransactionContext(issueCtx, deployment.TransactionContext{
|
||||
TaskID: task.ID, AttemptID: attempt.ID, AttemptNumber: attempt.AttemptNumber,
|
||||
})
|
||||
result, resultErr = execute(issueCtx, id)
|
||||
cancel()
|
||||
// 取消只停止外部操作,结果保存仍使用独立且有限时的收尾上下文。
|
||||
finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
if recordErr := w.tasks.RecordTaskDeploymentResults(finishCtx, task.ID, attempt.ID, number, taskDeploymentResults(result.Deployment)); recordErr != nil {
|
||||
resultErr = errors.Join(resultErr, recordErr)
|
||||
}
|
||||
finished := time.Now().UTC()
|
||||
status, message := certificateTaskOutcome(result, resultErr)
|
||||
recordErr := w.tasks.FinishAttempt(finishCtx, attempt.ID, status, message, &finished)
|
||||
finishCancel()
|
||||
if recordErr != nil {
|
||||
return result, errors.Join(resultErr, recordErr)
|
||||
}
|
||||
if resultErr == nil || number == attempts || result.Certificate.Saved.CertificatePath != "" || errors.Is(resultErr, context.Canceled) || errors.Is(resultErr, context.DeadlineExceeded) {
|
||||
if resultErr == nil || number == attempts || result.Certificate.Saved.CertificatePath != "" || result.Deployment.Status == "partial" || result.Deployment.RecoveryStatus == "unconfirmed" || result.Deployment.RecoveryStatus == "failed" || errors.Is(resultErr, context.Canceled) || errors.Is(resultErr, context.DeadlineExceeded) {
|
||||
return result, resultErr
|
||||
}
|
||||
if err := w.tasks.UpdateTask(taskCtx, task.ID, "retry_wait", "retry_wait", "证书申请失败,等待重试", &started, nil); err != nil {
|
||||
return result, errors.Join(resultErr, err)
|
||||
// 取消只停止外部操作,结果保存仍使用独立且有限时的收尾上下文。
|
||||
finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
finished := time.Now().UTC()
|
||||
status, message := certificateTaskOutcome(result, resultErr)
|
||||
recordErr := w.tasks.FinishTaskAttempt(finishCtx, repository.TaskCompletion{
|
||||
TaskID: task.ID, AttemptID: attempt.ID, Status: status, Stage: stage,
|
||||
Message: message, StartedAt: &started, FinishedAt: finished, Report: &result.Deployment, Retry: true,
|
||||
})
|
||||
finishCancel()
|
||||
if recordErr != nil {
|
||||
completionFailed = true
|
||||
return result, errors.Join(resultErr, recordErr)
|
||||
}
|
||||
activeAttempt = ""
|
||||
if err := waitForManagementRetry(taskCtx, interval); err != nil {
|
||||
return result, errors.Join(resultErr, err)
|
||||
}
|
||||
|
||||
@@ -59,10 +59,14 @@ func assertCertificateTask(t *testing.T, a *App, status string, statuses ...stri
|
||||
}
|
||||
}
|
||||
var count int
|
||||
if err := a.Database.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM audit_logs WHERE action = ? AND resource_id = ? AND status = ?", item.TaskType, item.DomainID, status).Scan(&count); err != nil || count != 1 {
|
||||
auditStatus := status
|
||||
if auditStatus == "failed" {
|
||||
auditStatus = "error"
|
||||
}
|
||||
if err := a.Database.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM audit_logs WHERE action = ? AND resource_id = ? AND status = ?", item.TaskType, item.DomainID, auditStatus).Scan(&count); err != nil || count != 1 {
|
||||
t.Fatalf("审计记录缺失:%d %v", count, err)
|
||||
}
|
||||
if err := a.Database.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM audit_logs WHERE task_id = ? AND scope_type = 'project' AND project_id = ? AND operator_id = '' AND status IN ('running', ?)", item.ID, item.ProjectID, status).Scan(&count); err != nil || count != 2 {
|
||||
if err := a.Database.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM audit_logs WHERE task_id = ? AND scope_type = 'project' AND project_id = ? AND operator_id = '' AND status IN ('running', ?)", item.ID, item.ProjectID, auditStatus).Scan(&count); err != nil || count != 2 {
|
||||
t.Fatalf("后台任务开始与收尾审计关联缺失:%d,%v", count, err)
|
||||
}
|
||||
return item
|
||||
@@ -168,11 +172,11 @@ func (r failingCertificateTasks) CreateTask(ctx context.Context, task repository
|
||||
return r.TaskRepository.CreateTask(ctx, task)
|
||||
}
|
||||
|
||||
func (r failingCertificateTasks) FinishAttempt(ctx context.Context, id, status, message string, at *time.Time) error {
|
||||
func (r failingCertificateTasks) FinishTaskAttempt(ctx context.Context, completion repository.TaskCompletion) error {
|
||||
if r.failFinish {
|
||||
return errors.New("attempt-store-unavailable")
|
||||
}
|
||||
return r.TaskRepository.FinishAttempt(ctx, id, status, message, at)
|
||||
return r.TaskRepository.FinishTaskAttempt(ctx, completion)
|
||||
}
|
||||
|
||||
func TestCertificateTaskPersistenceFailuresStopExecutionOrRetries(t *testing.T) {
|
||||
@@ -320,11 +324,8 @@ type failingCertificateTerminal struct {
|
||||
failure error
|
||||
}
|
||||
|
||||
func (r failingCertificateTerminal) UpdateTask(ctx context.Context, id, status, stage, message string, started, finished *time.Time) error {
|
||||
if finished != nil {
|
||||
return r.failure
|
||||
}
|
||||
return r.TaskRepository.UpdateTask(ctx, id, status, stage, message, started, finished)
|
||||
func (r failingCertificateTerminal) FinishTaskAttempt(ctx context.Context, completion repository.TaskCompletion) error {
|
||||
return r.failure
|
||||
}
|
||||
|
||||
func TestCertificateAuditFailuresPersistFailedTaskWithoutReissue(t *testing.T) {
|
||||
@@ -333,6 +334,13 @@ func TestCertificateAuditFailuresPersistFailedTaskWithoutReissue(t *testing.T) {
|
||||
w, a := newCertificateTaskFixture(t)
|
||||
failure := errors.New("audit-store-unavailable")
|
||||
w.audit = failingCertificateAudit{AuditRepository: w.audit, failStatus: failStatus, failure: failure}
|
||||
if failStatus == "success" {
|
||||
// 完成审计已移入数据库事务,在真实写入点注入失败。
|
||||
if _, err := a.Database.DB.ExecContext(t.Context(), `CREATE TRIGGER reject_success_audit BEFORE INSERT ON audit_logs
|
||||
WHEN NEW.status = 'success' BEGIN SELECT RAISE(ABORT, 'audit-store-unavailable'); END`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
calls := 0
|
||||
var logged error
|
||||
w.logError = func(err error) { logged = err }
|
||||
@@ -344,15 +352,19 @@ func TestCertificateAuditFailuresPersistFailedTaskWithoutReissue(t *testing.T) {
|
||||
if failStatus == "running" {
|
||||
expectedCalls = 0
|
||||
}
|
||||
if !errors.Is(err, failure) || !errors.Is(logged, failure) || calls != expectedCalls {
|
||||
if err == nil || logged == nil || !strings.Contains(err.Error(), failure.Error()) || !strings.Contains(logged.Error(), failure.Error()) || calls != expectedCalls {
|
||||
t.Fatalf("calls=%d error=%v logged=%v", calls, err, logged)
|
||||
}
|
||||
tasks, listErr := a.Tasks.ListTasks(t.Context(), "task-domain", 10)
|
||||
if listErr != nil || len(tasks) != 1 || tasks[0].Status != "failed" || tasks[0].FinishedAt == nil {
|
||||
if listErr != nil || len(tasks) != 1 {
|
||||
t.Fatalf("tasks=%+v error=%v", tasks, listErr)
|
||||
}
|
||||
if failStatus == "success" && tasks[0].Stage != "record" {
|
||||
t.Fatalf("stage=%s", tasks[0].Stage)
|
||||
if failStatus == "success" {
|
||||
if tasks[0].Status != "running" || tasks[0].FinishedAt != nil {
|
||||
t.Fatalf("完成审计失败必须回滚终态:%+v", tasks[0])
|
||||
}
|
||||
} else if tasks[0].Status != "failed" || tasks[0].FinishedAt == nil {
|
||||
t.Fatalf("开始审计失败必须停止执行并收尾:%+v", tasks[0])
|
||||
}
|
||||
if strings.Contains(tasks[0].Message, failure.Error()) {
|
||||
t.Fatal("audit-store-unavailable")
|
||||
|
||||
Reference in new issue
Block a user