package app import ( "context" "errors" "strings" "testing" "time" "git.pcux.cn/xuans/Scert/backend/internal/certificate" "git.pcux.cn/xuans/Scert/backend/internal/deployment" "git.pcux.cn/xuans/Scert/backend/internal/domain" "git.pcux.cn/xuans/Scert/backend/internal/repository" ) func newCertificateTaskFixture(t *testing.T) (*certificateTaskWorkflow, *App) { t.Helper() a, err := New(t.Context(), readyApplicationConfig(t)) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = a.Close() }) item, err := domain.NewDomain("task.example.test", domain.CertificateSingle, nil) if err != nil { t.Fatal(err) } item.ID = "task-domain" if err := a.Repository.Create(t.Context(), item); err != nil { t.Fatal(err) } return &certificateTaskWorkflow{ tasks: a.Tasks, audit: repository.NewSQLAuditRepository(a.Database.DB), }, a } func assertCertificateTask(t *testing.T, a *App, status string, statuses ...string) repository.Task { t.Helper() items, err := a.Tasks.ListTasks(t.Context(), "task-domain", 10) if err != nil || len(items) != 1 { t.Fatalf("任务数量错误:%+v %v", items, err) } item := items[0] if item.ScopeType != repository.ScopeProject || item.ProjectID != domain.DefaultProjectID || item.OperatorID != "" { t.Fatalf("后台任务范围错误或伪造操作者:%+v", item) } if item.Status != status || item.StartedAt == nil || item.FinishedAt == nil { t.Fatalf("任务未正确收尾:%+v", item) } attempts, err := a.Tasks.ListAttempts(t.Context(), item.ID) if err != nil || len(attempts) != len(statuses) { t.Fatalf("尝试数量错误:%+v %v", attempts, err) } for i, attempt := range attempts { if attempt.AttemptNumber != i+1 || attempt.Status != statuses[i] || attempt.FinishedAt == nil { t.Fatalf("尝试结果错误:%+v", attempt) } if strings.Contains(attempt.Message, "secret-path") { t.Fatal("尝试记录泄露内部错误") } } var count int auditStatus := status if auditStatus == "failed" { auditStatus = "error" } if err := a.Database.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM audit_logs WHERE action = ? AND resource_id = ? AND status = ?", item.TaskType, item.DomainID, auditStatus).Scan(&count); err != nil || count != 1 { t.Fatalf("审计记录缺失:%d %v", count, err) } if err := a.Database.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM audit_logs WHERE task_id = ? AND scope_type = 'project' AND project_id = ? AND operator_id = '' AND status IN ('running', ?)", item.ID, item.ProjectID, auditStatus).Scan(&count); err != nil || count != 2 { t.Fatalf("后台任务开始与收尾审计关联缺失:%d,%v", count, err) } return item } func TestCertificateTasksRecordSuccessFailureAndCancellation(t *testing.T) { for _, status := range []string{"success", "failed", "cancelled"} { t.Run(status, func(t *testing.T) { w, a := newCertificateTaskFixture(t) ctx, cancel := context.WithCancel(t.Context()) defer cancel() _, err := w.run(ctx, "task-domain", "renewal_run", "certificate_issue", 1, 0, func(context.Context, string) (CertificateWorkflowResult, error) { if status == "cancelled" { cancel() return CertificateWorkflowResult{}, context.Canceled } if status == "failed" { return CertificateWorkflowResult{}, errors.New("secret-path") } return CertificateWorkflowResult{}, nil }) if (err != nil) != (status != "success") { t.Fatalf("执行结果错误:%v", err) } assertCertificateTask(t, a, status, status) }) } } func TestCertificateTasksRecordOperatorFromWorkflowContext(t *testing.T) { w, a := newCertificateTaskFixture(t) ctx := repository.WithTaskOperator(t.Context(), "operator-1") _, err := w.run(ctx, "task-domain", "certificate_validate", "certificate_validate", 1, 0, func(context.Context, string) (CertificateWorkflowResult, error) { return CertificateWorkflowResult{}, nil }) if err != nil { t.Fatalf("带操作者的证书任务执行失败:%v", err) } items, err := a.Tasks.ListTasks(t.Context(), "task-domain", 10) if err != nil || len(items) != 1 || items[0].OperatorID != "operator-1" { t.Fatalf("任务创建人记录错误:%+v %v", items, err) } var operators []string rows, err := a.Database.DB.QueryContext(t.Context(), "SELECT operator_id FROM audit_logs WHERE task_id = ? ORDER BY created_at, id", items[0].ID) if err != nil { t.Fatalf("读取任务审计失败:%v", err) } defer rows.Close() for rows.Next() { var operator string if err := rows.Scan(&operator); err != nil { t.Fatalf("读取任务审计操作者失败:%v", err) } operators = append(operators, operator) } if err := rows.Err(); err != nil { t.Fatalf("遍历任务审计失败:%v", err) } if len(operators) != 2 || operators[0] != "operator-1" || operators[1] != "operator-1" { t.Fatalf("任务审计创建人记录错误:%v", operators) } } func TestCertificateTasksKeepRetriesInOneTask(t *testing.T) { w, a := newCertificateTaskFixture(t) calls := 0 _, err := w.run(t.Context(), "task-domain", "renewal_run", "certificate_issue", 3, time.Nanosecond, func(context.Context, string) (CertificateWorkflowResult, error) { calls++ if calls < 3 { return CertificateWorkflowResult{}, errors.New("secret-path") } return CertificateWorkflowResult{}, nil }) if err != nil || calls != 3 { t.Fatalf("重试次数错误:%d %v", calls, err) } assertCertificateTask(t, a, "success", "failed", "failed", "success") } func TestCertificateTasksDoNotReissueAfterDeploymentFailure(t *testing.T) { w, a := newCertificateTaskFixture(t) calls := 0 _, err := w.run(t.Context(), "task-domain", "renewal_run", "certificate_issue", 3, time.Nanosecond, func(context.Context, string) (CertificateWorkflowResult, error) { calls++ return CertificateWorkflowResult{Certificate: certificate.IssueResult{Saved: certificate.SavedResource{CertificatePath: "cert.pem"}}}, errors.New("secret-path") }) if err == nil || calls != 1 { t.Fatalf("部署失败不应重复签发:%d %v", calls, err) } assertCertificateTask(t, a, "failed", "failed") } type failingCertificateTasks struct { repository.TaskRepository failCreate bool failFinish bool } func (r failingCertificateTasks) CreateTask(ctx context.Context, task repository.Task) error { if r.failCreate { return errors.New("task-store-unavailable") } return r.TaskRepository.CreateTask(ctx, task) } func (r failingCertificateTasks) FinishTaskAttempt(ctx context.Context, completion repository.TaskCompletion) error { if r.failFinish { return errors.New("attempt-store-unavailable") } return r.TaskRepository.FinishTaskAttempt(ctx, completion) } func TestCertificateTaskPersistenceFailuresStopExecutionOrRetries(t *testing.T) { for _, failCreate := range []bool{true, false} { w, a := newCertificateTaskFixture(t) w.tasks = failingCertificateTasks{TaskRepository: a.Tasks, failCreate: failCreate, failFinish: !failCreate} calls := 0 _, err := w.run(t.Context(), "task-domain", "renewal_run", "certificate_issue", 3, time.Nanosecond, func(context.Context, string) (CertificateWorkflowResult, error) { calls++ return CertificateWorkflowResult{}, errors.New("secret-path") }) if err == nil || (failCreate && calls != 0) || (!failCreate && calls != 1) { t.Fatalf("记录失败未阻止副作用或重复执行:%d %v", calls, err) } } } func TestCertificateTaskEntryPointsRecordRenewalAndDeployment(t *testing.T) { for _, renew := range []bool{true, false} { w, a := newCertificateTaskFixture(t) service := &workflowCertificateService{result: certificate.IssueResult{ DomainID: "task-domain", Saved: certificate.SavedResource{CertificatePath: "cert.pem"}, }} w.workflow = &CertificateWorkflow{ Certificates: service, Domains: &workflowDomainState{}, Servers: workflowServerLister{servers: []deployment.Server{{ID: "server"}}}, Deployment: &workflowDeployer{report: deployment.Report{Total: 1, Success: 1, Status: "success"}}, CurrentCertificate: func(context.Context, string) (certificate.IssueResult, error) { return service.result, nil }, } var err error if renew { err = w.Renew(t.Context(), "task-domain", false, certificate.RenewalPolicy{}) } else { _, err = w.DeployCurrent(t.Context(), "task-domain") } if err != nil || service.called != renew { t.Fatalf("任务入口执行错误:%v", err) } item := assertCertificateTask(t, a, "success", "success") if (renew && item.TaskType != "renewal_run") || (!renew && item.TaskType != "certificate_deploy") { t.Fatalf("任务类型错误:%+v", item) } } } type restoringWorkflowCertificateService struct { workflowCertificateService restored certificate.IssueResult } func (s *restoringWorkflowCertificateService) RestoreVersion(context.Context, string, string) (certificate.IssueResult, error) { s.called = true return s.restored, nil } func TestCertificateTaskEntryPointsRecordImportRestoreAndValidate(t *testing.T) { t.Run("import", func(t *testing.T) { w, a := newCertificateTaskFixture(t) service := &workflowCertificateService{result: certificate.IssueResult{ DomainID: "task-domain", Domains: []string{"task.example.test"}, Saved: certificate.SavedResource{CertificatePath: "cert.pem"}, }} w.workflow = &CertificateWorkflow{ Certificates: service, Domains: &workflowDomainState{}, Servers: workflowServerLister{servers: []deployment.Server{{ID: "server"}}}, Deployment: &workflowDeployer{report: deployment.Report{Total: 1, Success: 1, Status: "success"}}, } _, err := w.Import(t.Context(), certificate.ImportRequest{DomainID: "task-domain", Certificate: "cert.pem", PrivateKey: "key.pem"}) if err != nil || !service.called { t.Fatalf("证书导入任务入口执行错误:%v", err) } item := assertCertificateTask(t, a, "success", "success") if item.TaskType != "certificate_import" { t.Fatalf("证书导入任务类型错误:%+v", item) } }) t.Run("restore", func(t *testing.T) { w, a := newCertificateTaskFixture(t) service := &restoringWorkflowCertificateService{workflowCertificateService: workflowCertificateService{ result: certificate.IssueResult{DomainID: "task-domain"}, }, restored: certificate.IssueResult{DomainID: "task-domain"}} w.workflow = &CertificateWorkflow{Certificates: service} _, err := w.Restore(t.Context(), "task-domain", "version-1") if err != nil || !service.called { t.Fatalf("证书恢复任务入口执行错误:%v", err) } item := assertCertificateTask(t, a, "success", "success") if item.TaskType != "certificate_restore" { t.Fatalf("证书恢复任务类型错误:%+v", item) } }) t.Run("validate", func(t *testing.T) { w, a := newCertificateTaskFixture(t) service := &workflowCertificateService{} w.workflow = &CertificateWorkflow{ CurrentCertificate: func(context.Context, string) (certificate.IssueResult, error) { service.called = true return certificate.IssueResult{DomainID: "task-domain"}, nil }, } _, _, err := w.Validate(t.Context(), "task-domain") if err == nil || !service.called { t.Fatal("证书校验任务入口未返回预期失败") } item := assertCertificateTask(t, a, "failed", "failed") if item.TaskType != "certificate_validate" { t.Fatalf("证书校验任务类型错误:%+v", item) } }) } func TestCertificateTaskRejectsInvalidExecutionConfiguration(t *testing.T) { w, a := newCertificateTaskFixture(t) _, err := w.run(t.Context(), "task-domain", "certificate_validate", "certificate_validate", 0, 0, func(context.Context, string) (CertificateWorkflowResult, error) { return CertificateWorkflowResult{}, nil }) if err == nil || !strings.Contains(err.Error(), "至少执行一次") { t.Fatalf("无效执行次数未被拒绝:%v", err) } items, listErr := a.Tasks.ListTasks(t.Context(), "task-domain", 10) if listErr != nil || len(items) != 0 { t.Fatalf("无效任务配置不应创建任务:%+v,%v", items, listErr) } } type failingCertificateAudit struct { repository.AuditRepository failStatus string failure error } func (r failingCertificateAudit) CreateAuditLog(ctx context.Context, item repository.AuditLog) error { if item.Status == r.failStatus { return r.failure } return r.AuditRepository.CreateAuditLog(ctx, item) } type failingCertificateTerminal struct { repository.TaskRepository failure error } func (r failingCertificateTerminal) FinishTaskAttempt(ctx context.Context, completion repository.TaskCompletion) error { return r.failure } func TestCertificateAuditFailuresPersistFailedTaskWithoutReissue(t *testing.T) { for _, failStatus := range []string{"running", "success"} { t.Run(failStatus, func(t *testing.T) { w, a := newCertificateTaskFixture(t) failure := errors.New("audit-store-unavailable") w.audit = failingCertificateAudit{AuditRepository: w.audit, failStatus: failStatus, failure: failure} if failStatus == "success" { // 完成审计已移入数据库事务,在真实写入点注入失败。 if _, err := a.Database.DB.ExecContext(t.Context(), `CREATE TRIGGER reject_success_audit BEFORE INSERT ON audit_logs WHEN NEW.status = 'success' BEGIN SELECT RAISE(ABORT, 'audit-store-unavailable'); END`); err != nil { t.Fatal(err) } } calls := 0 var logged error w.logError = func(err error) { logged = err } _, err := w.run(t.Context(), "task-domain", "renewal_run", "certificate_issue", 3, time.Nanosecond, func(context.Context, string) (CertificateWorkflowResult, error) { calls++ return CertificateWorkflowResult{}, nil }) expectedCalls := 1 if failStatus == "running" { expectedCalls = 0 } if err == nil || logged == nil || !strings.Contains(err.Error(), failure.Error()) || !strings.Contains(logged.Error(), failure.Error()) || calls != expectedCalls { t.Fatalf("calls=%d error=%v logged=%v", calls, err, logged) } tasks, listErr := a.Tasks.ListTasks(t.Context(), "task-domain", 10) if listErr != nil || len(tasks) != 1 { t.Fatalf("tasks=%+v error=%v", tasks, listErr) } if failStatus == "success" { if tasks[0].Status != "running" || tasks[0].FinishedAt != nil { t.Fatalf("完成审计失败必须回滚终态:%+v", tasks[0]) } } else if tasks[0].Status != "failed" || tasks[0].FinishedAt == nil { t.Fatalf("开始审计失败必须停止执行并收尾:%+v", tasks[0]) } if strings.Contains(tasks[0].Message, failure.Error()) { t.Fatal("audit-store-unavailable") } }) } } func TestCertificateTerminalPersistenceFailureIsReturnedWithoutReissue(t *testing.T) { w, a := newCertificateTaskFixture(t) failure := errors.New("task-terminal-unavailable") w.tasks = failingCertificateTerminal{TaskRepository: a.Tasks, failure: failure} calls := 0 var logged error w.logError = func(err error) { logged = err } _, err := w.run(t.Context(), "task-domain", "renewal_run", "certificate_issue", 3, time.Nanosecond, func(context.Context, string) (CertificateWorkflowResult, error) { calls++ return CertificateWorkflowResult{}, nil }) if calls != 1 || !errors.Is(err, failure) || !errors.Is(logged, failure) { t.Fatalf("calls=%d error=%v logged=%v", calls, err, logged) } } func TestCertificateTasksBindCancellationHook(t *testing.T) { w, a := newCertificateTaskFixture(t) registerCh := make(chan context.CancelFunc, 1) unregisterCh := make(chan string, 1) ctx := repository.WithTaskCancellationHooks(t.Context(), repository.TaskCancellationHooks{ Register: func(_ string, cancel context.CancelFunc) { registerCh <- cancel }, Unregister: func(taskID string) { unregisterCh <- taskID }, }) started := make(chan struct{}) finished := make(chan error, 1) go func() { _, err := w.run(ctx, "task-domain", "certificate_validate", "certificate_validate", 1, 0, func(ctx context.Context, _ string) (CertificateWorkflowResult, error) { close(started) <-ctx.Done() return CertificateWorkflowResult{}, ctx.Err() }) finished <- err }() <-started cancel := <-registerCh cancel() if err := <-finished; !errors.Is(err, context.Canceled) { t.Fatalf("task cancellation error = %v", err) } select { case <-unregisterCh: case <-time.After(time.Second): t.Fatal("task cancellation hook was not unregistered") } assertCertificateTask(t, a, "cancelled", "cancelled") } func TestCertificateTasksRecordPartialDeploymentFailure(t *testing.T) { w, a := newCertificateTaskFixture(t) _, err := w.run(t.Context(), "task-domain", "renewal_run", "certificate_issue", 1, 0, func(context.Context, string) (CertificateWorkflowResult, error) { return CertificateWorkflowResult{ Deployment: deployment.Report{Total: 2, Success: 1, Failed: 1, Status: "partial"}, }, errors.New("partial deployment") }) if err == nil { t.Fatal("partial deployment error should remain visible to the caller") } assertCertificateTask(t, a, "partial_success", "partial_success") } func TestCertificateTaskPersistsDeploymentResults(t *testing.T) { w, a := newCertificateTaskFixture(t) w.workflow = &CertificateWorkflow{ Certificates: &workflowCertificateService{result: certificate.IssueResult{DomainID: "task-domain"}}, Domains: &workflowDomainState{}, Servers: workflowServerLister{servers: []deployment.Server{{ID: "server-1"}, {ID: "server-2"}}}, CurrentCertificate: func(context.Context, string) (certificate.IssueResult, error) { return certificate.IssueResult{DomainID: "task-domain", Saved: certificate.SavedResource{CertificatePath: "cert.pem"}}, nil }, Deployment: &workflowDeployer{report: deployment.Report{ Total: 2, Success: 1, Failed: 1, Status: "partial", Results: []deployment.Result{ {ServerID: "server-1", Server: "第一台服务器", Status: "success", Message: "部署成功"}, {ServerID: "server-2", Server: "第二台服务器", Status: "failed", Message: "部署失败"}, }, }}, } _, err := w.DeployCurrent(t.Context(), "task-domain") if err == nil { t.Fatal("部分部署失败应返回错误") } items, err := a.Tasks.ListTasks(t.Context(), "task-domain", 10) if err != nil || len(items) != 1 { t.Fatalf("任务读取失败:%+v %v", items, err) } results, err := a.Tasks.ListTaskDeploymentResults(t.Context(), items[0].ID) if err != nil || len(results) != 2 || results[0].AttemptID == "" || results[1].Status != "failed" { t.Fatalf("部署结果未持久化:%+v %v", results, err) } }