Scert/backend/internal/app/certificate_tasks_test.go

package app

import (
	"context"
	"errors"
	"strings"
	"testing"
	"time"

	"git.pcux.cn/xuans/Scert/backend/internal/certificate"
	"git.pcux.cn/xuans/Scert/backend/internal/deployment"
	"git.pcux.cn/xuans/Scert/backend/internal/domain"
	"git.pcux.cn/xuans/Scert/backend/internal/repository"
)

func newCertificateTaskFixture(t *testing.T) (*certificateTaskWorkflow, *App) {
	t.Helper()
	a, err := New(t.Context(), readyApplicationConfig(t))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = a.Close() })
	item, err := domain.NewDomain("task.example.test", domain.CertificateSingle, nil)
	if err != nil {
		t.Fatal(err)
	}
	item.ID = "task-domain"
	if err := a.Repository.Create(t.Context(), item); err != nil {
		t.Fatal(err)
	}
	return &certificateTaskWorkflow{
		tasks: a.Tasks, audit: repository.NewSQLAuditRepository(a.Database.DB),
	}, a
}

func assertCertificateTask(t *testing.T, a *App, status string, statuses ...string) repository.Task {
	t.Helper()
	items, err := a.Tasks.ListTasks(t.Context(), "task-domain", 10)
	if err != nil || len(items) != 1 {
		t.Fatalf("任务数量错误：%+v %v", items, err)
	}
	item := items[0]
	if item.ScopeType != repository.ScopeProject || item.ProjectID != domain.DefaultProjectID || item.OperatorID != "" {
		t.Fatalf("后台任务范围错误或伪造操作者：%+v", item)
	}
	if item.Status != status || item.StartedAt == nil || item.FinishedAt == nil {
		t.Fatalf("任务未正确收尾：%+v", item)
	}
	attempts, err := a.Tasks.ListAttempts(t.Context(), item.ID)
	if err != nil || len(attempts) != len(statuses) {
		t.Fatalf("尝试数量错误：%+v %v", attempts, err)
	}
	for i, attempt := range attempts {
		if attempt.AttemptNumber != i+1 || attempt.Status != statuses[i] || attempt.FinishedAt == nil {
			t.Fatalf("尝试结果错误：%+v", attempt)
		}
		if strings.Contains(attempt.Message, "secret-path") {
			t.Fatal("尝试记录泄露内部错误")
		}
	}
	var count int
	auditStatus := status
	if auditStatus == "failed" {
		auditStatus = "error"
	}
	if err := a.Database.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM audit_logs WHERE action = ? AND resource_id = ? AND status = ?", item.TaskType, item.DomainID, auditStatus).Scan(&count); err != nil || count != 1 {
		t.Fatalf("审计记录缺失：%d %v", count, err)
	}
	if err := a.Database.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM audit_logs WHERE task_id = ? AND scope_type = 'project' AND project_id = ? AND operator_id = '' AND status IN ('running', ?)", item.ID, item.ProjectID, auditStatus).Scan(&count); err != nil || count != 2 {
		t.Fatalf("后台任务开始与收尾审计关联缺失：%d，%v", count, err)
	}
	return item
}

func TestCertificateTasksRecordSuccessFailureAndCancellation(t *testing.T) {
	for _, status := range []string{"success", "failed", "cancelled"} {
		t.Run(status, func(t *testing.T) {
			w, a := newCertificateTaskFixture(t)
			ctx, cancel := context.WithCancel(t.Context())
			defer cancel()
			_, err := w.run(ctx, "task-domain", "renewal_run", "certificate_issue", 1, 0, func(context.Context, string) (CertificateWorkflowResult, error) {
				if status == "cancelled" {
					cancel()
					return CertificateWorkflowResult{}, context.Canceled
				}
				if status == "failed" {
					return CertificateWorkflowResult{}, errors.New("secret-path")
				}
				return CertificateWorkflowResult{}, nil
			})
			if (err != nil) != (status != "success") {
				t.Fatalf("执行结果错误：%v", err)
			}
			assertCertificateTask(t, a, status, status)
		})
	}
}

func TestCertificateTasksRecordOperatorFromWorkflowContext(t *testing.T) {
	w, a := newCertificateTaskFixture(t)
	ctx := repository.WithTaskOperator(t.Context(), "operator-1")
	_, err := w.run(ctx, "task-domain", "certificate_validate", "certificate_validate", 1, 0, func(context.Context, string) (CertificateWorkflowResult, error) {
		return CertificateWorkflowResult{}, nil
	})
	if err != nil {
		t.Fatalf("带操作者的证书任务执行失败：%v", err)
	}
	items, err := a.Tasks.ListTasks(t.Context(), "task-domain", 10)
	if err != nil || len(items) != 1 || items[0].OperatorID != "operator-1" {
		t.Fatalf("任务创建人记录错误：%+v %v", items, err)
	}
	var operators []string
	rows, err := a.Database.DB.QueryContext(t.Context(), "SELECT operator_id FROM audit_logs WHERE task_id = ? ORDER BY created_at, id", items[0].ID)
	if err != nil {
		t.Fatalf("读取任务审计失败：%v", err)
	}
	defer rows.Close()
	for rows.Next() {
		var operator string
		if err := rows.Scan(&operator); err != nil {
			t.Fatalf("读取任务审计操作者失败：%v", err)
		}
		operators = append(operators, operator)
	}
	if err := rows.Err(); err != nil {
		t.Fatalf("遍历任务审计失败：%v", err)
	}
	if len(operators) != 2 || operators[0] != "operator-1" || operators[1] != "operator-1" {
		t.Fatalf("任务审计创建人记录错误：%v", operators)
	}
}

func TestCertificateTasksKeepRetriesInOneTask(t *testing.T) {
	w, a := newCertificateTaskFixture(t)
	calls := 0
	_, err := w.run(t.Context(), "task-domain", "renewal_run", "certificate_issue", 3, time.Nanosecond, func(context.Context, string) (CertificateWorkflowResult, error) {
		calls++
		if calls < 3 {
			return CertificateWorkflowResult{}, errors.New("secret-path")
		}
		return CertificateWorkflowResult{}, nil
	})
	if err != nil || calls != 3 {
		t.Fatalf("重试次数错误：%d %v", calls, err)
	}
	assertCertificateTask(t, a, "success", "failed", "failed", "success")
}

func TestCertificateTasksDoNotReissueAfterDeploymentFailure(t *testing.T) {
	w, a := newCertificateTaskFixture(t)
	calls := 0
	_, err := w.run(t.Context(), "task-domain", "renewal_run", "certificate_issue", 3, time.Nanosecond, func(context.Context, string) (CertificateWorkflowResult, error) {
		calls++
		return CertificateWorkflowResult{Certificate: certificate.IssueResult{Saved: certificate.SavedResource{CertificatePath: "cert.pem"}}}, errors.New("secret-path")
	})
	if err == nil || calls != 1 {
		t.Fatalf("部署失败不应重复签发：%d %v", calls, err)
	}
	assertCertificateTask(t, a, "failed", "failed")
}

type failingCertificateTasks struct {
	repository.TaskRepository
	failCreate bool
	failFinish bool
}

func (r failingCertificateTasks) CreateTask(ctx context.Context, task repository.Task) error {
	if r.failCreate {
		return errors.New("task-store-unavailable")
	}
	return r.TaskRepository.CreateTask(ctx, task)
}

func (r failingCertificateTasks) FinishTaskAttempt(ctx context.Context, completion repository.TaskCompletion) error {
	if r.failFinish {
		return errors.New("attempt-store-unavailable")
	}
	return r.TaskRepository.FinishTaskAttempt(ctx, completion)
}

func TestCertificateTaskPersistenceFailuresStopExecutionOrRetries(t *testing.T) {
	for _, failCreate := range []bool{true, false} {
		w, a := newCertificateTaskFixture(t)
		w.tasks = failingCertificateTasks{TaskRepository: a.Tasks, failCreate: failCreate, failFinish: !failCreate}
		calls := 0
		_, err := w.run(t.Context(), "task-domain", "renewal_run", "certificate_issue", 3, time.Nanosecond, func(context.Context, string) (CertificateWorkflowResult, error) {
			calls++
			return CertificateWorkflowResult{}, errors.New("secret-path")
		})
		if err == nil || (failCreate && calls != 0) || (!failCreate && calls != 1) {
			t.Fatalf("记录失败未阻止副作用或重复执行：%d %v", calls, err)
		}
	}
}

func TestCertificateTaskEntryPointsRecordRenewalAndDeployment(t *testing.T) {
	for _, renew := range []bool{true, false} {
		w, a := newCertificateTaskFixture(t)
		service := &workflowCertificateService{result: certificate.IssueResult{
			DomainID: "task-domain", Saved: certificate.SavedResource{CertificatePath: "cert.pem"},
		}}
		w.workflow = &CertificateWorkflow{
			Certificates: service, Domains: &workflowDomainState{},
			Servers:            workflowServerLister{servers: []deployment.Server{{ID: "server"}}},
			Deployment:         &workflowDeployer{report: deployment.Report{Total: 1, Success: 1, Status: "success"}},
			CurrentCertificate: func(context.Context, string) (certificate.IssueResult, error) { return service.result, nil },
		}
		var err error
		if renew {
			err = w.Renew(t.Context(), "task-domain", false, certificate.RenewalPolicy{})
		} else {
			_, err = w.DeployCurrent(t.Context(), "task-domain")
		}
		if err != nil || service.called != renew {
			t.Fatalf("任务入口执行错误：%v", err)
		}
		item := assertCertificateTask(t, a, "success", "success")
		if (renew && item.TaskType != "renewal_run") || (!renew && item.TaskType != "certificate_deploy") {
			t.Fatalf("任务类型错误：%+v", item)
		}
	}
}

type restoringWorkflowCertificateService struct {
	workflowCertificateService
	restored certificate.IssueResult
}

func (s *restoringWorkflowCertificateService) RestoreVersion(context.Context, string, string) (certificate.IssueResult, error) {
	s.called = true
	return s.restored, nil
}

func TestCertificateTaskEntryPointsRecordImportRestoreAndValidate(t *testing.T) {
	t.Run("import", func(t *testing.T) {
		w, a := newCertificateTaskFixture(t)
		service := &workflowCertificateService{result: certificate.IssueResult{
			DomainID: "task-domain",
			Domains:  []string{"task.example.test"},
			Saved:    certificate.SavedResource{CertificatePath: "cert.pem"},
		}}
		w.workflow = &CertificateWorkflow{
			Certificates: service,
			Domains:      &workflowDomainState{},
			Servers:      workflowServerLister{servers: []deployment.Server{{ID: "server"}}},
			Deployment:   &workflowDeployer{report: deployment.Report{Total: 1, Success: 1, Status: "success"}},
		}
		_, err := w.Import(t.Context(), certificate.ImportRequest{DomainID: "task-domain", Certificate: "cert.pem", PrivateKey: "key.pem"})
		if err != nil || !service.called {
			t.Fatalf("证书导入任务入口执行错误：%v", err)
		}
		item := assertCertificateTask(t, a, "success", "success")
		if item.TaskType != "certificate_import" {
			t.Fatalf("证书导入任务类型错误：%+v", item)
		}
	})

	t.Run("restore", func(t *testing.T) {
		w, a := newCertificateTaskFixture(t)
		service := &restoringWorkflowCertificateService{workflowCertificateService: workflowCertificateService{
			result: certificate.IssueResult{DomainID: "task-domain"},
		}, restored: certificate.IssueResult{DomainID: "task-domain"}}
		w.workflow = &CertificateWorkflow{Certificates: service}
		_, err := w.Restore(t.Context(), "task-domain", "version-1")
		if err != nil || !service.called {
			t.Fatalf("证书恢复任务入口执行错误：%v", err)
		}
		item := assertCertificateTask(t, a, "success", "success")
		if item.TaskType != "certificate_restore" {
			t.Fatalf("证书恢复任务类型错误：%+v", item)
		}
	})

	t.Run("validate", func(t *testing.T) {
		w, a := newCertificateTaskFixture(t)
		service := &workflowCertificateService{}
		w.workflow = &CertificateWorkflow{
			CurrentCertificate: func(context.Context, string) (certificate.IssueResult, error) {
				service.called = true
				return certificate.IssueResult{DomainID: "task-domain"}, nil
			},
		}
		_, _, err := w.Validate(t.Context(), "task-domain")
		if err == nil || !service.called {
			t.Fatal("证书校验任务入口未返回预期失败")
		}
		item := assertCertificateTask(t, a, "failed", "failed")
		if item.TaskType != "certificate_validate" {
			t.Fatalf("证书校验任务类型错误：%+v", item)
		}
	})
}

func TestCertificateTaskRejectsInvalidExecutionConfiguration(t *testing.T) {
	w, a := newCertificateTaskFixture(t)
	_, err := w.run(t.Context(), "task-domain", "certificate_validate", "certificate_validate", 0, 0, func(context.Context, string) (CertificateWorkflowResult, error) {
		return CertificateWorkflowResult{}, nil
	})
	if err == nil || !strings.Contains(err.Error(), "至少执行一次") {
		t.Fatalf("无效执行次数未被拒绝：%v", err)
	}
	items, listErr := a.Tasks.ListTasks(t.Context(), "task-domain", 10)
	if listErr != nil || len(items) != 0 {
		t.Fatalf("无效任务配置不应创建任务：%+v，%v", items, listErr)
	}
}

type failingCertificateAudit struct {
	repository.AuditRepository
	failStatus string
	failure    error
}

func (r failingCertificateAudit) CreateAuditLog(ctx context.Context, item repository.AuditLog) error {
	if item.Status == r.failStatus {
		return r.failure
	}
	return r.AuditRepository.CreateAuditLog(ctx, item)
}

type failingCertificateTerminal struct {
	repository.TaskRepository
	failure error
}

func (r failingCertificateTerminal) FinishTaskAttempt(ctx context.Context, completion repository.TaskCompletion) error {
	return r.failure
}

func TestCertificateAuditFailuresPersistFailedTaskWithoutReissue(t *testing.T) {
	for _, failStatus := range []string{"running", "success"} {
		t.Run(failStatus, func(t *testing.T) {
			w, a := newCertificateTaskFixture(t)
			failure := errors.New("audit-store-unavailable")
			w.audit = failingCertificateAudit{AuditRepository: w.audit, failStatus: failStatus, failure: failure}
			if failStatus == "success" {
				// 完成审计已移入数据库事务，在真实写入点注入失败。
				if _, err := a.Database.DB.ExecContext(t.Context(), `CREATE TRIGGER reject_success_audit BEFORE INSERT ON audit_logs
					WHEN NEW.status = 'success' BEGIN SELECT RAISE(ABORT, 'audit-store-unavailable'); END`); err != nil {
					t.Fatal(err)
				}
			}
			calls := 0
			var logged error
			w.logError = func(err error) { logged = err }
			_, err := w.run(t.Context(), "task-domain", "renewal_run", "certificate_issue", 3, time.Nanosecond, func(context.Context, string) (CertificateWorkflowResult, error) {
				calls++
				return CertificateWorkflowResult{}, nil
			})
			expectedCalls := 1
			if failStatus == "running" {
				expectedCalls = 0
			}
			if err == nil || logged == nil || !strings.Contains(err.Error(), failure.Error()) || !strings.Contains(logged.Error(), failure.Error()) || calls != expectedCalls {
				t.Fatalf("calls=%d error=%v logged=%v", calls, err, logged)
			}
			tasks, listErr := a.Tasks.ListTasks(t.Context(), "task-domain", 10)
			if listErr != nil || len(tasks) != 1 {
				t.Fatalf("tasks=%+v error=%v", tasks, listErr)
			}
			if failStatus == "success" {
				if tasks[0].Status != "running" || tasks[0].FinishedAt != nil {
					t.Fatalf("完成审计失败必须回滚终态：%+v", tasks[0])
				}
			} else if tasks[0].Status != "failed" || tasks[0].FinishedAt == nil {
				t.Fatalf("开始审计失败必须停止执行并收尾：%+v", tasks[0])
			}
			if strings.Contains(tasks[0].Message, failure.Error()) {
				t.Fatal("audit-store-unavailable")
			}
		})
	}
}

func TestCertificateTerminalPersistenceFailureIsReturnedWithoutReissue(t *testing.T) {
	w, a := newCertificateTaskFixture(t)
	failure := errors.New("task-terminal-unavailable")
	w.tasks = failingCertificateTerminal{TaskRepository: a.Tasks, failure: failure}
	calls := 0
	var logged error
	w.logError = func(err error) { logged = err }
	_, err := w.run(t.Context(), "task-domain", "renewal_run", "certificate_issue", 3, time.Nanosecond, func(context.Context, string) (CertificateWorkflowResult, error) {
		calls++
		return CertificateWorkflowResult{}, nil
	})
	if calls != 1 || !errors.Is(err, failure) || !errors.Is(logged, failure) {
		t.Fatalf("calls=%d error=%v logged=%v", calls, err, logged)
	}
}

func TestCertificateTasksBindCancellationHook(t *testing.T) {
	w, a := newCertificateTaskFixture(t)
	registerCh := make(chan context.CancelFunc, 1)
	unregisterCh := make(chan string, 1)
	ctx := repository.WithTaskCancellationHooks(t.Context(), repository.TaskCancellationHooks{
		Register:   func(_ string, cancel context.CancelFunc) { registerCh <- cancel },
		Unregister: func(taskID string) { unregisterCh <- taskID },
	})
	started := make(chan struct{})
	finished := make(chan error, 1)
	go func() {
		_, err := w.run(ctx, "task-domain", "certificate_validate", "certificate_validate", 1, 0, func(ctx context.Context, _ string) (CertificateWorkflowResult, error) {
			close(started)
			<-ctx.Done()
			return CertificateWorkflowResult{}, ctx.Err()
		})
		finished <- err
	}()
	<-started
	cancel := <-registerCh
	cancel()
	if err := <-finished; !errors.Is(err, context.Canceled) {
		t.Fatalf("task cancellation error = %v", err)
	}
	select {
	case <-unregisterCh:
	case <-time.After(time.Second):
		t.Fatal("task cancellation hook was not unregistered")
	}
	assertCertificateTask(t, a, "cancelled", "cancelled")
}

func TestCertificateTasksRecordPartialDeploymentFailure(t *testing.T) {
	w, a := newCertificateTaskFixture(t)
	_, err := w.run(t.Context(), "task-domain", "renewal_run", "certificate_issue", 1, 0, func(context.Context, string) (CertificateWorkflowResult, error) {
		return CertificateWorkflowResult{
			Deployment: deployment.Report{Total: 2, Success: 1, Failed: 1, Status: "partial"},
		}, errors.New("partial deployment")
	})
	if err == nil {
		t.Fatal("partial deployment error should remain visible to the caller")
	}
	assertCertificateTask(t, a, "partial_success", "partial_success")
}

func TestCertificateTaskPersistsDeploymentResults(t *testing.T) {
	w, a := newCertificateTaskFixture(t)
	w.workflow = &CertificateWorkflow{
		Certificates: &workflowCertificateService{result: certificate.IssueResult{DomainID: "task-domain"}},
		Domains:      &workflowDomainState{},
		Servers:      workflowServerLister{servers: []deployment.Server{{ID: "server-1"}, {ID: "server-2"}}},
		CurrentCertificate: func(context.Context, string) (certificate.IssueResult, error) {
			return certificate.IssueResult{DomainID: "task-domain", Saved: certificate.SavedResource{CertificatePath: "cert.pem"}}, nil
		},
		Deployment: &workflowDeployer{report: deployment.Report{
			Total: 2, Success: 1, Failed: 1, Status: "partial",
			Results: []deployment.Result{
				{ServerID: "server-1", Server: "第一台服务器", Status: "success", Message: "部署成功"},
				{ServerID: "server-2", Server: "第二台服务器", Status: "failed", Message: "部署失败"},
			},
		}},
	}
	_, err := w.DeployCurrent(t.Context(), "task-domain")
	if err == nil {
		t.Fatal("部分部署失败应返回错误")
	}
	items, err := a.Tasks.ListTasks(t.Context(), "task-domain", 10)
	if err != nil || len(items) != 1 {
		t.Fatalf("任务读取失败：%+v %v", items, err)
	}
	results, err := a.Tasks.ListTaskDeploymentResults(t.Context(), items[0].ID)
	if err != nil || len(results) != 2 || results[0].AttemptID == "" || results[1].Status != "failed" {
		t.Fatalf("部署结果未持久化：%+v %v", results, err)
	}
}