完善服务重启任务恢复
This commit is contained in:
parent
2f39b9c721
commit
6f96887f56
@ -87,6 +87,13 @@ func NewWithConfigPath(ctx context.Context, cfg config.Config, configPath string
|
||||
}
|
||||
}
|
||||
taskRepo := repository.NewSQLTaskRepository(db.DB)
|
||||
if recovered, err := taskRepo.RecoverInterruptedTasks(ctx, time.Now().UTC()); err != nil {
|
||||
_ = db.Close()
|
||||
_ = logger.Close()
|
||||
return nil, fmt.Errorf("\u6062\u590d\u670d\u52a1\u91cd\u542f\u524d\u672a\u5b8c\u6210\u4efb\u52a1\u5931\u8d25\uff1a%w", err)
|
||||
} else if recovered > 0 {
|
||||
logger.Warn("\u670d\u52a1\u91cd\u542f\u524d\u672a\u5b8c\u6210\u4efb\u52a1\u5df2\u6807\u8bb0\u4e3a\u5931\u8d25", "count", recovered)
|
||||
}
|
||||
auditRepo := repository.NewSQLAuditRepository(db.DB)
|
||||
sshKeys := sshkey.NewService(secretStore)
|
||||
accounts := certificate.NewAccountRepository(secretStore)
|
||||
|
||||
@ -45,6 +45,7 @@ type TaskRepository interface {
|
||||
ListTasks(context.Context, string, int) ([]Task, error)
|
||||
UpdateTask(context.Context, string, string, string, string, *time.Time, *time.Time) error
|
||||
CancelTask(context.Context, string, string, *time.Time) error
|
||||
RecoverInterruptedTasks(context.Context, time.Time) (int, error)
|
||||
FindActiveTaskByDomain(context.Context, string, string) (Task, error)
|
||||
FindTaskByIdempotencyKey(context.Context, string, string) (Task, error)
|
||||
CreateAttempt(context.Context, TaskAttempt) error
|
||||
@ -208,6 +209,48 @@ func (r *SQLTaskRepository) CancelTask(ctx context.Context, id, message string,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecoverInterruptedTasks 将服务重启前未完成的任务标记为失败,避免任务永久停留在执行状态。
|
||||
func (r *SQLTaskRepository) RecoverInterruptedTasks(ctx context.Context, finishedAt time.Time) (int, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return 0, errors.New("\u6570\u636e\u5e93\u8fde\u63a5\u4e0d\u80fd\u4e3a\u7a7a")
|
||||
}
|
||||
if finishedAt.IsZero() {
|
||||
finishedAt = time.Now().UTC()
|
||||
} else {
|
||||
finishedAt = finishedAt.UTC()
|
||||
}
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("\u5f00\u59cb\u6062\u590d\u4e2d\u65ad\u4efb\u52a1\u4e8b\u52a1\u5931\u8d25\uff1a%w", err)
|
||||
}
|
||||
rollback := func() {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE tasks
|
||||
SET status = 'failed', stage = 'service_restart', message = ?, finished_at = ?, updated_at = ?
|
||||
WHERE status IN ('queued', 'running')`, "\u670d\u52a1\u91cd\u542f\u5bfc\u81f4\u4efb\u52a1\u4e2d\u65ad\uff0c\u8bf7\u91cd\u65b0\u6267\u884c", finishedAt, finishedAt)
|
||||
if err != nil {
|
||||
rollback()
|
||||
return 0, fmt.Errorf("\u6062\u590d\u4e2d\u65ad\u4efb\u52a1\u5931\u8d25\uff1a%w", err)
|
||||
}
|
||||
count, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
rollback()
|
||||
return 0, fmt.Errorf("\u8bfb\u53d6\u4e2d\u65ad\u4efb\u52a1\u6570\u91cf\u5931\u8d25\uff1a%w", err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE task_attempts
|
||||
SET status = 'failed', message = ?, finished_at = ?
|
||||
WHERE status IN ('queued', 'running')`, "\u670d\u52a1\u91cd\u542f\u5bfc\u81f4\u4efb\u52a1\u4e2d\u65ad\uff0c\u8bf7\u91cd\u65b0\u6267\u884c", finishedAt); err != nil {
|
||||
rollback()
|
||||
return 0, fmt.Errorf("\u6062\u590d\u4e2d\u65ad\u4efb\u52a1\u5c1d\u8bd5\u5931\u8d25\uff1a%w", err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, fmt.Errorf("\u63d0\u4ea4\u4e2d\u65ad\u4efb\u52a1\u6062\u590d\u4e8b\u52a1\u5931\u8d25\uff1a%w", err)
|
||||
}
|
||||
return int(count), nil
|
||||
}
|
||||
|
||||
func (r *SQLTaskRepository) FindActiveTaskByDomain(ctx context.Context, domainID, taskType string) (Task, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return Task{}, errors.New("数据库连接不能为空")
|
||||
|
||||
@ -153,3 +153,52 @@ func TestSQLTaskRepositoryDoesNotOverwriteCancelledTask(t *testing.T) {
|
||||
t.Fatalf("已取消任务状态被覆盖:%+v %v", task, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLTaskRepositoryRecoversInterruptedTasksAndAttempts(t *testing.T) {
|
||||
repository, db := newTaskRepositoryTest(t)
|
||||
defer db.Close()
|
||||
ctx := context.Background()
|
||||
created := time.Date(2026, 8, 30, 9, 0, 0, 0, time.UTC)
|
||||
for _, item := range []Task{
|
||||
{ID: "task-queued", DomainID: "domain-1", TaskType: "certificate_issue", Status: "queued", Stage: "queued", Message: "\u7b49\u5f85\u6267\u884c", CreatedAt: created, UpdatedAt: created},
|
||||
{ID: "task-running", DomainID: "domain-2", TaskType: "certificate_issue", Status: "running", Stage: "certificate_issue", Message: "\u6b63\u5728\u6267\u884c", CreatedAt: created, UpdatedAt: created},
|
||||
{ID: "task-success", DomainID: "domain-3", TaskType: "certificate_issue", Status: "success", Stage: "certificate_issue", Message: "\u5df2\u5b8c\u6210", CreatedAt: created, UpdatedAt: created},
|
||||
} {
|
||||
if err := repository.CreateTask(ctx, item); err != nil {
|
||||
t.Fatalf("\u521b\u5efa\u6062\u590d\u6d4b\u8bd5\u4efb\u52a1\u5931\u8d25\uff1a%v", err)
|
||||
}
|
||||
}
|
||||
if err := repository.CreateAttempt(ctx, TaskAttempt{
|
||||
ID: "attempt-running", TaskID: "task-running", AttemptNumber: 1, Stage: "certificate_issue", Status: "running", Message: "\u6b63\u5728\u6267\u884c", CreatedAt: created,
|
||||
}); err != nil {
|
||||
t.Fatalf("\u521b\u5efa\u6062\u590d\u6d4b\u8bd5\u5c1d\u8bd5\u5931\u8d25\uff1a%v", err)
|
||||
}
|
||||
if err := repository.CreateAttempt(ctx, TaskAttempt{
|
||||
ID: "attempt-success", TaskID: "task-success", AttemptNumber: 1, Stage: "certificate_issue", Status: "success", Message: "\u5df2\u5b8c\u6210", CreatedAt: created,
|
||||
}); err != nil {
|
||||
t.Fatalf("\u521b\u5efa\u5df2\u5b8c\u6210\u6d4b\u8bd5\u5c1d\u8bd5\u5931\u8d25\uff1a%v", err)
|
||||
}
|
||||
finished := time.Date(2026, 8, 30, 10, 0, 0, 0, time.UTC)
|
||||
count, err := repository.RecoverInterruptedTasks(ctx, finished)
|
||||
if err != nil || count != 2 {
|
||||
t.Fatalf("\u6062\u590d\u4e2d\u65ad\u4efb\u52a1\u6570\u91cf\u9519\u8bef\uff1a%d %v", count, err)
|
||||
}
|
||||
for _, id := range []string{"task-queued", "task-running"} {
|
||||
item, getErr := repository.GetTask(ctx, id)
|
||||
if getErr != nil || item.Status != "failed" || item.Stage != "service_restart" || item.FinishedAt == nil || !item.FinishedAt.Equal(finished) {
|
||||
t.Fatalf("\u4e2d\u65ad\u4efb\u52a1\u6062\u590d\u7ed3\u679c\u9519\u8bef\uff1a%+v %v", item, getErr)
|
||||
}
|
||||
}
|
||||
completed, err := repository.GetTask(ctx, "task-success")
|
||||
if err != nil || completed.Status != "success" {
|
||||
t.Fatalf("\u5df2\u5b8c\u6210\u4efb\u52a1\u4e0d\u5e94\u88ab\u6062\u590d\u903b\u8f91\u4fee\u6539\uff1a%+v %v", completed, err)
|
||||
}
|
||||
attempts, err := repository.ListAttempts(ctx, "task-running")
|
||||
if err != nil || len(attempts) != 1 || attempts[0].Status != "failed" || attempts[0].FinishedAt == nil || !attempts[0].FinishedAt.Equal(finished) {
|
||||
t.Fatalf("\u8fd0\u884c\u4e2d\u4efb\u52a1\u5c1d\u8bd5\u6062\u590d\u7ed3\u679c\u9519\u8bef\uff1a%+v %v", attempts, err)
|
||||
}
|
||||
completedAttempts, err := repository.ListAttempts(ctx, "task-success")
|
||||
if err != nil || len(completedAttempts) != 1 || completedAttempts[0].Status != "success" {
|
||||
t.Fatalf("\u5df2\u5b8c\u6210\u4efb\u52a1\u5c1d\u8bd5\u4e0d\u5e94\u88ab\u6062\u590d\u903b\u8f91\u4fee\u6539\uff1a%+v %v", completedAttempts, err)
|
||||
}
|
||||
}
|
||||
|
||||
2
desktop/frontend/dist/index.html
vendored
2
desktop/frontend/dist/index.html
vendored
@ -5,7 +5,7 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<link rel="icon" type="image/png" href="/Scert.png" />
|
||||
<title>Scert SSL 证书管理</title>
|
||||
<script type="module" crossorigin src="/assets/index-DLHbbTXZ.js"></script>
|
||||
<script type="module" crossorigin src="/assets/index-bULaHqae.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-E9dMZuvh.css">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
@ -2958,3 +2958,20 @@ Linux Ubuntu 22.04 上的 `Scert-admin` 手动启停、无开机自启动、无
|
||||
- [✓] 后端 `go test ./...`、`go test -race ./...`、`go vet ./...` 和 `go mod verify` 已通过。
|
||||
- [✓] 前端 `npm.cmd run build` 已通过。
|
||||
- [!] 尚未进行真实 ACME、DNS 服务商、远程 SSH、生产 HTTPS 和多进程并发验收;这些仍需授权后的实机测试。
|
||||
|
||||
|
||||
## 七十三、服务重启任务恢复(2026-08-30)
|
||||
|
||||
### 73.1 本轮需求清单
|
||||
|
||||
- `[✓]` 服务启动时自动扫描数据库中仍处于排队或执行状态的任务。
|
||||
- `[✓]` 将服务重启前未完成的任务标记为失败,并记录“服务重启恢复”执行阶段。
|
||||
- `[✓]` 同步关闭未完成任务尝试,写入统一的中断说明和完成时间。
|
||||
- `[✓]` 已完成和已取消任务不受启动恢复逻辑影响。
|
||||
- `[✓]` 管理后台为“服务重启恢复”阶段补充中文显示名称。
|
||||
|
||||
### 73.2 验证状态
|
||||
|
||||
- `[x]` SQLite 任务仓储专项测试通过,覆盖排队任务、执行任务、执行尝试和已完成任务保护。
|
||||
- `[ ]` 等待本轮完整后端竞态测试、静态检查、前端构建和桌面版测试复核。
|
||||
- `[!]` 多进程同时启动时的跨进程任务调度仍不是本轮实现范围;服务重启恢复只负责清理持久化的未完成状态。
|
||||
|
||||
@ -137,6 +137,7 @@ export function useDashboard() {
|
||||
renewal_run: "执行自动续期",
|
||||
};
|
||||
const taskStageLabels = {
|
||||
service_restart: "服务重启恢复",
|
||||
queued: "排队中",
|
||||
certificate_issue: "申请证书",
|
||||
certificate_validate: "校验证书",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user