完善服务重启任务恢复

This commit is contained in:
xuans 2026-08-31 00:30:29 +08:00
parent 2f39b9c721
commit 6f96887f56
6 changed files with 118 additions and 1 deletions

View File

@ -87,6 +87,13 @@ func NewWithConfigPath(ctx context.Context, cfg config.Config, configPath string
} }
} }
taskRepo := repository.NewSQLTaskRepository(db.DB) taskRepo := repository.NewSQLTaskRepository(db.DB)
if recovered, err := taskRepo.RecoverInterruptedTasks(ctx, time.Now().UTC()); err != nil {
_ = db.Close()
_ = logger.Close()
return nil, fmt.Errorf("\u6062\u590d\u670d\u52a1\u91cd\u542f\u524d\u672a\u5b8c\u6210\u4efb\u52a1\u5931\u8d25\uff1a%w", err)
} else if recovered > 0 {
logger.Warn("\u670d\u52a1\u91cd\u542f\u524d\u672a\u5b8c\u6210\u4efb\u52a1\u5df2\u6807\u8bb0\u4e3a\u5931\u8d25", "count", recovered)
}
auditRepo := repository.NewSQLAuditRepository(db.DB) auditRepo := repository.NewSQLAuditRepository(db.DB)
sshKeys := sshkey.NewService(secretStore) sshKeys := sshkey.NewService(secretStore)
accounts := certificate.NewAccountRepository(secretStore) accounts := certificate.NewAccountRepository(secretStore)

View File

@ -45,6 +45,7 @@ type TaskRepository interface {
ListTasks(context.Context, string, int) ([]Task, error) ListTasks(context.Context, string, int) ([]Task, error)
UpdateTask(context.Context, string, string, string, string, *time.Time, *time.Time) error UpdateTask(context.Context, string, string, string, string, *time.Time, *time.Time) error
CancelTask(context.Context, string, string, *time.Time) error CancelTask(context.Context, string, string, *time.Time) error
RecoverInterruptedTasks(context.Context, time.Time) (int, error)
FindActiveTaskByDomain(context.Context, string, string) (Task, error) FindActiveTaskByDomain(context.Context, string, string) (Task, error)
FindTaskByIdempotencyKey(context.Context, string, string) (Task, error) FindTaskByIdempotencyKey(context.Context, string, string) (Task, error)
CreateAttempt(context.Context, TaskAttempt) error CreateAttempt(context.Context, TaskAttempt) error
@ -208,6 +209,48 @@ func (r *SQLTaskRepository) CancelTask(ctx context.Context, id, message string,
} }
return nil return nil
} }
// RecoverInterruptedTasks 将服务重启前未完成的任务标记为失败,避免任务永久停留在执行状态。
func (r *SQLTaskRepository) RecoverInterruptedTasks(ctx context.Context, finishedAt time.Time) (int, error) {
if r == nil || r.db == nil {
return 0, errors.New("\u6570\u636e\u5e93\u8fde\u63a5\u4e0d\u80fd\u4e3a\u7a7a")
}
if finishedAt.IsZero() {
finishedAt = time.Now().UTC()
} else {
finishedAt = finishedAt.UTC()
}
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return 0, fmt.Errorf("\u5f00\u59cb\u6062\u590d\u4e2d\u65ad\u4efb\u52a1\u4e8b\u52a1\u5931\u8d25\uff1a%w", err)
}
rollback := func() {
_ = tx.Rollback()
}
result, err := tx.ExecContext(ctx, `UPDATE tasks
SET status = 'failed', stage = 'service_restart', message = ?, finished_at = ?, updated_at = ?
WHERE status IN ('queued', 'running')`, "\u670d\u52a1\u91cd\u542f\u5bfc\u81f4\u4efb\u52a1\u4e2d\u65ad\uff0c\u8bf7\u91cd\u65b0\u6267\u884c", finishedAt, finishedAt)
if err != nil {
rollback()
return 0, fmt.Errorf("\u6062\u590d\u4e2d\u65ad\u4efb\u52a1\u5931\u8d25\uff1a%w", err)
}
count, err := result.RowsAffected()
if err != nil {
rollback()
return 0, fmt.Errorf("\u8bfb\u53d6\u4e2d\u65ad\u4efb\u52a1\u6570\u91cf\u5931\u8d25\uff1a%w", err)
}
if _, err := tx.ExecContext(ctx, `UPDATE task_attempts
SET status = 'failed', message = ?, finished_at = ?
WHERE status IN ('queued', 'running')`, "\u670d\u52a1\u91cd\u542f\u5bfc\u81f4\u4efb\u52a1\u4e2d\u65ad\uff0c\u8bf7\u91cd\u65b0\u6267\u884c", finishedAt); err != nil {
rollback()
return 0, fmt.Errorf("\u6062\u590d\u4e2d\u65ad\u4efb\u52a1\u5c1d\u8bd5\u5931\u8d25\uff1a%w", err)
}
if err := tx.Commit(); err != nil {
return 0, fmt.Errorf("\u63d0\u4ea4\u4e2d\u65ad\u4efb\u52a1\u6062\u590d\u4e8b\u52a1\u5931\u8d25\uff1a%w", err)
}
return int(count), nil
}
func (r *SQLTaskRepository) FindActiveTaskByDomain(ctx context.Context, domainID, taskType string) (Task, error) { func (r *SQLTaskRepository) FindActiveTaskByDomain(ctx context.Context, domainID, taskType string) (Task, error) {
if r == nil || r.db == nil { if r == nil || r.db == nil {
return Task{}, errors.New("数据库连接不能为空") return Task{}, errors.New("数据库连接不能为空")

View File

@ -153,3 +153,52 @@ func TestSQLTaskRepositoryDoesNotOverwriteCancelledTask(t *testing.T) {
t.Fatalf("已取消任务状态被覆盖:%+v %v", task, err) t.Fatalf("已取消任务状态被覆盖:%+v %v", task, err)
} }
} }
func TestSQLTaskRepositoryRecoversInterruptedTasksAndAttempts(t *testing.T) {
repository, db := newTaskRepositoryTest(t)
defer db.Close()
ctx := context.Background()
created := time.Date(2026, 8, 30, 9, 0, 0, 0, time.UTC)
for _, item := range []Task{
{ID: "task-queued", DomainID: "domain-1", TaskType: "certificate_issue", Status: "queued", Stage: "queued", Message: "\u7b49\u5f85\u6267\u884c", CreatedAt: created, UpdatedAt: created},
{ID: "task-running", DomainID: "domain-2", TaskType: "certificate_issue", Status: "running", Stage: "certificate_issue", Message: "\u6b63\u5728\u6267\u884c", CreatedAt: created, UpdatedAt: created},
{ID: "task-success", DomainID: "domain-3", TaskType: "certificate_issue", Status: "success", Stage: "certificate_issue", Message: "\u5df2\u5b8c\u6210", CreatedAt: created, UpdatedAt: created},
} {
if err := repository.CreateTask(ctx, item); err != nil {
t.Fatalf("\u521b\u5efa\u6062\u590d\u6d4b\u8bd5\u4efb\u52a1\u5931\u8d25\uff1a%v", err)
}
}
if err := repository.CreateAttempt(ctx, TaskAttempt{
ID: "attempt-running", TaskID: "task-running", AttemptNumber: 1, Stage: "certificate_issue", Status: "running", Message: "\u6b63\u5728\u6267\u884c", CreatedAt: created,
}); err != nil {
t.Fatalf("\u521b\u5efa\u6062\u590d\u6d4b\u8bd5\u5c1d\u8bd5\u5931\u8d25\uff1a%v", err)
}
if err := repository.CreateAttempt(ctx, TaskAttempt{
ID: "attempt-success", TaskID: "task-success", AttemptNumber: 1, Stage: "certificate_issue", Status: "success", Message: "\u5df2\u5b8c\u6210", CreatedAt: created,
}); err != nil {
t.Fatalf("\u521b\u5efa\u5df2\u5b8c\u6210\u6d4b\u8bd5\u5c1d\u8bd5\u5931\u8d25\uff1a%v", err)
}
finished := time.Date(2026, 8, 30, 10, 0, 0, 0, time.UTC)
count, err := repository.RecoverInterruptedTasks(ctx, finished)
if err != nil || count != 2 {
t.Fatalf("\u6062\u590d\u4e2d\u65ad\u4efb\u52a1\u6570\u91cf\u9519\u8bef\uff1a%d %v", count, err)
}
for _, id := range []string{"task-queued", "task-running"} {
item, getErr := repository.GetTask(ctx, id)
if getErr != nil || item.Status != "failed" || item.Stage != "service_restart" || item.FinishedAt == nil || !item.FinishedAt.Equal(finished) {
t.Fatalf("\u4e2d\u65ad\u4efb\u52a1\u6062\u590d\u7ed3\u679c\u9519\u8bef\uff1a%+v %v", item, getErr)
}
}
completed, err := repository.GetTask(ctx, "task-success")
if err != nil || completed.Status != "success" {
t.Fatalf("\u5df2\u5b8c\u6210\u4efb\u52a1\u4e0d\u5e94\u88ab\u6062\u590d\u903b\u8f91\u4fee\u6539\uff1a%+v %v", completed, err)
}
attempts, err := repository.ListAttempts(ctx, "task-running")
if err != nil || len(attempts) != 1 || attempts[0].Status != "failed" || attempts[0].FinishedAt == nil || !attempts[0].FinishedAt.Equal(finished) {
t.Fatalf("\u8fd0\u884c\u4e2d\u4efb\u52a1\u5c1d\u8bd5\u6062\u590d\u7ed3\u679c\u9519\u8bef\uff1a%+v %v", attempts, err)
}
completedAttempts, err := repository.ListAttempts(ctx, "task-success")
if err != nil || len(completedAttempts) != 1 || completedAttempts[0].Status != "success" {
t.Fatalf("\u5df2\u5b8c\u6210\u4efb\u52a1\u5c1d\u8bd5\u4e0d\u5e94\u88ab\u6062\u590d\u903b\u8f91\u4fee\u6539\uff1a%+v %v", completedAttempts, err)
}
}

View File

@ -5,7 +5,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="icon" type="image/png" href="/Scert.png" /> <link rel="icon" type="image/png" href="/Scert.png" />
<title>Scert SSL 证书管理</title> <title>Scert SSL 证书管理</title>
<script type="module" crossorigin src="/assets/index-DLHbbTXZ.js"></script> <script type="module" crossorigin src="/assets/index-bULaHqae.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-E9dMZuvh.css"> <link rel="stylesheet" crossorigin href="/assets/index-E9dMZuvh.css">
</head> </head>
<body> <body>

View File

@ -2958,3 +2958,20 @@ Linux Ubuntu 22.04 上的 `Scert-admin` 手动启停、无开机自启动、无
- [✓] 后端 `go test ./...`、`go test -race ./...`、`go vet ./...` 和 `go mod verify` 已通过。 - [✓] 后端 `go test ./...`、`go test -race ./...`、`go vet ./...` 和 `go mod verify` 已通过。
- [✓] 前端 `npm.cmd run build` 已通过。 - [✓] 前端 `npm.cmd run build` 已通过。
- [!] 尚未进行真实 ACME、DNS 服务商、远程 SSH、生产 HTTPS 和多进程并发验收;这些仍需授权后的实机测试。 - [!] 尚未进行真实 ACME、DNS 服务商、远程 SSH、生产 HTTPS 和多进程并发验收;这些仍需授权后的实机测试。
## 七十三、服务重启任务恢复(2026-08-30)
### 73.1 本轮需求清单
- `[✓]` 服务启动时自动扫描数据库中仍处于排队或执行状态的任务。
- `[✓]` 将服务重启前未完成的任务标记为失败,并记录“服务重启恢复”执行阶段。
- `[✓]` 同步关闭未完成任务尝试,写入统一的中断说明和完成时间。
- `[✓]` 已完成和已取消任务不受启动恢复逻辑影响。
- `[✓]` 管理后台为“服务重启恢复”阶段补充中文显示名称。
### 73.2 验证状态
- `[x]` SQLite 任务仓储专项测试通过,覆盖排队任务、执行任务、执行尝试和已完成任务保护。
- `[ ]` 等待本轮完整后端竞态测试、静态检查、前端构建和桌面版测试复核。
- `[!]` 多进程同时启动时的跨进程任务调度仍不是本轮实现范围;服务重启恢复只负责清理持久化的未完成状态。

View File

@ -137,6 +137,7 @@ export function useDashboard() {
renewal_run: "执行自动续期", renewal_run: "执行自动续期",
}; };
const taskStageLabels = { const taskStageLabels = {
service_restart: "服务重启恢复",
queued: "排队中", queued: "排队中",
certificate_issue: "申请证书", certificate_issue: "申请证书",
certificate_validate: "校验证书", certificate_validate: "校验证书",