完善配置快照失败恢复提示与重试测试
Some checks are pending
Security Checks / security (push) Waiting to run

This commit is contained in:
smail 2026-08-30 22:55:41 +08:00
parent 476c85ac56
commit e603295df3
7 changed files with 118 additions and 5 deletions

View File

@ -849,3 +849,14 @@ go vet ./...
- **回退验证**:未删除或覆盖既有配置、数据和保留的 `.codex*` 文件;若复审不通过,可回退本次提交。
- **结束状态**:待复审
- **复审人**:待人工复审。
---
## 2026-08-30 配置快照发布失败重试整改记录
- **本轮状态**?`[x] 已完成?等待复审`
- **实施内容**?统一配置启动加载?配置应用?配置回滚?数据库切换和插件 Secret 迁移的运行时快照发布入口?当磁盘事务已成功提交但运行时快照暂不可用时?保留可重试状态并将配置服务标记为降级?
- **接口提示**?新增稳定错误码 `CONFIG_RUNTIME_SNAPSHOT_UNAVAILABLE`?返回 HTTP 503?并明确提示配置已保存?运行时尚未切换?可稍后重试应用?
- **新增测试**?覆盖应用阶段快照发布失败后的磁盘代次?运行时旧快照?降级状态和再次应用成功?保留回滚票据失败重试及并发保护测试?
- **验证结果**?服务端 `go test ./... -count=1 -timeout 30m` 和 `go vet ./...` 通过?未将真实 MySQL?PostgreSQL?Ubuntu?Windows?浏览器和生产部署验收标记为完成?
- **遗留边界**?事务提交后快照发布失败的跨进程恢复仍依赖服务重载或后续重试?真实多数据库复制?校验?原子切换和部署联调继续按实机清单验收?

View File

@ -1260,3 +1260,14 @@ CG-053~CG-056 → AD-066 → CG-057
- **回退验证**:只新增回滚票据状态和错误映射,未删除既有配置管理路径;可通过回退提交 `acf0322` 恢复上一版本。
- **结束状态**:待复审
- **复审人**:待人工复审。
---
## 2026-08-30 配置快照发布失败重试整改记录
- **本轮状态**?`[x] 已完成?等待复审`
- **实施内容**?统一配置启动加载?配置应用?配置回滚?数据库切换和插件 Secret 迁移的运行时快照发布入口?当磁盘事务已成功提交但运行时快照暂不可用时?保留可重试状态并将配置服务标记为降级?
- **接口提示**?新增稳定错误码 `CONFIG_RUNTIME_SNAPSHOT_UNAVAILABLE`?返回 HTTP 503?并明确提示配置已保存?运行时尚未切换?可稍后重试应用?
- **新增测试**?覆盖应用阶段快照发布失败后的磁盘代次?运行时旧快照?降级状态和再次应用成功?保留回滚票据失败重试及并发保护测试?
- **验证结果**?服务端 `go test ./... -count=1 -timeout 30m` 和 `go vet ./...` 通过?未将真实 MySQL?PostgreSQL?Ubuntu?Windows?浏览器和生产部署验收标记为完成?
- **遗留边界**?事务提交后快照发布失败的跨进程恢复仍依赖服务重载或后续重试?真实多数据库复制?校验?原子切换和部署联调继续按实机清单验收?

View File

@ -77,7 +77,7 @@ func (m *ConfigManager) SwitchDatabase(ctx context.Context, expectedGeneration u
if err := m.transactions.Commit(current, candidate); err != nil {
return err
}
if err := m.snapshots.Publish(candidate); err != nil {
if err := m.publishSnapshot(candidate); err != nil {
return err
}
now := time.Now().UTC()

View File

@ -217,7 +217,7 @@ func (m *ConfigManager) Load(ctx context.Context) error {
if err := contextError(ctx); err != nil {
return err
}
if err := m.snapshots.Publish(runtime); err != nil {
if err := m.publishSnapshot(runtime); err != nil {
return err
}
now := runtime.LoadedAt
@ -395,7 +395,7 @@ func (m *ConfigManager) apply(ctx context.Context, generation uint64, actor Audi
if err := m.transactions.Commit(current, candidate); err != nil {
return nil, err
}
if err := m.snapshots.Publish(candidate); err != nil {
if err := m.publishSnapshot(candidate); err != nil {
return nil, err
}
now := time.Now().UTC()
@ -554,6 +554,25 @@ func rollbackTicketRetryable(err error) bool {
errors.Is(err, ErrRuntimeSnapshotUnavailable)
}
func (m *ConfigManager) publishSnapshot(candidate RuntimeConfig) error {
if m == nil || m.snapshots == nil {
return ErrRuntimeSnapshotUnavailable
}
if err := m.snapshots.Publish(candidate); err != nil {
if errors.Is(err, ErrRuntimeSnapshotUnavailable) {
m.mu.Lock()
m.status.SetupState = SetupStateDegraded
m.status.DegradedReasons = []PublicIssue{{
Code: "RUNTIME_SNAPSHOT_UNAVAILABLE",
Message: "配置已保存,但运行时配置尚未切换,请重试应用",
}}
m.mu.Unlock()
}
return err
}
return nil
}
func (m *ConfigManager) rollback(ctx context.Context, generation uint64, actor AuditActor, confirmation StepUpConfirmation) (*ApplyResult, error) {
result, _, err := m.rollbackWithState(ctx, generation, actor, confirmation)
return result, err
@ -619,7 +638,7 @@ func (m *ConfigManager) rollbackWithState(ctx context.Context, generation uint64
if err := m.transactions.Commit(current, candidate); err != nil {
return nil, false, err
}
if err := m.snapshots.Publish(candidate); err != nil {
if err := m.publishSnapshot(candidate); err != nil {
return nil, false, err
}
now := time.Now().UTC()
@ -942,7 +961,7 @@ func (m *ConfigManager) MigrateLegacyPluginSecrets(ctx context.Context, importer
if err := m.transactions.Commit(current, candidate); err != nil {
return false, err
}
if err := m.snapshots.Publish(candidate); err != nil {
if err := m.publishSnapshot(candidate); err != nil {
return false, err
}
now := time.Now().UTC()

View File

@ -1505,6 +1505,58 @@ func (s *flakyRuntimeSnapshotStore) Snapshot() (RuntimeConfig, error) {
return cloneRuntimeConfig(s.current), nil
}
func TestConfigManagerApplyCanRetryAfterSnapshotFailure(t *testing.T) {
root := t.TempDir()
writeRuntimeFixture(t, root, validRuntimeConfig())
options := ConfigLoaderOptions{Validation: RuntimeValidationOptions{HasSuperAdmin: true, Production: true}}
manager, err := NewConfigManager(root, options)
if err != nil {
t.Fatal(err)
}
updatedEmail := manager.Snapshot().Email
updatedEmail.Policy.MaxMessageBytes = 8 << 20
payload, err := json.Marshal(updatedEmail)
if err != nil {
t.Fatal(err)
}
updated, err := manager.Update(context.Background(), UpdateRequest{
Section: SectionEmail, ExpectedGeneration: 1, Payload: payload, Actor: authorizedConfigActor(),
})
if err != nil {
t.Fatal(err)
}
manager.snapshots = &flakyRuntimeSnapshotStore{current: manager.Snapshot(), failures: 1}
if _, err := manager.Apply(context.Background(), updated.Generation); !errors.Is(err, ErrRuntimeSnapshotUnavailable) {
t.Fatalf("first apply error = %v, want snapshot unavailable", err)
}
status := manager.Status()
if status.SetupState != SetupStateDegraded || len(status.DegradedReasons) != 1 || status.DegradedReasons[0].Code != "RUNTIME_SNAPSHOT_UNAVAILABLE" {
t.Fatalf("degraded status after snapshot failure = %+v", status)
}
if manager.Snapshot().Generation != 1 {
t.Fatalf("runtime snapshot generation after failed publish = %d, want 1", manager.Snapshot().Generation)
}
persisted, err := NewConfigLoader(root, options).Load()
if err != nil {
t.Fatal(err)
}
if persisted.Generation != 2 || persisted.Email.Policy.MaxMessageBytes != 8<<20 {
t.Fatalf("persisted candidate after snapshot failure = %+v", persisted)
}
applied, err := manager.Apply(context.Background(), updated.Generation)
if err != nil {
t.Fatal(err)
}
if applied.Generation != 2 || manager.Snapshot().Generation != 2 || manager.Snapshot().Email.Policy.MaxMessageBytes != 8<<20 {
t.Fatalf("retried apply result = %+v, runtime = %+v", applied, manager.Snapshot())
}
if manager.Status().SetupState == SetupStateDegraded || len(manager.Status().DegradedReasons) != 0 {
t.Fatalf("status was not recovered after retry: %+v", manager.Status())
}
}
func TestConfigManagerRollbackTicketCanRetryAfterSnapshotFailure(t *testing.T) {
root := t.TempDir()
writeRuntimeFixture(t, root, validRuntimeConfig())

View File

@ -57,6 +57,10 @@ func configErrorDetails(err error) (string, int, string, map[string]string) {
code = "CONFIG_ROLLBACK_IN_PROGRESS"
status = http.StatusConflict
message = "配置回滚正在执行,请稍后查看结果"
case errors.Is(err, config.ErrRuntimeSnapshotUnavailable):
code = "CONFIG_RUNTIME_SNAPSHOT_UNAVAILABLE"
status = http.StatusServiceUnavailable
message = "配置已保存,但运行时配置尚未切换,请重试应用"
case errors.Is(err, config.ErrGenerationConflict):
code = "CONFIG_GENERATION_CONFLICT"
status = http.StatusConflict

View File

@ -169,6 +169,22 @@ func TestConfigErrorDetailsMapsDatabaseMigrationFailures(t *testing.T) {
}
}
func TestConfigErrorDetailsMapsRuntimeSnapshotUnavailable(t *testing.T) {
code, status, message, fields := configErrorDetails(config.ErrRuntimeSnapshotUnavailable)
if code != "CONFIG_RUNTIME_SNAPSHOT_UNAVAILABLE" {
t.Fatalf("error code = %q", code)
}
if status != http.StatusServiceUnavailable {
t.Fatalf("status = %d", status)
}
if message != "配置已保存,但运行时配置尚未切换,请重试应用" {
t.Fatalf("message = %q", message)
}
if fields != nil {
t.Fatalf("fields = %+v, want nil", fields)
}
}
func TestConfigErrorDetailsMapsRollbackInProgress(t *testing.T) {
code, status, message, _ := configErrorDetails(config.ErrRollbackInProgress)
if code != "CONFIG_ROLLBACK_IN_PROGRESS" || status != http.StatusConflict {