完善DNS清理重试与容量边界
This commit is contained in:
parent
4a4946979b
commit
1aaa16d103
@ -101,6 +101,7 @@ type DNSVerificationManager struct {
|
||||
now func() time.Time
|
||||
sessions map[string]dnsVerificationSession
|
||||
expiredSession map[string]expiredDNSVerificationSession
|
||||
pendingCleanup map[string]struct{}
|
||||
inFlight int
|
||||
}
|
||||
|
||||
@ -108,7 +109,9 @@ func NewDNSVerificationManager(tester DNSCredentialTester, store SecretWriter) *
|
||||
deleter, _ := store.(SecretDeleter)
|
||||
return &DNSVerificationManager{
|
||||
tester: tester, store: store, deleter: deleter, now: time.Now,
|
||||
sessions: make(map[string]dnsVerificationSession), expiredSession: make(map[string]expiredDNSVerificationSession),
|
||||
sessions: make(map[string]dnsVerificationSession),
|
||||
expiredSession: make(map[string]expiredDNSVerificationSession),
|
||||
pendingCleanup: make(map[string]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
@ -116,6 +119,7 @@ func (m *DNSVerificationManager) Verify(ctx context.Context, input DNSVerificati
|
||||
if m == nil || m.tester == nil || m.store == nil {
|
||||
return DNSVerificationResult{}, errors.New("DNS 在线验证服务未初始化")
|
||||
}
|
||||
m.retryPendingCredentialCleanup()
|
||||
providerID := strings.ToLower(strings.TrimSpace(input.ProviderID))
|
||||
testDomain := normalizeTestDomain(input.TestDomain)
|
||||
if providerID == "" {
|
||||
@ -164,7 +168,7 @@ func (m *DNSVerificationManager) Verify(ctx context.Context, input DNSVerificati
|
||||
}
|
||||
credentialRef := fmt.Sprintf("secret://dns/%s/%s", providerID, credentialID)
|
||||
if err := m.store.Put(ctx, credentialRef, cleanedCredentials); err != nil {
|
||||
cleanupErr := m.deleteCredential(credentialRef)
|
||||
cleanupErr := m.deleteOrQueueCredential(credentialRef)
|
||||
if cleanupErr != nil {
|
||||
m.logCleanupFailure()
|
||||
return DNSVerificationResult{}, errors.New("加密保存 DNS 凭据失败，临时凭据回收未确认")
|
||||
@ -172,7 +176,7 @@ func (m *DNSVerificationManager) Verify(ctx context.Context, input DNSVerificati
|
||||
return DNSVerificationResult{}, fmt.Errorf("加密保存 DNS 凭据失败：%w", err)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
cleanupErr := m.deleteCredential(credentialRef)
|
||||
cleanupErr := m.deleteOrQueueCredential(credentialRef)
|
||||
if cleanupErr != nil {
|
||||
m.logCleanupFailure()
|
||||
return DNSVerificationResult{TestReport: testReport}, errors.Join(
|
||||
@ -243,6 +247,7 @@ func (m *DNSVerificationManager) Discard(token string) error {
|
||||
if m == nil {
|
||||
return errors.New("DNS 在线验证服务未初始化")
|
||||
}
|
||||
m.retryPendingCredentialCleanup()
|
||||
token = strings.TrimSpace(token)
|
||||
if token == "" {
|
||||
return errors.New("DNS 验证令牌不能为空")
|
||||
@ -277,6 +282,7 @@ func (m *DNSVerificationManager) DiscardAllExcept(protectedReferences ...string)
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.retryPendingCredentialCleanup()
|
||||
protected := make(map[string]bool, len(protectedReferences))
|
||||
for _, reference := range protectedReferences {
|
||||
if reference = strings.TrimSpace(reference); reference != "" {
|
||||
@ -309,6 +315,7 @@ func (m *DNSVerificationManager) validateSession(token, providerID, credentialRe
|
||||
if token == "" {
|
||||
return errors.New("必须先完成 DNS API 在线验证")
|
||||
}
|
||||
m.retryPendingCredentialCleanup()
|
||||
now := m.now().UTC()
|
||||
m.mu.Lock()
|
||||
expiredRefs := m.clearExpiredLocked(now)
|
||||
@ -341,6 +348,7 @@ func (m *DNSVerificationManager) resolveSession(token string, claim bool) (dnsVe
|
||||
if token == "" {
|
||||
return dnsVerificationSession{}, errors.New("必须先完成 DNS API 在线验证")
|
||||
}
|
||||
m.retryPendingCredentialCleanup()
|
||||
now := m.now().UTC()
|
||||
m.mu.Lock()
|
||||
expiredRefs := m.clearExpiredLocked(now)
|
||||
@ -368,7 +376,7 @@ func (m *DNSVerificationManager) reserveVerificationSlot() error {
|
||||
m.deleteCredentials(expiredRefs)
|
||||
return errors.New("DNS 在线验证请求过多，请稍后重试")
|
||||
}
|
||||
if len(m.sessions)+m.inFlight >= dnsVerificationSessionCapacity {
|
||||
if len(m.sessions)+len(m.expiredSession)+m.inFlight >= dnsVerificationSessionCapacity {
|
||||
m.mu.Unlock()
|
||||
m.deleteCredentials(expiredRefs)
|
||||
return errors.New("DNS 验证令牌数量已达上限，请等待现有验证完成或过期后重试")
|
||||
@ -458,6 +466,52 @@ func (m *DNSVerificationManager) forgetExpiredSession(reference string) {
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *DNSVerificationManager) rememberPendingCleanup(reference string) {
|
||||
if m == nil || strings.TrimSpace(reference) == "" {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.pendingCleanup[reference] = struct{}{}
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *DNSVerificationManager) forgetPendingCleanup(reference string) {
|
||||
if m == nil || strings.TrimSpace(reference) == "" {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
delete(m.pendingCleanup, reference)
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *DNSVerificationManager) retryPendingCredentialCleanup() {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
references := make([]string, 0, len(m.pendingCleanup))
|
||||
for reference := range m.pendingCleanup {
|
||||
references = append(references, reference)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
for _, reference := range references {
|
||||
if err := m.deleteCredential(reference); err != nil {
|
||||
m.logCleanupFailure()
|
||||
continue
|
||||
}
|
||||
m.forgetPendingCleanup(reference)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *DNSVerificationManager) deleteOrQueueCredential(reference string) error {
|
||||
if err := m.deleteCredential(reference); err != nil {
|
||||
m.rememberPendingCleanup(reference)
|
||||
return err
|
||||
}
|
||||
m.forgetPendingCleanup(reference)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *DNSVerificationManager) logCleanupFailure() {
|
||||
log.Print("DNS 临时凭据回收失败")
|
||||
}
|
||||
|
||||
@ -66,6 +66,11 @@ type retryableSecretWriter struct {
|
||||
failDelete bool
|
||||
}
|
||||
|
||||
type cancellingRetryableSecretWriter struct {
|
||||
retryableSecretWriter
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (s *cancellingSecretWriter) Put(ctx context.Context, reference string, value map[string]string) error {
|
||||
if err := s.memorySecretWriter.Put(ctx, reference, value); err != nil {
|
||||
return err
|
||||
@ -81,6 +86,14 @@ func (s *retryableSecretWriter) Delete(ctx context.Context, reference string) er
|
||||
return s.memorySecretWriter.Delete(ctx, reference)
|
||||
}
|
||||
|
||||
func (s *cancellingRetryableSecretWriter) Put(ctx context.Context, reference string, value map[string]string) error {
|
||||
if err := s.retryableSecretWriter.Put(ctx, reference, value); err != nil {
|
||||
return err
|
||||
}
|
||||
s.cancel()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memorySecretWriter) Put(_ context.Context, reference string, value map[string]string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
@ -290,6 +303,62 @@ func TestDNSVerificationManagerDeletesCredentialWhenPutReportsFailure(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSVerificationManagerRetriesCleanupAfterPutFailure(t *testing.T) {
|
||||
store := &retryableSecretWriter{failDelete: true}
|
||||
store.putErr = errors.New("write failed")
|
||||
manager := NewDNSVerificationManager(fakeDNSTester{}, store)
|
||||
_, err := manager.Verify(t.Context(), DNSVerificationInput{
|
||||
ProviderID: "cloudflare", TestDomain: "example.com",
|
||||
Credentials: map[string]string{"api_token": "secret"},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("保存失败时应返回错误")
|
||||
}
|
||||
if len(store.values) != 1 || len(manager.pendingCleanup) != 1 {
|
||||
t.Fatalf("删除失败后应保留待清理凭据：values=%d pending=%d", len(store.values), len(manager.pendingCleanup))
|
||||
}
|
||||
|
||||
store.putErr = nil
|
||||
store.failDelete = false
|
||||
if _, err := manager.Verify(t.Context(), DNSVerificationInput{
|
||||
ProviderID: "cloudflare", TestDomain: "example.com",
|
||||
Credentials: map[string]string{"api_token": "next-secret"},
|
||||
}); err != nil {
|
||||
t.Fatalf("下一次验证应先重试待清理凭据：%v", err)
|
||||
}
|
||||
if len(manager.pendingCleanup) != 0 || len(store.values) != 1 {
|
||||
t.Fatalf("待清理凭据重试后状态错误：pending=%d values=%d", len(manager.pendingCleanup), len(store.values))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSVerificationManagerRetriesCleanupAfterCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
store := &cancellingRetryableSecretWriter{
|
||||
retryableSecretWriter: retryableSecretWriter{failDelete: true},
|
||||
cancel: cancel,
|
||||
}
|
||||
manager := NewDNSVerificationManager(fakeDNSTester{}, store)
|
||||
|
||||
_, err := manager.Verify(ctx, DNSVerificationInput{
|
||||
ProviderID: "cloudflare", TestDomain: "example.com",
|
||||
Credentials: map[string]string{"api_token": "secret"},
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("保存后取消应返回 context.Canceled：%v", err)
|
||||
}
|
||||
if len(store.values) != 1 || len(manager.pendingCleanup) != 1 {
|
||||
t.Fatalf("取消后的删除失败应进入待清理状态：values=%d pending=%d", len(store.values), len(manager.pendingCleanup))
|
||||
}
|
||||
|
||||
store.failDelete = false
|
||||
if _, err := manager.Resolve("missing-token"); err == nil {
|
||||
t.Fatal("不存在的令牌应返回错误")
|
||||
}
|
||||
if len(store.values) != 0 || len(manager.pendingCleanup) != 0 {
|
||||
t.Fatalf("取消后的凭据重试未完成：values=%d pending=%d", len(store.values), len(manager.pendingCleanup))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSVerificationManagerRejectsAlreadyCancelledContextBeforeTesting(t *testing.T) {
|
||||
tester := &countingDNSTester{}
|
||||
store := &memorySecretWriter{}
|
||||
@ -423,6 +492,29 @@ func TestDNSVerificationManagerRejectsCapacityBeforeTestingOrStoring(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSVerificationManagerCountsExpiredCleanupAgainstCapacity(t *testing.T) {
|
||||
tester := &countingDNSTester{}
|
||||
store := &memorySecretWriter{}
|
||||
manager := NewDNSVerificationManager(tester, store)
|
||||
manager.sessions = make(map[string]dnsVerificationSession, dnsVerificationSessionCapacity-1)
|
||||
manager.expiredSession = make(map[string]expiredDNSVerificationSession, 1)
|
||||
expiresAt := time.Now().UTC().Add(time.Hour)
|
||||
for index := 0; index < dnsVerificationSessionCapacity-1; index++ {
|
||||
manager.sessions[string(rune(index))] = dnsVerificationSession{CredentialRef: "secret://existing", ExpiresAt: expiresAt}
|
||||
}
|
||||
manager.expiredSession["secret://pending"] = expiredDNSVerificationSession{
|
||||
Token: "pending-token",
|
||||
Session: dnsVerificationSession{CredentialRef: "secret://pending", ExpiresAt: expiresAt},
|
||||
}
|
||||
|
||||
_, err := manager.Verify(t.Context(), DNSVerificationInput{
|
||||
ProviderID: "cloudflare", TestDomain: "example.com", Credentials: map[string]string{"api_token": "secret"},
|
||||
})
|
||||
if err == nil || tester.CallCount() != 0 || len(store.values) != 0 {
|
||||
t.Fatalf("待清理凭据应计入容量限制：err=%v calls=%d values=%v", err, tester.CallCount(), store.values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSVerificationManagerRejectsInFlightCapacityBeforeTestingOrStoring(t *testing.T) {
|
||||
tester := &countingDNSTester{}
|
||||
store := &memorySecretWriter{}
|
||||
|
||||
@ -3780,3 +3780,13 @@ Linux Ubuntu 22.04 上的 `Scert-admin` 手动启停、无开机自启动、无
|
||||
- [~] 进行中：DNS 初始化回滚矩阵、令牌生命周期、其他 P0-10 任务链、跨数据库实机验证和统一复审。
|
||||
- [ ] 未开始：P0-12、真实数据库清理初始化、密码 SSH、远端多服务器部署及最终生产门禁。
|
||||
- [!] 不得标记 `[✓]`，总体验收仍为 **P0-03：完成，待复审**。
|
||||
|
||||
## 本轮开发进度（2026-09-30，P0-10 DNS 清理状态与容量边界）
|
||||
|
||||
总体验收继续保持 **P0-03：完成，待复审**。本轮补齐 DNS 临时凭据在保存失败、请求取消和过期清理失败时的待清理状态，并将清理中的过期会话计入容量限制。
|
||||
|
||||
- [x] 已完成，等待复审：临时凭据删除失败时保留可重试引用，后续管理操作自动重试。
|
||||
- [x] 已完成，等待复审：待清理过期会话占用验证容量，防止清理竞态导致超限。
|
||||
- [x] 已完成：回归测试、竞态测试、全量测试、静态检查和差异检查通过。
|
||||
- [~] 继续执行：DNS 初始化回滚、令牌消费、P0-10 任务链、三数据库实机矩阵和统一复审。
|
||||
- [!] 不新增 `[✓]`，不得以本地测试结果替代真实生产验收。
|
||||
@ -1085,3 +1085,13 @@ P0-05 从 `[!]` 调整为 `[~]`，并未标记完成；P0-04 剩余门禁和全
|
||||
- [~] 继续执行：DNS 初始化回滚、令牌消费、任务链、三数据库实机矩阵和统一复审。
|
||||
- [ ] 尚未执行：P0-12、真实生产数据库清理、密码 SSH、远端部署、真实 HTTPS／DNS／ACME 和最终生产门禁。
|
||||
- [!] 不新增 `[✓]`；总体验收继续为 **P0-03：完成，待复审**。
|
||||
|
||||
## 本轮执行记录（2026-09-30，P0-10 DNS 清理状态与容量边界）
|
||||
|
||||
本轮开发仍处于 **P0-03：完成，待复审**，未进入生产部署验收通过状态。
|
||||
|
||||
- [x] 待清理凭据：保存失败或请求取消后的删除失败引用进入重试集合，后续操作自动执行补偿清理。
|
||||
- [x] 容量边界：过期会话清理失败并等待重试期间，继续计入会话容量，避免并发超限。
|
||||
- [x] 验证结果：`go test ./...`、`go test -race ./internal/setup`、`go vet ./...`、`git diff --check` 已通过。
|
||||
- [~] 继续执行：初始化回滚、令牌消费、任务链、三数据库实机矩阵、生产环境测试和统一复审。
|
||||
- [!] 本地自动化验证不得替代密码 SSH、远端部署、真实 DNS／ACME 和最终生产门禁；不得新增 `[✓]`。
|
||||
@ -1733,3 +1733,14 @@ P0-11 生产验收门禁
|
||||
- [~] 进行中：DNS 初始化回滚、令牌消费、其他 P0-10 任务链、跨数据库实机矩阵和统一复审。
|
||||
- [ ] 未开始：P0-12 备份恢复、密码 SSH、远端部署、真实 HTTPS／DNS／ACME 和最终生产门禁。
|
||||
- [!] 总体验收不得升级：仍为 **P0-03：完成，待复审**，本轮没有新增 `[✓]` 复审通过项。
|
||||
|
||||
## 本轮开发进度（2026-09-30，P0-10 DNS 清理状态与容量边界）
|
||||
|
||||
总体验收位置继续保持 **P0-03：完成，待复审**；本轮仅补齐 DNS 临时凭据清理状态和并发容量边界，不代表 P0-10 统一复审通过或生产部署验收通过。
|
||||
|
||||
- [x] 已完成，等待复审：保存失败或请求取消后的临时凭据删除失败时，记录待清理引用，并在后续验证、令牌解析或清理操作前自动重试。
|
||||
- [x] 已完成，等待复审：过期清理失败的会话和待清理状态均计入验证会话容量，避免并发新建验证超过容量上限。
|
||||
- [x] 已完成：新增保存失败重试、取消后重试和容量边界回归测试；`go test ./...`、`go test -race ./internal/setup`、`go vet ./...` 和 `git diff --check` 均通过。
|
||||
- [~] 进行中：DNS 初始化回滚、令牌消费、其他 P0-10 任务链、跨数据库实机矩阵和统一复审。
|
||||
- [ ] 未开始：P0-12、真实生产数据库清理与初始化、密码 SSH、远端多服务器部署、真实 HTTPS／DNS／ACME、Linux／Windows 服务运行验证和最终生产门禁。
|
||||
- [!] 不得标记 `[✓]`；总体验收仍为 **P0-03：完成，待复审**。
|
||||
@ -657,3 +657,14 @@ P0-06 与 P0-07 均维持 `[~]`，六阶段中的隔离、生产和最终验收
|
||||
- [x] 已新增对应回归测试，并通过后端全量测试、静态检查和差异检查。
|
||||
- [~] DNS 初始化回滚、令牌消费及 P0-10 其他任务链继续开发和统一复审。
|
||||
- [!] 本地自动化结果不得替代真实 DNS／ACME、密码 SSH、远端多服务器部署和最终生产门禁；不得新增 `[✓]`。
|
||||
|
||||
## 本轮规格执行记录（2026-09-30，P0-10 DNS 清理状态与容量边界）
|
||||
|
||||
本轮仅完成 DNS 临时凭据清理状态和容量边界的开发与本地回归，总体验收继续保持 **P0-03：完成，待复审**。
|
||||
|
||||
- [x] 删除失败的临时凭据必须保留待清理引用，并在后续管理操作中重试，不能形成不可追踪残留。
|
||||
- [x] 请求取消后若临时凭据回收失败，后续操作必须能够完成补偿清理。
|
||||
- [x] 清理失败的过期会话必须计入验证容量，避免恢复会话时突破容量上限。
|
||||
- [x] 已新增对应回归测试，并通过全量测试、竞态测试、静态检查和差异检查。
|
||||
- [~] DNS 初始化回滚、令牌消费及 P0-10 其他任务链、实机矩阵和统一复审继续进行。
|
||||
- [!] 本地结果不得替代真实 DNS／ACME、密码 SSH、远端多服务器部署和最终生产门禁；不得新增 `[✓]`。
|
||||
Loading…
Reference in New Issue
Block a user