完善DNS清理重试与容量边界
This commit is contained in:
1 parent
4a4946979b
commit
1aaa16d103
6 files changed
+192
-4
No files matched your search
@@ -101,6 +101,7 @@ type DNSVerificationManager struct {
|
||||
now func() time.Time
|
||||
sessions map[string]dnsVerificationSession
|
||||
expiredSession map[string]expiredDNSVerificationSession
|
||||
pendingCleanup map[string]struct{}
|
||||
inFlight int
|
||||
}
|
||||
|
||||
@@ -108,7 +109,9 @@ func NewDNSVerificationManager(tester DNSCredentialTester, store SecretWriter) *
|
||||
deleter, _ := store.(SecretDeleter)
|
||||
return &DNSVerificationManager{
|
||||
tester: tester, store: store, deleter: deleter, now: time.Now,
|
||||
sessions: make(map[string]dnsVerificationSession), expiredSession: make(map[string]expiredDNSVerificationSession),
|
||||
sessions: make(map[string]dnsVerificationSession),
|
||||
expiredSession: make(map[string]expiredDNSVerificationSession),
|
||||
pendingCleanup: make(map[string]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,6 +119,7 @@ func (m *DNSVerificationManager) Verify(ctx context.Context, input DNSVerificati
|
||||
if m == nil || m.tester == nil || m.store == nil {
|
||||
return DNSVerificationResult{}, errors.New("DNS 在线验证服务未初始化")
|
||||
}
|
||||
m.retryPendingCredentialCleanup()
|
||||
providerID := strings.ToLower(strings.TrimSpace(input.ProviderID))
|
||||
testDomain := normalizeTestDomain(input.TestDomain)
|
||||
if providerID == "" {
|
||||
@@ -164,7 +168,7 @@ func (m *DNSVerificationManager) Verify(ctx context.Context, input DNSVerificati
|
||||
}
|
||||
credentialRef := fmt.Sprintf("secret://dns/%s/%s", providerID, credentialID)
|
||||
if err := m.store.Put(ctx, credentialRef, cleanedCredentials); err != nil {
|
||||
cleanupErr := m.deleteCredential(credentialRef)
|
||||
cleanupErr := m.deleteOrQueueCredential(credentialRef)
|
||||
if cleanupErr != nil {
|
||||
m.logCleanupFailure()
|
||||
return DNSVerificationResult{}, errors.New("加密保存 DNS 凭据失败,临时凭据回收未确认")
|
||||
@@ -172,7 +176,7 @@ func (m *DNSVerificationManager) Verify(ctx context.Context, input DNSVerificati
|
||||
return DNSVerificationResult{}, fmt.Errorf("加密保存 DNS 凭据失败:%w", err)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
cleanupErr := m.deleteCredential(credentialRef)
|
||||
cleanupErr := m.deleteOrQueueCredential(credentialRef)
|
||||
if cleanupErr != nil {
|
||||
m.logCleanupFailure()
|
||||
return DNSVerificationResult{TestReport: testReport}, errors.Join(
|
||||
@@ -243,6 +247,7 @@ func (m *DNSVerificationManager) Discard(token string) error {
|
||||
if m == nil {
|
||||
return errors.New("DNS 在线验证服务未初始化")
|
||||
}
|
||||
m.retryPendingCredentialCleanup()
|
||||
token = strings.TrimSpace(token)
|
||||
if token == "" {
|
||||
return errors.New("DNS 验证令牌不能为空")
|
||||
@@ -277,6 +282,7 @@ func (m *DNSVerificationManager) DiscardAllExcept(protectedReferences ...string)
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.retryPendingCredentialCleanup()
|
||||
protected := make(map[string]bool, len(protectedReferences))
|
||||
for _, reference := range protectedReferences {
|
||||
if reference = strings.TrimSpace(reference); reference != "" {
|
||||
@@ -309,6 +315,7 @@ func (m *DNSVerificationManager) validateSession(token, providerID, credentialRe
|
||||
if token == "" {
|
||||
return errors.New("必须先完成 DNS API 在线验证")
|
||||
}
|
||||
m.retryPendingCredentialCleanup()
|
||||
now := m.now().UTC()
|
||||
m.mu.Lock()
|
||||
expiredRefs := m.clearExpiredLocked(now)
|
||||
@@ -341,6 +348,7 @@ func (m *DNSVerificationManager) resolveSession(token string, claim bool) (dnsVe
|
||||
if token == "" {
|
||||
return dnsVerificationSession{}, errors.New("必须先完成 DNS API 在线验证")
|
||||
}
|
||||
m.retryPendingCredentialCleanup()
|
||||
now := m.now().UTC()
|
||||
m.mu.Lock()
|
||||
expiredRefs := m.clearExpiredLocked(now)
|
||||
@@ -368,7 +376,7 @@ func (m *DNSVerificationManager) reserveVerificationSlot() error {
|
||||
m.deleteCredentials(expiredRefs)
|
||||
return errors.New("DNS 在线验证请求过多,请稍后重试")
|
||||
}
|
||||
if len(m.sessions)+m.inFlight >= dnsVerificationSessionCapacity {
|
||||
if len(m.sessions)+len(m.expiredSession)+m.inFlight >= dnsVerificationSessionCapacity {
|
||||
m.mu.Unlock()
|
||||
m.deleteCredentials(expiredRefs)
|
||||
return errors.New("DNS 验证令牌数量已达上限,请等待现有验证完成或过期后重试")
|
||||
@@ -458,6 +466,52 @@ func (m *DNSVerificationManager) forgetExpiredSession(reference string) {
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *DNSVerificationManager) rememberPendingCleanup(reference string) {
|
||||
if m == nil || strings.TrimSpace(reference) == "" {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.pendingCleanup[reference] = struct{}{}
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *DNSVerificationManager) forgetPendingCleanup(reference string) {
|
||||
if m == nil || strings.TrimSpace(reference) == "" {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
delete(m.pendingCleanup, reference)
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *DNSVerificationManager) retryPendingCredentialCleanup() {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
references := make([]string, 0, len(m.pendingCleanup))
|
||||
for reference := range m.pendingCleanup {
|
||||
references = append(references, reference)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
for _, reference := range references {
|
||||
if err := m.deleteCredential(reference); err != nil {
|
||||
m.logCleanupFailure()
|
||||
continue
|
||||
}
|
||||
m.forgetPendingCleanup(reference)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *DNSVerificationManager) deleteOrQueueCredential(reference string) error {
|
||||
if err := m.deleteCredential(reference); err != nil {
|
||||
m.rememberPendingCleanup(reference)
|
||||
return err
|
||||
}
|
||||
m.forgetPendingCleanup(reference)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *DNSVerificationManager) logCleanupFailure() {
|
||||
log.Print("DNS 临时凭据回收失败")
|
||||
}
|
||||
|
||||
@@ -66,6 +66,11 @@ type retryableSecretWriter struct {
|
||||
failDelete bool
|
||||
}
|
||||
|
||||
type cancellingRetryableSecretWriter struct {
|
||||
retryableSecretWriter
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (s *cancellingSecretWriter) Put(ctx context.Context, reference string, value map[string]string) error {
|
||||
if err := s.memorySecretWriter.Put(ctx, reference, value); err != nil {
|
||||
return err
|
||||
@@ -81,6 +86,14 @@ func (s *retryableSecretWriter) Delete(ctx context.Context, reference string) er
|
||||
return s.memorySecretWriter.Delete(ctx, reference)
|
||||
}
|
||||
|
||||
func (s *cancellingRetryableSecretWriter) Put(ctx context.Context, reference string, value map[string]string) error {
|
||||
if err := s.retryableSecretWriter.Put(ctx, reference, value); err != nil {
|
||||
return err
|
||||
}
|
||||
s.cancel()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memorySecretWriter) Put(_ context.Context, reference string, value map[string]string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
@@ -290,6 +303,62 @@ func TestDNSVerificationManagerDeletesCredentialWhenPutReportsFailure(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSVerificationManagerRetriesCleanupAfterPutFailure(t *testing.T) {
|
||||
store := &retryableSecretWriter{failDelete: true}
|
||||
store.putErr = errors.New("write failed")
|
||||
manager := NewDNSVerificationManager(fakeDNSTester{}, store)
|
||||
_, err := manager.Verify(t.Context(), DNSVerificationInput{
|
||||
ProviderID: "cloudflare", TestDomain: "example.com",
|
||||
Credentials: map[string]string{"api_token": "secret"},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("保存失败时应返回错误")
|
||||
}
|
||||
if len(store.values) != 1 || len(manager.pendingCleanup) != 1 {
|
||||
t.Fatalf("删除失败后应保留待清理凭据:values=%d pending=%d", len(store.values), len(manager.pendingCleanup))
|
||||
}
|
||||
|
||||
store.putErr = nil
|
||||
store.failDelete = false
|
||||
if _, err := manager.Verify(t.Context(), DNSVerificationInput{
|
||||
ProviderID: "cloudflare", TestDomain: "example.com",
|
||||
Credentials: map[string]string{"api_token": "next-secret"},
|
||||
}); err != nil {
|
||||
t.Fatalf("下一次验证应先重试待清理凭据:%v", err)
|
||||
}
|
||||
if len(manager.pendingCleanup) != 0 || len(store.values) != 1 {
|
||||
t.Fatalf("待清理凭据重试后状态错误:pending=%d values=%d", len(manager.pendingCleanup), len(store.values))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSVerificationManagerRetriesCleanupAfterCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
store := &cancellingRetryableSecretWriter{
|
||||
retryableSecretWriter: retryableSecretWriter{failDelete: true},
|
||||
cancel: cancel,
|
||||
}
|
||||
manager := NewDNSVerificationManager(fakeDNSTester{}, store)
|
||||
|
||||
_, err := manager.Verify(ctx, DNSVerificationInput{
|
||||
ProviderID: "cloudflare", TestDomain: "example.com",
|
||||
Credentials: map[string]string{"api_token": "secret"},
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("保存后取消应返回 context.Canceled:%v", err)
|
||||
}
|
||||
if len(store.values) != 1 || len(manager.pendingCleanup) != 1 {
|
||||
t.Fatalf("取消后的删除失败应进入待清理状态:values=%d pending=%d", len(store.values), len(manager.pendingCleanup))
|
||||
}
|
||||
|
||||
store.failDelete = false
|
||||
if _, err := manager.Resolve("missing-token"); err == nil {
|
||||
t.Fatal("不存在的令牌应返回错误")
|
||||
}
|
||||
if len(store.values) != 0 || len(manager.pendingCleanup) != 0 {
|
||||
t.Fatalf("取消后的凭据重试未完成:values=%d pending=%d", len(store.values), len(manager.pendingCleanup))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSVerificationManagerRejectsAlreadyCancelledContextBeforeTesting(t *testing.T) {
|
||||
tester := &countingDNSTester{}
|
||||
store := &memorySecretWriter{}
|
||||
@@ -423,6 +492,29 @@ func TestDNSVerificationManagerRejectsCapacityBeforeTestingOrStoring(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSVerificationManagerCountsExpiredCleanupAgainstCapacity(t *testing.T) {
|
||||
tester := &countingDNSTester{}
|
||||
store := &memorySecretWriter{}
|
||||
manager := NewDNSVerificationManager(tester, store)
|
||||
manager.sessions = make(map[string]dnsVerificationSession, dnsVerificationSessionCapacity-1)
|
||||
manager.expiredSession = make(map[string]expiredDNSVerificationSession, 1)
|
||||
expiresAt := time.Now().UTC().Add(time.Hour)
|
||||
for index := 0; index < dnsVerificationSessionCapacity-1; index++ {
|
||||
manager.sessions[string(rune(index))] = dnsVerificationSession{CredentialRef: "secret://existing", ExpiresAt: expiresAt}
|
||||
}
|
||||
manager.expiredSession["secret://pending"] = expiredDNSVerificationSession{
|
||||
Token: "pending-token",
|
||||
Session: dnsVerificationSession{CredentialRef: "secret://pending", ExpiresAt: expiresAt},
|
||||
}
|
||||
|
||||
_, err := manager.Verify(t.Context(), DNSVerificationInput{
|
||||
ProviderID: "cloudflare", TestDomain: "example.com", Credentials: map[string]string{"api_token": "secret"},
|
||||
})
|
||||
if err == nil || tester.CallCount() != 0 || len(store.values) != 0 {
|
||||
t.Fatalf("待清理凭据应计入容量限制:err=%v calls=%d values=%v", err, tester.CallCount(), store.values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSVerificationManagerRejectsInFlightCapacityBeforeTestingOrStoring(t *testing.T) {
|
||||
tester := &countingDNSTester{}
|
||||
store := &memorySecretWriter{}
|
||||
|
||||
Reference in new issue
Block a user