修复证书部署校验重试与状态恢复
This commit is contained in:
parent
1951d48493
commit
aaaf7c3a81
@ -14,16 +14,24 @@ import (
|
||||
)
|
||||
|
||||
type CertificateWorkflow struct {
|
||||
Certificates certificateService
|
||||
Domains domainDeploymentRepository
|
||||
Servers serverLister
|
||||
Deployment savedDeployer
|
||||
Notifier notification.Notifier
|
||||
NotifyError func(error)
|
||||
RemoteVerifier func(context.Context, string, []string) (certificate.ValidationResult, error)
|
||||
CurrentCertificate func(context.Context, string) (certificate.IssueResult, error)
|
||||
Certificates certificateService
|
||||
Domains domainDeploymentRepository
|
||||
Servers serverLister
|
||||
Deployment savedDeployer
|
||||
Notifier notification.Notifier
|
||||
NotifyError func(error)
|
||||
RemoteVerifier func(context.Context, string, []string) (certificate.ValidationResult, error)
|
||||
RemoteVerificationAttempts int
|
||||
RemoteVerificationInterval time.Duration
|
||||
RemoteVerificationWait func(context.Context, time.Duration) error
|
||||
CurrentCertificate func(context.Context, string) (certificate.IssueResult, error)
|
||||
}
|
||||
|
||||
const (
|
||||
defaultRemoteVerificationAttempts = 6
|
||||
defaultRemoteVerificationInterval = 500 * time.Millisecond
|
||||
)
|
||||
|
||||
type certificateService interface {
|
||||
Issue(context.Context, string) (certificate.IssueResult, error)
|
||||
Import(context.Context, certificate.ImportRequest) (certificate.IssueResult, error)
|
||||
@ -95,6 +103,8 @@ func (w *CertificateWorkflow) DeployCurrent(ctx context.Context, domainID string
|
||||
report, err := w.deploy(ctx, result)
|
||||
if err != nil {
|
||||
w.markDeploymentError(result)
|
||||
} else if updateErr := w.Domains.UpdateCertificateState(ctx, result.DomainID, "valid", &result.Saved.NotAfter); updateErr != nil {
|
||||
err = fmt.Errorf("证书部署成功，但更新域名证书状态失败：%w", updateErr)
|
||||
}
|
||||
status := "success"
|
||||
if err != nil {
|
||||
@ -181,13 +191,58 @@ func (w *CertificateWorkflow) deploy(ctx context.Context, result certificate.Iss
|
||||
if verificationDomain == "" {
|
||||
return report, errors.New("无法确定部署后的 HTTPS 校验域名")
|
||||
}
|
||||
if _, err := w.RemoteVerifier(ctx, "https://"+verificationDomain, result.Domains); err != nil {
|
||||
if err := w.verifyRemoteDeployment(ctx, "https://"+verificationDomain, result.Domains); err != nil {
|
||||
return report, fmt.Errorf("部署后 HTTPS 校验失败：%w", err)
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (w *CertificateWorkflow) verifyRemoteDeployment(ctx context.Context, endpoint string, expectedDomains []string) error {
|
||||
attempts := w.RemoteVerificationAttempts
|
||||
if attempts <= 0 {
|
||||
attempts = defaultRemoteVerificationAttempts
|
||||
}
|
||||
interval := w.RemoteVerificationInterval
|
||||
if interval <= 0 {
|
||||
interval = defaultRemoteVerificationInterval
|
||||
}
|
||||
wait := w.RemoteVerificationWait
|
||||
if wait == nil {
|
||||
wait = waitForRemoteVerification
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= attempts; attempt++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.RemoteVerifier(ctx, endpoint, expectedDomains); err == nil {
|
||||
return nil
|
||||
} else {
|
||||
lastErr = err
|
||||
}
|
||||
if attempt == attempts {
|
||||
break
|
||||
}
|
||||
if err := wait(ctx, interval); err != nil {
|
||||
return fmt.Errorf("等待 HTTPS 校验重试时被中断：%w", err)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("重试 %d 次后仍未通过：%w", attempts, lastErr)
|
||||
}
|
||||
|
||||
func waitForRemoteVerification(ctx context.Context, interval time.Duration) error {
|
||||
timer := time.NewTimer(interval)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func verificationName(item domain.Domain, issuedDomains []string) string {
|
||||
for _, value := range issuedDomains {
|
||||
value = strings.TrimSpace(value)
|
||||
|
||||
@ -2,6 +2,8 @@ package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -40,9 +42,12 @@ type workflowDeployer struct {
|
||||
}
|
||||
|
||||
type workflowDomainState struct {
|
||||
id string
|
||||
status string
|
||||
item domain.Domain
|
||||
id string
|
||||
status string
|
||||
expiresAt *time.Time
|
||||
updateErr error
|
||||
updateCalled int
|
||||
item domain.Domain
|
||||
}
|
||||
|
||||
func (s *workflowDomainState) GetByID(_ context.Context, id string) (domain.Domain, error) {
|
||||
@ -59,8 +64,16 @@ func (s *workflowDomainState) GetByID(_ context.Context, id string) (domain.Doma
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (s *workflowDomainState) UpdateCertificateState(_ context.Context, id, status string, _ *time.Time) error {
|
||||
func (s *workflowDomainState) UpdateCertificateState(_ context.Context, id, status string, expiresAt *time.Time) error {
|
||||
s.updateCalled++
|
||||
if s.updateErr != nil {
|
||||
return s.updateErr
|
||||
}
|
||||
s.id, s.status = id, status
|
||||
if expiresAt != nil {
|
||||
value := *expiresAt
|
||||
s.expiresAt = &value
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -143,3 +156,158 @@ func TestCertificateWorkflowResolvesSameServerForDifferentDomains(t *testing.T)
|
||||
t.Fatalf("共享服务器的域名部署路径发生覆盖：%s %s", firstPath, secondPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCertificateWorkflowRetriesRemoteVerificationUntilSuccess(t *testing.T) {
|
||||
deployer := &workflowDeployer{report: deployment.Report{Total: 1, Success: 1, Status: "success"}}
|
||||
state := &workflowDomainState{item: domain.Domain{ID: "domain-1", Name: "example.com", CertificateType: domain.CertificateSingle}}
|
||||
verificationCalls := 0
|
||||
waitCalls := 0
|
||||
workflow := &CertificateWorkflow{
|
||||
Certificates: &workflowCertificateService{result: certificate.IssueResult{DomainID: "domain-1", Domains: []string{"example.com"}}},
|
||||
Domains: state,
|
||||
Servers: workflowServerLister{servers: []deployment.Server{{ID: "server-1"}}},
|
||||
Deployment: deployer,
|
||||
RemoteVerificationAttempts: 4,
|
||||
RemoteVerificationInterval: 10 * time.Millisecond,
|
||||
RemoteVerificationWait: func(ctx context.Context, interval time.Duration) error {
|
||||
waitCalls++
|
||||
if interval != 10*time.Millisecond {
|
||||
t.Fatalf("HTTPS 校验重试间隔错误：%s", interval)
|
||||
}
|
||||
return ctx.Err()
|
||||
},
|
||||
RemoteVerifier: func(_ context.Context, endpoint string, expectedDomains []string) (certificate.ValidationResult, error) {
|
||||
verificationCalls++
|
||||
if endpoint != "https://example.com" || len(expectedDomains) != 1 || expectedDomains[0] != "example.com" {
|
||||
t.Fatalf("HTTPS 校验参数错误：%s %+v", endpoint, expectedDomains)
|
||||
}
|
||||
if verificationCalls < 3 {
|
||||
return certificate.ValidationResult{}, errors.New("仍返回旧证书")
|
||||
}
|
||||
return certificate.ValidationResult{}, nil
|
||||
},
|
||||
}
|
||||
|
||||
if _, err := workflow.IssueWithReport(context.Background(), "domain-1"); err != nil {
|
||||
t.Fatalf("HTTPS 校验重试后应成功：%v", err)
|
||||
}
|
||||
if verificationCalls != 3 || waitCalls != 2 {
|
||||
t.Fatalf("HTTPS 校验重试次数错误：校验 %d 次，等待 %d 次", verificationCalls, waitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCertificateWorkflowRemoteVerificationStopsAfterConfiguredAttempts(t *testing.T) {
|
||||
deployer := &workflowDeployer{report: deployment.Report{Total: 1, Success: 1, Status: "success"}}
|
||||
state := &workflowDomainState{item: domain.Domain{ID: "domain-1", Name: "example.com", CertificateType: domain.CertificateSingle}}
|
||||
verificationCalls := 0
|
||||
waitCalls := 0
|
||||
workflow := &CertificateWorkflow{
|
||||
Certificates: &workflowCertificateService{result: certificate.IssueResult{DomainID: "domain-1", Domains: []string{"example.com"}}},
|
||||
Domains: state,
|
||||
Servers: workflowServerLister{servers: []deployment.Server{{ID: "server-1"}}},
|
||||
Deployment: deployer,
|
||||
RemoteVerificationAttempts: 3,
|
||||
RemoteVerificationWait: func(context.Context, time.Duration) error {
|
||||
waitCalls++
|
||||
return nil
|
||||
},
|
||||
RemoteVerifier: func(context.Context, string, []string) (certificate.ValidationResult, error) {
|
||||
verificationCalls++
|
||||
return certificate.ValidationResult{}, errors.New("证书永久不匹配")
|
||||
},
|
||||
}
|
||||
|
||||
_, err := workflow.IssueWithReport(context.Background(), "domain-1")
|
||||
if err == nil || !strings.Contains(err.Error(), "重试 3 次后仍未通过") {
|
||||
t.Fatalf("永久校验错误应在有限重试后返回：%v", err)
|
||||
}
|
||||
if verificationCalls != 3 || waitCalls != 2 {
|
||||
t.Fatalf("永久错误的重试次数错误：校验 %d 次，等待 %d 次", verificationCalls, waitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCertificateWorkflowRemoteVerificationWaitHonorsContextCancellation(t *testing.T) {
|
||||
deployer := &workflowDeployer{report: deployment.Report{Total: 1, Success: 1, Status: "success"}}
|
||||
state := &workflowDomainState{item: domain.Domain{ID: "domain-1", Name: "example.com", CertificateType: domain.CertificateSingle}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
verificationCalls := 0
|
||||
workflow := &CertificateWorkflow{
|
||||
Certificates: &workflowCertificateService{result: certificate.IssueResult{DomainID: "domain-1", Domains: []string{"example.com"}}},
|
||||
Domains: state,
|
||||
Servers: workflowServerLister{servers: []deployment.Server{{ID: "server-1"}}},
|
||||
Deployment: deployer,
|
||||
RemoteVerificationAttempts: 6,
|
||||
RemoteVerificationInterval: time.Minute,
|
||||
RemoteVerifier: func(context.Context, string, []string) (certificate.ValidationResult, error) {
|
||||
verificationCalls++
|
||||
cancel()
|
||||
return certificate.ValidationResult{}, errors.New("等待新证书")
|
||||
},
|
||||
}
|
||||
|
||||
_, err := workflow.IssueWithReport(ctx, "domain-1")
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("取消上下文后应立即终止 HTTPS 校验重试：%v", err)
|
||||
}
|
||||
if verificationCalls != 1 {
|
||||
t.Fatalf("取消后不应继续校验，实际调用 %d 次", verificationCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCertificateWorkflowDeployCurrentRestoresValidState(t *testing.T) {
|
||||
notAfter := time.Now().UTC().Add(30 * 24 * time.Hour).Truncate(time.Second)
|
||||
state := &workflowDomainState{
|
||||
status: "error",
|
||||
item: domain.Domain{ID: "domain-1", Name: "example.com", CertificateType: domain.CertificateSingle},
|
||||
}
|
||||
workflow := &CertificateWorkflow{
|
||||
CurrentCertificate: func(context.Context, string) (certificate.IssueResult, error) {
|
||||
return certificate.IssueResult{
|
||||
DomainID: "domain-1",
|
||||
Domains: []string{"example.com"},
|
||||
Saved: certificate.SavedResource{NotAfter: notAfter},
|
||||
}, nil
|
||||
},
|
||||
Domains: state,
|
||||
Servers: workflowServerLister{servers: []deployment.Server{{ID: "server-1"}}},
|
||||
Deployment: &workflowDeployer{report: deployment.Report{Total: 1, Success: 1, Status: "success"}},
|
||||
RemoteVerifier: func(context.Context, string, []string) (certificate.ValidationResult, error) {
|
||||
return certificate.ValidationResult{}, nil
|
||||
},
|
||||
}
|
||||
|
||||
result, err := workflow.DeployCurrent(context.Background(), "domain-1")
|
||||
if err != nil {
|
||||
t.Fatalf("当前证书重新部署应成功：%v", err)
|
||||
}
|
||||
if result.Deployment.Status != "success" || state.id != "domain-1" || state.status != "valid" || state.updateCalled != 1 {
|
||||
t.Fatalf("成功部署后域名状态未恢复：%+v %+v", result.Deployment, state)
|
||||
}
|
||||
if state.expiresAt == nil || !state.expiresAt.Equal(notAfter) {
|
||||
t.Fatalf("成功部署后证书到期时间未更新：%v", state.expiresAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCertificateWorkflowDeployCurrentReturnsStateUpdateError(t *testing.T) {
|
||||
state := &workflowDomainState{
|
||||
status: "error",
|
||||
updateErr: errors.New("数据库不可写"),
|
||||
item: domain.Domain{ID: "domain-1", Name: "example.com", CertificateType: domain.CertificateSingle},
|
||||
}
|
||||
workflow := &CertificateWorkflow{
|
||||
CurrentCertificate: func(context.Context, string) (certificate.IssueResult, error) {
|
||||
return certificate.IssueResult{DomainID: "domain-1"}, nil
|
||||
},
|
||||
Domains: state,
|
||||
Servers: workflowServerLister{servers: []deployment.Server{{ID: "server-1"}}},
|
||||
Deployment: &workflowDeployer{report: deployment.Report{Total: 1, Success: 1, Status: "success"}},
|
||||
}
|
||||
|
||||
result, err := workflow.DeployCurrent(context.Background(), "domain-1")
|
||||
if err == nil || !strings.Contains(err.Error(), "更新域名证书状态失败") {
|
||||
t.Fatalf("域名状态更新失败时应返回明确错误：%v", err)
|
||||
}
|
||||
if result.Deployment.Status != "success" || state.updateCalled != 1 || state.status != "error" {
|
||||
t.Fatalf("状态更新失败时应保留部署报告和原状态，且不能静默忽略：%+v %+v", result.Deployment, state)
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user