This commit is contained in:
parent
d534f0187d
commit
acf03222e9
@ -17,6 +17,7 @@ import (
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
ErrGenerationConflict = errors.New("configuration generation conflict")
|
ErrGenerationConflict = errors.New("configuration generation conflict")
|
||||||
|
ErrRollbackInProgress = errors.New("configuration rollback is already in progress")
|
||||||
ErrNoBackup = errors.New("configuration backup not found")
|
ErrNoBackup = errors.New("configuration backup not found")
|
||||||
ErrManagerStorage = errors.New("configuration manager storage unavailable")
|
ErrManagerStorage = errors.New("configuration manager storage unavailable")
|
||||||
)
|
)
|
||||||
|
|||||||
@ -96,6 +96,7 @@ type rollbackTicket struct {
|
|||||||
SessionID string
|
SessionID string
|
||||||
TargetGeneration uint64
|
TargetGeneration uint64
|
||||||
ExpiresAt time.Time
|
ExpiresAt time.Time
|
||||||
|
InFlight bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type LegacyPluginSecret struct {
|
type LegacyPluginSecret struct {
|
||||||
@ -524,31 +525,60 @@ func (m *ConfigManager) RollbackWithTicket(ctx context.Context, token string, ac
|
|||||||
m.ticketMu.Unlock()
|
m.ticketMu.Unlock()
|
||||||
return nil, ErrAuthorizationRequired
|
return nil, ErrAuthorizationRequired
|
||||||
}
|
}
|
||||||
delete(m.rollbackTickets, token)
|
if ticket.InFlight {
|
||||||
|
m.ticketMu.Unlock()
|
||||||
|
return nil, ErrRollbackInProgress
|
||||||
|
}
|
||||||
|
ticket.InFlight = true
|
||||||
|
m.rollbackTickets[token] = ticket
|
||||||
m.ticketMu.Unlock()
|
m.ticketMu.Unlock()
|
||||||
return m.rollback(ctx, ticket.TargetGeneration, actor, StepUpConfirmation{Confirmed: true, Method: "ticket"})
|
|
||||||
|
result, applied, err := m.rollbackWithState(ctx, ticket.TargetGeneration, actor, StepUpConfirmation{Confirmed: true, Method: "ticket"})
|
||||||
|
m.ticketMu.Lock()
|
||||||
|
if err == nil || applied || !rollbackTicketRetryable(err) {
|
||||||
|
delete(m.rollbackTickets, token)
|
||||||
|
} else if current, exists := m.rollbackTickets[token]; exists {
|
||||||
|
current.InFlight = false
|
||||||
|
m.rollbackTickets[token] = current
|
||||||
|
}
|
||||||
|
m.ticketMu.Unlock()
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func rollbackTicketRetryable(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return errors.Is(err, context.Canceled) ||
|
||||||
|
errors.Is(err, context.DeadlineExceeded) ||
|
||||||
|
errors.Is(err, ErrRuntimeSnapshotUnavailable)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *ConfigManager) rollback(ctx context.Context, generation uint64, actor AuditActor, confirmation StepUpConfirmation) (*ApplyResult, error) {
|
func (m *ConfigManager) rollback(ctx context.Context, generation uint64, actor AuditActor, confirmation StepUpConfirmation) (*ApplyResult, error) {
|
||||||
|
result, _, err := m.rollbackWithState(ctx, generation, actor, confirmation)
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *ConfigManager) rollbackWithState(ctx context.Context, generation uint64, actor AuditActor, confirmation StepUpConfirmation) (*ApplyResult, bool, error) {
|
||||||
if err := contextError(ctx); err != nil {
|
if err := contextError(ctx); err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
if m == nil {
|
if m == nil {
|
||||||
return nil, ErrManagerStorage
|
return nil, false, ErrManagerStorage
|
||||||
}
|
}
|
||||||
if generation == 0 {
|
if generation == 0 {
|
||||||
return nil, ErrGenerationConflict
|
return nil, false, ErrGenerationConflict
|
||||||
}
|
}
|
||||||
if err := m.authorize(ctx, actor, ConfigOperationRollback, SectionRoot); err != nil {
|
if err := m.authorize(ctx, actor, ConfigOperationRollback, SectionRoot); err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
if confirmation.Method != "ticket" {
|
if confirmation.Method != "ticket" {
|
||||||
if err := m.confirm(ctx, actor, confirmation, ConfigOperationRollback, SectionRoot); err != nil {
|
if err := m.confirm(ctx, actor, confirmation, ConfigOperationRollback, SectionRoot); err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := contextError(ctx); err != nil {
|
if err := contextError(ctx); err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
m.writeMu.Lock()
|
m.writeMu.Lock()
|
||||||
defer m.writeMu.Unlock()
|
defer m.writeMu.Unlock()
|
||||||
@ -557,40 +587,40 @@ func (m *ConfigManager) rollback(ctx context.Context, generation uint64, actor A
|
|||||||
pending := m.pending != nil
|
pending := m.pending != nil
|
||||||
m.mu.RUnlock()
|
m.mu.RUnlock()
|
||||||
if pending || generation >= current.Generation {
|
if pending || generation >= current.Generation {
|
||||||
return nil, ErrGenerationConflict
|
return nil, false, ErrGenerationConflict
|
||||||
}
|
}
|
||||||
candidate, err := m.transactions.LoadBackup(generation)
|
candidate, err := m.transactions.LoadBackup(generation)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
if err := contextError(ctx); err != nil {
|
if err := contextError(ctx); err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
if err := ValidateRuntimeSecrets(m.vault, candidate); err != nil {
|
if err := ValidateRuntimeSecrets(m.vault, candidate); err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
setCandidateGeneration(&candidate, current.Generation+1)
|
setCandidateGeneration(&candidate, current.Generation+1)
|
||||||
candidate.Root.UpdatedAt = time.Now().UTC()
|
candidate.Root.UpdatedAt = time.Now().UTC()
|
||||||
if err := ValidateRuntimeConfig(candidate, m.options); err != nil {
|
if err := ValidateRuntimeConfig(candidate, m.options); err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
if err := m.snapshots.Validate(candidate); err != nil {
|
if err := m.snapshots.Validate(candidate); err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
if err := contextError(ctx); err != nil {
|
if err := contextError(ctx); err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
if _, _, err := m.transactions.CreateBackupIfDue(current, time.Now().UTC()); err != nil {
|
if _, _, err := m.transactions.CreateBackupIfDue(current, time.Now().UTC()); err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
if err := contextError(ctx); err != nil {
|
if err := contextError(ctx); err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
if err := m.transactions.Commit(current, candidate); err != nil {
|
if err := m.transactions.Commit(current, candidate); err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
if err := m.snapshots.Publish(candidate); err != nil {
|
if err := m.snapshots.Publish(candidate); err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
@ -615,9 +645,9 @@ func (m *ConfigManager) rollback(ctx context.Context, generation uint64, actor A
|
|||||||
Result: "success",
|
Result: "success",
|
||||||
CreatedAt: now,
|
CreatedAt: now,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return nil, err
|
return nil, true, err
|
||||||
}
|
}
|
||||||
return &ApplyResult{Snapshot: cloneRuntimeConfig(candidate), Generation: candidate.Generation, AppliedAt: now}, nil
|
return &ApplyResult{Snapshot: cloneRuntimeConfig(candidate), Generation: candidate.Generation, AppliedAt: now}, true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *ConfigManager) authorize(ctx context.Context, actor AuditActor, operation ConfigOperation, section Section) error {
|
func (m *ConfigManager) authorize(ctx context.Context, actor AuditActor, operation ConfigOperation, section Section) error {
|
||||||
|
|||||||
@ -7,6 +7,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -1476,3 +1477,75 @@ func TestConfigManagerSnapshotResultsAreIndependentCopies(t *testing.T) {
|
|||||||
t.Fatal("apply result exposed manager-owned active data")
|
t.Fatal("apply result exposed manager-owned active data")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type flakyRuntimeSnapshotStore struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
current RuntimeConfig
|
||||||
|
failures int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *flakyRuntimeSnapshotStore) Validate(RuntimeConfig) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *flakyRuntimeSnapshotStore) Publish(candidate RuntimeConfig) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.failures > 0 {
|
||||||
|
s.failures--
|
||||||
|
return ErrRuntimeSnapshotUnavailable
|
||||||
|
}
|
||||||
|
s.current = cloneRuntimeConfig(candidate)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *flakyRuntimeSnapshotStore) Snapshot() (RuntimeConfig, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
return cloneRuntimeConfig(s.current), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigManagerRollbackTicketCanRetryAfterSnapshotFailure(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeRuntimeFixture(t, root, validRuntimeConfig())
|
||||||
|
manager, err := NewConfigManager(root, ConfigLoaderOptions{Validation: RuntimeValidationOptions{HasSuperAdmin: true, Production: true}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
updatedEmail := manager.Snapshot().Email
|
||||||
|
updatedEmail.Policy.MaxMessageBytes = 8 << 20
|
||||||
|
payload, err := json.Marshal(updatedEmail)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
updated, err := manager.Update(context.Background(), UpdateRequest{Section: SectionEmail, ExpectedGeneration: 1, Payload: payload, Actor: authorizedConfigActor()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := manager.Apply(context.Background(), updated.Generation); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
manager.snapshots = &flakyRuntimeSnapshotStore{current: manager.Snapshot(), failures: 1}
|
||||||
|
actor := authorizedConfigActor()
|
||||||
|
ticket, err := manager.IssueRollbackTicket(context.Background(), 1, actor, StepUpConfirmation{Confirmed: true, Method: "webauthn"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := manager.RollbackWithTicket(context.Background(), ticket, actor); !errors.Is(err, ErrRuntimeSnapshotUnavailable) {
|
||||||
|
t.Fatalf("first rollback error = %v, want snapshot unavailable", err)
|
||||||
|
}
|
||||||
|
stored, exists := manager.rollbackTickets[ticket]
|
||||||
|
if !exists || stored.InFlight {
|
||||||
|
t.Fatalf("rollback ticket state after retryable failure = %+v, exists=%v", stored, exists)
|
||||||
|
}
|
||||||
|
result, err := manager.RollbackWithTicket(context.Background(), ticket, actor)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Generation != 3 || result.Snapshot.Email.Policy.MaxMessageBytes == 8<<20 {
|
||||||
|
t.Fatalf("retry rollback result = %+v", result)
|
||||||
|
}
|
||||||
|
if _, err := manager.RollbackWithTicket(context.Background(), ticket, actor); !errors.Is(err, ErrAuthorizationRequired) {
|
||||||
|
t.Fatalf("replayed retry ticket error = %v, want authorization required", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -53,6 +53,10 @@ func configErrorDetails(err error) (string, int, string, map[string]string) {
|
|||||||
code = "CONFIG_MANAGER_UNAVAILABLE"
|
code = "CONFIG_MANAGER_UNAVAILABLE"
|
||||||
status = http.StatusServiceUnavailable
|
status = http.StatusServiceUnavailable
|
||||||
message = "配置服务暂时不可用,请稍后重试"
|
message = "配置服务暂时不可用,请稍后重试"
|
||||||
|
case errors.Is(err, config.ErrRollbackInProgress):
|
||||||
|
code = "CONFIG_ROLLBACK_IN_PROGRESS"
|
||||||
|
status = http.StatusConflict
|
||||||
|
message = "配置回滚正在执行,请稍后查看结果"
|
||||||
case errors.Is(err, config.ErrGenerationConflict):
|
case errors.Is(err, config.ErrGenerationConflict):
|
||||||
code = "CONFIG_GENERATION_CONFLICT"
|
code = "CONFIG_GENERATION_CONFLICT"
|
||||||
status = http.StatusConflict
|
status = http.StatusConflict
|
||||||
|
|||||||
@ -2,6 +2,7 @@ package controllers
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@ -167,3 +168,13 @@ func TestConfigErrorDetailsMapsDatabaseMigrationFailures(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfigErrorDetailsMapsRollbackInProgress(t *testing.T) {
|
||||||
|
code, status, message, _ := configErrorDetails(config.ErrRollbackInProgress)
|
||||||
|
if code != "CONFIG_ROLLBACK_IN_PROGRESS" || status != http.StatusConflict {
|
||||||
|
t.Fatalf("rollback in progress mapping = %q/%d, want conflict", code, status)
|
||||||
|
}
|
||||||
|
if message == "" || strings.Contains(message, "rollback in progress") {
|
||||||
|
t.Fatalf("rollback in progress message is not a Chinese public message: %q", message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user