修复证书重载失败后未自动恢复运行证书的问题
This commit is contained in:
parent
7a08dfe27d
commit
2e6d992b98
102
backend/internal/deployment/reload_recovery_test.go
Normal file
102
backend/internal/deployment/reload_recovery_test.go
Normal file
@ -0,0 +1,102 @@
|
|||||||
|
package deployment
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestVerifiedReloadFailureRestoresRuntimeBeforeUnlock(t *testing.T) {
|
||||||
|
for _, failure := range []string{"exit_status", "transport_error"} {
|
||||||
|
t.Run(failure, func(t *testing.T) {
|
||||||
|
servers := []Server{testServer("first").Normalize(), testServer("second").Normalize()}
|
||||||
|
clients := map[string]*memoryClient{}
|
||||||
|
originals := map[string]map[string]memoryFile{}
|
||||||
|
for _, server := range servers {
|
||||||
|
client := newMemoryClient()
|
||||||
|
clients[server.ID] = client
|
||||||
|
originals[server.ID] = seedVerifiedFiles(client, server)
|
||||||
|
}
|
||||||
|
if failure == "exit_status" {
|
||||||
|
clients["second"].runResults = []CommandResult{{}, {ExitCode: 1}}
|
||||||
|
} else {
|
||||||
|
clients["second"].runErrors = []error{nil, errors.New("reload response lost")}
|
||||||
|
}
|
||||||
|
journal := &memoryTransactionJournal{}
|
||||||
|
service := NewServiceWithJournal(memoryFactory{clients: clients}, journal)
|
||||||
|
ctx := WithTransactionContext(t.Context(), TransactionContext{TaskID: "task", AttemptID: "attempt", AttemptNumber: 1})
|
||||||
|
recoveryCalled := false
|
||||||
|
report, err := service.DeployVerified(ctx, []Target{{Server: servers[0]}, {Server: servers[1]}}, testArtifact(t), Verification{
|
||||||
|
Verify: func(context.Context) error { t.Fatal("failed reload must not verify deployment"); return nil },
|
||||||
|
VerifyRecovery: func(ctx context.Context) error {
|
||||||
|
recoveryCalled = true
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
t.Fatal("recovery context is cancelled")
|
||||||
|
}
|
||||||
|
for _, server := range servers {
|
||||||
|
client := clients[server.ID]
|
||||||
|
assertVerifiedFiles(t, client, originals[server.ID])
|
||||||
|
assertVerifiedLocks(t, client, server, true)
|
||||||
|
commands := [][]string{server.ValidationCommand, server.ReloadCommand, server.ValidationCommand, server.ReloadCommand}
|
||||||
|
if !reflect.DeepEqual(client.commands, commands) {
|
||||||
|
t.Fatalf("old runtime was not reloaded: %v", client.commands)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil || report.Status != "failed" || report.RecoveryStatus != "success" || !recoveryCalled || report.Success != 0 || report.Failed != 2 {
|
||||||
|
t.Fatalf("unexpected recovery report: %+v, %v", report, err)
|
||||||
|
}
|
||||||
|
if len(journal.finished) != 2 {
|
||||||
|
t.Fatalf("unexpected journal: %+v", journal.finished)
|
||||||
|
}
|
||||||
|
for _, record := range journal.finished {
|
||||||
|
if record.status != TransactionStatusRolledBack {
|
||||||
|
t.Fatalf("unexpected transaction: %+v", record)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for index, server := range servers {
|
||||||
|
assertVerifiedLocks(t, clients[server.ID], server, false)
|
||||||
|
if report.Results[index].FailureCode != FailureRecoveryRestored {
|
||||||
|
t.Fatalf("unexpected result: %+v", report.Results[index])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVerifiedReloadFailureKeepsLocksWithoutRecoveryProof(t *testing.T) {
|
||||||
|
for _, scenario := range []struct {
|
||||||
|
name string
|
||||||
|
recovery func(context.Context) error
|
||||||
|
restoreExit int
|
||||||
|
wantRecovery string
|
||||||
|
}{
|
||||||
|
{"missing_verifier", nil, 0, "unconfirmed"},
|
||||||
|
{"wrong_runtime", func(context.Context) error { return errors.New("old fingerprint not served") }, 0, "unconfirmed"},
|
||||||
|
{"restore_reload_failed", func(context.Context) error { t.Fatal("failed restore must not verify"); return nil }, 1, "failed"},
|
||||||
|
} {
|
||||||
|
t.Run(scenario.name, func(t *testing.T) {
|
||||||
|
client, server := newMemoryClient(), testServer("failed-reload")
|
||||||
|
original := seedVerifiedFiles(client, server)
|
||||||
|
client.runResults = []CommandResult{{}, {ExitCode: 1}, {}, {ExitCode: scenario.restoreExit}}
|
||||||
|
journal := &memoryTransactionJournal{}
|
||||||
|
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
|
||||||
|
ctx := WithTransactionContext(t.Context(), TransactionContext{TaskID: "task", AttemptID: "attempt", AttemptNumber: 1})
|
||||||
|
report, err := service.DeployVerified(ctx, []Target{{Server: server}}, testArtifact(t), Verification{
|
||||||
|
Verify: func(context.Context) error { t.Fatal("failed reload must not verify deployment"); return nil },
|
||||||
|
VerifyRecovery: scenario.recovery,
|
||||||
|
})
|
||||||
|
if err == nil || report.Status != "failed" || report.RecoveryStatus != scenario.wantRecovery {
|
||||||
|
t.Fatalf("unexpected report: %+v, %v", report, err)
|
||||||
|
}
|
||||||
|
assertVerifiedFiles(t, client, original)
|
||||||
|
assertVerifiedLocks(t, client, server, true)
|
||||||
|
if len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRecoveryRequired {
|
||||||
|
t.Fatalf("unexpected journal: %+v", journal.finished)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -447,7 +447,8 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
commandResult, err := client.Run(ctx, server.ReloadCommand)
|
commandResult, err := client.Run(ctx, server.ReloadCommand)
|
||||||
if err != nil || commandResult.ExitCode != 0 {
|
reloadFailed := err != nil || commandResult.ExitCode != 0
|
||||||
|
if reloadFailed && pending == nil {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
err = fmt.Errorf("退出码 %d", commandResult.ExitCode)
|
err = fmt.Errorf("退出码 %d", commandResult.ExitCode)
|
||||||
}
|
}
|
||||||
@ -458,6 +459,11 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
|||||||
result.Status = "success"
|
result.Status = "success"
|
||||||
result.FailureCode = ""
|
result.FailureCode = ""
|
||||||
result.Message = "证书已推送并重载,等待站点校验"
|
result.Message = "证书已推送并重载,等待站点校验"
|
||||||
|
if reloadFailed {
|
||||||
|
result.Status = "failed"
|
||||||
|
result.FailureCode = FailureRuntimeUnknown
|
||||||
|
result.Message = publicFailureMessages[FailureRuntimeUnknown]
|
||||||
|
}
|
||||||
closed := false
|
closed := false
|
||||||
finished := false
|
finished := false
|
||||||
transaction := &pendingDeployment{index: index}
|
transaction := &pendingDeployment{index: index}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user