From 2e6d992b985556f334fa237dd8b69cd44ec23c14 Mon Sep 17 00:00:00 2001 From: xuans Date: Thu, 1 Oct 2026 21:31:12 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=AF=81=E4=B9=A6=E9=87=8D?= =?UTF-8?q?=E8=BD=BD=E5=A4=B1=E8=B4=A5=E5=90=8E=E6=9C=AA=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E6=81=A2=E5=A4=8D=E8=BF=90=E8=A1=8C=E8=AF=81=E4=B9=A6=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../deployment/reload_recovery_test.go | 102 ++++++++++++++++++ backend/internal/deployment/service.go | 8 +- 2 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 backend/internal/deployment/reload_recovery_test.go diff --git a/backend/internal/deployment/reload_recovery_test.go b/backend/internal/deployment/reload_recovery_test.go new file mode 100644 index 0000000..dbb5802 --- /dev/null +++ b/backend/internal/deployment/reload_recovery_test.go @@ -0,0 +1,102 @@ +package deployment + +import ( + "context" + "errors" + "reflect" + "testing" +) + +func TestVerifiedReloadFailureRestoresRuntimeBeforeUnlock(t *testing.T) { + for _, failure := range []string{"exit_status", "transport_error"} { + t.Run(failure, func(t *testing.T) { + servers := []Server{testServer("first").Normalize(), testServer("second").Normalize()} + clients := map[string]*memoryClient{} + originals := map[string]map[string]memoryFile{} + for _, server := range servers { + client := newMemoryClient() + clients[server.ID] = client + originals[server.ID] = seedVerifiedFiles(client, server) + } + if failure == "exit_status" { + clients["second"].runResults = []CommandResult{{}, {ExitCode: 1}} + } else { + clients["second"].runErrors = []error{nil, errors.New("reload response lost")} + } + journal := &memoryTransactionJournal{} + service := NewServiceWithJournal(memoryFactory{clients: clients}, journal) + ctx := WithTransactionContext(t.Context(), TransactionContext{TaskID: "task", AttemptID: "attempt", AttemptNumber: 1}) + recoveryCalled := false + report, err := service.DeployVerified(ctx, []Target{{Server: servers[0]}, {Server: servers[1]}}, testArtifact(t), Verification{ + Verify: func(context.Context) error { t.Fatal("failed reload must not verify deployment"); return nil }, + VerifyRecovery: func(ctx context.Context) error { + recoveryCalled = true + if ctx.Err() != nil { + t.Fatal("recovery context is cancelled") + } + for _, server := range servers { + client := clients[server.ID] + assertVerifiedFiles(t, client, originals[server.ID]) + assertVerifiedLocks(t, client, server, true) + commands := [][]string{server.ValidationCommand, server.ReloadCommand, server.ValidationCommand, server.ReloadCommand} + if !reflect.DeepEqual(client.commands, commands) { + t.Fatalf("old runtime was not reloaded: %v", client.commands) + } + } + return nil + }, + }) + if err == nil || report.Status != "failed" || report.RecoveryStatus != "success" || !recoveryCalled || report.Success != 0 || report.Failed != 2 { + t.Fatalf("unexpected recovery report: %+v, %v", report, err) + } + if len(journal.finished) != 2 { + t.Fatalf("unexpected journal: %+v", journal.finished) + } + for _, record := range journal.finished { + if record.status != TransactionStatusRolledBack { + t.Fatalf("unexpected transaction: %+v", record) + } + } + for index, server := range servers { + assertVerifiedLocks(t, clients[server.ID], server, false) + if report.Results[index].FailureCode != FailureRecoveryRestored { + t.Fatalf("unexpected result: %+v", report.Results[index]) + } + } + }) + } +} + +func TestVerifiedReloadFailureKeepsLocksWithoutRecoveryProof(t *testing.T) { + for _, scenario := range []struct { + name string + recovery func(context.Context) error + restoreExit int + wantRecovery string + }{ + {"missing_verifier", nil, 0, "unconfirmed"}, + {"wrong_runtime", func(context.Context) error { return errors.New("old fingerprint not served") }, 0, "unconfirmed"}, + {"restore_reload_failed", func(context.Context) error { t.Fatal("failed restore must not verify"); return nil }, 1, "failed"}, + } { + t.Run(scenario.name, func(t *testing.T) { + client, server := newMemoryClient(), testServer("failed-reload") + original := seedVerifiedFiles(client, server) + client.runResults = []CommandResult{{}, {ExitCode: 1}, {}, {ExitCode: scenario.restoreExit}} + journal := &memoryTransactionJournal{} + service := NewServiceWithJournal(fixedClientFactory{client}, journal) + ctx := WithTransactionContext(t.Context(), TransactionContext{TaskID: "task", AttemptID: "attempt", AttemptNumber: 1}) + report, err := service.DeployVerified(ctx, []Target{{Server: server}}, testArtifact(t), Verification{ + Verify: func(context.Context) error { t.Fatal("failed reload must not verify deployment"); return nil }, + VerifyRecovery: scenario.recovery, + }) + if err == nil || report.Status != "failed" || report.RecoveryStatus != scenario.wantRecovery { + t.Fatalf("unexpected report: %+v, %v", report, err) + } + assertVerifiedFiles(t, client, original) + assertVerifiedLocks(t, client, server, true) + if len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRecoveryRequired { + t.Fatalf("unexpected journal: %+v", journal.finished) + } + }) + } +} diff --git a/backend/internal/deployment/service.go b/backend/internal/deployment/service.go index aa642e8..c8dc55c 100644 --- a/backend/internal/deployment/service.go +++ b/backend/internal/deployment/service.go @@ -447,7 +447,8 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac } } commandResult, err := client.Run(ctx, server.ReloadCommand) - if err != nil || commandResult.ExitCode != 0 { + reloadFailed := err != nil || commandResult.ExitCode != 0 + if reloadFailed && pending == nil { if err == nil { err = fmt.Errorf("退出码 %d", commandResult.ExitCode) } @@ -458,6 +459,11 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac result.Status = "success" result.FailureCode = "" result.Message = "证书已推送并重载，等待站点校验" + if reloadFailed { + result.Status = "failed" + result.FailureCode = FailureRuntimeUnknown + result.Message = publicFailureMessages[FailureRuntimeUnknown] + } closed := false finished := false transaction := &pendingDeployment{index: index}