103 lines
4.3 KiB
Go
103 lines
4.3 KiB
Go
package deployment
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"reflect"
|
|
"testing"
|
|
)
|
|
|
|
func TestVerifiedReloadFailureRestoresRuntimeBeforeUnlock(t *testing.T) {
|
|
for _, failure := range []string{"exit_status", "transport_error"} {
|
|
t.Run(failure, func(t *testing.T) {
|
|
servers := []Server{testServer("first").Normalize(), testServer("second").Normalize()}
|
|
clients := map[string]*memoryClient{}
|
|
originals := map[string]map[string]memoryFile{}
|
|
for _, server := range servers {
|
|
client := newMemoryClient()
|
|
clients[server.ID] = client
|
|
originals[server.ID] = seedVerifiedFiles(client, server)
|
|
}
|
|
if failure == "exit_status" {
|
|
clients["second"].runResults = []CommandResult{{}, {ExitCode: 1}}
|
|
} else {
|
|
clients["second"].runErrors = []error{nil, errors.New("reload response lost")}
|
|
}
|
|
journal := &memoryTransactionJournal{}
|
|
service := NewServiceWithJournal(memoryFactory{clients: clients}, journal)
|
|
ctx := WithTransactionContext(t.Context(), TransactionContext{TaskID: "task", AttemptID: "attempt", AttemptNumber: 1})
|
|
recoveryCalled := false
|
|
report, err := service.DeployVerified(ctx, []Target{{Server: servers[0]}, {Server: servers[1]}}, testArtifact(t), Verification{
|
|
Verify: func(context.Context) error { t.Fatal("failed reload must not verify deployment"); return nil },
|
|
VerifyRecovery: func(ctx context.Context) error {
|
|
recoveryCalled = true
|
|
if ctx.Err() != nil {
|
|
t.Fatal("recovery context is cancelled")
|
|
}
|
|
for _, server := range servers {
|
|
client := clients[server.ID]
|
|
assertVerifiedFiles(t, client, originals[server.ID])
|
|
assertVerifiedLocks(t, client, server, true)
|
|
commands := [][]string{server.ValidationCommand, server.ReloadCommand, server.ValidationCommand, server.ReloadCommand}
|
|
if !reflect.DeepEqual(client.commands, commands) {
|
|
t.Fatalf("old runtime was not reloaded: %v", client.commands)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
})
|
|
if err == nil || report.Status != "failed" || report.RecoveryStatus != "success" || !recoveryCalled || report.Success != 0 || report.Failed != 2 {
|
|
t.Fatalf("unexpected recovery report: %+v, %v", report, err)
|
|
}
|
|
if len(journal.finished) != 2 {
|
|
t.Fatalf("unexpected journal: %+v", journal.finished)
|
|
}
|
|
for _, record := range journal.finished {
|
|
if record.status != TransactionStatusRolledBack {
|
|
t.Fatalf("unexpected transaction: %+v", record)
|
|
}
|
|
}
|
|
for index, server := range servers {
|
|
assertVerifiedLocks(t, clients[server.ID], server, false)
|
|
if report.Results[index].FailureCode != FailureRecoveryRestored {
|
|
t.Fatalf("unexpected result: %+v", report.Results[index])
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVerifiedReloadFailureKeepsLocksWithoutRecoveryProof(t *testing.T) {
|
|
for _, scenario := range []struct {
|
|
name string
|
|
recovery func(context.Context) error
|
|
restoreExit int
|
|
wantRecovery string
|
|
}{
|
|
{"missing_verifier", nil, 0, "unconfirmed"},
|
|
{"wrong_runtime", func(context.Context) error { return errors.New("old fingerprint not served") }, 0, "unconfirmed"},
|
|
{"restore_reload_failed", func(context.Context) error { t.Fatal("failed restore must not verify"); return nil }, 1, "failed"},
|
|
} {
|
|
t.Run(scenario.name, func(t *testing.T) {
|
|
client, server := newMemoryClient(), testServer("failed-reload")
|
|
original := seedVerifiedFiles(client, server)
|
|
client.runResults = []CommandResult{{}, {ExitCode: 1}, {}, {ExitCode: scenario.restoreExit}}
|
|
journal := &memoryTransactionJournal{}
|
|
service := NewServiceWithJournal(fixedClientFactory{client}, journal)
|
|
ctx := WithTransactionContext(t.Context(), TransactionContext{TaskID: "task", AttemptID: "attempt", AttemptNumber: 1})
|
|
report, err := service.DeployVerified(ctx, []Target{{Server: server}}, testArtifact(t), Verification{
|
|
Verify: func(context.Context) error { t.Fatal("failed reload must not verify deployment"); return nil },
|
|
VerifyRecovery: scenario.recovery,
|
|
})
|
|
if err == nil || report.Status != "failed" || report.RecoveryStatus != scenario.wantRecovery {
|
|
t.Fatalf("unexpected report: %+v, %v", report, err)
|
|
}
|
|
assertVerifiedFiles(t, client, original)
|
|
assertVerifiedLocks(t, client, server, true)
|
|
if len(journal.finished) != 1 || journal.finished[0].status != TransactionStatusRecoveryRequired {
|
|
t.Fatalf("unexpected journal: %+v", journal.finished)
|
|
}
|
|
})
|
|
}
|
|
}
|