检查点:保存双层证书协调进展并准备新主机验收
This commit is contained in:
1 parent
48cee12c4a
commit
2c0d0e8eda
15 files changed
+1190
-9
No files matched your search
@@ -0,0 +1,97 @@
|
||||
package nginxactivation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
const coordinationName = "coordination.json"
|
||||
|
||||
type coordinationRecord struct {
|
||||
Version int `json:"version"`
|
||||
Phase string `json:"phase"`
|
||||
BaselineID string `json:"baseline_id,omitempty"`
|
||||
ProxyID string `json:"proxy_id,omitempty"`
|
||||
Previous ManagedCertificate `json:"previous"`
|
||||
Candidate ManagedCertificate `json:"candidate"`
|
||||
}
|
||||
|
||||
func (value ManagedCertificate) validate() error {
|
||||
hash, err := hex.DecodeString(value.LeafHash)
|
||||
if err != nil || len(hash) != 32 || hex.EncodeToString(hash) != value.LeafHash || value.Mode != "proxy" {
|
||||
return errors.New("管理证书协调摘要或模式无效")
|
||||
}
|
||||
if err := errors.Join((Endpoint{Address: value.ProxyAddress, Hostname: value.Hostname}).validate(), (Endpoint{Address: value.UpstreamAddress, Hostname: value.Hostname}).validate()); err != nil {
|
||||
return err
|
||||
}
|
||||
if value.ProxyAddress == value.UpstreamAddress {
|
||||
return errors.New("管理证书代理与上游地址不能相同")
|
||||
}
|
||||
for _, path := range []string{value.CertificatePath, value.PrivateKeyPath} {
|
||||
if !filepath.IsAbs(path) || filepath.Clean(path) != path || path == "/" || len(path) > 4096 {
|
||||
return errors.New("管理证书协调路径无效")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (record coordinationRecord) validate() error {
|
||||
if record.Version != 1 || (record.BaselineID != "" && !validID(record.BaselineID)) || (record.ProxyID != "" && !validID(record.ProxyID)) {
|
||||
return errors.New("管理证书协调记录版本或编号无效")
|
||||
}
|
||||
switch record.Phase {
|
||||
case "prepared":
|
||||
case "staged", "upstream", "committed", "rolled_back":
|
||||
if record.ProxyID == "" {
|
||||
return errors.New("管理证书协调记录缺少代理事务编号")
|
||||
}
|
||||
default:
|
||||
return errors.New("管理证书协调记录阶段无效")
|
||||
}
|
||||
if err := errors.Join(record.Previous.validate(), record.Candidate.validate()); err != nil {
|
||||
return err
|
||||
}
|
||||
previous, candidate := record.Previous, record.Candidate
|
||||
if previous.LeafHash == candidate.LeafHash || previous.CertificatePath == candidate.CertificatePath || previous.PrivateKeyPath == candidate.PrivateKeyPath {
|
||||
return errors.New("候选证书必须使用独立的新版本材料")
|
||||
}
|
||||
candidate.CertificateFiles, candidate.LeafHash = previous.CertificateFiles, previous.LeafHash
|
||||
if candidate != previous || (record.ProxyID != "" && record.ProxyID == record.BaselineID) {
|
||||
return errors.New("管理证书协调记录包含不一致的策略或事务编号")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readCoordination(root *os.Root) (coordinationRecord, error) {
|
||||
var record coordinationRecord
|
||||
data, err := readRegular(root, coordinationName, 64<<10)
|
||||
if err != nil {
|
||||
return record, err
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&record); err != nil {
|
||||
return record, err
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return record, errors.New("管理证书协调记录包含多余内容")
|
||||
}
|
||||
return record, record.validate()
|
||||
}
|
||||
|
||||
func saveCoordination(root *os.Root, record coordinationRecord) error {
|
||||
if err := record.validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.Marshal(record)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeAtomic(root, coordinationName, data)
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
package nginxactivation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/processlock"
|
||||
)
|
||||
|
||||
var ErrCoordinationPending = errors.New("管理证书协调仍未完成,已阻止再次申请证书")
|
||||
|
||||
type CertificateFiles struct {
|
||||
CertificatePath string `json:"certificate_path"`
|
||||
PrivateKeyPath string `json:"private_key_path"`
|
||||
}
|
||||
|
||||
type ManagedCertificate struct {
|
||||
CertificateFiles
|
||||
Hostname string `json:"hostname"`
|
||||
Mode string `json:"mode"`
|
||||
LeafHash string `json:"leaf_hash"`
|
||||
ProxyAddress string `json:"proxy_address"`
|
||||
UpstreamAddress string `json:"upstream_address"`
|
||||
}
|
||||
|
||||
type ManagedSession interface {
|
||||
Snapshot() (ManagedCertificate, error)
|
||||
Activate(CertificateFiles, string) error
|
||||
}
|
||||
|
||||
// ManagedUpstream 的回调必须独占所有证书切换及相关配置写入,但不能阻塞 TLS 和就绪探测。
|
||||
type ManagedUpstream interface {
|
||||
WithCertificate(context.Context, func(ManagedSession) error) error
|
||||
}
|
||||
|
||||
type ProxyClient interface {
|
||||
Ready(context.Context) (ReadyState, error)
|
||||
State(context.Context) (State, error)
|
||||
Stage(context.Context, []byte, []byte) (string, error)
|
||||
Commit(context.Context, string) error
|
||||
Reconcile(context.Context, string) (State, error)
|
||||
}
|
||||
|
||||
// Coordinator 的目录归应用用户所有,与 root 激活器的专用目录分离。
|
||||
type Coordinator struct {
|
||||
Directory string
|
||||
Proxy ProxyClient
|
||||
Upstream ManagedUpstream
|
||||
}
|
||||
|
||||
func (c Coordinator) Ready(ctx context.Context) error {
|
||||
return c.withSession(ctx, func(root *os.Root, session ManagedSession) error {
|
||||
if _, err := c.recover(ctx, root, session); err != nil {
|
||||
return err
|
||||
}
|
||||
current, err := session.Snapshot()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.ready(ctx, current)
|
||||
})
|
||||
}
|
||||
|
||||
func (c Coordinator) Recover(ctx context.Context) error {
|
||||
return c.withSession(ctx, func(root *os.Root, session ManagedSession) error {
|
||||
_, err := c.recover(ctx, root, session)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func (c Coordinator) Activate(ctx context.Context, files CertificateFiles) error {
|
||||
return c.withSession(ctx, func(root *os.Root, session ManagedSession) (resultErr error) {
|
||||
if _, err := c.recover(ctx, root, session); err != nil {
|
||||
return err
|
||||
}
|
||||
previous, err := session.Snapshot()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.ready(ctx, previous); err != nil {
|
||||
return err
|
||||
}
|
||||
chain, err := readManagedMaterial(files.CertificatePath, 1<<20)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key, err := readManagedMaterial(files.PrivateKeyPath, 64<<10)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer clear(key)
|
||||
pair, err := tls.X509KeyPair(chain, key)
|
||||
if err != nil {
|
||||
return errors.New("候选管理证书与私钥不匹配")
|
||||
}
|
||||
candidate := previous
|
||||
candidate.CertificateFiles, candidate.LeafHash = files, leafHash(pair.Certificate[0])
|
||||
baseline, err := c.Proxy.State(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
record := coordinationRecord{Version: 1, Phase: "prepared", BaselineID: baseline.ID, Previous: previous, Candidate: candidate}
|
||||
if err := record.validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := saveCoordination(root, record); err != nil {
|
||||
return err
|
||||
}
|
||||
// 请求取消或应答丢失仍在同一独占区内恢复;不能把网络失败当成尚未执行。
|
||||
defer func() {
|
||||
if resultErr != nil {
|
||||
recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 100*time.Second)
|
||||
defer cancel()
|
||||
_, recoveryErr := c.recover(recoveryCtx, root, session)
|
||||
resultErr = errors.Join(resultErr, recoveryErr)
|
||||
}
|
||||
}()
|
||||
id, err := c.Proxy.Stage(ctx, chain, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !validID(id) || id == record.BaselineID {
|
||||
return errors.New("代理返回的暂存事务编号无效")
|
||||
}
|
||||
record.ProxyID, record.Phase = id, "staged"
|
||||
if err := saveCoordination(root, record); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := session.Activate(files, candidate.LeafHash); err != nil {
|
||||
return err
|
||||
}
|
||||
record.Phase = "upstream"
|
||||
if err := saveCoordination(root, record); err != nil {
|
||||
return err
|
||||
}
|
||||
commitErr := c.Proxy.Commit(ctx, id)
|
||||
verifyCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 100*time.Second)
|
||||
defer cancel()
|
||||
completed, err := c.recover(verifyCtx, root, session)
|
||||
if err != nil || completed.Phase != "committed" {
|
||||
return errors.Join(commitErr, err, ErrCoordinationPending)
|
||||
}
|
||||
// 只有同一事务终态及实时双层入口均已验证,才接受丢失提交应答后的成功。
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (c Coordinator) recover(ctx context.Context, root *os.Root, session ManagedSession) (coordinationRecord, error) {
|
||||
record, err := readCoordination(root)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return coordinationRecord{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return record, err
|
||||
}
|
||||
if record.Phase == "committed" || record.Phase == "rolled_back" {
|
||||
return record, nil
|
||||
}
|
||||
current, err := session.Snapshot()
|
||||
if err != nil {
|
||||
return record, errors.Join(ErrCoordinationPending, err)
|
||||
}
|
||||
if current != record.Previous && current != record.Candidate {
|
||||
return record, errors.Join(ErrCoordinationPending, errors.New("当前上游不属于持久化协调记录,拒绝覆盖"))
|
||||
}
|
||||
state, err := c.Proxy.State(ctx)
|
||||
if err != nil {
|
||||
return record, errors.Join(ErrCoordinationPending, err)
|
||||
}
|
||||
if !validID(state.ID) || state.ID == record.BaselineID ||
|
||||
(record.ProxyID != "" && state.ID != record.ProxyID) ||
|
||||
state.PreviousHash != record.Previous.LeafHash || state.CandidateHash != record.Candidate.LeafHash {
|
||||
// 尚未观测到对应事务时保留意图;延迟到达的暂存请求仍可能执行,不能清除或重放。
|
||||
return record, errors.Join(ErrCoordinationPending, errors.New("尚未确认对应的代理事务,保留协调记录"))
|
||||
}
|
||||
record.ProxyID = state.ID
|
||||
if state.Status == "pending" || state.Status == "staged" {
|
||||
state, err = c.Proxy.Reconcile(ctx, state.ID)
|
||||
if err != nil {
|
||||
return record, errors.Join(ErrCoordinationPending, err)
|
||||
}
|
||||
}
|
||||
if state.ID != record.ProxyID || state.PreviousHash != record.Previous.LeafHash || state.CandidateHash != record.Candidate.LeafHash {
|
||||
return record, errors.Join(ErrCoordinationPending, errors.New("代理恢复结果不属于当前事务"))
|
||||
}
|
||||
if (state.Status != "committed" || current != record.Candidate) && (state.Status != "rolled_back" || current != record.Previous) {
|
||||
return record, errors.Join(ErrCoordinationPending, errors.New("代理终态与实际上游证书不一致"))
|
||||
}
|
||||
if err := c.ready(ctx, current); err != nil {
|
||||
return record, errors.Join(ErrCoordinationPending, err)
|
||||
}
|
||||
record.Phase = state.Status
|
||||
return record, saveCoordination(root, record)
|
||||
}
|
||||
|
||||
func (c Coordinator) ready(ctx context.Context, expected ManagedCertificate) error {
|
||||
if err := expected.validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
ready, err := c.Proxy.Ready(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ready.Hostname != expected.Hostname || ready.LeafHash != expected.LeafHash || ready.ProxyAddress != expected.ProxyAddress || ready.UpstreamAddress != expected.UpstreamAddress {
|
||||
return errors.New("本地激活策略或双层入口与应用管理配置不一致")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c Coordinator) withSession(ctx context.Context, operation func(*os.Root, ManagedSession) error) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if c.Proxy == nil || c.Upstream == nil {
|
||||
return errors.New("管理证书协调器尚未配置")
|
||||
}
|
||||
if err := validateDirectory(c.Directory); err != nil {
|
||||
return err
|
||||
}
|
||||
lockPath := filepath.Join(c.Directory, ".coordination.lock")
|
||||
if info, err := os.Lstat(lockPath); err == nil {
|
||||
if err := validateStateFile(info, 4096); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
lock, err := processlock.Acquire(lockPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer lock.Close()
|
||||
root, err := os.OpenRoot(c.Directory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer root.Close()
|
||||
return c.Upstream.WithCertificate(ctx, func(session ManagedSession) error {
|
||||
return operation(root, session)
|
||||
})
|
||||
}
|
||||
|
||||
func readManagedMaterial(path string, limit int64) ([]byte, error) {
|
||||
if !filepath.IsAbs(path) || filepath.Clean(path) != path {
|
||||
return nil, errors.New("管理证书材料必须使用规范绝对路径")
|
||||
}
|
||||
if err := validateTrustedParents(filepath.Dir(path), uint32(os.Geteuid())); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root, err := os.OpenRoot(filepath.Dir(path))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer root.Close()
|
||||
data, err := readRegular(root, filepath.Base(path), limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取候选管理证书材料失败:%w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
package nginxactivation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type coordinatedTestUpstream struct {
|
||||
mu sync.Mutex
|
||||
current ManagedCertificate
|
||||
readErr error
|
||||
activate func(CertificateFiles, string) error
|
||||
}
|
||||
|
||||
func (u *coordinatedTestUpstream) WithCertificate(ctx context.Context, operation func(ManagedSession) error) error {
|
||||
if !u.mu.TryLock() {
|
||||
return errors.New("测试上游正被独占")
|
||||
}
|
||||
defer u.mu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return operation(u)
|
||||
}
|
||||
|
||||
func (u *coordinatedTestUpstream) Snapshot() (ManagedCertificate, error) {
|
||||
return u.current, u.readErr
|
||||
}
|
||||
|
||||
func (u *coordinatedTestUpstream) Activate(files CertificateFiles, hash string) error {
|
||||
return u.activate(files, hash)
|
||||
}
|
||||
|
||||
type coordinatedTestProxy struct {
|
||||
ProxyClient
|
||||
stage func(context.Context, []byte, []byte) (string, error)
|
||||
commit func(context.Context, string) error
|
||||
}
|
||||
|
||||
func (p coordinatedTestProxy) Stage(ctx context.Context, chain, key []byte) (string, error) {
|
||||
if p.stage != nil {
|
||||
return p.stage(ctx, chain, key)
|
||||
}
|
||||
return p.ProxyClient.Stage(ctx, chain, key)
|
||||
}
|
||||
|
||||
func (p coordinatedTestProxy) Commit(ctx context.Context, id string) error {
|
||||
if p.commit != nil {
|
||||
return p.commit(ctx, id)
|
||||
}
|
||||
return p.ProxyClient.Commit(ctx, id)
|
||||
}
|
||||
|
||||
type coordinatorFixture struct {
|
||||
*rotationFixture
|
||||
coordinator Coordinator
|
||||
session *coordinatedTestUpstream
|
||||
previous ManagedCertificate
|
||||
candidate ManagedCertificate
|
||||
}
|
||||
|
||||
func newCoordinatorFixture(t *testing.T) *coordinatorFixture {
|
||||
t.Helper()
|
||||
f := &coordinatorFixture{rotationFixture: newRotationFixture(t)}
|
||||
materials := t.TempDir()
|
||||
files := CertificateFiles{CertificatePath: filepath.Join(materials, "candidate.pem"), PrivateKeyPath: filepath.Join(materials, "candidate.key")}
|
||||
writeTestFile(t, files.CertificatePath, f.rotationFixture.candidate.chain)
|
||||
writeTestFile(t, files.PrivateKeyPath, f.rotationFixture.candidate.key)
|
||||
f.previous = ManagedCertificate{
|
||||
CertificateFiles: CertificateFiles{CertificatePath: filepath.Join(materials, "old.pem"), PrivateKeyPath: filepath.Join(materials, "old.key")},
|
||||
Hostname: testHostname, Mode: "proxy", LeafHash: leafHash(f.old.pair.Certificate[0]),
|
||||
ProxyAddress: f.proxy.Address, UpstreamAddress: f.upstream.Address,
|
||||
}
|
||||
writeTestFile(t, f.previous.CertificatePath, f.old.chain)
|
||||
writeTestFile(t, f.previous.PrivateKeyPath, f.old.key)
|
||||
f.candidate = f.previous
|
||||
f.candidate.CertificateFiles, f.candidate.LeafHash = files, leafHash(f.rotationFixture.candidate.pair.Certificate[0])
|
||||
f.session = &coordinatedTestUpstream{current: f.previous}
|
||||
f.session.activate = func(actual CertificateFiles, hash string) error {
|
||||
if actual != files || hash != f.candidate.LeafHash {
|
||||
return errors.New("协调器未固定候选材料")
|
||||
}
|
||||
f.session.current = f.candidate
|
||||
f.upstream.active.Store(&f.rotationFixture.candidate.pair)
|
||||
return nil
|
||||
}
|
||||
directory := t.TempDir()
|
||||
if err := os.Chmod(directory, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.coordinator = Coordinator{Directory: directory, Proxy: f.rotation, Upstream: f.session}
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *coordinatorFixture) record(t *testing.T) coordinationRecord {
|
||||
t.Helper()
|
||||
root, err := os.OpenRoot(f.coordinator.Directory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer root.Close()
|
||||
value, err := readCoordination(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func (f *coordinatorFixture) save(t *testing.T, record coordinationRecord) {
|
||||
t.Helper()
|
||||
root, err := os.OpenRoot(f.coordinator.Directory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer root.Close()
|
||||
if err := saveCoordination(root, record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinatorCommitsOverLocalIPC(t *testing.T) {
|
||||
f := newCoordinatorFixture(t)
|
||||
client, cancel, done := startLocalService(t, f.rotation, uint32(os.Geteuid()))
|
||||
defer stopLocalService(t, cancel, done)
|
||||
f.coordinator.Proxy = client
|
||||
if err := f.coordinator.Ready(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.coordinator.Activate(t.Context(), f.candidate.CertificateFiles); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if record := f.record(t); record.Phase != "committed" || record.ProxyID == "" {
|
||||
t.Fatal("双层提交未持久化", record.Phase)
|
||||
}
|
||||
if err := f.coordinator.Ready(t.Context()); err != nil {
|
||||
t.Fatal("提交后的双层入口未就绪", err)
|
||||
}
|
||||
if err := f.coordinator.Recover(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.reloader.reloadCount != 1 {
|
||||
t.Fatal("已完成事务不应再次重载")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinatorRecoversLostResponses(t *testing.T) {
|
||||
for _, operation := range []string{"stage", "commit"} {
|
||||
t.Run(operation, func(t *testing.T) {
|
||||
f := newCoordinatorFixture(t)
|
||||
proxy := coordinatedTestProxy{ProxyClient: f.rotation}
|
||||
if operation == "stage" {
|
||||
proxy.stage = func(ctx context.Context, chain, key []byte) (string, error) {
|
||||
if _, err := f.rotation.Stage(ctx, chain, key); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "", errors.New("注入暂存应答丢失")
|
||||
}
|
||||
} else {
|
||||
proxy.commit = func(ctx context.Context, id string) error {
|
||||
if err := f.rotation.Commit(ctx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
return errors.New("注入提交应答丢失")
|
||||
}
|
||||
}
|
||||
f.coordinator.Proxy = proxy
|
||||
err := f.coordinator.Activate(t.Context(), f.candidate.CertificateFiles)
|
||||
want := "committed"
|
||||
if operation == "stage" {
|
||||
want = "rolled_back"
|
||||
if err == nil || f.session.current != f.previous {
|
||||
t.Fatal("暂存应答丢失后不应切换上游", err)
|
||||
}
|
||||
} else if err != nil || f.session.current != f.candidate {
|
||||
t.Fatal("应验证已提交的同一事务", err)
|
||||
}
|
||||
if record := f.record(t); record.Phase != want || record.ProxyID == "" {
|
||||
t.Fatal("应答丢失后的终态错误", record.Phase)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinatorRecoversRestartWindows(t *testing.T) {
|
||||
for _, phase := range []string{"prepared", "staged", "saved_before_snapshot", "upstream", "commit_before_record"} {
|
||||
t.Run(phase, func(t *testing.T) {
|
||||
f := newCoordinatorFixture(t)
|
||||
record := coordinationRecord{Version: 1, Phase: "prepared", Previous: f.previous, Candidate: f.candidate}
|
||||
f.save(t, record)
|
||||
id, err := f.rotation.Stage(t.Context(), f.rotationFixture.candidate.chain, f.rotationFixture.candidate.key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "rolled_back"
|
||||
if phase != "prepared" {
|
||||
record.Phase, record.ProxyID = "staged", id
|
||||
f.save(t, record)
|
||||
}
|
||||
if phase == "saved_before_snapshot" {
|
||||
f.session.readErr = errors.New("配置已保存但运行快照仍旧")
|
||||
if err := f.coordinator.Recover(t.Context()); !errors.Is(err, ErrCoordinationPending) {
|
||||
t.Fatal("混合状态下不能回退或提交", err)
|
||||
}
|
||||
f.assertStatus(t, "staged")
|
||||
f.session.readErr = nil
|
||||
}
|
||||
if phase == "saved_before_snapshot" || phase == "upstream" || phase == "commit_before_record" {
|
||||
if err := f.session.Activate(f.candidate.CertificateFiles, f.candidate.LeafHash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want = "committed"
|
||||
}
|
||||
if phase == "upstream" || phase == "commit_before_record" {
|
||||
record.Phase = "upstream"
|
||||
f.save(t, record)
|
||||
}
|
||||
if phase == "commit_before_record" {
|
||||
if err := f.rotation.Commit(t.Context(), id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// 新实例只依靠同一目录的持久化记录和重启后实际提供的证书。
|
||||
restarted := Coordinator{Directory: f.coordinator.Directory, Proxy: f.rotation, Upstream: f.session}
|
||||
if err := restarted.Recover(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if record := f.record(t); record.Phase != want || record.ProxyID != id {
|
||||
t.Fatal("重启恢复终态错误", record.Phase)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinatorKeepsUnobservedStagePending(t *testing.T) {
|
||||
f := newCoordinatorFixture(t)
|
||||
calls := 0
|
||||
f.coordinator.Proxy = coordinatedTestProxy{ProxyClient: f.rotation, stage: func(context.Context, []byte, []byte) (string, error) {
|
||||
calls++
|
||||
return "", errors.New("暂存请求是否到达尚不可知")
|
||||
}}
|
||||
if err := f.coordinator.Activate(t.Context(), f.candidate.CertificateFiles); !errors.Is(err, ErrCoordinationPending) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 2 {
|
||||
if err := f.coordinator.Ready(t.Context()); !errors.Is(err, ErrCoordinationPending) {
|
||||
t.Fatal("未观测到事务不能允许再次申请", err)
|
||||
}
|
||||
if err := f.coordinator.Activate(t.Context(), f.candidate.CertificateFiles); !errors.Is(err, ErrCoordinationPending) {
|
||||
t.Fatal("未观测到事务不能重放暂存", err)
|
||||
}
|
||||
}
|
||||
if calls != 1 || f.record(t).Phase != "prepared" {
|
||||
t.Fatal("不能重放请求或清除不确定意图")
|
||||
}
|
||||
if _, err := f.rotation.Stage(t.Context(), f.rotationFixture.candidate.chain, f.rotationFixture.candidate.key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.coordinator.Recover(t.Context()); err != nil {
|
||||
t.Fatal("迟到事务应在观测后恢复", err)
|
||||
}
|
||||
if f.record(t).Phase != "rolled_back" {
|
||||
t.Fatal("迟到暂存没有回退")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinatorRefusesUnrelatedStateAndPolicy(t *testing.T) {
|
||||
for _, change := range []string{"hostname", "proxy", "upstream", "leaf", "mode"} {
|
||||
t.Run(change, func(t *testing.T) {
|
||||
f := newCoordinatorFixture(t)
|
||||
switch change {
|
||||
case "hostname":
|
||||
f.session.current.Hostname = "other.example.test"
|
||||
case "proxy":
|
||||
f.session.current.ProxyAddress = "127.0.0.1:1"
|
||||
case "upstream":
|
||||
f.session.current.UpstreamAddress = "127.0.0.1:2"
|
||||
case "leaf":
|
||||
f.session.current.LeafHash = f.candidate.LeafHash
|
||||
case "mode":
|
||||
f.session.current.Mode = "direct"
|
||||
}
|
||||
if err := f.coordinator.Ready(t.Context()); err == nil {
|
||||
t.Fatal("错误策略不能通过签发前检查")
|
||||
}
|
||||
})
|
||||
}
|
||||
f := newCoordinatorFixture(t)
|
||||
f.save(t, coordinationRecord{Version: 1, Phase: "prepared", Previous: f.previous, Candidate: f.candidate})
|
||||
f.session.current.Mode = "direct"
|
||||
if err := f.coordinator.Recover(t.Context()); !errors.Is(err, ErrCoordinationPending) {
|
||||
t.Fatal("不能因模式切换忽略未决事务", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinatorSerializesWholeLifecycle(t *testing.T) {
|
||||
f := newCoordinatorFixture(t)
|
||||
started, release := make(chan struct{}), make(chan struct{})
|
||||
defer close(release)
|
||||
f.reloader.check = func(number int) error {
|
||||
if number == 2 {
|
||||
close(started)
|
||||
<-release
|
||||
}
|
||||
return nil
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
go func() { done <- f.coordinator.Activate(ctx, f.candidate.CertificateFiles) }()
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("协调未开始")
|
||||
}
|
||||
second := f.coordinator
|
||||
if err := second.Recover(ctx); err == nil {
|
||||
t.Fatal("独立协调实例不能在生命周期中途恢复")
|
||||
}
|
||||
if err := second.Ready(ctx); err == nil {
|
||||
t.Fatal("独立协调实例不能开始下一次申请")
|
||||
}
|
||||
go func() { release <- struct{}{} }()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user