检查点:保存双层证书协调进展并准备新主机验收

This commit is contained in:
xuans committed 2026-09-20 01:02:16 +08:00
1 parent 48cee12c4a
commit 2c0d0e8eda
15 files changed
+1190 -9

No files matched your search

@@ -0,0 +1,97 @@
package nginxactivation
import (
"bytes"
"encoding/hex"
"encoding/json"
"errors"
"io"
"os"
"path/filepath"
)
const coordinationName = "coordination.json"
type coordinationRecord struct {
Version int `json:"version"`
Phase string `json:"phase"`
BaselineID string `json:"baseline_id,omitempty"`
ProxyID string `json:"proxy_id,omitempty"`
Previous ManagedCertificate `json:"previous"`
Candidate ManagedCertificate `json:"candidate"`
}
func (value ManagedCertificate) validate() error {
hash, err := hex.DecodeString(value.LeafHash)
if err != nil || len(hash) != 32 || hex.EncodeToString(hash) != value.LeafHash || value.Mode != "proxy" {
return errors.New("管理证书协调摘要或模式无效")
}
if err := errors.Join((Endpoint{Address: value.ProxyAddress, Hostname: value.Hostname}).validate(), (Endpoint{Address: value.UpstreamAddress, Hostname: value.Hostname}).validate()); err != nil {
return err
}
if value.ProxyAddress == value.UpstreamAddress {
return errors.New("管理证书代理与上游地址不能相同")
}
for _, path := range []string{value.CertificatePath, value.PrivateKeyPath} {
if !filepath.IsAbs(path) || filepath.Clean(path) != path || path == "/" || len(path) > 4096 {
return errors.New("管理证书协调路径无效")
}
}
return nil
}
func (record coordinationRecord) validate() error {
if record.Version != 1 || (record.BaselineID != "" && !validID(record.BaselineID)) || (record.ProxyID != "" && !validID(record.ProxyID)) {
return errors.New("管理证书协调记录版本或编号无效")
}
switch record.Phase {
case "prepared":
case "staged", "upstream", "committed", "rolled_back":
if record.ProxyID == "" {
return errors.New("管理证书协调记录缺少代理事务编号")
}
default:
return errors.New("管理证书协调记录阶段无效")
}
if err := errors.Join(record.Previous.validate(), record.Candidate.validate()); err != nil {
return err
}
previous, candidate := record.Previous, record.Candidate
if previous.LeafHash == candidate.LeafHash || previous.CertificatePath == candidate.CertificatePath || previous.PrivateKeyPath == candidate.PrivateKeyPath {
return errors.New("候选证书必须使用独立的新版本材料")
}
candidate.CertificateFiles, candidate.LeafHash = previous.CertificateFiles, previous.LeafHash
if candidate != previous || (record.ProxyID != "" && record.ProxyID == record.BaselineID) {
return errors.New("管理证书协调记录包含不一致的策略或事务编号")
}
return nil
}
func readCoordination(root *os.Root) (coordinationRecord, error) {
var record coordinationRecord
data, err := readRegular(root, coordinationName, 64<<10)
if err != nil {
return record, err
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&record); err != nil {
return record, err
}
var extra any
if err := decoder.Decode(&extra); err != io.EOF {
return record, errors.New("管理证书协调记录包含多余内容")
}
return record, record.validate()
}
func saveCoordination(root *os.Root, record coordinationRecord) error {
if err := record.validate(); err != nil {
return err
}
data, err := json.Marshal(record)
if err != nil {
return err
}
return writeAtomic(root, coordinationName, data)
}
@@ -0,0 +1,268 @@
package nginxactivation
import (
"context"
"crypto/tls"
"errors"
"fmt"
"os"
"path/filepath"
"time"
"git.pcux.cn/xuans/Scert/backend/internal/processlock"
)
var ErrCoordinationPending = errors.New("管理证书协调仍未完成,已阻止再次申请证书")
type CertificateFiles struct {
CertificatePath string `json:"certificate_path"`
PrivateKeyPath string `json:"private_key_path"`
}
type ManagedCertificate struct {
CertificateFiles
Hostname string `json:"hostname"`
Mode string `json:"mode"`
LeafHash string `json:"leaf_hash"`
ProxyAddress string `json:"proxy_address"`
UpstreamAddress string `json:"upstream_address"`
}
type ManagedSession interface {
Snapshot() (ManagedCertificate, error)
Activate(CertificateFiles, string) error
}
// ManagedUpstream 的回调必须独占所有证书切换及相关配置写入,但不能阻塞 TLS 和就绪探测。
type ManagedUpstream interface {
WithCertificate(context.Context, func(ManagedSession) error) error
}
type ProxyClient interface {
Ready(context.Context) (ReadyState, error)
State(context.Context) (State, error)
Stage(context.Context, []byte, []byte) (string, error)
Commit(context.Context, string) error
Reconcile(context.Context, string) (State, error)
}
// Coordinator 的目录归应用用户所有,与 root 激活器的专用目录分离。
type Coordinator struct {
Directory string
Proxy ProxyClient
Upstream ManagedUpstream
}
func (c Coordinator) Ready(ctx context.Context) error {
return c.withSession(ctx, func(root *os.Root, session ManagedSession) error {
if _, err := c.recover(ctx, root, session); err != nil {
return err
}
current, err := session.Snapshot()
if err != nil {
return err
}
return c.ready(ctx, current)
})
}
func (c Coordinator) Recover(ctx context.Context) error {
return c.withSession(ctx, func(root *os.Root, session ManagedSession) error {
_, err := c.recover(ctx, root, session)
return err
})
}
func (c Coordinator) Activate(ctx context.Context, files CertificateFiles) error {
return c.withSession(ctx, func(root *os.Root, session ManagedSession) (resultErr error) {
if _, err := c.recover(ctx, root, session); err != nil {
return err
}
previous, err := session.Snapshot()
if err != nil {
return err
}
if err := c.ready(ctx, previous); err != nil {
return err
}
chain, err := readManagedMaterial(files.CertificatePath, 1<<20)
if err != nil {
return err
}
key, err := readManagedMaterial(files.PrivateKeyPath, 64<<10)
if err != nil {
return err
}
defer clear(key)
pair, err := tls.X509KeyPair(chain, key)
if err != nil {
return errors.New("候选管理证书与私钥不匹配")
}
candidate := previous
candidate.CertificateFiles, candidate.LeafHash = files, leafHash(pair.Certificate[0])
baseline, err := c.Proxy.State(ctx)
if err != nil {
return err
}
record := coordinationRecord{Version: 1, Phase: "prepared", BaselineID: baseline.ID, Previous: previous, Candidate: candidate}
if err := record.validate(); err != nil {
return err
}
if err := saveCoordination(root, record); err != nil {
return err
}
// 请求取消或应答丢失仍在同一独占区内恢复;不能把网络失败当成尚未执行。
defer func() {
if resultErr != nil {
recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 100*time.Second)
defer cancel()
_, recoveryErr := c.recover(recoveryCtx, root, session)
resultErr = errors.Join(resultErr, recoveryErr)
}
}()
id, err := c.Proxy.Stage(ctx, chain, key)
if err != nil {
return err
}
if !validID(id) || id == record.BaselineID {
return errors.New("代理返回的暂存事务编号无效")
}
record.ProxyID, record.Phase = id, "staged"
if err := saveCoordination(root, record); err != nil {
return err
}
if err := ctx.Err(); err != nil {
return err
}
if err := session.Activate(files, candidate.LeafHash); err != nil {
return err
}
record.Phase = "upstream"
if err := saveCoordination(root, record); err != nil {
return err
}
commitErr := c.Proxy.Commit(ctx, id)
verifyCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 100*time.Second)
defer cancel()
completed, err := c.recover(verifyCtx, root, session)
if err != nil || completed.Phase != "committed" {
return errors.Join(commitErr, err, ErrCoordinationPending)
}
// 只有同一事务终态及实时双层入口均已验证,才接受丢失提交应答后的成功。
return nil
})
}
func (c Coordinator) recover(ctx context.Context, root *os.Root, session ManagedSession) (coordinationRecord, error) {
record, err := readCoordination(root)
if errors.Is(err, os.ErrNotExist) {
return coordinationRecord{}, nil
}
if err != nil {
return record, err
}
if record.Phase == "committed" || record.Phase == "rolled_back" {
return record, nil
}
current, err := session.Snapshot()
if err != nil {
return record, errors.Join(ErrCoordinationPending, err)
}
if current != record.Previous && current != record.Candidate {
return record, errors.Join(ErrCoordinationPending, errors.New("当前上游不属于持久化协调记录,拒绝覆盖"))
}
state, err := c.Proxy.State(ctx)
if err != nil {
return record, errors.Join(ErrCoordinationPending, err)
}
if !validID(state.ID) || state.ID == record.BaselineID ||
(record.ProxyID != "" && state.ID != record.ProxyID) ||
state.PreviousHash != record.Previous.LeafHash || state.CandidateHash != record.Candidate.LeafHash {
// 尚未观测到对应事务时保留意图;延迟到达的暂存请求仍可能执行,不能清除或重放。
return record, errors.Join(ErrCoordinationPending, errors.New("尚未确认对应的代理事务,保留协调记录"))
}
record.ProxyID = state.ID
if state.Status == "pending" || state.Status == "staged" {
state, err = c.Proxy.Reconcile(ctx, state.ID)
if err != nil {
return record, errors.Join(ErrCoordinationPending, err)
}
}
if state.ID != record.ProxyID || state.PreviousHash != record.Previous.LeafHash || state.CandidateHash != record.Candidate.LeafHash {
return record, errors.Join(ErrCoordinationPending, errors.New("代理恢复结果不属于当前事务"))
}
if (state.Status != "committed" || current != record.Candidate) && (state.Status != "rolled_back" || current != record.Previous) {
return record, errors.Join(ErrCoordinationPending, errors.New("代理终态与实际上游证书不一致"))
}
if err := c.ready(ctx, current); err != nil {
return record, errors.Join(ErrCoordinationPending, err)
}
record.Phase = state.Status
return record, saveCoordination(root, record)
}
func (c Coordinator) ready(ctx context.Context, expected ManagedCertificate) error {
if err := expected.validate(); err != nil {
return err
}
ready, err := c.Proxy.Ready(ctx)
if err != nil {
return err
}
if ready.Hostname != expected.Hostname || ready.LeafHash != expected.LeafHash || ready.ProxyAddress != expected.ProxyAddress || ready.UpstreamAddress != expected.UpstreamAddress {
return errors.New("本地激活策略或双层入口与应用管理配置不一致")
}
return nil
}
func (c Coordinator) withSession(ctx context.Context, operation func(*os.Root, ManagedSession) error) error {
if err := ctx.Err(); err != nil {
return err
}
if c.Proxy == nil || c.Upstream == nil {
return errors.New("管理证书协调器尚未配置")
}
if err := validateDirectory(c.Directory); err != nil {
return err
}
lockPath := filepath.Join(c.Directory, ".coordination.lock")
if info, err := os.Lstat(lockPath); err == nil {
if err := validateStateFile(info, 4096); err != nil {
return err
}
} else if !errors.Is(err, os.ErrNotExist) {
return err
}
lock, err := processlock.Acquire(lockPath)
if err != nil {
return err
}
defer lock.Close()
root, err := os.OpenRoot(c.Directory)
if err != nil {
return err
}
defer root.Close()
return c.Upstream.WithCertificate(ctx, func(session ManagedSession) error {
return operation(root, session)
})
}
func readManagedMaterial(path string, limit int64) ([]byte, error) {
if !filepath.IsAbs(path) || filepath.Clean(path) != path {
return nil, errors.New("管理证书材料必须使用规范绝对路径")
}
if err := validateTrustedParents(filepath.Dir(path), uint32(os.Geteuid())); err != nil {
return nil, err
}
root, err := os.OpenRoot(filepath.Dir(path))
if err != nil {
return nil, err
}
defer root.Close()
data, err := readRegular(root, filepath.Base(path), limit)
if err != nil {
return nil, fmt.Errorf("读取候选管理证书材料失败:%w", err)
}
return data, nil
}
@@ -0,0 +1,331 @@
package nginxactivation
import (
"context"
"errors"
"os"
"path/filepath"
"sync"
"testing"
"time"
)
type coordinatedTestUpstream struct {
mu sync.Mutex
current ManagedCertificate
readErr error
activate func(CertificateFiles, string) error
}
func (u *coordinatedTestUpstream) WithCertificate(ctx context.Context, operation func(ManagedSession) error) error {
if !u.mu.TryLock() {
return errors.New("测试上游正被独占")
}
defer u.mu.Unlock()
if err := ctx.Err(); err != nil {
return err
}
return operation(u)
}
func (u *coordinatedTestUpstream) Snapshot() (ManagedCertificate, error) {
return u.current, u.readErr
}
func (u *coordinatedTestUpstream) Activate(files CertificateFiles, hash string) error {
return u.activate(files, hash)
}
type coordinatedTestProxy struct {
ProxyClient
stage func(context.Context, []byte, []byte) (string, error)
commit func(context.Context, string) error
}
func (p coordinatedTestProxy) Stage(ctx context.Context, chain, key []byte) (string, error) {
if p.stage != nil {
return p.stage(ctx, chain, key)
}
return p.ProxyClient.Stage(ctx, chain, key)
}
func (p coordinatedTestProxy) Commit(ctx context.Context, id string) error {
if p.commit != nil {
return p.commit(ctx, id)
}
return p.ProxyClient.Commit(ctx, id)
}
type coordinatorFixture struct {
*rotationFixture
coordinator Coordinator
session *coordinatedTestUpstream
previous ManagedCertificate
candidate ManagedCertificate
}
func newCoordinatorFixture(t *testing.T) *coordinatorFixture {
t.Helper()
f := &coordinatorFixture{rotationFixture: newRotationFixture(t)}
materials := t.TempDir()
files := CertificateFiles{CertificatePath: filepath.Join(materials, "candidate.pem"), PrivateKeyPath: filepath.Join(materials, "candidate.key")}
writeTestFile(t, files.CertificatePath, f.rotationFixture.candidate.chain)
writeTestFile(t, files.PrivateKeyPath, f.rotationFixture.candidate.key)
f.previous = ManagedCertificate{
CertificateFiles: CertificateFiles{CertificatePath: filepath.Join(materials, "old.pem"), PrivateKeyPath: filepath.Join(materials, "old.key")},
Hostname: testHostname, Mode: "proxy", LeafHash: leafHash(f.old.pair.Certificate[0]),
ProxyAddress: f.proxy.Address, UpstreamAddress: f.upstream.Address,
}
writeTestFile(t, f.previous.CertificatePath, f.old.chain)
writeTestFile(t, f.previous.PrivateKeyPath, f.old.key)
f.candidate = f.previous
f.candidate.CertificateFiles, f.candidate.LeafHash = files, leafHash(f.rotationFixture.candidate.pair.Certificate[0])
f.session = &coordinatedTestUpstream{current: f.previous}
f.session.activate = func(actual CertificateFiles, hash string) error {
if actual != files || hash != f.candidate.LeafHash {
return errors.New("协调器未固定候选材料")
}
f.session.current = f.candidate
f.upstream.active.Store(&f.rotationFixture.candidate.pair)
return nil
}
directory := t.TempDir()
if err := os.Chmod(directory, 0o700); err != nil {
t.Fatal(err)
}
f.coordinator = Coordinator{Directory: directory, Proxy: f.rotation, Upstream: f.session}
return f
}
func (f *coordinatorFixture) record(t *testing.T) coordinationRecord {
t.Helper()
root, err := os.OpenRoot(f.coordinator.Directory)
if err != nil {
t.Fatal(err)
}
defer root.Close()
value, err := readCoordination(root)
if err != nil {
t.Fatal(err)
}
return value
}
func (f *coordinatorFixture) save(t *testing.T, record coordinationRecord) {
t.Helper()
root, err := os.OpenRoot(f.coordinator.Directory)
if err != nil {
t.Fatal(err)
}
defer root.Close()
if err := saveCoordination(root, record); err != nil {
t.Fatal(err)
}
}
func TestCoordinatorCommitsOverLocalIPC(t *testing.T) {
f := newCoordinatorFixture(t)
client, cancel, done := startLocalService(t, f.rotation, uint32(os.Geteuid()))
defer stopLocalService(t, cancel, done)
f.coordinator.Proxy = client
if err := f.coordinator.Ready(t.Context()); err != nil {
t.Fatal(err)
}
if err := f.coordinator.Activate(t.Context(), f.candidate.CertificateFiles); err != nil {
t.Fatal(err)
}
if record := f.record(t); record.Phase != "committed" || record.ProxyID == "" {
t.Fatal("双层提交未持久化", record.Phase)
}
if err := f.coordinator.Ready(t.Context()); err != nil {
t.Fatal("提交后的双层入口未就绪", err)
}
if err := f.coordinator.Recover(t.Context()); err != nil {
t.Fatal(err)
}
if f.reloader.reloadCount != 1 {
t.Fatal("已完成事务不应再次重载")
}
}
func TestCoordinatorRecoversLostResponses(t *testing.T) {
for _, operation := range []string{"stage", "commit"} {
t.Run(operation, func(t *testing.T) {
f := newCoordinatorFixture(t)
proxy := coordinatedTestProxy{ProxyClient: f.rotation}
if operation == "stage" {
proxy.stage = func(ctx context.Context, chain, key []byte) (string, error) {
if _, err := f.rotation.Stage(ctx, chain, key); err != nil {
return "", err
}
return "", errors.New("注入暂存应答丢失")
}
} else {
proxy.commit = func(ctx context.Context, id string) error {
if err := f.rotation.Commit(ctx, id); err != nil {
return err
}
return errors.New("注入提交应答丢失")
}
}
f.coordinator.Proxy = proxy
err := f.coordinator.Activate(t.Context(), f.candidate.CertificateFiles)
want := "committed"
if operation == "stage" {
want = "rolled_back"
if err == nil || f.session.current != f.previous {
t.Fatal("暂存应答丢失后不应切换上游", err)
}
} else if err != nil || f.session.current != f.candidate {
t.Fatal("应验证已提交的同一事务", err)
}
if record := f.record(t); record.Phase != want || record.ProxyID == "" {
t.Fatal("应答丢失后的终态错误", record.Phase)
}
})
}
}
func TestCoordinatorRecoversRestartWindows(t *testing.T) {
for _, phase := range []string{"prepared", "staged", "saved_before_snapshot", "upstream", "commit_before_record"} {
t.Run(phase, func(t *testing.T) {
f := newCoordinatorFixture(t)
record := coordinationRecord{Version: 1, Phase: "prepared", Previous: f.previous, Candidate: f.candidate}
f.save(t, record)
id, err := f.rotation.Stage(t.Context(), f.rotationFixture.candidate.chain, f.rotationFixture.candidate.key)
if err != nil {
t.Fatal(err)
}
want := "rolled_back"
if phase != "prepared" {
record.Phase, record.ProxyID = "staged", id
f.save(t, record)
}
if phase == "saved_before_snapshot" {
f.session.readErr = errors.New("配置已保存但运行快照仍旧")
if err := f.coordinator.Recover(t.Context()); !errors.Is(err, ErrCoordinationPending) {
t.Fatal("混合状态下不能回退或提交", err)
}
f.assertStatus(t, "staged")
f.session.readErr = nil
}
if phase == "saved_before_snapshot" || phase == "upstream" || phase == "commit_before_record" {
if err := f.session.Activate(f.candidate.CertificateFiles, f.candidate.LeafHash); err != nil {
t.Fatal(err)
}
want = "committed"
}
if phase == "upstream" || phase == "commit_before_record" {
record.Phase = "upstream"
f.save(t, record)
}
if phase == "commit_before_record" {
if err := f.rotation.Commit(t.Context(), id); err != nil {
t.Fatal(err)
}
}
// 新实例只依靠同一目录的持久化记录和重启后实际提供的证书。
restarted := Coordinator{Directory: f.coordinator.Directory, Proxy: f.rotation, Upstream: f.session}
if err := restarted.Recover(t.Context()); err != nil {
t.Fatal(err)
}
if record := f.record(t); record.Phase != want || record.ProxyID != id {
t.Fatal("重启恢复终态错误", record.Phase)
}
})
}
}
func TestCoordinatorKeepsUnobservedStagePending(t *testing.T) {
f := newCoordinatorFixture(t)
calls := 0
f.coordinator.Proxy = coordinatedTestProxy{ProxyClient: f.rotation, stage: func(context.Context, []byte, []byte) (string, error) {
calls++
return "", errors.New("暂存请求是否到达尚不可知")
}}
if err := f.coordinator.Activate(t.Context(), f.candidate.CertificateFiles); !errors.Is(err, ErrCoordinationPending) {
t.Fatal(err)
}
for range 2 {
if err := f.coordinator.Ready(t.Context()); !errors.Is(err, ErrCoordinationPending) {
t.Fatal("未观测到事务不能允许再次申请", err)
}
if err := f.coordinator.Activate(t.Context(), f.candidate.CertificateFiles); !errors.Is(err, ErrCoordinationPending) {
t.Fatal("未观测到事务不能重放暂存", err)
}
}
if calls != 1 || f.record(t).Phase != "prepared" {
t.Fatal("不能重放请求或清除不确定意图")
}
if _, err := f.rotation.Stage(t.Context(), f.rotationFixture.candidate.chain, f.rotationFixture.candidate.key); err != nil {
t.Fatal(err)
}
if err := f.coordinator.Recover(t.Context()); err != nil {
t.Fatal("迟到事务应在观测后恢复", err)
}
if f.record(t).Phase != "rolled_back" {
t.Fatal("迟到暂存没有回退")
}
}
func TestCoordinatorRefusesUnrelatedStateAndPolicy(t *testing.T) {
for _, change := range []string{"hostname", "proxy", "upstream", "leaf", "mode"} {
t.Run(change, func(t *testing.T) {
f := newCoordinatorFixture(t)
switch change {
case "hostname":
f.session.current.Hostname = "other.example.test"
case "proxy":
f.session.current.ProxyAddress = "127.0.0.1:1"
case "upstream":
f.session.current.UpstreamAddress = "127.0.0.1:2"
case "leaf":
f.session.current.LeafHash = f.candidate.LeafHash
case "mode":
f.session.current.Mode = "direct"
}
if err := f.coordinator.Ready(t.Context()); err == nil {
t.Fatal("错误策略不能通过签发前检查")
}
})
}
f := newCoordinatorFixture(t)
f.save(t, coordinationRecord{Version: 1, Phase: "prepared", Previous: f.previous, Candidate: f.candidate})
f.session.current.Mode = "direct"
if err := f.coordinator.Recover(t.Context()); !errors.Is(err, ErrCoordinationPending) {
t.Fatal("不能因模式切换忽略未决事务", err)
}
}
func TestCoordinatorSerializesWholeLifecycle(t *testing.T) {
f := newCoordinatorFixture(t)
started, release := make(chan struct{}), make(chan struct{})
defer close(release)
f.reloader.check = func(number int) error {
if number == 2 {
close(started)
<-release
}
return nil
}
done := make(chan error, 1)
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
go func() { done <- f.coordinator.Activate(ctx, f.candidate.CertificateFiles) }()
select {
case <-started:
case <-time.After(3 * time.Second):
t.Fatal("协调未开始")
}
second := f.coordinator
if err := second.Recover(ctx); err == nil {
t.Fatal("独立协调实例不能在生命周期中途恢复")
}
if err := second.Ready(ctx); err == nil {
t.Fatal("独立协调实例不能开始下一次申请")
}
go func() { release <- struct{}{} }()
if err := <-done; err != nil {
t.Fatal(err)
}
}