补齐真实应用持久化与Nginx双层切换集成用例

This commit is contained in:
xuans committed 2026-09-20 03:07:49 +08:00
1 parent 53ed3d6021
commit c7d5344329
3 files changed
+618

No files matched your search

@@ -0,0 +1,231 @@
package app
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"git.pcux.cn/xuans/Scert/backend/internal/config"
"git.pcux.cn/xuans/Scert/backend/internal/nginxactivation"
)
// 系统根证书缓存仅在子进程内加载测试 CA,不改生产依赖或主机信任库。
func TestApplicationNginxIntegration(t *testing.T) {
binary := os.Getenv("SCERT_TEST_NGINX_BINARY")
if binary == "" {
t.Skip("未指定隔离验收使用的 Nginx 可执行文件")
}
if !filepath.IsAbs(binary) {
t.Fatal("测试 Nginx 路径必须是绝对路径")
}
if scenario := os.Getenv("SCERT_APP_NGINX_CHILD"); scenario != "" {
runApplicationNginxScenario(t, binary, scenario)
return
}
executable, err := os.Executable()
if err != nil {
t.Fatal(err)
}
for _, scenario := range []string{"normal", "stage_response", "commit_response", "restart_staged", "restart_upstream"} {
t.Run(scenario, func(t *testing.T) {
f := newApplicationTLSFixture(t)
ctx, cancel := context.WithTimeout(t.Context(), 120*time.Second)
defer cancel()
command := exec.CommandContext(ctx, executable, "-test.run=^TestApplicationNginxIntegration$", "-test.v", "-test.timeout=90s")
command.Env = append(os.Environ(),
"SCERT_APP_NGINX_CHILD="+scenario, "SCERT_APP_NGINX_ROOT="+f.root,
"SSL_CERT_FILE="+f.caPath, "SSL_CERT_DIR="+filepath.Join(f.root, "empty-roots"),
"GODEBUG=x509usefallbackroots=0",
)
if output, err := command.CombinedOutput(); err != nil {
t.Fatalf("隔离应用与 Nginx 测试失败:%v\n%s", err, output)
}
})
}
}
type applicationFaultProxy struct {
nginxactivation.ProxyClient
scenario string
pending bool
}
func (p *applicationFaultProxy) StageRequest(ctx context.Context, id string, chain, key []byte) (string, error) {
result, err := p.ProxyClient.StageRequest(ctx, id, chain, key)
if err != nil {
return result, err
}
if p.scenario == "stage_response" || p.scenario == "restart_staged" {
p.pending = p.scenario == "restart_staged"
return "", errors.New("注入暂存应答丢失")
}
return result, nil
}
func (p *applicationFaultProxy) Commit(ctx context.Context, id string) error {
if p.scenario == "restart_upstream" {
p.pending = true
return errors.New("注入上游持久化后辅助服务不可达")
}
if err := p.ProxyClient.Commit(ctx, id); err != nil {
return err
}
if p.scenario == "commit_response" {
return errors.New("注入提交应答丢失")
}
return nil
}
func (p *applicationFaultProxy) ResolveRequest(ctx context.Context, id string) (nginxactivation.State, error) {
if p.pending {
return nginxactivation.State{}, errors.New("注入恢复查询不可达,保留未完成事务")
}
return p.ProxyClient.ResolveRequest(ctx, id)
}
func runApplicationNginxScenario(t *testing.T, binary, scenario string) {
f := loadApplicationTLSFixture(t, os.Getenv("SCERT_APP_NGINX_ROOT"))
application, stop := f.start(t)
f.seed(t, application)
rotation := startApplicationNginx(t, f, binary)
client := startApplicationActivationIPC(t, rotation)
proxy := &applicationFaultProxy{ProxyClient: client, scenario: scenario}
activator := coordinatedManagementActivator{
server: application.HTTP, directory: filepath.Join(f.config.Paths.Data, "management-activation"), proxy: proxy,
}
state, err := application.HTTP.ManagementCertificate()
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second)
defer cancel()
if err := activator.Ready(ctx, state); err != nil {
t.Fatal(err)
}
err = activator.Activate(ctx, state, f.candidate)
want, leaf := "committed", f.newLeaf
if scenario == "stage_response" || scenario == "restart_staged" {
want, leaf = "rolled_back", f.oldLeaf
}
if scenario == "stage_response" || strings.HasPrefix(scenario, "restart_") {
if err == nil {
t.Fatal("注入故障后不应报告切换成功")
}
} else if err != nil {
t.Fatal(err)
}
if strings.HasPrefix(scenario, "restart_") {
phase := "prepared"
if scenario == "restart_upstream" {
phase = "upstream"
}
assertApplicationCoordination(t, activator.directory, phase, false)
pending, err := client.State(ctx)
if err != nil || pending.Status != "staged" {
t.Fatal("重建前必须保留真实代理未完成事务", pending, err)
}
} else {
assertApplicationCoordination(t, activator.directory, want, true)
}
proxyLeaf := leaf
if scenario == "restart_staged" {
proxyLeaf = f.newLeaf
}
if _, err := f.proxy.Check(ctx, proxyLeaf); err != nil {
t.Fatal("重建前代理证书与预期阶段不一致", err)
}
if _, err := f.upstream.Check(ctx, leaf); err != nil {
t.Fatal("重建前应用监听证书与预期阶段不一致", err)
}
assertApplicationTLSConfig(t, f, want)
stop()
restarted, _ := f.start(t)
recovered := coordinatedManagementActivator{server: restarted.HTTP, directory: activator.directory, proxy: client}
// 使用实际应用恢复工作循环,首次完成后停止,避免与后续断言抢占独占会话。
completed := make(chan error, 1)
restarted.startManagementRecovery(func(ctx context.Context) error {
err := recovered.coordinator(nil).Recover(ctx)
completed <- err
return err
})
select {
case err := <-completed:
if err != nil {
t.Fatal("应用恢复工作循环失败", err)
}
case <-ctx.Done():
t.Fatal("应用恢复工作循环超时", ctx.Err())
}
restarted.managementStop()
<-restarted.managementDone
assertApplicationCoordination(t, recovered.directory, want, true)
assertApplicationTLSConfig(t, f, want)
for _, endpoint := range []nginxactivation.Endpoint{f.proxy, f.upstream} {
if err := endpoint.Wait(ctx, leaf); err != nil {
t.Fatal("重建应用后真实双层证书未收敛", err)
}
}
actual, err := client.State(ctx)
if err != nil || actual.Status != want {
t.Fatal("代理最终事务状态错误", actual, err)
}
var snapshot nginxactivation.ManagedCertificate
if err := (managementUpstream{server: restarted.HTTP}).WithCertificate(ctx, func(session nginxactivation.ManagedSession) error {
var err error
snapshot, err = session.Snapshot()
return err
}); err != nil || snapshot.LeafHash != fmt.Sprintf("%x", sha256.Sum256(leaf)) {
t.Fatal("重建后应用管理快照错误", err)
}
current, err := restarted.HTTP.ManagementCertificate()
if err != nil {
t.Fatal(err)
}
if err := recovered.Ready(ctx, current); err != nil {
t.Fatal("恢复后未允许后续续期", err)
}
f.assertDatabase(t, restarted)
}
func assertApplicationTLSConfig(t *testing.T, f *applicationTLSFixture, phase string) {
t.Helper()
actual, err := config.Load(f.path)
if err != nil {
t.Fatal(err)
}
want := f.config
if phase == "committed" {
want.Server.TLSCert, want.Server.TLSKey = f.candidate.FullChainPath, f.candidate.PrivateKeyPath
}
if !reflect.DeepEqual(actual, want) {
t.Fatal("双层协调改变了证书路径以外的配置,或证书路径未按终态持久化")
}
}
func assertApplicationCoordination(t *testing.T, directory, phase string, terminal bool) {
t.Helper()
content, err := os.ReadFile(filepath.Join(directory, "coordination.json"))
if err != nil {
t.Fatal(err)
}
var record struct {
Phase string `json:"phase"`
RequestID string `json:"request_id"`
ProxyID string `json:"proxy_id"`
}
if err := json.Unmarshal(content, &record); err != nil {
t.Fatal(err)
}
if record.Phase != phase || record.RequestID == "" || (terminal && record.RequestID != record.ProxyID) {
t.Fatalf("应用协调记录错误:%+v,预期阶段 %s", record, phase)
}
}