补齐真实应用持久化与Nginx双层切换集成用例
This commit is contained in:
1 parent
53ed3d6021
commit
c7d5344329
3 files changed
+618
No files matched your search
@@ -0,0 +1,231 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/config"
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/nginxactivation"
|
||||
)
|
||||
|
||||
// 系统根证书缓存仅在子进程内加载测试 CA,不改生产依赖或主机信任库。
|
||||
func TestApplicationNginxIntegration(t *testing.T) {
|
||||
binary := os.Getenv("SCERT_TEST_NGINX_BINARY")
|
||||
if binary == "" {
|
||||
t.Skip("未指定隔离验收使用的 Nginx 可执行文件")
|
||||
}
|
||||
if !filepath.IsAbs(binary) {
|
||||
t.Fatal("测试 Nginx 路径必须是绝对路径")
|
||||
}
|
||||
if scenario := os.Getenv("SCERT_APP_NGINX_CHILD"); scenario != "" {
|
||||
runApplicationNginxScenario(t, binary, scenario)
|
||||
return
|
||||
}
|
||||
executable, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, scenario := range []string{"normal", "stage_response", "commit_response", "restart_staged", "restart_upstream"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
f := newApplicationTLSFixture(t)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 120*time.Second)
|
||||
defer cancel()
|
||||
command := exec.CommandContext(ctx, executable, "-test.run=^TestApplicationNginxIntegration$", "-test.v", "-test.timeout=90s")
|
||||
command.Env = append(os.Environ(),
|
||||
"SCERT_APP_NGINX_CHILD="+scenario, "SCERT_APP_NGINX_ROOT="+f.root,
|
||||
"SSL_CERT_FILE="+f.caPath, "SSL_CERT_DIR="+filepath.Join(f.root, "empty-roots"),
|
||||
"GODEBUG=x509usefallbackroots=0",
|
||||
)
|
||||
if output, err := command.CombinedOutput(); err != nil {
|
||||
t.Fatalf("隔离应用与 Nginx 测试失败:%v\n%s", err, output)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type applicationFaultProxy struct {
|
||||
nginxactivation.ProxyClient
|
||||
scenario string
|
||||
pending bool
|
||||
}
|
||||
|
||||
func (p *applicationFaultProxy) StageRequest(ctx context.Context, id string, chain, key []byte) (string, error) {
|
||||
result, err := p.ProxyClient.StageRequest(ctx, id, chain, key)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if p.scenario == "stage_response" || p.scenario == "restart_staged" {
|
||||
p.pending = p.scenario == "restart_staged"
|
||||
return "", errors.New("注入暂存应答丢失")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *applicationFaultProxy) Commit(ctx context.Context, id string) error {
|
||||
if p.scenario == "restart_upstream" {
|
||||
p.pending = true
|
||||
return errors.New("注入上游持久化后辅助服务不可达")
|
||||
}
|
||||
if err := p.ProxyClient.Commit(ctx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
if p.scenario == "commit_response" {
|
||||
return errors.New("注入提交应答丢失")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *applicationFaultProxy) ResolveRequest(ctx context.Context, id string) (nginxactivation.State, error) {
|
||||
if p.pending {
|
||||
return nginxactivation.State{}, errors.New("注入恢复查询不可达,保留未完成事务")
|
||||
}
|
||||
return p.ProxyClient.ResolveRequest(ctx, id)
|
||||
}
|
||||
|
||||
func runApplicationNginxScenario(t *testing.T, binary, scenario string) {
|
||||
f := loadApplicationTLSFixture(t, os.Getenv("SCERT_APP_NGINX_ROOT"))
|
||||
application, stop := f.start(t)
|
||||
f.seed(t, application)
|
||||
rotation := startApplicationNginx(t, f, binary)
|
||||
client := startApplicationActivationIPC(t, rotation)
|
||||
proxy := &applicationFaultProxy{ProxyClient: client, scenario: scenario}
|
||||
activator := coordinatedManagementActivator{
|
||||
server: application.HTTP, directory: filepath.Join(f.config.Paths.Data, "management-activation"), proxy: proxy,
|
||||
}
|
||||
state, err := application.HTTP.ManagementCertificate()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second)
|
||||
defer cancel()
|
||||
if err := activator.Ready(ctx, state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = activator.Activate(ctx, state, f.candidate)
|
||||
want, leaf := "committed", f.newLeaf
|
||||
if scenario == "stage_response" || scenario == "restart_staged" {
|
||||
want, leaf = "rolled_back", f.oldLeaf
|
||||
}
|
||||
if scenario == "stage_response" || strings.HasPrefix(scenario, "restart_") {
|
||||
if err == nil {
|
||||
t.Fatal("注入故障后不应报告切换成功")
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.HasPrefix(scenario, "restart_") {
|
||||
phase := "prepared"
|
||||
if scenario == "restart_upstream" {
|
||||
phase = "upstream"
|
||||
}
|
||||
assertApplicationCoordination(t, activator.directory, phase, false)
|
||||
pending, err := client.State(ctx)
|
||||
if err != nil || pending.Status != "staged" {
|
||||
t.Fatal("重建前必须保留真实代理未完成事务", pending, err)
|
||||
}
|
||||
} else {
|
||||
assertApplicationCoordination(t, activator.directory, want, true)
|
||||
}
|
||||
proxyLeaf := leaf
|
||||
if scenario == "restart_staged" {
|
||||
proxyLeaf = f.newLeaf
|
||||
}
|
||||
if _, err := f.proxy.Check(ctx, proxyLeaf); err != nil {
|
||||
t.Fatal("重建前代理证书与预期阶段不一致", err)
|
||||
}
|
||||
if _, err := f.upstream.Check(ctx, leaf); err != nil {
|
||||
t.Fatal("重建前应用监听证书与预期阶段不一致", err)
|
||||
}
|
||||
assertApplicationTLSConfig(t, f, want)
|
||||
stop()
|
||||
restarted, _ := f.start(t)
|
||||
recovered := coordinatedManagementActivator{server: restarted.HTTP, directory: activator.directory, proxy: client}
|
||||
// 使用实际应用恢复工作循环,首次完成后停止,避免与后续断言抢占独占会话。
|
||||
completed := make(chan error, 1)
|
||||
restarted.startManagementRecovery(func(ctx context.Context) error {
|
||||
err := recovered.coordinator(nil).Recover(ctx)
|
||||
completed <- err
|
||||
return err
|
||||
})
|
||||
select {
|
||||
case err := <-completed:
|
||||
if err != nil {
|
||||
t.Fatal("应用恢复工作循环失败", err)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("应用恢复工作循环超时", ctx.Err())
|
||||
}
|
||||
restarted.managementStop()
|
||||
<-restarted.managementDone
|
||||
assertApplicationCoordination(t, recovered.directory, want, true)
|
||||
assertApplicationTLSConfig(t, f, want)
|
||||
for _, endpoint := range []nginxactivation.Endpoint{f.proxy, f.upstream} {
|
||||
if err := endpoint.Wait(ctx, leaf); err != nil {
|
||||
t.Fatal("重建应用后真实双层证书未收敛", err)
|
||||
}
|
||||
}
|
||||
actual, err := client.State(ctx)
|
||||
if err != nil || actual.Status != want {
|
||||
t.Fatal("代理最终事务状态错误", actual, err)
|
||||
}
|
||||
var snapshot nginxactivation.ManagedCertificate
|
||||
if err := (managementUpstream{server: restarted.HTTP}).WithCertificate(ctx, func(session nginxactivation.ManagedSession) error {
|
||||
var err error
|
||||
snapshot, err = session.Snapshot()
|
||||
return err
|
||||
}); err != nil || snapshot.LeafHash != fmt.Sprintf("%x", sha256.Sum256(leaf)) {
|
||||
t.Fatal("重建后应用管理快照错误", err)
|
||||
}
|
||||
current, err := restarted.HTTP.ManagementCertificate()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recovered.Ready(ctx, current); err != nil {
|
||||
t.Fatal("恢复后未允许后续续期", err)
|
||||
}
|
||||
f.assertDatabase(t, restarted)
|
||||
}
|
||||
|
||||
func assertApplicationTLSConfig(t *testing.T, f *applicationTLSFixture, phase string) {
|
||||
t.Helper()
|
||||
actual, err := config.Load(f.path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := f.config
|
||||
if phase == "committed" {
|
||||
want.Server.TLSCert, want.Server.TLSKey = f.candidate.FullChainPath, f.candidate.PrivateKeyPath
|
||||
}
|
||||
if !reflect.DeepEqual(actual, want) {
|
||||
t.Fatal("双层协调改变了证书路径以外的配置,或证书路径未按终态持久化")
|
||||
}
|
||||
}
|
||||
|
||||
func assertApplicationCoordination(t *testing.T, directory, phase string, terminal bool) {
|
||||
t.Helper()
|
||||
content, err := os.ReadFile(filepath.Join(directory, "coordination.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var record struct {
|
||||
Phase string `json:"phase"`
|
||||
RequestID string `json:"request_id"`
|
||||
ProxyID string `json:"proxy_id"`
|
||||
}
|
||||
if err := json.Unmarshal(content, &record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if record.Phase != phase || record.RequestID == "" || (terminal && record.RequestID != record.ProxyID) {
|
||||
t.Fatalf("应用协调记录错误:%+v,预期阶段 %s", record, phase)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user