修复权限失效放行并加固证书推送事务和超时回滚

This commit is contained in:
xuans committed 2026-09-15 20:00:38 +08:00
1 parent d74f74b631
commit 05e05a1a23
15 files changed
+861 -178

No files matched your search

+53
View File
@@ -0,0 +1,53 @@
package deployment
import (
"bytes"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"time"
)
// validateArtifact 是传输前的完整性检查;域名覆盖和环境信任策略由证书服务负责。
func validateArtifact(artifact Artifact) error {
if len(artifact.Certificate) == 0 || len(artifact.FullChain) == 0 || len(artifact.PrivateKey) == 0 {
return errors.New("证书部署文件不完整")
}
certificate, err := tls.X509KeyPair(artifact.Certificate, artifact.PrivateKey)
if err != nil {
return fmt.Errorf("部署证书与私钥不匹配:%w", err)
}
chain, err := tls.X509KeyPair(artifact.FullChain, artifact.PrivateKey)
if err != nil {
return fmt.Errorf("部署证书链与私钥不匹配:%w", err)
}
if !bytes.Equal(certificate.Certificate[0], chain.Certificate[0]) {
return errors.New("部署证书与完整证书链的叶子证书不一致")
}
var previous *x509.Certificate
remaining := bytes.TrimSpace(artifact.FullChain)
now := time.Now()
for len(remaining) > 0 {
block, rest := pem.Decode(remaining)
if block == nil || block.Type != "CERTIFICATE" || !bytes.HasPrefix(remaining, []byte("-----BEGIN CERTIFICATE-----")) {
return errors.New("部署证书链包含无效 PEM 内容")
}
cert, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return fmt.Errorf("解析部署证书链失败:%w", err)
}
if now.Before(cert.NotBefore) || !now.Before(cert.NotAfter) {
return errors.New("部署证书链包含尚未生效或已过期的证书")
}
if previous != nil {
if err := previous.CheckSignatureFrom(cert); err != nil {
return fmt.Errorf("部署证书链签名或顺序无效:%w", err)
}
}
previous = cert
remaining = bytes.TrimSpace(rest)
}
return nil
}
+28
View File
@@ -0,0 +1,28 @@
package deployment
import "sync"
const commandOutputLimit = 64 << 10
// commandOutput 限制远程标准输出和错误输出占用,防止异常命令耗尽管理端内存。
type commandOutput struct {
mu sync.Mutex
data []byte
}
func (b *commandOutput) Write(data []byte) (int, error) {
b.mu.Lock()
defer b.mu.Unlock()
remaining := commandOutputLimit - len(b.data)
if remaining > len(data) {
remaining = len(data)
}
b.data = append(b.data, data[:remaining]...)
return len(data), nil
}
func (b *commandOutput) Bytes() []byte {
b.mu.Lock()
defer b.mu.Unlock()
return append([]byte(nil), b.data...)
}
@@ -0,0 +1,146 @@
package deployment
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"io"
"os"
"path"
"path/filepath"
"strings"
"testing"
"time"
"git.pcux.cn/xuans/Scert/backend/internal/config"
"git.pcux.cn/xuans/Scert/backend/internal/secretstore"
)
// TestRemoteDeploymentIsolatedAcceptance 仅在显式配置后运行,不启动业务数据库、DNS 或 ACME。
// 每次使用全新隔离目录,保留验收产物;不执行目标机已有服务的校验、重载或重启。
func TestRemoteDeploymentIsolatedAcceptance(t *testing.T) {
if os.Getenv("SCERT_ACCEPTANCE_CONFIRM") != "isolated-directory" {
t.Skip("未启用隔离实机部署验收")
}
cfg, err := config.Load(os.Getenv("SCERT_ACCEPTANCE_CONFIG"))
if err != nil {
t.Fatal(err)
}
for _, name := range []string{"master.key", "secrets.json"} {
info, err := os.Lstat(filepath.Join(cfg.Paths.Config, name))
if err != nil || !info.Mode().IsRegular() {
t.Fatalf("现有凭据文件不可用:%s", name)
}
}
store, err := secretstore.Open(cfg.Paths.Config)
if err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(os.Getenv("SCERT_ACCEPTANCE_TARGET"))
if err != nil {
t.Fatal(err)
}
var server Server
if err := json.Unmarshal(data, &server); err != nil {
t.Fatal(err)
}
root := "/etc/ssl/scert-acceptance-" + time.Now().UTC().Format("20060102-150405.000000000")
server.CertificatePath = path.Join(root, "cert.pem")
server.CertificateChainPath = path.Join(root, "fullchain.pem")
server.PrivateKeyPath = path.Join(root, "key.pem")
server.ValidationCommand = []string{"/usr/bin/true"}
server.ReloadCommand = []string{"/usr/bin/true"}
factory := SSHFactory{Secrets: store, Timeout: 10 * time.Second}
client, err := factory.Connect(t.Context(), server)
if err != nil {
t.Fatal(err)
}
exists, err := client.Exists(t.Context(), root)
_ = client.Close()
if err != nil || exists {
t.Fatalf("隔离目录必须尚不存在:exists=%t err=%v", exists, err)
}
t.Logf("隔离目标:%s,目录:%s;命令使用 true,不代表真实服务重载验收", server.Address, root)
service := NewService(factory)
original := testArtifact(t)
updated := testArtifact(t)
for index, artifact := range []Artifact{original, updated} {
report := service.Deploy(t.Context(), []Target{{Server: server}}, artifact)
if report.Status != "success" {
t.Fatalf("第 %d 次隔离部署失败:%+v", index+1, report)
}
t.Logf("隔离部署 %d 成功,事务:%s", index+1, report.Results[0].BackupID)
}
verify := func() {
t.Helper()
client, err := factory.Connect(t.Context(), server)
if err != nil {
t.Fatal(err)
}
defer client.Close()
adapter := client.(*sshClientAdapter)
for filename, expected := range map[string][]byte{"cert.pem": updated.Certificate, "fullchain.pem": updated.FullChain, "key.pem": updated.PrivateKey} {
file, err := adapter.sftp.Open(path.Join(root, filename))
if err != nil {
t.Fatal(err)
}
actual, err := io.ReadAll(file)
_ = file.Close()
if err != nil || !bytes.Equal(actual, expected) {
t.Fatalf("文件内容不一致:%s %v", filename, err)
}
info, err := adapter.sftp.Stat(path.Join(root, filename))
if err != nil {
t.Fatal(err)
}
wantMode := os.FileMode(0o644)
if filename == "key.pem" {
wantMode = 0o600
}
if info.Mode().Perm() != wantMode {
t.Fatalf("文件权限错误:%s %o", filename, info.Mode().Perm())
}
if filename != "key.pem" {
t.Logf("内容与权限验证:%s SHA256=%x mode=%o", filename, sha256.Sum256(actual), wantMode)
}
}
files, err := adapter.sftp.ReadDir(root)
if err != nil {
t.Fatal(err)
}
for _, file := range files {
if strings.Contains(file.Name(), ".scert-tmp-") {
t.Fatalf("残留临时文件:%s", file.Name())
}
if strings.HasPrefix(file.Name(), "key.pem") && file.Mode().Perm() != 0o600 {
t.Fatal("私钥或私钥备份权限不安全")
}
}
}
verify()
for _, scenario := range []string{"validation_failure", "reload_failure", "validation_timeout"} {
candidate := server
ctx := t.Context()
cancel := func() {}
switch scenario {
case "validation_failure":
candidate.ValidationCommand = []string{"/usr/bin/false"}
case "reload_failure":
candidate.ReloadCommand = []string{"/usr/bin/false"}
case "validation_timeout":
candidate.ValidationCommand = []string{"/usr/bin/sleep", "30"}
ctx, cancel = context.WithTimeout(ctx, 4*time.Second)
}
report := service.Deploy(ctx, []Target{{Server: candidate}}, original)
cancel()
if report.Status != "failed" || !strings.Contains(report.Results[0].Message, "文件已回滚") || strings.Contains(report.Results[0].Message, "自动回滚失败") {
t.Fatalf("故障场景未安全回滚 %s:%+v", scenario, report)
}
if scenario == "validation_timeout" && !strings.Contains(report.Results[0].Message, "deadline exceeded") {
t.Fatalf("必须实际触发超时:%+v", report)
}
t.Logf("故障场景 %s:%s", scenario, report.Results[0].Message)
verify()
}
}
+40 -15
View File
@@ -109,6 +109,8 @@ func (s *Service) DeploySaved(ctx context.Context, targets []Target, saved Saved
}
func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifact) Result {
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
server = server.Normalize()
result := Result{ServerID: server.ID, Server: server.Name, Status: "failed"}
if err := server.Validate(); err != nil {
@@ -120,7 +122,7 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
result.Message = fmt.Sprintf("连接服务器失败:%v", err)
return result
}
defer client.Close()
defer func() { _ = client.Close() }()
paths := []remoteFile{
{path: RemotePath(server.CertificatePath), data: artifact.Certificate, mode: 0o644},
{path: RemotePath(server.CertificateChainPath), data: artifact.FullChain, mode: 0o644},
@@ -137,6 +139,7 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
return result
}
backupID := s.backupID()
result.BackupID = backupID
type backup struct{ original, saved string }
backups := make([]backup, 0, len(paths))
temporary := make([]string, 0, len(paths))
@@ -144,6 +147,14 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
rollback := func() error {
rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
defer cancel()
if state, ok := client.(interface{ Disconnected() bool }); ok && state.Disconnected() {
_ = client.Close()
reconnected, err := s.factory.Connect(rollbackContext, server)
if err != nil {
return fmt.Errorf("重新连接服务器以回滚失败:%w", err)
}
client = reconnected
}
var rollbackErrors []error
for _, path := range temporary {
if err := client.Remove(rollbackContext, path); err != nil {
@@ -157,15 +168,31 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
}
for index := len(backups) - 1; index >= 0; index-- {
item := backups[index]
// 备份请求超时可能丢失响应,重连后以目标文件是否存在核实实际结果。
exists, err := client.Exists(rollbackContext, item.saved)
if err != nil {
rollbackErrors = append(rollbackErrors, fmt.Errorf("检查备份文件 %s 失败:%w", item.saved, err))
continue
}
if !exists {
continue
}
if err := client.Rename(rollbackContext, item.saved, item.original); err != nil {
rollbackErrors = append(rollbackErrors, fmt.Errorf("恢复备份文件 %s 失败:%w", item.original, err))
}
}
return errors.Join(rollbackErrors...)
}
reloadAttempted := false
failed := func(message string) Result {
if rollbackErr := rollback(); rollbackErr != nil {
message += fmt.Sprintf(";自动回滚失败:%v", rollbackErr)
} else {
message += ";文件已回滚"
}
if reloadAttempted {
// 命令可能已部分执行;文件回滚并不证明运行中的服务已经恢复。
message += ";服务运行状态未确认,请检查服务并在确认旧配置后重新加载"
}
result.Message = message
return result
@@ -174,27 +201,31 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
if err := client.MkdirAll(ctx, parentPath(file.path)); err != nil {
return failed(fmt.Sprintf("创建远程目录失败:%v", err))
}
temporaryPath := file.path + ".scert-tmp-" + backupID
// 上传失败也可能留下半份私钥,必须在开始上传前登记清理路径。
temporary = append(temporary, temporaryPath)
if err := client.Upload(ctx, temporaryPath, file.data, file.mode); err != nil {
return failed(fmt.Sprintf("上传远程证书失败:%v", err))
}
}
// 所有文件上传完成后才触碰当前版本,避免上传失败造成不必要的证书替换。
for _, file := range paths {
exists, err := client.Exists(ctx, file.path)
if err != nil {
return failed(fmt.Sprintf("检查远程文件失败:%v", err))
}
if exists {
backupPath := file.path + ".scert-backup-" + backupID
backups = append(backups, backup{original: file.path, saved: backupPath})
if err := client.Rename(ctx, file.path, backupPath); err != nil {
return failed(fmt.Sprintf("备份远程证书失败:%v", err))
}
backups = append(backups, backup{original: file.path, saved: backupPath})
}
temporaryPath := file.path + ".scert-tmp-" + backupID
if err := client.Upload(ctx, temporaryPath, file.data, file.mode); err != nil {
return failed(fmt.Sprintf("上传远程证书失败:%v", err))
}
temporary = append(temporary, temporaryPath)
installed = append(installed, file.path)
if err := client.Rename(ctx, temporaryPath, file.path); err != nil {
return failed(fmt.Sprintf("替换远程证书失败:%v", err))
}
temporary = temporary[:len(temporary)-1]
installed = append(installed, file.path)
}
validationResult, err := client.Run(ctx, server.ValidationCommand)
if err != nil || validationResult.ExitCode != 0 {
@@ -203,6 +234,7 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
}
return failed(fmt.Sprintf("配置校验失败:%v", err))
}
reloadAttempted = true
commandResult, err := client.Run(ctx, server.ReloadCommand)
if err != nil || commandResult.ExitCode != 0 {
if err == nil {
@@ -237,13 +269,6 @@ func validateDeploymentPaths(files []remoteFile) error {
return nil
}
func validateArtifact(artifact Artifact) error {
if len(artifact.Certificate) == 0 || len(artifact.FullChain) == 0 || len(artifact.PrivateKey) == 0 {
return errors.New("证书部署文件不完整")
}
return nil
}
func (s *Service) backupID() string {
var random [6]byte
_, _ = rand.Read(random[:])
+83 -10
View File
@@ -1,8 +1,15 @@
package deployment
import (
"bytes"
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"encoding/pem"
"errors"
"math/big"
"strings"
"testing"
"time"
@@ -38,6 +45,7 @@ func (c *memoryClient) Exists(_ context.Context, path string) (bool, error) {
func (c *memoryClient) Upload(_ context.Context, path string, data []byte, mode uint32) error {
if c.uploadError != "" && strings.Contains(path, c.uploadError) {
c.files[path] = memoryFile{data: append([]byte(nil), data[:len(data)/2]...), mode: mode}
return errors.New("模拟上传失败")
}
c.files[path] = memoryFile{data: append([]byte(nil), data...), mode: mode}
@@ -105,11 +113,12 @@ func TestDeployReplacesFilesAndKeepsBackup(t *testing.T) {
client.files[server.PrivateKeyPath] = memoryFile{data: []byte("old-key"), mode: 0o600}
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
service.now = func() time.Time { return time.Date(2026, 8, 27, 1, 2, 3, 4, time.UTC) }
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact())
artifact := testArtifact(t)
report := service.Deploy(context.Background(), []Target{{Server: server}}, artifact)
if report.Status != "success" || report.Success != 1 {
t.Fatalf("部署结果错误:%+v", report)
}
if string(client.files[server.CertificatePath].data) != "new-cert" || client.files[server.PrivateKeyPath].mode != 0o600 {
if !bytes.Equal(client.files[server.CertificatePath].data, artifact.Certificate) || client.files[server.PrivateKeyPath].mode != 0o600 {
t.Fatalf("远程证书内容或权限错误:%+v", client.files)
}
if len(client.commands) != 2 || len(client.commands[0]) != 2 || client.commands[0][1] != "-t" || len(client.commands[1]) != 3 || client.commands[1][0] != "nginx" {
@@ -132,7 +141,7 @@ func TestDeployRollsBackWhenReloadFails(t *testing.T) {
server := testServer("server-1")
client.files[server.CertificatePath] = memoryFile{data: []byte("old-cert"), mode: 0o644}
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact())
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact(t))
if report.Status != "failed" || report.Failed != 1 {
t.Fatalf("reload 失败结果错误:%+v", report)
}
@@ -157,7 +166,7 @@ func TestDeployRollsBackWhenValidationFailsWithoutReload(t *testing.T) {
client.files[server.CertificateChainPath] = memoryFile{data: []byte("old-chain"), mode: 0o644}
client.files[server.PrivateKeyPath] = memoryFile{data: []byte("old-key"), mode: 0o600}
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact())
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact(t))
if report.Status != "failed" || report.Failed != 1 || !strings.Contains(report.Results[0].Message, "配置校验失败") {
t.Fatalf("配置校验失败结果错误:%+v", report)
}
@@ -181,7 +190,7 @@ func TestDeployReportsRollbackFailure(t *testing.T) {
server := testServer("server-1")
client.files[server.CertificatePath] = memoryFile{data: []byte("old-cert"), mode: 0o644}
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact())
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact(t))
if report.Status != "failed" || report.Failed != 1 {
t.Fatalf("回滚失败结果错误:%+v", report)
}
@@ -197,19 +206,41 @@ func TestDeployReportsPartialSuccess(t *testing.T) {
firstServer := testServer("first")
secondServer := testServer("second")
service := NewService(memoryFactory{clients: map[string]*memoryClient{"first": first, "second": second}})
report := service.Deploy(context.Background(), []Target{{Server: firstServer}, {Server: secondServer}}, testArtifact())
report := service.Deploy(context.Background(), []Target{{Server: firstServer}, {Server: secondServer}}, testArtifact(t))
if report.Status != "partial" || report.Success != 1 || report.Failed != 1 {
t.Fatalf("部分成功汇总错误:%+v", report)
}
}
func TestDeployCleansPartialPrivateKeyUploadWithoutReplacingCurrentFiles(t *testing.T) {
client := newMemoryClient()
client.uploadError = "key.pem"
server := testServer("partial-key")
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
client.files[path] = memoryFile{data: []byte("original"), mode: 0o600}
}
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
report := service.Deploy(t.Context(), []Target{{Server: server}}, testArtifact(t))
if report.Status != "failed" || report.Results[0].BackupID == "" {
t.Fatalf("失败报告必须包含事务标识:%+v", report)
}
if len(client.files) != 3 || len(client.commands) != 0 {
t.Fatalf("部分上传失败不得残留临时文件或执行命令:files=%v commands=%v", client.files, client.commands)
}
for path, file := range client.files {
if string(file.data) != "original" || file.mode != 0o600 {
t.Fatalf("上传阶段不得替换原文件:%s", path)
}
}
}
func TestDeployRejectsDuplicateCertificatePaths(t *testing.T) {
client := newMemoryClient()
server := testServer("server-1")
server.PrivateKeyPath = server.CertificatePath
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact())
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact(t))
if report.Status != "failed" || report.Failed != 1 {
t.Fatalf("重复远程路径应直接失败:%+v", report)
}
@@ -227,7 +258,7 @@ func TestDeployRejectsRelativeCertificatePaths(t *testing.T) {
server.CertificatePath = "cert.pem"
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact())
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact(t))
if report.Status != "failed" || !strings.Contains(report.Results[0].Message, "必须是绝对路径") {
t.Fatalf("相对远程路径应被拒绝:%+v", report)
}
@@ -242,6 +273,48 @@ func testServer(id string) Server {
}
}
func testArtifact() Artifact {
return Artifact{Certificate: []byte("new-cert"), FullChain: []byte("new-chain"), PrivateKey: []byte("new-key")}
func testArtifact(t *testing.T) Artifact {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
template := &x509.Certificate{
SerialNumber: big.NewInt(time.Now().UnixNano()), DNSNames: []string{"example.com"},
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
}
der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
privateKey, err := x509.MarshalPKCS8PrivateKey(key)
if err != nil {
t.Fatal(err)
}
cert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
return Artifact{Certificate: cert, FullChain: append([]byte(nil), cert...), PrivateKey: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateKey})}
}
func TestDeployRejectsCorruptArtifactsBeforeConnecting(t *testing.T) {
for _, scenario := range []string{"invalid_pem", "wrong_key", "wrong_leaf", "garbage_chain"} {
t.Run(scenario, func(t *testing.T) {
artifact := testArtifact(t)
switch scenario {
case "invalid_pem":
artifact.Certificate = []byte("invalid")
case "wrong_key":
artifact.PrivateKey = testArtifact(t).PrivateKey
case "wrong_leaf":
artifact.FullChain = testArtifact(t).FullChain
case "garbage_chain":
artifact.FullChain = append(artifact.FullChain, []byte("invalid")...)
}
service := NewService(memoryFactory{})
report := service.Deploy(t.Context(), []Target{{Server: testServer("invalid")}}, artifact)
if report.Status != "failed" {
t.Fatalf("无效产物未被拒绝:%+v", report)
}
})
}
}
+102 -54
View File
@@ -8,6 +8,7 @@ import (
"net"
"os"
"strings"
"sync/atomic"
"time"
"github.com/pkg/sftp"
@@ -64,12 +65,16 @@ func (f SSHFactory) Connect(ctx context.Context, server Server) (Client, error)
if err != nil {
return nil, err
}
sftpClient, err := sftp.NewClient(sshClient)
adapter := &sshClientAdapter{ssh: sshClient}
sftpClient, err := sshOperation(timeoutContext, adapter, func() (*sftp.Client, error) {
return sftp.NewClient(sshClient)
})
if err != nil {
_ = sshClient.Close()
return nil, fmt.Errorf("创建 SFTP 客户端失败:%w", err)
}
return &sshClientAdapter{ssh: sshClient, sftp: sftpClient}, nil
adapter.sftp = sftpClient
return adapter, nil
}
func (f SSHFactory) secureHostKeyAlgorithms() ([]string, error) {
@@ -138,7 +143,13 @@ func connectSSHClient(
return nil
},
}
stopCancel := context.AfterFunc(ctx, func() { _ = connection.Close() })
clientConn, channels, requests, handshakeErr := ssh.NewClientConn(connection, address, config)
stopCancel()
if err := ctx.Err(); err != nil {
_ = connection.Close()
return nil, fmt.Errorf("SSH 握手已取消:%w", err)
}
if handshakeErr == nil {
if err := connection.SetDeadline(time.Time{}); err != nil {
_ = clientConn.Close()
@@ -174,64 +185,96 @@ func connectSSHClient(
}
type sshClientAdapter struct {
ssh *ssh.Client
sftp *sftp.Client
ssh *ssh.Client
sftp *sftp.Client
disconnected atomic.Bool
}
// sshOperation 取消时关闭传输并等待操作退出,避免回滚与仍在写入的操作并行。
// 已断开的连接由部署事务使用独立回滚上下文重新建立,仍严格校验主机指纹。
func sshOperation[T any](ctx context.Context, c *sshClientAdapter, operation func() (T, error)) (T, error) {
var zero T
if err := ctx.Err(); err != nil {
return zero, err
}
type outcome struct {
value T
err error
}
done := make(chan outcome, 1)
go func() {
value, err := operation()
done <- outcome{value, err}
}()
select {
case result := <-done:
if err := ctx.Err(); err != nil {
c.disconnected.Store(true)
_ = c.ssh.Close()
return zero, err
}
return result.value, result.err
case <-ctx.Done():
c.disconnected.Store(true)
_ = c.ssh.Close()
<-done
return zero, ctx.Err()
}
}
func (c *sshClientAdapter) Disconnected() bool { return c.disconnected.Load() }
func (c *sshClientAdapter) fileOperation(ctx context.Context, operation func() error) error {
_, err := sshOperation(ctx, c, func() (struct{}, error) { return struct{}{}, operation() })
return err
}
func (c *sshClientAdapter) MkdirAll(ctx context.Context, path string) error {
if err := ctx.Err(); err != nil {
return err
}
return c.sftp.MkdirAll(path)
return c.fileOperation(ctx, func() error { return c.sftp.MkdirAll(path) })
}
func (c *sshClientAdapter) Exists(ctx context.Context, path string) (bool, error) {
if err := ctx.Err(); err != nil {
return false, err
}
_, err := c.sftp.Stat(path)
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
return err == nil, err
return sshOperation(ctx, c, func() (bool, error) {
_, err := c.sftp.Stat(path)
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
return err == nil, err
})
}
func (c *sshClientAdapter) Upload(ctx context.Context, path string, data []byte, mode uint32) error {
if err := ctx.Err(); err != nil {
return err
}
file, err := c.sftp.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
if err != nil {
return err
}
closeFile := func() error { return file.Close() }
if _, err := file.Write(data); err != nil {
_ = closeFile()
return err
}
if err := file.Chmod(os.FileMode(mode)); err != nil {
_ = closeFile()
return err
}
return closeFile()
return c.fileOperation(ctx, func() error {
// 只创建新临时文件,不跟随已有符号链接,也不截断已存在的文件。
file, err := c.sftp.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL)
if err != nil {
return err
}
// 先收紧空文件权限,再写入任何证书或私钥内容。
if err := file.Chmod(os.FileMode(mode)); err != nil {
_ = file.Close()
return err
}
if _, err := file.Write(data); err != nil {
_ = file.Close()
return err
}
return file.Close()
})
}
func (c *sshClientAdapter) Rename(ctx context.Context, source, target string) error {
if err := ctx.Err(); err != nil {
return err
}
return c.sftp.Rename(source, target)
return c.fileOperation(ctx, func() error { return c.sftp.Rename(source, target) })
}
func (c *sshClientAdapter) Remove(ctx context.Context, path string) error {
if err := ctx.Err(); err != nil {
return c.fileOperation(ctx, func() error {
err := c.sftp.Remove(path)
if errors.Is(err, os.ErrNotExist) {
return nil
}
return err
}
err := c.sftp.Remove(path)
if errors.Is(err, os.ErrNotExist) {
return nil
}
return err
})
}
func (c *sshClientAdapter) Run(ctx context.Context, command []string) (CommandResult, error) {
@@ -241,12 +284,20 @@ func (c *sshClientAdapter) Run(ctx context.Context, command []string) (CommandRe
if err := validateCommand(command); err != nil {
return CommandResult{}, err
}
return sshOperation(ctx, c, func() (CommandResult, error) { return c.runCommand(command) })
}
func (c *sshClientAdapter) runCommand(command []string) (CommandResult, error) {
session, err := c.ssh.NewSession()
if err != nil {
return CommandResult{}, err
}
defer session.Close()
output, err := session.CombinedOutput(joinShellArgs(command))
outputBuffer := &commandOutput{}
session.Stdout = outputBuffer
session.Stderr = outputBuffer
err = session.Run(joinShellArgs(command))
output := outputBuffer.Bytes()
if err != nil {
var exitError *ssh.ExitError
if errors.As(err, &exitError) {
@@ -254,21 +305,18 @@ func (c *sshClientAdapter) Run(ctx context.Context, command []string) (CommandRe
}
return CommandResult{Output: output, ExitCode: -1}, err
}
if err := ctx.Err(); err != nil {
return CommandResult{Output: output, ExitCode: -1}, err
}
return CommandResult{Output: output}, nil
}
func (c *sshClientAdapter) Close() error {
var first error
if c.sftp != nil {
if err := c.sftp.Close(); err != nil {
first = err
}
}
// 先关闭传输,避免无响应的 SFTP 服务阻塞关闭过程。
if c.ssh != nil {
if err := c.ssh.Close(); err != nil && first == nil {
c.disconnected.Store(true)
first = c.ssh.Close()
}
if c.sftp != nil {
if err := c.sftp.Close(); err != nil && first == nil {
first = err
}
}
+21 -10
View File
@@ -182,17 +182,27 @@ func TestSSHFactoryFiltersUnsupportedAndDuplicateHostKeyAlgorithms(t *testing.T)
}
type testSSHServer struct {
listener net.Listener
accepted atomic.Int32
host string
port int
config *ssh.ServerConfig
listener net.Listener
accepted atomic.Int32
host string
port int
config *ssh.ServerConfig
channelHandler func(ssh.Channel, <-chan *ssh.Request)
}
func startTestSSHServer(
t *testing.T,
authorize func(ssh.PublicKey) error,
hostSigners ...ssh.Signer,
) *testSSHServer {
return startTestSSHServerWithHandler(t, authorize, serveTestSFTP, hostSigners...)
}
func startTestSSHServerWithHandler(
t *testing.T,
authorize func(ssh.PublicKey) error,
handler func(ssh.Channel, <-chan *ssh.Request),
hostSigners ...ssh.Signer,
) *testSSHServer {
t.Helper()
config := &ssh.ServerConfig{
@@ -218,10 +228,11 @@ func startTestSSHServer(
t.Fatalf("解析测试 SSH 端口失败:%v", err)
}
server := &testSSHServer{
listener: listener,
host: host,
port: port,
config: config,
listener: listener,
host: host,
port: port,
config: config,
channelHandler: handler,
}
t.Cleanup(func() {
listener.Close()
@@ -257,7 +268,7 @@ func (s *testSSHServer) serveConnection(connection net.Conn) {
if err != nil {
continue
}
go serveTestSFTP(channel, requests)
go s.channelHandler(channel, requests)
}
}
@@ -0,0 +1,255 @@
package deployment
import (
"bytes"
"context"
"errors"
"io"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/pkg/sftp"
"golang.org/x/crypto/ssh"
)
type observedSFTP struct {
handlers sftp.Handlers
mode atomic.Uint32
writes atomic.Int32
unsafeWrite atomic.Bool
exclusive atomic.Bool
denyMode bool
}
func (h *observedSFTP) Filewrite(request *sftp.Request) (io.WriterAt, error) {
h.exclusive.Store(request.Pflags().Excl && !request.Pflags().Trunc)
writer, err := h.handlers.FilePut.Filewrite(request)
if err != nil {
return nil, err
}
return &observedWriter{WriterAt: writer, owner: h}, nil
}
func (h *observedSFTP) Filecmd(request *sftp.Request) error {
if request.Method == "Setstat" {
if h.denyMode {
return errors.New("测试拒绝权限设置")
}
h.mode.Store(request.Attributes().Mode & 0o777)
}
return h.handlers.FileCmd.Filecmd(request)
}
type observedWriter struct {
io.WriterAt
owner *observedSFTP
}
func (w *observedWriter) WriteAt(data []byte, offset int64) (int, error) {
w.owner.writes.Add(1)
if w.owner.mode.Load() != 0o600 {
w.owner.unsafeWrite.Store(true)
}
return w.WriterAt.WriteAt(data, offset)
}
func (w *observedWriter) Close() error {
if closer, ok := w.WriterAt.(io.Closer); ok {
return closer.Close()
}
return nil
}
func safetyFactory(t *testing.T, handler func(ssh.Channel, <-chan *ssh.Request)) (SSHFactory, Server) {
t.Helper()
_, privateKey := newTestSSHSigner(t)
hostSigner, _ := newTestSSHSigner(t)
server := startTestSSHServerWithHandler(t, func(ssh.PublicKey) error { return nil }, handler, hostSigner)
model := testSSHServerModel(server, "secret://ssh/safety", ssh.FingerprintSHA256(hostSigner.PublicKey()))
return SSHFactory{
Secrets: staticSecretReader{model.PrivateKeyRef: {"private_key": privateKey}},
Timeout: time.Second, HostKeyAlgorithms: []string{ssh.KeyAlgoED25519},
}, model
}
func requestSFTPHandler(handlers sftp.Handlers, stallVersion, stallCommand bool) func(ssh.Channel, <-chan *ssh.Request) {
return func(channel ssh.Channel, requests <-chan *ssh.Request) {
defer channel.Close()
for request := range requests {
switch request.Type {
case "subsystem":
_ = request.Reply(true, nil)
if stallVersion {
_, _ = io.Copy(io.Discard, channel)
return
}
server := sftp.NewRequestServer(channel, handlers)
_ = server.Serve()
_ = server.Close()
return
case "exec":
_ = request.Reply(true, nil)
if stallCommand {
for request := range requests {
_ = request.Reply(false, nil)
}
return
}
_, _ = channel.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{0}))
return
default:
_ = request.Reply(false, nil)
}
}
}
}
func TestSSHUploadRestrictsPermissionsBeforeWritingAndRejectsExistingFile(t *testing.T) {
for _, denyMode := range []bool{false, true} {
t.Run(map[bool]string{false: "success", true: "chmod_denied"}[denyMode], func(t *testing.T) {
observer := &observedSFTP{handlers: sftp.InMemHandler(), denyMode: denyMode}
handlers := observer.handlers
handlers.FilePut = observer
handlers.FileCmd = observer
factory, server := safetyFactory(t, requestSFTPHandler(handlers, false, false))
client, err := factory.Connect(t.Context(), server)
if err != nil {
t.Fatal(err)
}
defer client.Close()
err = client.Upload(t.Context(), "/key.tmp", []byte("private-test-key"), 0o600)
if denyMode {
if err == nil || observer.writes.Load() != 0 {
t.Fatal("权限设置失败后不得写入私钥")
}
return
}
if err != nil {
t.Fatal(err)
}
if observer.unsafeWrite.Load() || observer.writes.Load() == 0 || !observer.exclusive.Load() {
t.Fatal("上传必须独占创建并在权限收紧后才写入")
}
if err := client.Upload(t.Context(), "/key.tmp", []byte("overwrite"), 0o600); err == nil {
t.Fatal("不得覆盖已有上传目标")
}
})
}
}
func TestSSHFactoryCancelsUnresponsiveSFTPInitialization(t *testing.T) {
factory, server := safetyFactory(t, requestSFTPHandler(sftp.InMemHandler(), true, false))
ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
defer cancel()
start := time.Now()
if _, err := factory.Connect(ctx, server); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("SFTP 握手无响应必须超时:%v", err)
}
if time.Since(start) > 2*time.Second {
t.Fatal("SFTP 握手取消不及时")
}
}
type stalledLister struct{ release <-chan struct{} }
func (l stalledLister) Filelist(*sftp.Request) (sftp.ListerAt, error) {
<-l.release
return nil, errors.New("测试结束")
}
func TestSSHOperationsCancelUnresponsiveServer(t *testing.T) {
for _, operation := range []string{"stat", "command"} {
t.Run(operation, func(t *testing.T) {
handlers := sftp.InMemHandler()
release := make(chan struct{})
defer close(release)
if operation == "stat" {
handlers.FileList = stalledLister{release: release}
}
factory, server := safetyFactory(t, requestSFTPHandler(handlers, false, true))
client, err := factory.Connect(t.Context(), server)
if err != nil {
t.Fatal(err)
}
defer client.Close()
ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
defer cancel()
start := time.Now()
if operation == "stat" {
_, err = client.Exists(ctx, "/test")
} else {
_, err = client.Run(ctx, []string{"sleep", "60"})
}
if !errors.Is(err, context.DeadlineExceeded) || time.Since(start) > 2*time.Second {
t.Fatalf("无响应操作未及时取消:%v", err)
}
if !client.(*sshClientAdapter).Disconnected() {
t.Fatal("取消后必须标记传输已断开以触发重连回滚")
}
})
}
}
func TestSSHDeploymentReconnectsAndRollsBackAfterCommandTimeout(t *testing.T) {
handlers := sftp.InMemHandler()
factory, server := safetyFactory(t, requestSFTPHandler(handlers, false, true))
client, err := factory.Connect(t.Context(), server)
if err != nil {
t.Fatal(err)
}
if err := client.MkdirAll(t.Context(), "/tmp"); err != nil {
t.Fatal(err)
}
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
if err := client.Upload(t.Context(), path, []byte("original"), 0o600); err != nil {
t.Fatal(err)
}
}
_ = client.Close()
ctx, cancel := context.WithTimeout(t.Context(), 400*time.Millisecond)
defer cancel()
report := NewService(factory).Deploy(ctx, []Target{{Server: server}}, testArtifact(t))
if !errors.Is(ctx.Err(), context.DeadlineExceeded) {
t.Fatal("测试必须实际触发命令超时,而不是直接执行失败")
}
if report.Status != "failed" || !strings.Contains(report.Results[0].Message, "文件已回滚") || strings.Contains(report.Results[0].Message, "自动回滚失败") {
t.Fatalf("超时后应重连完成文件回滚:%+v", report)
}
client, err = factory.Connect(t.Context(), server)
if err != nil {
t.Fatal(err)
}
defer client.Close()
adapter := client.(*sshClientAdapter)
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
file, err := adapter.sftp.Open(path)
if err != nil {
t.Fatal(err)
}
data, err := io.ReadAll(file)
_ = file.Close()
if err != nil || !bytes.Equal(data, []byte("original")) {
t.Fatalf("原文件未恢复:%s %v", path, err)
}
}
files, err := adapter.sftp.ReadDir("/tmp")
if err != nil {
t.Fatal(err)
}
if len(files) != 3 {
t.Fatalf("回滚后残留事务文件:%v", files)
}
}
func TestCommandOutputIsBounded(t *testing.T) {
buffer := &commandOutput{}
data := bytes.Repeat([]byte("x"), commandOutputLimit*3)
if written, err := buffer.Write(data); written != len(data) || err != nil {
t.Fatal("截断不应阻塞远程输出")
}
if len(buffer.Bytes()) != commandOutputLimit {
t.Fatal("远程输出未限制内存占用")
}
}