修复权限失效放行并加固证书推送事务和超时回滚
This commit is contained in:
1 parent
d74f74b631
commit
05e05a1a23
15 files changed
+861
-178
No files matched your search
@@ -0,0 +1,53 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// validateArtifact 是传输前的完整性检查;域名覆盖和环境信任策略由证书服务负责。
|
||||
func validateArtifact(artifact Artifact) error {
|
||||
if len(artifact.Certificate) == 0 || len(artifact.FullChain) == 0 || len(artifact.PrivateKey) == 0 {
|
||||
return errors.New("证书部署文件不完整")
|
||||
}
|
||||
certificate, err := tls.X509KeyPair(artifact.Certificate, artifact.PrivateKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("部署证书与私钥不匹配:%w", err)
|
||||
}
|
||||
chain, err := tls.X509KeyPair(artifact.FullChain, artifact.PrivateKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("部署证书链与私钥不匹配:%w", err)
|
||||
}
|
||||
if !bytes.Equal(certificate.Certificate[0], chain.Certificate[0]) {
|
||||
return errors.New("部署证书与完整证书链的叶子证书不一致")
|
||||
}
|
||||
var previous *x509.Certificate
|
||||
remaining := bytes.TrimSpace(artifact.FullChain)
|
||||
now := time.Now()
|
||||
for len(remaining) > 0 {
|
||||
block, rest := pem.Decode(remaining)
|
||||
if block == nil || block.Type != "CERTIFICATE" || !bytes.HasPrefix(remaining, []byte("-----BEGIN CERTIFICATE-----")) {
|
||||
return errors.New("部署证书链包含无效 PEM 内容")
|
||||
}
|
||||
cert, err := x509.ParseCertificate(block.Bytes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("解析部署证书链失败:%w", err)
|
||||
}
|
||||
if now.Before(cert.NotBefore) || !now.Before(cert.NotAfter) {
|
||||
return errors.New("部署证书链包含尚未生效或已过期的证书")
|
||||
}
|
||||
if previous != nil {
|
||||
if err := previous.CheckSignatureFrom(cert); err != nil {
|
||||
return fmt.Errorf("部署证书链签名或顺序无效:%w", err)
|
||||
}
|
||||
}
|
||||
previous = cert
|
||||
remaining = bytes.TrimSpace(rest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package deployment
|
||||
|
||||
import "sync"
|
||||
|
||||
const commandOutputLimit = 64 << 10
|
||||
|
||||
// commandOutput 限制远程标准输出和错误输出占用,防止异常命令耗尽管理端内存。
|
||||
type commandOutput struct {
|
||||
mu sync.Mutex
|
||||
data []byte
|
||||
}
|
||||
|
||||
func (b *commandOutput) Write(data []byte) (int, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
remaining := commandOutputLimit - len(b.data)
|
||||
if remaining > len(data) {
|
||||
remaining = len(data)
|
||||
}
|
||||
b.data = append(b.data, data[:remaining]...)
|
||||
return len(data), nil
|
||||
}
|
||||
|
||||
func (b *commandOutput) Bytes() []byte {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return append([]byte(nil), b.data...)
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/config"
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/secretstore"
|
||||
)
|
||||
|
||||
// TestRemoteDeploymentIsolatedAcceptance 仅在显式配置后运行,不启动业务数据库、DNS 或 ACME。
|
||||
// 每次使用全新隔离目录,保留验收产物;不执行目标机已有服务的校验、重载或重启。
|
||||
func TestRemoteDeploymentIsolatedAcceptance(t *testing.T) {
|
||||
if os.Getenv("SCERT_ACCEPTANCE_CONFIRM") != "isolated-directory" {
|
||||
t.Skip("未启用隔离实机部署验收")
|
||||
}
|
||||
cfg, err := config.Load(os.Getenv("SCERT_ACCEPTANCE_CONFIG"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"master.key", "secrets.json"} {
|
||||
info, err := os.Lstat(filepath.Join(cfg.Paths.Config, name))
|
||||
if err != nil || !info.Mode().IsRegular() {
|
||||
t.Fatalf("现有凭据文件不可用:%s", name)
|
||||
}
|
||||
}
|
||||
store, err := secretstore.Open(cfg.Paths.Config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(os.Getenv("SCERT_ACCEPTANCE_TARGET"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var server Server
|
||||
if err := json.Unmarshal(data, &server); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root := "/etc/ssl/scert-acceptance-" + time.Now().UTC().Format("20060102-150405.000000000")
|
||||
server.CertificatePath = path.Join(root, "cert.pem")
|
||||
server.CertificateChainPath = path.Join(root, "fullchain.pem")
|
||||
server.PrivateKeyPath = path.Join(root, "key.pem")
|
||||
server.ValidationCommand = []string{"/usr/bin/true"}
|
||||
server.ReloadCommand = []string{"/usr/bin/true"}
|
||||
factory := SSHFactory{Secrets: store, Timeout: 10 * time.Second}
|
||||
client, err := factory.Connect(t.Context(), server)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exists, err := client.Exists(t.Context(), root)
|
||||
_ = client.Close()
|
||||
if err != nil || exists {
|
||||
t.Fatalf("隔离目录必须尚不存在:exists=%t err=%v", exists, err)
|
||||
}
|
||||
t.Logf("隔离目标:%s,目录:%s;命令使用 true,不代表真实服务重载验收", server.Address, root)
|
||||
service := NewService(factory)
|
||||
original := testArtifact(t)
|
||||
updated := testArtifact(t)
|
||||
for index, artifact := range []Artifact{original, updated} {
|
||||
report := service.Deploy(t.Context(), []Target{{Server: server}}, artifact)
|
||||
if report.Status != "success" {
|
||||
t.Fatalf("第 %d 次隔离部署失败:%+v", index+1, report)
|
||||
}
|
||||
t.Logf("隔离部署 %d 成功,事务:%s", index+1, report.Results[0].BackupID)
|
||||
}
|
||||
verify := func() {
|
||||
t.Helper()
|
||||
client, err := factory.Connect(t.Context(), server)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.Close()
|
||||
adapter := client.(*sshClientAdapter)
|
||||
for filename, expected := range map[string][]byte{"cert.pem": updated.Certificate, "fullchain.pem": updated.FullChain, "key.pem": updated.PrivateKey} {
|
||||
file, err := adapter.sftp.Open(path.Join(root, filename))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
actual, err := io.ReadAll(file)
|
||||
_ = file.Close()
|
||||
if err != nil || !bytes.Equal(actual, expected) {
|
||||
t.Fatalf("文件内容不一致:%s %v", filename, err)
|
||||
}
|
||||
info, err := adapter.sftp.Stat(path.Join(root, filename))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantMode := os.FileMode(0o644)
|
||||
if filename == "key.pem" {
|
||||
wantMode = 0o600
|
||||
}
|
||||
if info.Mode().Perm() != wantMode {
|
||||
t.Fatalf("文件权限错误:%s %o", filename, info.Mode().Perm())
|
||||
}
|
||||
if filename != "key.pem" {
|
||||
t.Logf("内容与权限验证:%s SHA256=%x mode=%o", filename, sha256.Sum256(actual), wantMode)
|
||||
}
|
||||
}
|
||||
files, err := adapter.sftp.ReadDir(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, file := range files {
|
||||
if strings.Contains(file.Name(), ".scert-tmp-") {
|
||||
t.Fatalf("残留临时文件:%s", file.Name())
|
||||
}
|
||||
if strings.HasPrefix(file.Name(), "key.pem") && file.Mode().Perm() != 0o600 {
|
||||
t.Fatal("私钥或私钥备份权限不安全")
|
||||
}
|
||||
}
|
||||
}
|
||||
verify()
|
||||
for _, scenario := range []string{"validation_failure", "reload_failure", "validation_timeout"} {
|
||||
candidate := server
|
||||
ctx := t.Context()
|
||||
cancel := func() {}
|
||||
switch scenario {
|
||||
case "validation_failure":
|
||||
candidate.ValidationCommand = []string{"/usr/bin/false"}
|
||||
case "reload_failure":
|
||||
candidate.ReloadCommand = []string{"/usr/bin/false"}
|
||||
case "validation_timeout":
|
||||
candidate.ValidationCommand = []string{"/usr/bin/sleep", "30"}
|
||||
ctx, cancel = context.WithTimeout(ctx, 4*time.Second)
|
||||
}
|
||||
report := service.Deploy(ctx, []Target{{Server: candidate}}, original)
|
||||
cancel()
|
||||
if report.Status != "failed" || !strings.Contains(report.Results[0].Message, "文件已回滚") || strings.Contains(report.Results[0].Message, "自动回滚失败") {
|
||||
t.Fatalf("故障场景未安全回滚 %s:%+v", scenario, report)
|
||||
}
|
||||
if scenario == "validation_timeout" && !strings.Contains(report.Results[0].Message, "deadline exceeded") {
|
||||
t.Fatalf("必须实际触发超时:%+v", report)
|
||||
}
|
||||
t.Logf("故障场景 %s:%s", scenario, report.Results[0].Message)
|
||||
verify()
|
||||
}
|
||||
}
|
||||
@@ -109,6 +109,8 @@ func (s *Service) DeploySaved(ctx context.Context, targets []Target, saved Saved
|
||||
}
|
||||
|
||||
func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifact) Result {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
server = server.Normalize()
|
||||
result := Result{ServerID: server.ID, Server: server.Name, Status: "failed"}
|
||||
if err := server.Validate(); err != nil {
|
||||
@@ -120,7 +122,7 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
result.Message = fmt.Sprintf("连接服务器失败:%v", err)
|
||||
return result
|
||||
}
|
||||
defer client.Close()
|
||||
defer func() { _ = client.Close() }()
|
||||
paths := []remoteFile{
|
||||
{path: RemotePath(server.CertificatePath), data: artifact.Certificate, mode: 0o644},
|
||||
{path: RemotePath(server.CertificateChainPath), data: artifact.FullChain, mode: 0o644},
|
||||
@@ -137,6 +139,7 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
return result
|
||||
}
|
||||
backupID := s.backupID()
|
||||
result.BackupID = backupID
|
||||
type backup struct{ original, saved string }
|
||||
backups := make([]backup, 0, len(paths))
|
||||
temporary := make([]string, 0, len(paths))
|
||||
@@ -144,6 +147,14 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
rollback := func() error {
|
||||
rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
|
||||
defer cancel()
|
||||
if state, ok := client.(interface{ Disconnected() bool }); ok && state.Disconnected() {
|
||||
_ = client.Close()
|
||||
reconnected, err := s.factory.Connect(rollbackContext, server)
|
||||
if err != nil {
|
||||
return fmt.Errorf("重新连接服务器以回滚失败:%w", err)
|
||||
}
|
||||
client = reconnected
|
||||
}
|
||||
var rollbackErrors []error
|
||||
for _, path := range temporary {
|
||||
if err := client.Remove(rollbackContext, path); err != nil {
|
||||
@@ -157,15 +168,31 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
}
|
||||
for index := len(backups) - 1; index >= 0; index-- {
|
||||
item := backups[index]
|
||||
// 备份请求超时可能丢失响应,重连后以目标文件是否存在核实实际结果。
|
||||
exists, err := client.Exists(rollbackContext, item.saved)
|
||||
if err != nil {
|
||||
rollbackErrors = append(rollbackErrors, fmt.Errorf("检查备份文件 %s 失败:%w", item.saved, err))
|
||||
continue
|
||||
}
|
||||
if !exists {
|
||||
continue
|
||||
}
|
||||
if err := client.Rename(rollbackContext, item.saved, item.original); err != nil {
|
||||
rollbackErrors = append(rollbackErrors, fmt.Errorf("恢复备份文件 %s 失败:%w", item.original, err))
|
||||
}
|
||||
}
|
||||
return errors.Join(rollbackErrors...)
|
||||
}
|
||||
reloadAttempted := false
|
||||
failed := func(message string) Result {
|
||||
if rollbackErr := rollback(); rollbackErr != nil {
|
||||
message += fmt.Sprintf(";自动回滚失败:%v", rollbackErr)
|
||||
} else {
|
||||
message += ";文件已回滚"
|
||||
}
|
||||
if reloadAttempted {
|
||||
// 命令可能已部分执行;文件回滚并不证明运行中的服务已经恢复。
|
||||
message += ";服务运行状态未确认,请检查服务并在确认旧配置后重新加载"
|
||||
}
|
||||
result.Message = message
|
||||
return result
|
||||
@@ -174,27 +201,31 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
if err := client.MkdirAll(ctx, parentPath(file.path)); err != nil {
|
||||
return failed(fmt.Sprintf("创建远程目录失败:%v", err))
|
||||
}
|
||||
temporaryPath := file.path + ".scert-tmp-" + backupID
|
||||
// 上传失败也可能留下半份私钥,必须在开始上传前登记清理路径。
|
||||
temporary = append(temporary, temporaryPath)
|
||||
if err := client.Upload(ctx, temporaryPath, file.data, file.mode); err != nil {
|
||||
return failed(fmt.Sprintf("上传远程证书失败:%v", err))
|
||||
}
|
||||
}
|
||||
// 所有文件上传完成后才触碰当前版本,避免上传失败造成不必要的证书替换。
|
||||
for _, file := range paths {
|
||||
exists, err := client.Exists(ctx, file.path)
|
||||
if err != nil {
|
||||
return failed(fmt.Sprintf("检查远程文件失败:%v", err))
|
||||
}
|
||||
if exists {
|
||||
backupPath := file.path + ".scert-backup-" + backupID
|
||||
backups = append(backups, backup{original: file.path, saved: backupPath})
|
||||
if err := client.Rename(ctx, file.path, backupPath); err != nil {
|
||||
return failed(fmt.Sprintf("备份远程证书失败:%v", err))
|
||||
}
|
||||
backups = append(backups, backup{original: file.path, saved: backupPath})
|
||||
}
|
||||
temporaryPath := file.path + ".scert-tmp-" + backupID
|
||||
if err := client.Upload(ctx, temporaryPath, file.data, file.mode); err != nil {
|
||||
return failed(fmt.Sprintf("上传远程证书失败:%v", err))
|
||||
}
|
||||
temporary = append(temporary, temporaryPath)
|
||||
installed = append(installed, file.path)
|
||||
if err := client.Rename(ctx, temporaryPath, file.path); err != nil {
|
||||
return failed(fmt.Sprintf("替换远程证书失败:%v", err))
|
||||
}
|
||||
temporary = temporary[:len(temporary)-1]
|
||||
installed = append(installed, file.path)
|
||||
}
|
||||
validationResult, err := client.Run(ctx, server.ValidationCommand)
|
||||
if err != nil || validationResult.ExitCode != 0 {
|
||||
@@ -203,6 +234,7 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
}
|
||||
return failed(fmt.Sprintf("配置校验失败:%v", err))
|
||||
}
|
||||
reloadAttempted = true
|
||||
commandResult, err := client.Run(ctx, server.ReloadCommand)
|
||||
if err != nil || commandResult.ExitCode != 0 {
|
||||
if err == nil {
|
||||
@@ -237,13 +269,6 @@ func validateDeploymentPaths(files []remoteFile) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateArtifact(artifact Artifact) error {
|
||||
if len(artifact.Certificate) == 0 || len(artifact.FullChain) == 0 || len(artifact.PrivateKey) == 0 {
|
||||
return errors.New("证书部署文件不完整")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) backupID() string {
|
||||
var random [6]byte
|
||||
_, _ = rand.Read(random[:])
|
||||
|
||||
@@ -1,8 +1,15 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"math/big"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -38,6 +45,7 @@ func (c *memoryClient) Exists(_ context.Context, path string) (bool, error) {
|
||||
|
||||
func (c *memoryClient) Upload(_ context.Context, path string, data []byte, mode uint32) error {
|
||||
if c.uploadError != "" && strings.Contains(path, c.uploadError) {
|
||||
c.files[path] = memoryFile{data: append([]byte(nil), data[:len(data)/2]...), mode: mode}
|
||||
return errors.New("模拟上传失败")
|
||||
}
|
||||
c.files[path] = memoryFile{data: append([]byte(nil), data...), mode: mode}
|
||||
@@ -105,11 +113,12 @@ func TestDeployReplacesFilesAndKeepsBackup(t *testing.T) {
|
||||
client.files[server.PrivateKeyPath] = memoryFile{data: []byte("old-key"), mode: 0o600}
|
||||
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
|
||||
service.now = func() time.Time { return time.Date(2026, 8, 27, 1, 2, 3, 4, time.UTC) }
|
||||
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact())
|
||||
artifact := testArtifact(t)
|
||||
report := service.Deploy(context.Background(), []Target{{Server: server}}, artifact)
|
||||
if report.Status != "success" || report.Success != 1 {
|
||||
t.Fatalf("部署结果错误:%+v", report)
|
||||
}
|
||||
if string(client.files[server.CertificatePath].data) != "new-cert" || client.files[server.PrivateKeyPath].mode != 0o600 {
|
||||
if !bytes.Equal(client.files[server.CertificatePath].data, artifact.Certificate) || client.files[server.PrivateKeyPath].mode != 0o600 {
|
||||
t.Fatalf("远程证书内容或权限错误:%+v", client.files)
|
||||
}
|
||||
if len(client.commands) != 2 || len(client.commands[0]) != 2 || client.commands[0][1] != "-t" || len(client.commands[1]) != 3 || client.commands[1][0] != "nginx" {
|
||||
@@ -132,7 +141,7 @@ func TestDeployRollsBackWhenReloadFails(t *testing.T) {
|
||||
server := testServer("server-1")
|
||||
client.files[server.CertificatePath] = memoryFile{data: []byte("old-cert"), mode: 0o644}
|
||||
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
|
||||
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact())
|
||||
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact(t))
|
||||
if report.Status != "failed" || report.Failed != 1 {
|
||||
t.Fatalf("reload 失败结果错误:%+v", report)
|
||||
}
|
||||
@@ -157,7 +166,7 @@ func TestDeployRollsBackWhenValidationFailsWithoutReload(t *testing.T) {
|
||||
client.files[server.CertificateChainPath] = memoryFile{data: []byte("old-chain"), mode: 0o644}
|
||||
client.files[server.PrivateKeyPath] = memoryFile{data: []byte("old-key"), mode: 0o600}
|
||||
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
|
||||
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact())
|
||||
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact(t))
|
||||
if report.Status != "failed" || report.Failed != 1 || !strings.Contains(report.Results[0].Message, "配置校验失败") {
|
||||
t.Fatalf("配置校验失败结果错误:%+v", report)
|
||||
}
|
||||
@@ -181,7 +190,7 @@ func TestDeployReportsRollbackFailure(t *testing.T) {
|
||||
server := testServer("server-1")
|
||||
client.files[server.CertificatePath] = memoryFile{data: []byte("old-cert"), mode: 0o644}
|
||||
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
|
||||
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact())
|
||||
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact(t))
|
||||
if report.Status != "failed" || report.Failed != 1 {
|
||||
t.Fatalf("回滚失败结果错误:%+v", report)
|
||||
}
|
||||
@@ -197,19 +206,41 @@ func TestDeployReportsPartialSuccess(t *testing.T) {
|
||||
firstServer := testServer("first")
|
||||
secondServer := testServer("second")
|
||||
service := NewService(memoryFactory{clients: map[string]*memoryClient{"first": first, "second": second}})
|
||||
report := service.Deploy(context.Background(), []Target{{Server: firstServer}, {Server: secondServer}}, testArtifact())
|
||||
report := service.Deploy(context.Background(), []Target{{Server: firstServer}, {Server: secondServer}}, testArtifact(t))
|
||||
if report.Status != "partial" || report.Success != 1 || report.Failed != 1 {
|
||||
t.Fatalf("部分成功汇总错误:%+v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeployCleansPartialPrivateKeyUploadWithoutReplacingCurrentFiles(t *testing.T) {
|
||||
client := newMemoryClient()
|
||||
client.uploadError = "key.pem"
|
||||
server := testServer("partial-key")
|
||||
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
|
||||
client.files[path] = memoryFile{data: []byte("original"), mode: 0o600}
|
||||
}
|
||||
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
|
||||
report := service.Deploy(t.Context(), []Target{{Server: server}}, testArtifact(t))
|
||||
if report.Status != "failed" || report.Results[0].BackupID == "" {
|
||||
t.Fatalf("失败报告必须包含事务标识:%+v", report)
|
||||
}
|
||||
if len(client.files) != 3 || len(client.commands) != 0 {
|
||||
t.Fatalf("部分上传失败不得残留临时文件或执行命令:files=%v commands=%v", client.files, client.commands)
|
||||
}
|
||||
for path, file := range client.files {
|
||||
if string(file.data) != "original" || file.mode != 0o600 {
|
||||
t.Fatalf("上传阶段不得替换原文件:%s", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeployRejectsDuplicateCertificatePaths(t *testing.T) {
|
||||
client := newMemoryClient()
|
||||
server := testServer("server-1")
|
||||
server.PrivateKeyPath = server.CertificatePath
|
||||
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
|
||||
|
||||
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact())
|
||||
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact(t))
|
||||
if report.Status != "failed" || report.Failed != 1 {
|
||||
t.Fatalf("重复远程路径应直接失败:%+v", report)
|
||||
}
|
||||
@@ -227,7 +258,7 @@ func TestDeployRejectsRelativeCertificatePaths(t *testing.T) {
|
||||
server.CertificatePath = "cert.pem"
|
||||
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
|
||||
|
||||
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact())
|
||||
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact(t))
|
||||
if report.Status != "failed" || !strings.Contains(report.Results[0].Message, "必须是绝对路径") {
|
||||
t.Fatalf("相对远程路径应被拒绝:%+v", report)
|
||||
}
|
||||
@@ -242,6 +273,48 @@ func testServer(id string) Server {
|
||||
}
|
||||
}
|
||||
|
||||
func testArtifact() Artifact {
|
||||
return Artifact{Certificate: []byte("new-cert"), FullChain: []byte("new-chain"), PrivateKey: []byte("new-key")}
|
||||
func testArtifact(t *testing.T) Artifact {
|
||||
t.Helper()
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
template := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(time.Now().UnixNano()), DNSNames: []string{"example.com"},
|
||||
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
privateKey, err := x509.MarshalPKCS8PrivateKey(key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||
return Artifact{Certificate: cert, FullChain: append([]byte(nil), cert...), PrivateKey: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateKey})}
|
||||
}
|
||||
|
||||
func TestDeployRejectsCorruptArtifactsBeforeConnecting(t *testing.T) {
|
||||
for _, scenario := range []string{"invalid_pem", "wrong_key", "wrong_leaf", "garbage_chain"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
artifact := testArtifact(t)
|
||||
switch scenario {
|
||||
case "invalid_pem":
|
||||
artifact.Certificate = []byte("invalid")
|
||||
case "wrong_key":
|
||||
artifact.PrivateKey = testArtifact(t).PrivateKey
|
||||
case "wrong_leaf":
|
||||
artifact.FullChain = testArtifact(t).FullChain
|
||||
case "garbage_chain":
|
||||
artifact.FullChain = append(artifact.FullChain, []byte("invalid")...)
|
||||
}
|
||||
service := NewService(memoryFactory{})
|
||||
report := service.Deploy(t.Context(), []Target{{Server: testServer("invalid")}}, artifact)
|
||||
if report.Status != "failed" {
|
||||
t.Fatalf("无效产物未被拒绝:%+v", report)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/sftp"
|
||||
@@ -64,12 +65,16 @@ func (f SSHFactory) Connect(ctx context.Context, server Server) (Client, error)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sftpClient, err := sftp.NewClient(sshClient)
|
||||
adapter := &sshClientAdapter{ssh: sshClient}
|
||||
sftpClient, err := sshOperation(timeoutContext, adapter, func() (*sftp.Client, error) {
|
||||
return sftp.NewClient(sshClient)
|
||||
})
|
||||
if err != nil {
|
||||
_ = sshClient.Close()
|
||||
return nil, fmt.Errorf("创建 SFTP 客户端失败:%w", err)
|
||||
}
|
||||
return &sshClientAdapter{ssh: sshClient, sftp: sftpClient}, nil
|
||||
adapter.sftp = sftpClient
|
||||
return adapter, nil
|
||||
}
|
||||
|
||||
func (f SSHFactory) secureHostKeyAlgorithms() ([]string, error) {
|
||||
@@ -138,7 +143,13 @@ func connectSSHClient(
|
||||
return nil
|
||||
},
|
||||
}
|
||||
stopCancel := context.AfterFunc(ctx, func() { _ = connection.Close() })
|
||||
clientConn, channels, requests, handshakeErr := ssh.NewClientConn(connection, address, config)
|
||||
stopCancel()
|
||||
if err := ctx.Err(); err != nil {
|
||||
_ = connection.Close()
|
||||
return nil, fmt.Errorf("SSH 握手已取消:%w", err)
|
||||
}
|
||||
if handshakeErr == nil {
|
||||
if err := connection.SetDeadline(time.Time{}); err != nil {
|
||||
_ = clientConn.Close()
|
||||
@@ -174,64 +185,96 @@ func connectSSHClient(
|
||||
}
|
||||
|
||||
type sshClientAdapter struct {
|
||||
ssh *ssh.Client
|
||||
sftp *sftp.Client
|
||||
ssh *ssh.Client
|
||||
sftp *sftp.Client
|
||||
disconnected atomic.Bool
|
||||
}
|
||||
|
||||
// sshOperation 取消时关闭传输并等待操作退出,避免回滚与仍在写入的操作并行。
|
||||
// 已断开的连接由部署事务使用独立回滚上下文重新建立,仍严格校验主机指纹。
|
||||
func sshOperation[T any](ctx context.Context, c *sshClientAdapter, operation func() (T, error)) (T, error) {
|
||||
var zero T
|
||||
if err := ctx.Err(); err != nil {
|
||||
return zero, err
|
||||
}
|
||||
type outcome struct {
|
||||
value T
|
||||
err error
|
||||
}
|
||||
done := make(chan outcome, 1)
|
||||
go func() {
|
||||
value, err := operation()
|
||||
done <- outcome{value, err}
|
||||
}()
|
||||
select {
|
||||
case result := <-done:
|
||||
if err := ctx.Err(); err != nil {
|
||||
c.disconnected.Store(true)
|
||||
_ = c.ssh.Close()
|
||||
return zero, err
|
||||
}
|
||||
return result.value, result.err
|
||||
case <-ctx.Done():
|
||||
c.disconnected.Store(true)
|
||||
_ = c.ssh.Close()
|
||||
<-done
|
||||
return zero, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *sshClientAdapter) Disconnected() bool { return c.disconnected.Load() }
|
||||
|
||||
func (c *sshClientAdapter) fileOperation(ctx context.Context, operation func() error) error {
|
||||
_, err := sshOperation(ctx, c, func() (struct{}, error) { return struct{}{}, operation() })
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *sshClientAdapter) MkdirAll(ctx context.Context, path string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return c.sftp.MkdirAll(path)
|
||||
return c.fileOperation(ctx, func() error { return c.sftp.MkdirAll(path) })
|
||||
}
|
||||
|
||||
func (c *sshClientAdapter) Exists(ctx context.Context, path string) (bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
_, err := c.sftp.Stat(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
return sshOperation(ctx, c, func() (bool, error) {
|
||||
_, err := c.sftp.Stat(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
})
|
||||
}
|
||||
|
||||
func (c *sshClientAdapter) Upload(ctx context.Context, path string, data []byte, mode uint32) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := c.sftp.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
closeFile := func() error { return file.Close() }
|
||||
if _, err := file.Write(data); err != nil {
|
||||
_ = closeFile()
|
||||
return err
|
||||
}
|
||||
if err := file.Chmod(os.FileMode(mode)); err != nil {
|
||||
_ = closeFile()
|
||||
return err
|
||||
}
|
||||
return closeFile()
|
||||
return c.fileOperation(ctx, func() error {
|
||||
// 只创建新临时文件,不跟随已有符号链接,也不截断已存在的文件。
|
||||
file, err := c.sftp.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 先收紧空文件权限,再写入任何证书或私钥内容。
|
||||
if err := file.Chmod(os.FileMode(mode)); err != nil {
|
||||
_ = file.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := file.Write(data); err != nil {
|
||||
_ = file.Close()
|
||||
return err
|
||||
}
|
||||
return file.Close()
|
||||
})
|
||||
}
|
||||
|
||||
func (c *sshClientAdapter) Rename(ctx context.Context, source, target string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return c.sftp.Rename(source, target)
|
||||
return c.fileOperation(ctx, func() error { return c.sftp.Rename(source, target) })
|
||||
}
|
||||
|
||||
func (c *sshClientAdapter) Remove(ctx context.Context, path string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return c.fileOperation(ctx, func() error {
|
||||
err := c.sftp.Remove(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
err := c.sftp.Remove(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func (c *sshClientAdapter) Run(ctx context.Context, command []string) (CommandResult, error) {
|
||||
@@ -241,12 +284,20 @@ func (c *sshClientAdapter) Run(ctx context.Context, command []string) (CommandRe
|
||||
if err := validateCommand(command); err != nil {
|
||||
return CommandResult{}, err
|
||||
}
|
||||
return sshOperation(ctx, c, func() (CommandResult, error) { return c.runCommand(command) })
|
||||
}
|
||||
|
||||
func (c *sshClientAdapter) runCommand(command []string) (CommandResult, error) {
|
||||
session, err := c.ssh.NewSession()
|
||||
if err != nil {
|
||||
return CommandResult{}, err
|
||||
}
|
||||
defer session.Close()
|
||||
output, err := session.CombinedOutput(joinShellArgs(command))
|
||||
outputBuffer := &commandOutput{}
|
||||
session.Stdout = outputBuffer
|
||||
session.Stderr = outputBuffer
|
||||
err = session.Run(joinShellArgs(command))
|
||||
output := outputBuffer.Bytes()
|
||||
if err != nil {
|
||||
var exitError *ssh.ExitError
|
||||
if errors.As(err, &exitError) {
|
||||
@@ -254,21 +305,18 @@ func (c *sshClientAdapter) Run(ctx context.Context, command []string) (CommandRe
|
||||
}
|
||||
return CommandResult{Output: output, ExitCode: -1}, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return CommandResult{Output: output, ExitCode: -1}, err
|
||||
}
|
||||
return CommandResult{Output: output}, nil
|
||||
}
|
||||
|
||||
func (c *sshClientAdapter) Close() error {
|
||||
var first error
|
||||
if c.sftp != nil {
|
||||
if err := c.sftp.Close(); err != nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
// 先关闭传输,避免无响应的 SFTP 服务阻塞关闭过程。
|
||||
if c.ssh != nil {
|
||||
if err := c.ssh.Close(); err != nil && first == nil {
|
||||
c.disconnected.Store(true)
|
||||
first = c.ssh.Close()
|
||||
}
|
||||
if c.sftp != nil {
|
||||
if err := c.sftp.Close(); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,17 +182,27 @@ func TestSSHFactoryFiltersUnsupportedAndDuplicateHostKeyAlgorithms(t *testing.T)
|
||||
}
|
||||
|
||||
type testSSHServer struct {
|
||||
listener net.Listener
|
||||
accepted atomic.Int32
|
||||
host string
|
||||
port int
|
||||
config *ssh.ServerConfig
|
||||
listener net.Listener
|
||||
accepted atomic.Int32
|
||||
host string
|
||||
port int
|
||||
config *ssh.ServerConfig
|
||||
channelHandler func(ssh.Channel, <-chan *ssh.Request)
|
||||
}
|
||||
|
||||
func startTestSSHServer(
|
||||
t *testing.T,
|
||||
authorize func(ssh.PublicKey) error,
|
||||
hostSigners ...ssh.Signer,
|
||||
) *testSSHServer {
|
||||
return startTestSSHServerWithHandler(t, authorize, serveTestSFTP, hostSigners...)
|
||||
}
|
||||
|
||||
func startTestSSHServerWithHandler(
|
||||
t *testing.T,
|
||||
authorize func(ssh.PublicKey) error,
|
||||
handler func(ssh.Channel, <-chan *ssh.Request),
|
||||
hostSigners ...ssh.Signer,
|
||||
) *testSSHServer {
|
||||
t.Helper()
|
||||
config := &ssh.ServerConfig{
|
||||
@@ -218,10 +228,11 @@ func startTestSSHServer(
|
||||
t.Fatalf("解析测试 SSH 端口失败:%v", err)
|
||||
}
|
||||
server := &testSSHServer{
|
||||
listener: listener,
|
||||
host: host,
|
||||
port: port,
|
||||
config: config,
|
||||
listener: listener,
|
||||
host: host,
|
||||
port: port,
|
||||
config: config,
|
||||
channelHandler: handler,
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
listener.Close()
|
||||
@@ -257,7 +268,7 @@ func (s *testSSHServer) serveConnection(connection net.Conn) {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
go serveTestSFTP(channel, requests)
|
||||
go s.channelHandler(channel, requests)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,255 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/sftp"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
type observedSFTP struct {
|
||||
handlers sftp.Handlers
|
||||
mode atomic.Uint32
|
||||
writes atomic.Int32
|
||||
unsafeWrite atomic.Bool
|
||||
exclusive atomic.Bool
|
||||
denyMode bool
|
||||
}
|
||||
|
||||
func (h *observedSFTP) Filewrite(request *sftp.Request) (io.WriterAt, error) {
|
||||
h.exclusive.Store(request.Pflags().Excl && !request.Pflags().Trunc)
|
||||
writer, err := h.handlers.FilePut.Filewrite(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &observedWriter{WriterAt: writer, owner: h}, nil
|
||||
}
|
||||
|
||||
func (h *observedSFTP) Filecmd(request *sftp.Request) error {
|
||||
if request.Method == "Setstat" {
|
||||
if h.denyMode {
|
||||
return errors.New("测试拒绝权限设置")
|
||||
}
|
||||
h.mode.Store(request.Attributes().Mode & 0o777)
|
||||
}
|
||||
return h.handlers.FileCmd.Filecmd(request)
|
||||
}
|
||||
|
||||
type observedWriter struct {
|
||||
io.WriterAt
|
||||
owner *observedSFTP
|
||||
}
|
||||
|
||||
func (w *observedWriter) WriteAt(data []byte, offset int64) (int, error) {
|
||||
w.owner.writes.Add(1)
|
||||
if w.owner.mode.Load() != 0o600 {
|
||||
w.owner.unsafeWrite.Store(true)
|
||||
}
|
||||
return w.WriterAt.WriteAt(data, offset)
|
||||
}
|
||||
|
||||
func (w *observedWriter) Close() error {
|
||||
if closer, ok := w.WriterAt.(io.Closer); ok {
|
||||
return closer.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func safetyFactory(t *testing.T, handler func(ssh.Channel, <-chan *ssh.Request)) (SSHFactory, Server) {
|
||||
t.Helper()
|
||||
_, privateKey := newTestSSHSigner(t)
|
||||
hostSigner, _ := newTestSSHSigner(t)
|
||||
server := startTestSSHServerWithHandler(t, func(ssh.PublicKey) error { return nil }, handler, hostSigner)
|
||||
model := testSSHServerModel(server, "secret://ssh/safety", ssh.FingerprintSHA256(hostSigner.PublicKey()))
|
||||
return SSHFactory{
|
||||
Secrets: staticSecretReader{model.PrivateKeyRef: {"private_key": privateKey}},
|
||||
Timeout: time.Second, HostKeyAlgorithms: []string{ssh.KeyAlgoED25519},
|
||||
}, model
|
||||
}
|
||||
|
||||
func requestSFTPHandler(handlers sftp.Handlers, stallVersion, stallCommand bool) func(ssh.Channel, <-chan *ssh.Request) {
|
||||
return func(channel ssh.Channel, requests <-chan *ssh.Request) {
|
||||
defer channel.Close()
|
||||
for request := range requests {
|
||||
switch request.Type {
|
||||
case "subsystem":
|
||||
_ = request.Reply(true, nil)
|
||||
if stallVersion {
|
||||
_, _ = io.Copy(io.Discard, channel)
|
||||
return
|
||||
}
|
||||
server := sftp.NewRequestServer(channel, handlers)
|
||||
_ = server.Serve()
|
||||
_ = server.Close()
|
||||
return
|
||||
case "exec":
|
||||
_ = request.Reply(true, nil)
|
||||
if stallCommand {
|
||||
for request := range requests {
|
||||
_ = request.Reply(false, nil)
|
||||
}
|
||||
return
|
||||
}
|
||||
_, _ = channel.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{0}))
|
||||
return
|
||||
default:
|
||||
_ = request.Reply(false, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSHUploadRestrictsPermissionsBeforeWritingAndRejectsExistingFile(t *testing.T) {
|
||||
for _, denyMode := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "success", true: "chmod_denied"}[denyMode], func(t *testing.T) {
|
||||
observer := &observedSFTP{handlers: sftp.InMemHandler(), denyMode: denyMode}
|
||||
handlers := observer.handlers
|
||||
handlers.FilePut = observer
|
||||
handlers.FileCmd = observer
|
||||
factory, server := safetyFactory(t, requestSFTPHandler(handlers, false, false))
|
||||
client, err := factory.Connect(t.Context(), server)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.Close()
|
||||
err = client.Upload(t.Context(), "/key.tmp", []byte("private-test-key"), 0o600)
|
||||
if denyMode {
|
||||
if err == nil || observer.writes.Load() != 0 {
|
||||
t.Fatal("权限设置失败后不得写入私钥")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if observer.unsafeWrite.Load() || observer.writes.Load() == 0 || !observer.exclusive.Load() {
|
||||
t.Fatal("上传必须独占创建并在权限收紧后才写入")
|
||||
}
|
||||
if err := client.Upload(t.Context(), "/key.tmp", []byte("overwrite"), 0o600); err == nil {
|
||||
t.Fatal("不得覆盖已有上传目标")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSHFactoryCancelsUnresponsiveSFTPInitialization(t *testing.T) {
|
||||
factory, server := safetyFactory(t, requestSFTPHandler(sftp.InMemHandler(), true, false))
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
if _, err := factory.Connect(ctx, server); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("SFTP 握手无响应必须超时:%v", err)
|
||||
}
|
||||
if time.Since(start) > 2*time.Second {
|
||||
t.Fatal("SFTP 握手取消不及时")
|
||||
}
|
||||
}
|
||||
|
||||
type stalledLister struct{ release <-chan struct{} }
|
||||
|
||||
func (l stalledLister) Filelist(*sftp.Request) (sftp.ListerAt, error) {
|
||||
<-l.release
|
||||
return nil, errors.New("测试结束")
|
||||
}
|
||||
|
||||
func TestSSHOperationsCancelUnresponsiveServer(t *testing.T) {
|
||||
for _, operation := range []string{"stat", "command"} {
|
||||
t.Run(operation, func(t *testing.T) {
|
||||
handlers := sftp.InMemHandler()
|
||||
release := make(chan struct{})
|
||||
defer close(release)
|
||||
if operation == "stat" {
|
||||
handlers.FileList = stalledLister{release: release}
|
||||
}
|
||||
factory, server := safetyFactory(t, requestSFTPHandler(handlers, false, true))
|
||||
client, err := factory.Connect(t.Context(), server)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.Close()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
if operation == "stat" {
|
||||
_, err = client.Exists(ctx, "/test")
|
||||
} else {
|
||||
_, err = client.Run(ctx, []string{"sleep", "60"})
|
||||
}
|
||||
if !errors.Is(err, context.DeadlineExceeded) || time.Since(start) > 2*time.Second {
|
||||
t.Fatalf("无响应操作未及时取消:%v", err)
|
||||
}
|
||||
if !client.(*sshClientAdapter).Disconnected() {
|
||||
t.Fatal("取消后必须标记传输已断开以触发重连回滚")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSHDeploymentReconnectsAndRollsBackAfterCommandTimeout(t *testing.T) {
|
||||
handlers := sftp.InMemHandler()
|
||||
factory, server := safetyFactory(t, requestSFTPHandler(handlers, false, true))
|
||||
client, err := factory.Connect(t.Context(), server)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := client.MkdirAll(t.Context(), "/tmp"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
|
||||
if err := client.Upload(t.Context(), path, []byte("original"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_ = client.Close()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 400*time.Millisecond)
|
||||
defer cancel()
|
||||
report := NewService(factory).Deploy(ctx, []Target{{Server: server}}, testArtifact(t))
|
||||
if !errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||
t.Fatal("测试必须实际触发命令超时,而不是直接执行失败")
|
||||
}
|
||||
if report.Status != "failed" || !strings.Contains(report.Results[0].Message, "文件已回滚") || strings.Contains(report.Results[0].Message, "自动回滚失败") {
|
||||
t.Fatalf("超时后应重连完成文件回滚:%+v", report)
|
||||
}
|
||||
client, err = factory.Connect(t.Context(), server)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.Close()
|
||||
adapter := client.(*sshClientAdapter)
|
||||
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
|
||||
file, err := adapter.sftp.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := io.ReadAll(file)
|
||||
_ = file.Close()
|
||||
if err != nil || !bytes.Equal(data, []byte("original")) {
|
||||
t.Fatalf("原文件未恢复:%s %v", path, err)
|
||||
}
|
||||
}
|
||||
files, err := adapter.sftp.ReadDir("/tmp")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(files) != 3 {
|
||||
t.Fatalf("回滚后残留事务文件:%v", files)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommandOutputIsBounded(t *testing.T) {
|
||||
buffer := &commandOutput{}
|
||||
data := bytes.Repeat([]byte("x"), commandOutputLimit*3)
|
||||
if written, err := buffer.Write(data); written != len(data) || err != nil {
|
||||
t.Fatal("截断不应阻塞远程输出")
|
||||
}
|
||||
if len(buffer.Bytes()) != commandOutputLimit {
|
||||
t.Fatal("远程输出未限制内存占用")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user