完成初始化证书事务和远程部署互斥并纳入发布构建脚本

This commit is contained in:
xuans committed 2026-09-15 20:17:34 +08:00
1 parent 05e05a1a23
commit a4dffb04d4
27 files changed
+1294 -25

No files matched your search

+2
View File
@@ -8,6 +8,8 @@ type CommandResult struct {
}
type Client interface {
Lock(ctx context.Context, path string) error
Unlock(ctx context.Context, path string) error
MkdirAll(ctx context.Context, path string) error
Exists(ctx context.Context, path string) (bool, error)
Upload(ctx context.Context, path string, data []byte, mode uint32) error
@@ -113,6 +113,9 @@ func TestRemoteDeploymentIsolatedAcceptance(t *testing.T) {
if strings.Contains(file.Name(), ".scert-tmp-") {
t.Fatalf("残留临时文件:%s", file.Name())
}
if strings.HasSuffix(file.Name(), ".scert-lock") {
t.Fatalf("残留部署锁:%s", file.Name())
}
if strings.HasPrefix(file.Name(), "key.pem") && file.Mode().Perm() != 0o600 {
t.Fatal("私钥或私钥备份权限不安全")
}
+64 -12
View File
@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"os"
"sort"
"strings"
"time"
)
@@ -108,11 +109,11 @@ func (s *Service) DeploySaved(ctx context.Context, targets []Target, saved Saved
return s.Deploy(ctx, targets, Artifact{Certificate: certificateData, FullChain: fullChainData, PrivateKey: privateKeyData}), nil
}
func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifact) Result {
func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifact) (result Result) {
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
server = server.Normalize()
result := Result{ServerID: server.ID, Server: server.Name, Status: "failed"}
result = Result{ServerID: server.ID, Server: server.Name, Status: "failed"}
if err := server.Validate(); err != nil {
result.Message = err.Error()
return result
@@ -138,6 +139,43 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
result.Message = err.Error()
return result
}
// 每个实际文件路径使用远程协作锁,阻止不同任务或进程的替换与回滚相互覆盖。
lockPaths := make([]string, 0, len(paths))
for _, file := range paths {
if err := client.MkdirAll(ctx, parentPath(file.path)); err != nil {
result.Message = fmt.Sprintf("创建远程目录失败:%v", err)
return result
}
lockPaths = append(lockPaths, file.path+".scert-lock")
}
sort.Strings(lockPaths)
heldLocks := make([]string, 0, len(lockPaths))
retainLocks := false
defer func() {
if len(heldLocks) == 0 || retainLocks {
return
}
releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
defer cancel()
if err := reconnectDeploymentClient(releaseCtx, s.factory, server, &client); err != nil {
result.Status = "failed"
result.Message += fmt.Sprintf(";释放部署锁失败,需人工检查:%v", err)
return
}
for index := len(heldLocks) - 1; index >= 0; index-- {
if err := client.Unlock(releaseCtx, heldLocks[index]); err != nil {
result.Status = "failed"
result.Message += fmt.Sprintf(";释放部署锁 %s 失败,需人工检查:%v", heldLocks[index], err)
}
}
}()
for _, lockPath := range lockPaths {
if err := client.Lock(ctx, lockPath); err != nil {
result.Message = fmt.Sprintf("无法取得部署锁 %s,可能存在并发部署或遗留锁;未修改当前证书,请人工核查:%v", lockPath, err)
return result
}
heldLocks = append(heldLocks, lockPath)
}
backupID := s.backupID()
result.BackupID = backupID
type backup struct{ original, saved string }
@@ -147,13 +185,8 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
rollback := func() error {
rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
defer cancel()
if state, ok := client.(interface{ Disconnected() bool }); ok && state.Disconnected() {
_ = client.Close()
reconnected, err := s.factory.Connect(rollbackContext, server)
if err != nil {
return fmt.Errorf("重新连接服务器以回滚失败:%w", err)
}
client = reconnected
if err := reconnectDeploymentClient(rollbackContext, s.factory, server, &client); err != nil {
return fmt.Errorf("重新连接服务器以回滚失败:%w", err)
}
var rollbackErrors []error
for _, path := range temporary {
@@ -187,6 +220,8 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
failed := func(message string) Result {
if rollbackErr := rollback(); rollbackErr != nil {
message += fmt.Sprintf(";自动回滚失败:%v", rollbackErr)
retainLocks = true
message += ";已保留本次部署锁,修复并确认文件状态后再由管理员释放"
} else {
message += ";文件已回滚"
}
@@ -198,13 +233,15 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
return result
}
for _, file := range paths {
if err := client.MkdirAll(ctx, parentPath(file.path)); err != nil {
return failed(fmt.Sprintf("创建远程目录失败:%v", err))
}
temporaryPath := file.path + ".scert-tmp-" + backupID
// 上传失败也可能留下半份私钥,必须在开始上传前登记清理路径。
temporary = append(temporary, temporaryPath)
if err := client.Upload(ctx, temporaryPath, file.data, file.mode); err != nil {
var notCreated *uploadNotCreatedError
if errors.As(err, &notCreated) {
temporary = temporary[:len(temporary)-1]
return failed(fmt.Sprintf("上传失败且未确认临时路径所有权,已保留 %s 供人工核查:%v", temporaryPath, err))
}
return failed(fmt.Sprintf("上传远程证书失败:%v", err))
}
}
@@ -216,6 +253,9 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
}
if exists {
backupPath := file.path + ".scert-backup-" + backupID
if exists, err := client.Exists(ctx, backupPath); err != nil || exists {
return failed(fmt.Sprintf("备份目标已存在或无法检查,已停止替换:%s:%v", backupPath, err))
}
backups = append(backups, backup{original: file.path, saved: backupPath})
if err := client.Rename(ctx, file.path, backupPath); err != nil {
return failed(fmt.Sprintf("备份远程证书失败:%v", err))
@@ -248,6 +288,18 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
return result
}
func reconnectDeploymentClient(ctx context.Context, factory ClientFactory, server Server, client *Client) error {
if state, ok := (*client).(interface{ Disconnected() bool }); ok && state.Disconnected() {
_ = (*client).Close()
reconnected, err := factory.Connect(ctx, server)
if err != nil {
return err
}
*client = reconnected
}
return nil
}
type remoteFile struct {
path string
data []byte
@@ -38,6 +38,19 @@ func newMemoryClient() *memoryClient {
func (c *memoryClient) MkdirAll(context.Context, string) error { return nil }
func (c *memoryClient) Lock(_ context.Context, path string) error {
if _, exists := c.files[path]; exists {
return errors.New("模拟锁已存在")
}
c.files[path] = memoryFile{}
return nil
}
func (c *memoryClient) Unlock(_ context.Context, path string) error {
delete(c.files, path)
return nil
}
func (c *memoryClient) Exists(_ context.Context, path string) (bool, error) {
_, ok := c.files[path]
return ok, nil
@@ -197,6 +210,11 @@ func TestDeployReportsRollbackFailure(t *testing.T) {
if !strings.Contains(report.Results[0].Message, "自动回滚失败") || !strings.Contains(report.Results[0].Message, "恢复备份文件") {
t.Fatalf("应报告自动回滚失败原因:%s", report.Results[0].Message)
}
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
if _, exists := client.files[path+".scert-lock"]; !exists {
t.Fatal("回滚失败后必须保留协作锁,阻止后续任务覆盖待恢复现场")
}
}
}
func TestDeployReportsPartialSuccess(t *testing.T) {
+27 -1
View File
@@ -213,6 +213,11 @@ func sshOperation[T any](ctx context.Context, c *sshClientAdapter, operation fun
_ = c.ssh.Close()
return zero, err
}
var networkErr *net.OpError
if errors.Is(result.err, io.EOF) || errors.Is(result.err, io.ErrUnexpectedEOF) || errors.Is(result.err, net.ErrClosed) || errors.As(result.err, &networkErr) {
c.disconnected.Store(true)
_ = c.ssh.Close()
}
return result.value, result.err
case <-ctx.Done():
c.disconnected.Store(true)
@@ -233,6 +238,15 @@ func (c *sshClientAdapter) MkdirAll(ctx context.Context, path string) error {
return c.fileOperation(ctx, func() error { return c.sftp.MkdirAll(path) })
}
// 远程原子建目录用于协作锁,跨进程共享;已有锁绝不自动抢占或清理。
func (c *sshClientAdapter) Lock(ctx context.Context, path string) error {
return c.fileOperation(ctx, func() error { return c.sftp.Mkdir(path) })
}
func (c *sshClientAdapter) Unlock(ctx context.Context, path string) error {
return c.fileOperation(ctx, func() error { return c.sftp.RemoveDirectory(path) })
}
func (c *sshClientAdapter) Exists(ctx context.Context, path string) (bool, error) {
return sshOperation(ctx, c, func() (bool, error) {
_, err := c.sftp.Stat(path)
@@ -244,12 +258,14 @@ func (c *sshClientAdapter) Exists(ctx context.Context, path string) (bool, error
}
func (c *sshClientAdapter) Upload(ctx context.Context, path string, data []byte, mode uint32) error {
return c.fileOperation(ctx, func() error {
var created atomic.Bool
err := c.fileOperation(ctx, func() error {
// 只创建新临时文件,不跟随已有符号链接,也不截断已存在的文件。
file, err := c.sftp.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL)
if err != nil {
return err
}
created.Store(true)
// 先收紧空文件权限,再写入任何证书或私钥内容。
if err := file.Chmod(os.FileMode(mode)); err != nil {
_ = file.Close()
@@ -261,8 +277,18 @@ func (c *sshClientAdapter) Upload(ctx context.Context, path string, data []byte,
}
return file.Close()
})
if err != nil && !created.Load() {
// 未确认取得所有权的路径不得交给事务删除,包括创建响应丢失的情况。
return &uploadNotCreatedError{err: err}
}
return err
}
type uploadNotCreatedError struct{ err error }
func (e *uploadNotCreatedError) Error() string { return e.err.Error() }
func (e *uploadNotCreatedError) Unwrap() error { return e.err }
func (c *sshClientAdapter) Rename(ctx context.Context, source, target string) error {
return c.fileOperation(ctx, func() error { return c.sftp.Rename(source, target) })
}
@@ -152,6 +152,36 @@ func TestSSHFactoryCancelsUnresponsiveSFTPInitialization(t *testing.T) {
}
}
func TestSSHDeploymentLockIsSharedAcrossConnections(t *testing.T) {
factory, server := safetyFactory(t, requestSFTPHandler(sftp.InMemHandler(), false, false))
first, err := factory.Connect(t.Context(), server)
if err != nil {
t.Fatal(err)
}
defer first.Close()
second, err := factory.Connect(t.Context(), server)
if err != nil {
t.Fatal(err)
}
defer second.Close()
const path = "/certificate.scert-lock"
if err := first.Lock(t.Context(), path); err != nil {
t.Fatal(err)
}
if err := second.Lock(t.Context(), path); err == nil {
t.Fatal("其他 SSH 连接不得取得同一个远程部署锁")
}
if err := first.Unlock(t.Context(), path); err != nil {
t.Fatal(err)
}
if err := second.Lock(t.Context(), path); err != nil {
t.Fatalf("释放锁后应允许后续部署:%v", err)
}
if err := second.Unlock(t.Context(), path); err != nil {
t.Fatal(err)
}
}
type stalledLister struct{ release <-chan struct{} }
func (l stalledLister) Filelist(*sftp.Request) (sftp.ListerAt, error) {
@@ -0,0 +1,110 @@
package deployment
import (
"context"
"errors"
"strings"
"testing"
)
func TestDeployLeavesExistingLockAndCertificateUntouched(t *testing.T) {
client := newMemoryClient()
server := testServer("locked")
lockPath := server.PrivateKeyPath + ".scert-lock"
client.files[lockPath] = memoryFile{data: []byte("other-transaction")}
client.files[server.CertificatePath] = memoryFile{data: []byte("current-certificate")}
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
report := service.Deploy(t.Context(), []Target{{Server: server}}, testArtifact(t))
if report.Status != "failed" || !strings.Contains(report.Results[0].Message, "部署锁") {
t.Fatalf("已有部署锁时必须停止:%+v", report)
}
if len(client.files) != 2 || string(client.files[lockPath].data) != "other-transaction" || string(client.files[server.CertificatePath].data) != "current-certificate" || len(client.commands) != 0 {
t.Fatal("锁冲突时不得覆盖证书、移除他人的锁或遗留自己取得的锁")
}
}
type collisionClient struct {
*memoryClient
collided string
}
func (c *collisionClient) Upload(_ context.Context, path string, _ []byte, _ uint32) error {
c.collided = path
c.files[path] = memoryFile{data: []byte("not-owned")}
return &uploadNotCreatedError{err: errors.New("测试独占创建冲突")}
}
type fixedClientFactory struct{ client Client }
func (f fixedClientFactory) Connect(context.Context, Server) (Client, error) { return f.client, nil }
func TestDeployDoesNotRemoveUnownedUploadPath(t *testing.T) {
client := &collisionClient{memoryClient: newMemoryClient()}
server := testServer("collision")
client.files[server.CertificatePath] = memoryFile{data: []byte("current")}
report := NewService(fixedClientFactory{client}).Deploy(t.Context(), []Target{{Server: server}}, testArtifact(t))
if report.Status != "failed" || !strings.Contains(report.Results[0].Message, "所有权") {
t.Fatalf("必须报告所有权不明确:%+v", report)
}
if string(client.files[client.collided].data) != "not-owned" || string(client.files[server.CertificatePath].data) != "current" || len(client.files) != 2 {
t.Fatal("上传冲突不得删除原有临时文件或修改当前证书")
}
}
type lostRenameClient struct {
*memoryClient
lostBackup bool
lost bool
closed bool
}
func (c *lostRenameClient) Disconnected() bool { return c.closed }
func (c *lostRenameClient) Rename(ctx context.Context, source, target string) error {
if err := c.memoryClient.Rename(ctx, source, target); err != nil {
return err
}
isBackup := strings.Contains(target, ".scert-backup-")
isInstall := strings.Contains(source, ".scert-tmp-")
if !c.lost && ((c.lostBackup && isBackup) || (!c.lostBackup && isInstall)) {
c.lost, c.closed = true, true
return context.DeadlineExceeded
}
return nil
}
type reconnectingMemoryFactory struct {
client *lostRenameClient
calls int
}
func (f *reconnectingMemoryFactory) Connect(context.Context, Server) (Client, error) {
f.calls++
f.client.closed = false
return f.client, nil
}
func TestDeployRecoversWhenRenameSucceededButReplyWasLost(t *testing.T) {
for _, backup := range []bool{true, false} {
t.Run(map[bool]string{true: "backup", false: "install"}[backup], func(t *testing.T) {
client := &lostRenameClient{memoryClient: newMemoryClient(), lostBackup: backup}
server := testServer("lost-reply")
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
client.files[path] = memoryFile{data: []byte("original"), mode: 0o600}
}
factory := &reconnectingMemoryFactory{client: client}
report := NewService(factory).Deploy(t.Context(), []Target{{Server: server}}, testArtifact(t))
if report.Status != "failed" || !strings.Contains(report.Results[0].Message, "文件已回滚") || factory.calls != 2 {
t.Fatalf("响应丢失后应重连并回滚:%+v,连接次数 %d", report, factory.calls)
}
if len(client.files) != 3 || len(client.commands) != 0 {
t.Fatal("回滚后不得残留临时文件或锁,也不得执行 reload")
}
for _, file := range client.files {
if string(file.data) != "original" || file.mode != 0o600 {
t.Fatal("响应丢失后原始文件内容或权限被破坏")
}
}
})
}
}