完成初始化证书事务和远程部署互斥并纳入发布构建脚本
This commit is contained in:
1 parent
05e05a1a23
commit
a4dffb04d4
27 files changed
+1294
-25
No files matched your search
@@ -8,6 +8,8 @@ type CommandResult struct {
|
||||
}
|
||||
|
||||
type Client interface {
|
||||
Lock(ctx context.Context, path string) error
|
||||
Unlock(ctx context.Context, path string) error
|
||||
MkdirAll(ctx context.Context, path string) error
|
||||
Exists(ctx context.Context, path string) (bool, error)
|
||||
Upload(ctx context.Context, path string, data []byte, mode uint32) error
|
||||
|
||||
@@ -113,6 +113,9 @@ func TestRemoteDeploymentIsolatedAcceptance(t *testing.T) {
|
||||
if strings.Contains(file.Name(), ".scert-tmp-") {
|
||||
t.Fatalf("残留临时文件:%s", file.Name())
|
||||
}
|
||||
if strings.HasSuffix(file.Name(), ".scert-lock") {
|
||||
t.Fatalf("残留部署锁:%s", file.Name())
|
||||
}
|
||||
if strings.HasPrefix(file.Name(), "key.pem") && file.Mode().Perm() != 0o600 {
|
||||
t.Fatal("私钥或私钥备份权限不安全")
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -108,11 +109,11 @@ func (s *Service) DeploySaved(ctx context.Context, targets []Target, saved Saved
|
||||
return s.Deploy(ctx, targets, Artifact{Certificate: certificateData, FullChain: fullChainData, PrivateKey: privateKeyData}), nil
|
||||
}
|
||||
|
||||
func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifact) Result {
|
||||
func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifact) (result Result) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
server = server.Normalize()
|
||||
result := Result{ServerID: server.ID, Server: server.Name, Status: "failed"}
|
||||
result = Result{ServerID: server.ID, Server: server.Name, Status: "failed"}
|
||||
if err := server.Validate(); err != nil {
|
||||
result.Message = err.Error()
|
||||
return result
|
||||
@@ -138,6 +139,43 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
result.Message = err.Error()
|
||||
return result
|
||||
}
|
||||
// 每个实际文件路径使用远程协作锁,阻止不同任务或进程的替换与回滚相互覆盖。
|
||||
lockPaths := make([]string, 0, len(paths))
|
||||
for _, file := range paths {
|
||||
if err := client.MkdirAll(ctx, parentPath(file.path)); err != nil {
|
||||
result.Message = fmt.Sprintf("创建远程目录失败:%v", err)
|
||||
return result
|
||||
}
|
||||
lockPaths = append(lockPaths, file.path+".scert-lock")
|
||||
}
|
||||
sort.Strings(lockPaths)
|
||||
heldLocks := make([]string, 0, len(lockPaths))
|
||||
retainLocks := false
|
||||
defer func() {
|
||||
if len(heldLocks) == 0 || retainLocks {
|
||||
return
|
||||
}
|
||||
releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
|
||||
defer cancel()
|
||||
if err := reconnectDeploymentClient(releaseCtx, s.factory, server, &client); err != nil {
|
||||
result.Status = "failed"
|
||||
result.Message += fmt.Sprintf(";释放部署锁失败,需人工检查:%v", err)
|
||||
return
|
||||
}
|
||||
for index := len(heldLocks) - 1; index >= 0; index-- {
|
||||
if err := client.Unlock(releaseCtx, heldLocks[index]); err != nil {
|
||||
result.Status = "failed"
|
||||
result.Message += fmt.Sprintf(";释放部署锁 %s 失败,需人工检查:%v", heldLocks[index], err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
for _, lockPath := range lockPaths {
|
||||
if err := client.Lock(ctx, lockPath); err != nil {
|
||||
result.Message = fmt.Sprintf("无法取得部署锁 %s,可能存在并发部署或遗留锁;未修改当前证书,请人工核查:%v", lockPath, err)
|
||||
return result
|
||||
}
|
||||
heldLocks = append(heldLocks, lockPath)
|
||||
}
|
||||
backupID := s.backupID()
|
||||
result.BackupID = backupID
|
||||
type backup struct{ original, saved string }
|
||||
@@ -147,13 +185,8 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
rollback := func() error {
|
||||
rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
|
||||
defer cancel()
|
||||
if state, ok := client.(interface{ Disconnected() bool }); ok && state.Disconnected() {
|
||||
_ = client.Close()
|
||||
reconnected, err := s.factory.Connect(rollbackContext, server)
|
||||
if err != nil {
|
||||
return fmt.Errorf("重新连接服务器以回滚失败:%w", err)
|
||||
}
|
||||
client = reconnected
|
||||
if err := reconnectDeploymentClient(rollbackContext, s.factory, server, &client); err != nil {
|
||||
return fmt.Errorf("重新连接服务器以回滚失败:%w", err)
|
||||
}
|
||||
var rollbackErrors []error
|
||||
for _, path := range temporary {
|
||||
@@ -187,6 +220,8 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
failed := func(message string) Result {
|
||||
if rollbackErr := rollback(); rollbackErr != nil {
|
||||
message += fmt.Sprintf(";自动回滚失败:%v", rollbackErr)
|
||||
retainLocks = true
|
||||
message += ";已保留本次部署锁,修复并确认文件状态后再由管理员释放"
|
||||
} else {
|
||||
message += ";文件已回滚"
|
||||
}
|
||||
@@ -198,13 +233,15 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
return result
|
||||
}
|
||||
for _, file := range paths {
|
||||
if err := client.MkdirAll(ctx, parentPath(file.path)); err != nil {
|
||||
return failed(fmt.Sprintf("创建远程目录失败:%v", err))
|
||||
}
|
||||
temporaryPath := file.path + ".scert-tmp-" + backupID
|
||||
// 上传失败也可能留下半份私钥,必须在开始上传前登记清理路径。
|
||||
temporary = append(temporary, temporaryPath)
|
||||
if err := client.Upload(ctx, temporaryPath, file.data, file.mode); err != nil {
|
||||
var notCreated *uploadNotCreatedError
|
||||
if errors.As(err, ¬Created) {
|
||||
temporary = temporary[:len(temporary)-1]
|
||||
return failed(fmt.Sprintf("上传失败且未确认临时路径所有权,已保留 %s 供人工核查:%v", temporaryPath, err))
|
||||
}
|
||||
return failed(fmt.Sprintf("上传远程证书失败:%v", err))
|
||||
}
|
||||
}
|
||||
@@ -216,6 +253,9 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
}
|
||||
if exists {
|
||||
backupPath := file.path + ".scert-backup-" + backupID
|
||||
if exists, err := client.Exists(ctx, backupPath); err != nil || exists {
|
||||
return failed(fmt.Sprintf("备份目标已存在或无法检查,已停止替换:%s:%v", backupPath, err))
|
||||
}
|
||||
backups = append(backups, backup{original: file.path, saved: backupPath})
|
||||
if err := client.Rename(ctx, file.path, backupPath); err != nil {
|
||||
return failed(fmt.Sprintf("备份远程证书失败:%v", err))
|
||||
@@ -248,6 +288,18 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
|
||||
return result
|
||||
}
|
||||
|
||||
func reconnectDeploymentClient(ctx context.Context, factory ClientFactory, server Server, client *Client) error {
|
||||
if state, ok := (*client).(interface{ Disconnected() bool }); ok && state.Disconnected() {
|
||||
_ = (*client).Close()
|
||||
reconnected, err := factory.Connect(ctx, server)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*client = reconnected
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type remoteFile struct {
|
||||
path string
|
||||
data []byte
|
||||
|
||||
@@ -38,6 +38,19 @@ func newMemoryClient() *memoryClient {
|
||||
|
||||
func (c *memoryClient) MkdirAll(context.Context, string) error { return nil }
|
||||
|
||||
func (c *memoryClient) Lock(_ context.Context, path string) error {
|
||||
if _, exists := c.files[path]; exists {
|
||||
return errors.New("模拟锁已存在")
|
||||
}
|
||||
c.files[path] = memoryFile{}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *memoryClient) Unlock(_ context.Context, path string) error {
|
||||
delete(c.files, path)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *memoryClient) Exists(_ context.Context, path string) (bool, error) {
|
||||
_, ok := c.files[path]
|
||||
return ok, nil
|
||||
@@ -197,6 +210,11 @@ func TestDeployReportsRollbackFailure(t *testing.T) {
|
||||
if !strings.Contains(report.Results[0].Message, "自动回滚失败") || !strings.Contains(report.Results[0].Message, "恢复备份文件") {
|
||||
t.Fatalf("应报告自动回滚失败原因:%s", report.Results[0].Message)
|
||||
}
|
||||
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
|
||||
if _, exists := client.files[path+".scert-lock"]; !exists {
|
||||
t.Fatal("回滚失败后必须保留协作锁,阻止后续任务覆盖待恢复现场")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeployReportsPartialSuccess(t *testing.T) {
|
||||
|
||||
@@ -213,6 +213,11 @@ func sshOperation[T any](ctx context.Context, c *sshClientAdapter, operation fun
|
||||
_ = c.ssh.Close()
|
||||
return zero, err
|
||||
}
|
||||
var networkErr *net.OpError
|
||||
if errors.Is(result.err, io.EOF) || errors.Is(result.err, io.ErrUnexpectedEOF) || errors.Is(result.err, net.ErrClosed) || errors.As(result.err, &networkErr) {
|
||||
c.disconnected.Store(true)
|
||||
_ = c.ssh.Close()
|
||||
}
|
||||
return result.value, result.err
|
||||
case <-ctx.Done():
|
||||
c.disconnected.Store(true)
|
||||
@@ -233,6 +238,15 @@ func (c *sshClientAdapter) MkdirAll(ctx context.Context, path string) error {
|
||||
return c.fileOperation(ctx, func() error { return c.sftp.MkdirAll(path) })
|
||||
}
|
||||
|
||||
// 远程原子建目录用于协作锁,跨进程共享;已有锁绝不自动抢占或清理。
|
||||
func (c *sshClientAdapter) Lock(ctx context.Context, path string) error {
|
||||
return c.fileOperation(ctx, func() error { return c.sftp.Mkdir(path) })
|
||||
}
|
||||
|
||||
func (c *sshClientAdapter) Unlock(ctx context.Context, path string) error {
|
||||
return c.fileOperation(ctx, func() error { return c.sftp.RemoveDirectory(path) })
|
||||
}
|
||||
|
||||
func (c *sshClientAdapter) Exists(ctx context.Context, path string) (bool, error) {
|
||||
return sshOperation(ctx, c, func() (bool, error) {
|
||||
_, err := c.sftp.Stat(path)
|
||||
@@ -244,12 +258,14 @@ func (c *sshClientAdapter) Exists(ctx context.Context, path string) (bool, error
|
||||
}
|
||||
|
||||
func (c *sshClientAdapter) Upload(ctx context.Context, path string, data []byte, mode uint32) error {
|
||||
return c.fileOperation(ctx, func() error {
|
||||
var created atomic.Bool
|
||||
err := c.fileOperation(ctx, func() error {
|
||||
// 只创建新临时文件,不跟随已有符号链接,也不截断已存在的文件。
|
||||
file, err := c.sftp.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
created.Store(true)
|
||||
// 先收紧空文件权限,再写入任何证书或私钥内容。
|
||||
if err := file.Chmod(os.FileMode(mode)); err != nil {
|
||||
_ = file.Close()
|
||||
@@ -261,8 +277,18 @@ func (c *sshClientAdapter) Upload(ctx context.Context, path string, data []byte,
|
||||
}
|
||||
return file.Close()
|
||||
})
|
||||
if err != nil && !created.Load() {
|
||||
// 未确认取得所有权的路径不得交给事务删除,包括创建响应丢失的情况。
|
||||
return &uploadNotCreatedError{err: err}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
type uploadNotCreatedError struct{ err error }
|
||||
|
||||
func (e *uploadNotCreatedError) Error() string { return e.err.Error() }
|
||||
func (e *uploadNotCreatedError) Unwrap() error { return e.err }
|
||||
|
||||
func (c *sshClientAdapter) Rename(ctx context.Context, source, target string) error {
|
||||
return c.fileOperation(ctx, func() error { return c.sftp.Rename(source, target) })
|
||||
}
|
||||
|
||||
@@ -152,6 +152,36 @@ func TestSSHFactoryCancelsUnresponsiveSFTPInitialization(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSHDeploymentLockIsSharedAcrossConnections(t *testing.T) {
|
||||
factory, server := safetyFactory(t, requestSFTPHandler(sftp.InMemHandler(), false, false))
|
||||
first, err := factory.Connect(t.Context(), server)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer first.Close()
|
||||
second, err := factory.Connect(t.Context(), server)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer second.Close()
|
||||
const path = "/certificate.scert-lock"
|
||||
if err := first.Lock(t.Context(), path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := second.Lock(t.Context(), path); err == nil {
|
||||
t.Fatal("其他 SSH 连接不得取得同一个远程部署锁")
|
||||
}
|
||||
if err := first.Unlock(t.Context(), path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := second.Lock(t.Context(), path); err != nil {
|
||||
t.Fatalf("释放锁后应允许后续部署:%v", err)
|
||||
}
|
||||
if err := second.Unlock(t.Context(), path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type stalledLister struct{ release <-chan struct{} }
|
||||
|
||||
func (l stalledLister) Filelist(*sftp.Request) (sftp.ListerAt, error) {
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
package deployment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDeployLeavesExistingLockAndCertificateUntouched(t *testing.T) {
|
||||
client := newMemoryClient()
|
||||
server := testServer("locked")
|
||||
lockPath := server.PrivateKeyPath + ".scert-lock"
|
||||
client.files[lockPath] = memoryFile{data: []byte("other-transaction")}
|
||||
client.files[server.CertificatePath] = memoryFile{data: []byte("current-certificate")}
|
||||
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
|
||||
report := service.Deploy(t.Context(), []Target{{Server: server}}, testArtifact(t))
|
||||
if report.Status != "failed" || !strings.Contains(report.Results[0].Message, "部署锁") {
|
||||
t.Fatalf("已有部署锁时必须停止:%+v", report)
|
||||
}
|
||||
if len(client.files) != 2 || string(client.files[lockPath].data) != "other-transaction" || string(client.files[server.CertificatePath].data) != "current-certificate" || len(client.commands) != 0 {
|
||||
t.Fatal("锁冲突时不得覆盖证书、移除他人的锁或遗留自己取得的锁")
|
||||
}
|
||||
}
|
||||
|
||||
type collisionClient struct {
|
||||
*memoryClient
|
||||
collided string
|
||||
}
|
||||
|
||||
func (c *collisionClient) Upload(_ context.Context, path string, _ []byte, _ uint32) error {
|
||||
c.collided = path
|
||||
c.files[path] = memoryFile{data: []byte("not-owned")}
|
||||
return &uploadNotCreatedError{err: errors.New("测试独占创建冲突")}
|
||||
}
|
||||
|
||||
type fixedClientFactory struct{ client Client }
|
||||
|
||||
func (f fixedClientFactory) Connect(context.Context, Server) (Client, error) { return f.client, nil }
|
||||
|
||||
func TestDeployDoesNotRemoveUnownedUploadPath(t *testing.T) {
|
||||
client := &collisionClient{memoryClient: newMemoryClient()}
|
||||
server := testServer("collision")
|
||||
client.files[server.CertificatePath] = memoryFile{data: []byte("current")}
|
||||
report := NewService(fixedClientFactory{client}).Deploy(t.Context(), []Target{{Server: server}}, testArtifact(t))
|
||||
if report.Status != "failed" || !strings.Contains(report.Results[0].Message, "所有权") {
|
||||
t.Fatalf("必须报告所有权不明确:%+v", report)
|
||||
}
|
||||
if string(client.files[client.collided].data) != "not-owned" || string(client.files[server.CertificatePath].data) != "current" || len(client.files) != 2 {
|
||||
t.Fatal("上传冲突不得删除原有临时文件或修改当前证书")
|
||||
}
|
||||
}
|
||||
|
||||
type lostRenameClient struct {
|
||||
*memoryClient
|
||||
lostBackup bool
|
||||
lost bool
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (c *lostRenameClient) Disconnected() bool { return c.closed }
|
||||
|
||||
func (c *lostRenameClient) Rename(ctx context.Context, source, target string) error {
|
||||
if err := c.memoryClient.Rename(ctx, source, target); err != nil {
|
||||
return err
|
||||
}
|
||||
isBackup := strings.Contains(target, ".scert-backup-")
|
||||
isInstall := strings.Contains(source, ".scert-tmp-")
|
||||
if !c.lost && ((c.lostBackup && isBackup) || (!c.lostBackup && isInstall)) {
|
||||
c.lost, c.closed = true, true
|
||||
return context.DeadlineExceeded
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type reconnectingMemoryFactory struct {
|
||||
client *lostRenameClient
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *reconnectingMemoryFactory) Connect(context.Context, Server) (Client, error) {
|
||||
f.calls++
|
||||
f.client.closed = false
|
||||
return f.client, nil
|
||||
}
|
||||
|
||||
func TestDeployRecoversWhenRenameSucceededButReplyWasLost(t *testing.T) {
|
||||
for _, backup := range []bool{true, false} {
|
||||
t.Run(map[bool]string{true: "backup", false: "install"}[backup], func(t *testing.T) {
|
||||
client := &lostRenameClient{memoryClient: newMemoryClient(), lostBackup: backup}
|
||||
server := testServer("lost-reply")
|
||||
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
|
||||
client.files[path] = memoryFile{data: []byte("original"), mode: 0o600}
|
||||
}
|
||||
factory := &reconnectingMemoryFactory{client: client}
|
||||
report := NewService(factory).Deploy(t.Context(), []Target{{Server: server}}, testArtifact(t))
|
||||
if report.Status != "failed" || !strings.Contains(report.Results[0].Message, "文件已回滚") || factory.calls != 2 {
|
||||
t.Fatalf("响应丢失后应重连并回滚:%+v,连接次数 %d", report, factory.calls)
|
||||
}
|
||||
if len(client.files) != 3 || len(client.commands) != 0 {
|
||||
t.Fatal("回滚后不得残留临时文件或锁,也不得执行 reload")
|
||||
}
|
||||
for _, file := range client.files {
|
||||
if string(file.data) != "original" || file.mode != 0o600 {
|
||||
t.Fatal("响应丢失后原始文件内容或权限被破坏")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user