xuans/Scert
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完善远程部署回滚错误处理

Browse Source
This commit is contained in:
xuans 2026-08-30 22:08:54 +08:00
parent e2b75d29ea
commit 0efeebfb88
2 changed files with 62 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
backend/internal/deployment/service.go
View File

@ -35,6 +35,8 @@ type Result struct {
BackupID string `json:"backup_id,omitempty"`
}
const deploymentRollbackTimeout = 30 * time.Second
type Report struct {
Total int `json:"total"`
Success int `json:"success"`
@ -135,50 +137,57 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
backups := make([]backup, 0, len(paths))
temporary := make([]string, 0, len(paths))
installed := make([]string, 0, len(paths))
rollback := func() {
rollback := func() error {
rollbackContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
defer cancel()
var rollbackErrors []error
for _, path := range temporary {
_ = client.Remove(context.Background(), path)
if err := client.Remove(rollbackContext, path); err != nil {
rollbackErrors = append(rollbackErrors, fmt.Errorf("删除临时文件 %s 失败：%w", path, err))
}
}
for _, path := range installed {
_ = client.Remove(context.Background(), path)
if err := client.Remove(rollbackContext, path); err != nil {
rollbackErrors = append(rollbackErrors, fmt.Errorf("删除新证书文件 %s 失败：%w", path, err))
}
}
for index := len(backups) - 1; index >= 0; index-- {
item := backups[index]
_ = client.Rename(context.Background(), item.saved, item.original)
if err := client.Rename(rollbackContext, item.saved, item.original); err != nil {
rollbackErrors = append(rollbackErrors, fmt.Errorf("恢复备份文件 %s 失败：%w", item.original, err))
}
}
return errors.Join(rollbackErrors...)
}
failed := func(message string) Result {
if rollbackErr := rollback(); rollbackErr != nil {
message += fmt.Sprintf("；自动回滚失败：%v", rollbackErr)
}
result.Message = message
return result
}
for _, file := range paths {
if err := client.MkdirAll(ctx, parentPath(file.path)); err != nil {
result.Message = fmt.Sprintf("创建远程目录失败：%v", err)
rollback()
return result
return failed(fmt.Sprintf("创建远程目录失败：%v", err))
}
exists, err := client.Exists(ctx, file.path)
if err != nil {
result.Message = fmt.Sprintf("检查远程文件失败：%v", err)
rollback()
return result
return failed(fmt.Sprintf("检查远程文件失败：%v", err))
}
if exists {
backupPath := file.path + ".scert-backup-" + backupID
if err := client.Rename(ctx, file.path, backupPath); err != nil {
result.Message = fmt.Sprintf("备份远程证书失败：%v", err)
rollback()
return result
return failed(fmt.Sprintf("备份远程证书失败：%v", err))
}
backups = append(backups, backup{original: file.path, saved: backupPath})
}
temporaryPath := file.path + ".scert-tmp-" + backupID
if err := client.Upload(ctx, temporaryPath, file.data, file.mode); err != nil {
result.Message = fmt.Sprintf("上传远程证书失败：%v", err)
rollback()
return result
return failed(fmt.Sprintf("上传远程证书失败：%v", err))
}
temporary = append(temporary, temporaryPath)
if err := client.Rename(ctx, temporaryPath, file.path); err != nil {
result.Message = fmt.Sprintf("替换远程证书失败：%v", err)
rollback()
return result
return failed(fmt.Sprintf("替换远程证书失败：%v", err))
}
temporary = temporary[:len(temporary)-1]
installed = append(installed, file.path)
@ -188,9 +197,7 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
if err == nil {
err = fmt.Errorf("退出码 %d", commandResult.ExitCode)
}
result.Message = fmt.Sprintf("reload 失败：%v", err)
rollback()
return result
return failed(fmt.Sprintf("reload 失败：%v", err))
}
result.Status = "success"
result.Message = "证书部署并 reload 成功"

38
backend/internal/deployment/service_test.go
View File

@ -14,11 +14,14 @@ type memoryFile struct {
}
type memoryClient struct {
files map[string]memoryFile
command []string
runResult CommandResult
runErr error
uploadError string
files map[string]memoryFile
command []string
runResult CommandResult
runErr error
uploadError string
removeError string
renameError string
restoreError bool
}
func newMemoryClient() *memoryClient {
@ -41,6 +44,12 @@ func (c *memoryClient) Upload(_ context.Context, path string, data []byte, mode
}
func (c *memoryClient) Rename(_ context.Context, source, target string) error {
if c.renameError != "" && (strings.Contains(source, c.renameError) || strings.Contains(target, c.renameError)) {
return errors.New("模拟重命名失败")
}
if c.restoreError && strings.Contains(source, ".scert-backup-") && !strings.Contains(target, ".scert-backup-") {
return errors.New("模拟恢复备份失败")
}
value, ok := c.files[source]
if !ok {
return errors.New("源文件不存在")
@ -51,6 +60,9 @@ func (c *memoryClient) Rename(_ context.Context, source, target string) error {
}
func (c *memoryClient) Remove(_ context.Context, path string) error {
if c.removeError != "" && strings.Contains(path, c.removeError) {
return errors.New("模拟删除失败")
}
delete(c.files, path)
return nil
}
@ -126,6 +138,22 @@ func TestDeployRollsBackWhenReloadFails(t *testing.T) {
}
}
func TestDeployReportsRollbackFailure(t *testing.T) {
client := newMemoryClient()
client.runResult.ExitCode = 1
client.restoreError = true
server := testServer("server-1")
client.files[server.CertificatePath] = memoryFile{data: []byte("old-cert"), mode: 0o644}
service := NewService(memoryFactory{clients: map[string]*memoryClient{server.ID: client}})
report := service.Deploy(context.Background(), []Target{{Server: server}}, testArtifact())
if report.Status != "failed" || report.Failed != 1 {
t.Fatalf("回滚失败结果错误：%+v", report)
}
if !strings.Contains(report.Results[0].Message, "自动回滚失败") || !strings.Contains(report.Results[0].Message, "恢复备份文件") {
t.Fatalf("应报告自动回滚失败原因：%s", report.Results[0].Message)
}
}
func TestDeployReportsPartialSuccess(t *testing.T) {
first := newMemoryClient()
second := newMemoryClient()