完成部署后校验失败恢复整改并更新实机进度

延后事务提交,校验失败时恢复目标文件、配置和服务状态;新增恢复边界回归、前端提示及生产测试记录。
This commit is contained in:
xuans committed 2026-09-30 23:14:42 +08:00
1 parent c623e2d324
commit 3cc8fb6470
13 files changed
+588 -58

No files matched your search

+31 -26
View File
@@ -3,36 +3,41 @@ package deployment
type FailureCode string
const (
FailureUnavailable FailureCode = "unavailable"
FailureArtifact FailureCode = "artifact_invalid"
FailureConfiguration FailureCode = "configuration_invalid"
FailureConnection FailureCode = "connection_failed"
FailureDirectory FailureCode = "directory_failed"
FailureLock FailureCode = "lock_unavailable"
FailureUpload FailureCode = "upload_failed"
FailureReplace FailureCode = "replace_failed"
FailureValidation FailureCode = "validation_failed"
FailureRollback FailureCode = "rollback_failed"
FailureRuntimeUnknown FailureCode = "runtime_unconfirmed"
FailureCleanup FailureCode = "cleanup_failed"
FailureLockRelease FailureCode = "lock_release_failed"
FailureUnavailable FailureCode = "unavailable"
FailureArtifact FailureCode = "artifact_invalid"
FailureConfiguration FailureCode = "configuration_invalid"
FailureConnection FailureCode = "connection_failed"
FailureDirectory FailureCode = "directory_failed"
FailureLock FailureCode = "lock_unavailable"
FailureUpload FailureCode = "upload_failed"
FailureReplace FailureCode = "replace_failed"
FailureValidation FailureCode = "validation_failed"
FailureRollback FailureCode = "rollback_failed"
FailureRuntimeUnknown FailureCode = "runtime_unconfirmed"
FailureCleanup FailureCode = "cleanup_failed"
FailureLockRelease FailureCode = "lock_release_failed"
FailureRecoveryPending FailureCode = "recovery_pending"
FailureRecoveryUnknown FailureCode = "recovery_unconfirmed"
FailureRecoveryRestored FailureCode = "recovery_restored"
)
// 只公开固定诊断,不从远端输出、错误文本或文件路径推断可展示的消息。
var publicFailureMessages = map[FailureCode]string{
FailureUnavailable: "部署服务不可用,请联系管理员检查服务配置",
FailureArtifact: "待部署的证书或私钥校验失败,尚未推送到服务器",
FailureConfiguration: "服务器部署配置无效,请检查认证、证书路径和命令配置",
FailureConnection: "连接部署服务器失败,请检查网络、SSH 认证及主机身份配置",
FailureDirectory: "无法创建证书目录,请检查部署账户的目录权限",
FailureLock: "无法取得部署锁,可能有任务正在执行或存在中断遗留;请管理员核查远端文件、部署锁及服务状态后再重试,勿直接删除锁",
FailureUpload: "证书上传失败,已尝试回滚;请检查受保护的服务日志后再重试",
FailureReplace: "证书文件替换失败,已尝试回滚;请检查受保护的服务日志后再重试",
FailureValidation: "服务配置校验失败,文件已回滚,未执行重载;请修正配置后再重试",
FailureRollback: "部署失败且文件回滚未完成,已保留部署锁;请管理员核查远端文件及服务状态,勿直接重试或删除锁",
FailureRuntimeUnknown: "服务重载失败,文件已回滚,但服务运行状态未确认;请管理员核对配置并恢复服务后再重试",
FailureCleanup: "证书已推送并重载,但旧文件清理未完成;请管理员核查现场,不能据此确认站点新证书已生效",
FailureLockRelease: "部署锁释放未完成;请管理员核查远端文件、部署锁及服务状态后再重试",
FailureRecoveryUnknown: "本次部署未完成,已恢复旧文件,但服务或站点恢复状态未确认,已保留部署锁;请管理员核查,勿直接重试或删除锁",
FailureRecoveryRestored: "本次部署未完成,旧文件已恢复并通过配置校验及重载,站点已恢复部署前证书;请核查失败原因后再重试",
FailureUnavailable: "部署服务不可用,请联系管理员检查服务配置",
FailureArtifact: "待部署的证书或私钥校验失败,尚未推送到服务器",
FailureConfiguration: "服务器部署配置无效,请检查认证、证书路径和命令配置",
FailureConnection: "连接部署服务器失败,请检查网络、SSH 认证及主机身份配置",
FailureDirectory: "无法创建证书目录,请检查部署账户的目录权限",
FailureLock: "无法取得部署锁,可能有任务正在执行或存在中断遗留;请管理员核查远端文件、部署锁及服务状态后再重试,勿直接删除锁",
FailureUpload: "证书上传失败,已尝试回滚;请检查受保护的服务日志后再重试",
FailureReplace: "证书文件替换失败,已尝试回滚;请检查受保护的服务日志后再重试",
FailureValidation: "服务配置校验失败,文件已回滚,未执行重载;请修正配置后再重试",
FailureRollback: "部署失败且文件回滚未完成,已保留部署锁;请管理员核查远端文件及服务状态,勿直接重试或删除锁",
FailureRuntimeUnknown: "服务重载失败,文件已回滚,但服务运行状态未确认;请管理员核对配置并恢复服务后再重试",
FailureCleanup: "证书已推送并重载,但旧文件清理未完成;请管理员核查现场,不能据此确认站点新证书已生效",
FailureLockRelease: "部署锁释放未完成;请管理员核查远端文件、部署锁及服务状态后再重试",
}
// PublicResult 保留结果事实,同时只允许固定错误码及中文诊断进入公开响应。
+85 -9
View File
@@ -47,6 +47,7 @@ type Report struct {
Results []Result `json:"results"`
// 站点校验与各服务器文件推送分开记录,不表示所有服务器均完成独立 HTTPS 验证。
VerificationStatus string `json:"verification_status,omitempty"`
RecoveryStatus string `json:"recovery_status,omitempty"`
}
type Service struct {
@@ -59,6 +60,10 @@ func NewService(factory ClientFactory) *Service {
}
func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifact) Report {
return s.deploy(ctx, targets, artifact, nil)
}
func (s *Service) deploy(ctx context.Context, targets []Target, artifact Artifact, pending *[]*pendingDeployment) Report {
report := Report{Total: len(targets), Results: make([]Result, 0, len(targets))}
if len(targets) == 0 {
report.Status = "failed"
@@ -80,8 +85,8 @@ func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifac
}
return report
}
for _, target := range targets {
result := s.deployOne(ctx, target.Server, artifact)
for index, target := range targets {
result := s.deployOne(ctx, target.Server, artifact, pending, index)
report.Results = append(report.Results, result)
if result.Status == "success" {
report.Success++
@@ -101,22 +106,30 @@ func (s *Service) Deploy(ctx context.Context, targets []Target, artifact Artifac
}
func (s *Service) DeploySaved(ctx context.Context, targets []Target, saved SavedFiles) (Report, error) {
artifact, err := loadArtifact(saved)
if err != nil {
return Report{}, err
}
return s.Deploy(ctx, targets, artifact), nil
}
func loadArtifact(saved SavedFiles) (Artifact, error) {
certificateData, err := os.ReadFile(saved.CertificatePath)
if err != nil {
return Report{}, fmt.Errorf("读取证书文件失败:%w", err)
return Artifact{}, fmt.Errorf("读取证书文件失败:%w", err)
}
fullChainData, err := os.ReadFile(saved.FullChainPath)
if err != nil {
return Report{}, fmt.Errorf("读取完整证书链失败:%w", err)
return Artifact{}, fmt.Errorf("读取完整证书链失败:%w", err)
}
privateKeyData, err := os.ReadFile(saved.PrivateKeyPath)
if err != nil {
return Report{}, fmt.Errorf("读取私钥文件失败:%w", err)
return Artifact{}, fmt.Errorf("读取私钥文件失败:%w", err)
}
return s.Deploy(ctx, targets, Artifact{Certificate: certificateData, FullChain: fullChainData, PrivateKey: privateKeyData}), nil
return Artifact{Certificate: certificateData, FullChain: fullChainData, PrivateKey: privateKeyData}, nil
}
func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifact) (result Result) {
func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifact, pending *[]*pendingDeployment, index int) (result Result) {
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
server = server.Normalize()
@@ -131,7 +144,12 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
result.Message = fmt.Sprintf("连接服务器失败:%v", err)
return result
}
defer func() { _ = client.Close() }()
handedOff := false
defer func() {
if !handedOff {
_ = client.Close()
}
}()
paths := []remoteFile{
{path: RemotePath(server.CertificatePath), data: artifact.Certificate, mode: 0o644},
{path: RemotePath(server.CertificateChainPath), data: artifact.FullChain, mode: 0o644},
@@ -160,7 +178,7 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
sort.Strings(lockPaths)
heldLocks := make([]string, 0, len(lockPaths))
retainLocks := false
defer func() {
releaseLocks := func() {
if len(heldLocks) == 0 || retainLocks {
return
}
@@ -179,6 +197,11 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
result.Message += fmt.Sprintf(";释放部署锁 %s 失败,需人工检查:%v", heldLocks[index], err)
}
}
}
defer func() {
if !handedOff {
releaseLocks()
}
}()
for _, lockPath := range lockPaths {
if err := client.Lock(ctx, lockPath); err != nil {
@@ -300,6 +323,59 @@ func (s *Service) deployOne(ctx context.Context, server Server, artifact Artifac
}
return failed(fmt.Sprintf("reload 失败:%v", err))
}
if pending != nil {
// 旧文件与锁在所有目标及站点校验结束前始终保留,禁止提前提交。
result.Status = "success"
result.FailureCode = ""
result.Message = "证书已推送并重载,等待站点校验"
closed := false
transaction := &pendingDeployment{index: index}
transaction.restore = func(recoveryContext context.Context) Result {
result.Status = "failed"
if err := rollback(); err != nil {
result.FailureCode = FailureRollback
result.Message = fmt.Sprintf("部署未提交,自动回滚失败:%v", err)
return result
}
result.FailureCode = FailureRecoveryUnknown
if err := reconnectDeploymentClient(recoveryContext, s.factory, server, &client); err != nil {
result.Message = fmt.Sprintf("旧文件已恢复,重新连接以恢复服务失败:%v", err)
return result
}
for _, command := range [][]string{server.ValidationCommand, server.ReloadCommand} {
outcome, err := client.Run(recoveryContext, command)
if err != nil || outcome.ExitCode != 0 {
result.Message = fmt.Sprintf("旧文件已恢复,但恢复配置校验或重载失败,退出码 %d:%v", outcome.ExitCode, err)
return result
}
}
result.FailureCode = FailureRecoveryPending
result.Message = "本次部署未提交,旧文件已恢复并通过配置校验及重载,等待站点恢复核验"
return result
}
transaction.commit = func(commitContext context.Context) Result {
if err := cleanupBackups(commitContext, client, backups, paths); err != nil {
result.Status = "failed"
result.FailureCode = FailureCleanup
result.Message = fmt.Sprintf("站点校验通过,但事务旧文件清理失败:%v", err)
return result
}
result.Message = "证书部署、配置校验并 reload 成功"
return result
}
transaction.finish = func(retain bool) Result {
if !closed {
retainLocks = retain
releaseLocks()
_ = client.Close()
closed = true
}
return result
}
handedOff = true
*pending = append(*pending, transaction)
return result
}
if err := cleanupBackups(ctx, client, backups, paths); err != nil {
result.Status = "failed"
result.FailureCode = FailureCleanup
+150
View File
@@ -0,0 +1,150 @@
package deployment
import (
"context"
"errors"
"fmt"
)
// Verification 将站点生效校验与恢复核验置于远端文件事务之内。
type Verification struct {
Verify func(context.Context) error
VerifyRecovery func(context.Context) error
}
type pendingDeployment struct {
index int
commit func(context.Context) Result
restore func(context.Context) Result
finish func(bool) Result
}
func (s *Service) DeploySavedVerified(ctx context.Context, targets []Target, saved SavedFiles, verification Verification) (Report, error) {
artifact, err := loadArtifact(saved)
if err != nil {
return Report{Status: "failed"}, err
}
return s.DeployVerified(ctx, targets, artifact, verification)
}
func (s *Service) DeployVerified(ctx context.Context, targets []Target, artifact Artifact, verification Verification) (Report, error) {
if verification.Verify == nil {
return Report{Status: "failed"}, errors.New("缺少部署后站点校验,未执行推送")
}
pending := make([]*pendingDeployment, 0, len(targets))
// 异常退出时关闭连接但保留现场;不得以释放锁掩盖未确认的事务。
defer func() {
for _, transaction := range pending {
transaction.finish(true)
}
}()
report := s.deploy(ctx, targets, artifact, &pending)
var deployErr error
if report.Status != "success" {
deployErr = fmt.Errorf("证书部署未全部成功:成功 %d 台,失败 %d 台", report.Success, report.Failed)
} else {
report.VerificationStatus = "failed"
deployErr = ctx.Err()
if deployErr == nil {
deployErr = verification.Verify(ctx)
}
if deployErr == nil {
deployErr = ctx.Err()
}
if deployErr != nil {
deployErr = fmt.Errorf("部署后 HTTPS 校验失败:%w", deployErr)
} else {
report.VerificationStatus = "success"
}
}
if deployErr != nil {
if len(pending) > 0 {
s.restorePending(ctx, &report, pending, verification)
}
summarizeReport(&report)
report.Status = "failed"
return report, deployErr
}
for _, transaction := range pending {
result := transaction.commit(ctx)
report.Results[transaction.index] = transaction.finish(result.Status != "success")
}
summarizeReport(&report)
if report.Status != "success" {
return report, errors.New("站点校验通过,但事务清理或释放部署锁未完成")
}
return report, nil
}
func (s *Service) restorePending(ctx context.Context, report *Report, pending []*pendingDeployment, verification Verification) {
report.RecoveryStatus = "unconfirmed"
for index := len(pending) - 1; index >= 0; index-- {
transaction := pending[index]
// 请求取消后仍使用独立有界上下文恢复,不重放签发或文件推送。
recoveryContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
report.Results[transaction.index] = transaction.restore(recoveryContext)
cancel()
}
restored := true
for _, result := range report.Results {
switch result.FailureCode {
case FailureRecoveryPending, FailureConfiguration, FailureConnection, FailureDirectory, FailureUpload, FailureReplace, FailureValidation:
// 这些失败要么未触碰目标,要么已完成文件回滚且未执行重载。
default:
restored = false
}
}
var recoveryErr error
if !restored {
report.RecoveryStatus = "failed"
} else if verification.VerifyRecovery != nil {
recoveryContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), deploymentRollbackTimeout)
err := verification.VerifyRecovery(recoveryContext)
if err == nil {
err = recoveryContext.Err()
}
cancel()
if err == nil {
report.RecoveryStatus = "success"
} else {
recoveryErr = err
}
}
for _, transaction := range pending {
result := transaction.finish(report.RecoveryStatus != "success")
if result.FailureCode == FailureLockRelease {
report.RecoveryStatus = "failed"
}
if result.FailureCode == FailureRecoveryPending {
if report.RecoveryStatus == "success" {
result.FailureCode = FailureRecoveryRestored
} else {
result.FailureCode = FailureRecoveryUnknown
}
result.Message = publicFailureMessages[result.FailureCode]
if recoveryErr != nil {
result.Message += fmt.Sprintf(";站点恢复核验失败:%v", recoveryErr)
}
}
report.Results[transaction.index] = result
}
}
func summarizeReport(report *Report) {
report.Success, report.Failed = 0, 0
for _, result := range report.Results {
if result.Status == "success" {
report.Success++
} else {
report.Failed++
}
}
switch {
case report.Total == 0 || report.Success == 0:
report.Status = "failed"
case report.Failed == 0:
report.Status = "success"
default:
report.Status = "partial"
}
}
@@ -0,0 +1,212 @@
package deployment
import (
"bytes"
"context"
"errors"
"reflect"
"strings"
"testing"
)
func seedVerifiedFiles(client *memoryClient, server Server) map[string]memoryFile {
files := map[string]memoryFile{
server.CertificatePath: {data: []byte("原证书"), mode: 0o644},
server.CertificateChainPath: {data: []byte("原证书链"), mode: 0o644},
server.PrivateKeyPath: {data: []byte("原私钥"), mode: 0o600},
}
for path, file := range files {
client.files[path] = file
}
return files
}
func assertVerifiedFiles(t *testing.T, client *memoryClient, expected map[string]memoryFile) {
t.Helper()
for path, file := range expected {
actual, ok := client.files[path]
if !ok || !bytes.Equal(actual.data, file.data) || actual.mode != file.mode {
t.Fatalf("未恢复原内容或权限:%s", path)
}
}
}
func assertVerifiedLocks(t *testing.T, client *memoryClient, server Server, held bool) {
t.Helper()
for _, path := range []string{server.CertificatePath, server.CertificateChainPath, server.PrivateKeyPath} {
_, exists := client.files[path+".scert-lock"]
if exists != held {
t.Fatalf("部署锁状态不符合预期:%s,锁定 %t", path, exists)
}
}
}
func TestVerifiedDeploymentDefersCommitUntilAllTargetsAndSitePass(t *testing.T) {
clients := map[string]*memoryClient{"first": newMemoryClient(), "second": newMemoryClient()}
servers := []Server{testServer("first"), testServer("second")}
for _, server := range servers {
seedVerifiedFiles(clients[server.ID], server)
}
artifact := testArtifact(t)
called := false
service := NewService(memoryFactory{clients: clients})
report, err := service.DeployVerified(t.Context(), []Target{{Server: servers[0]}, {Server: servers[1]}}, artifact, Verification{
Verify: func(context.Context) error {
called = true
for _, server := range servers {
client := clients[server.ID]
assertVerifiedLocks(t, client, server, true)
if !bytes.Equal(client.files[server.CertificatePath].data, artifact.Certificate) || len(client.commands) != 2 {
t.Fatal("必须在所有服务器推送和重载后才核验站点")
}
backups := 0
for path := range client.files {
if strings.Contains(path, ".scert-backup-") {
backups++
}
}
if backups != 3 {
t.Fatal("站点确认前必须保留三份事务旧文件")
}
}
return nil
},
VerifyRecovery: func(context.Context) error { t.Fatal("成功部署不得回滚"); return nil },
})
if err != nil || !called || report.Status != "success" || report.Success != 2 || report.VerificationStatus != "success" || report.RecoveryStatus != "" {
t.Fatalf("事务提交结果错误:%+v,%v", report, err)
}
for _, server := range servers {
client := clients[server.ID]
assertVerifiedLocks(t, client, server, false)
if len(client.files) != 3 {
t.Fatal("提交后不应残留事务文件")
}
}
}
func TestVerifiedDeploymentFailureRestoresAllTargetsBeforeRecoveryCheck(t *testing.T) {
clients := map[string]*memoryClient{"first": newMemoryClient(), "second": newMemoryClient()}
servers := []Server{testServer("first"), testServer("second")}
originals := map[string]map[string]memoryFile{}
for _, server := range servers {
originals[server.ID] = seedVerifiedFiles(clients[server.ID], server)
}
recovered := false
verificationErr := errors.New("公网仍返回错误证书")
report, err := NewService(memoryFactory{clients: clients}).DeployVerified(t.Context(), []Target{{Server: servers[0]}, {Server: servers[1]}}, testArtifact(t), Verification{
Verify: func(context.Context) error { return verificationErr },
VerifyRecovery: func(ctx context.Context) error {
recovered = true
if _, ok := ctx.Deadline(); !ok || ctx.Err() != nil {
t.Fatal("恢复核验必须有独立有效期限")
}
for _, server := range servers {
client := clients[server.ID]
assertVerifiedFiles(t, client, originals[server.ID])
assertVerifiedLocks(t, client, server, true)
expectedCommands := [][]string{{"nginx", "-t"}, {"nginx", "-s", "reload"}, {"nginx", "-t"}, {"nginx", "-s", "reload"}}
if !reflect.DeepEqual(client.commands, expectedCommands) {
t.Fatalf("必须恢复配置并重载后核验站点:实际=%#v,期望=%#v", client.commands, expectedCommands)
}
}
return nil
},
})
if !errors.Is(err, verificationErr) || !recovered || report.Status != "failed" || report.Success != 0 || report.Failed != 2 || report.VerificationStatus != "failed" || report.RecoveryStatus != "success" {
t.Fatalf("失败与恢复状态错误:%+v,%v", report, err)
}
for index, server := range servers {
assertVerifiedFiles(t, clients[server.ID], originals[server.ID])
assertVerifiedLocks(t, clients[server.ID], server, false)
if report.Results[index].FailureCode != FailureRecoveryRestored {
t.Fatalf("应保留部署失败并提供恢复诊断:%+v", report.Results[index])
}
}
}
func TestVerifiedDeploymentCannotClaimRecoveryOnIncompleteEvidence(t *testing.T) {
for _, scenario := range []struct {
name string
configure func(*memoryClient)
recovery func(context.Context) error
status string
code FailureCode
}{
{"missing_baseline", func(*memoryClient) {}, nil, "unconfirmed", FailureRecoveryUnknown},
{"wrong_runtime", func(*memoryClient) {}, func(context.Context) error { return errors.New("secret-path 旧站点身份不符") }, "unconfirmed", FailureRecoveryUnknown},
{"restore_failed", func(c *memoryClient) { c.restoreError = true }, nil, "failed", FailureRollback},
{"validation_failed", func(c *memoryClient) { c.runResults = []CommandResult{{}, {}, {ExitCode: 1}} }, nil, "failed", FailureRecoveryUnknown},
{"reload_failed", func(c *memoryClient) { c.runResults = []CommandResult{{}, {}, {}, {ExitCode: 1}} }, nil, "failed", FailureRecoveryUnknown},
} {
t.Run(scenario.name, func(t *testing.T) {
client, server := newMemoryClient(), testServer("recovery")
seedVerifiedFiles(client, server)
scenario.configure(client)
report, err := NewService(fixedClientFactory{client}).DeployVerified(t.Context(), []Target{{Server: server}}, testArtifact(t), Verification{
Verify: func(context.Context) error { return errors.New("部署后校验失败") }, VerifyRecovery: scenario.recovery,
})
if err == nil || report.RecoveryStatus != scenario.status || report.Results[0].FailureCode != scenario.code {
t.Fatalf("不得夸大恢复能力:%+v,%v", report, err)
}
assertVerifiedLocks(t, client, server, true)
public := PublicResult(report.Results[0])
if strings.Contains(public.Message, "secret-path") || public.Message == "证书部署失败" {
t.Fatalf("恢复诊断应安全且明确:%+v", public)
}
})
}
}
func TestVerifiedDeploymentCancellationStillRestoresWithBoundedContext(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
client, server := newMemoryClient(), testServer("cancel")
original := seedVerifiedFiles(client, server)
report, err := NewService(fixedClientFactory{client}).DeployVerified(ctx, []Target{{Server: server}}, testArtifact(t), Verification{
Verify: func(context.Context) error { cancel(); return nil },
VerifyRecovery: func(ctx context.Context) error {
if ctx.Err() != nil {
t.Fatal("请求取消不得取消恢复核验")
}
assertVerifiedFiles(t, client, original)
return nil
},
})
if !errors.Is(err, context.Canceled) || report.Status != "failed" || report.RecoveryStatus != "success" || len(client.commands) != 4 {
t.Fatalf("取消后应恢复但不得转为部署成功:%+v,%v", report, err)
}
assertVerifiedLocks(t, client, server, false)
}
func TestVerifiedDeploymentPartialPreparationDoesNotCommitEarlierTarget(t *testing.T) {
client, first := newMemoryClient(), testServer("first")
second := testServer("second")
original := seedVerifiedFiles(client, first)
service := NewService(memoryFactory{clients: map[string]*memoryClient{"first": client}, errors: map[string]error{"second": errors.New("连接失败")}})
report, err := service.DeployVerified(t.Context(), []Target{{Server: first}, {Server: second}}, testArtifact(t), Verification{
Verify: func(context.Context) error { t.Fatal("部分准备失败不得进行提交校验"); return nil },
VerifyRecovery: func(context.Context) error { assertVerifiedFiles(t, client, original); return nil },
})
if err == nil || report.Status != "failed" || report.Success != 0 || report.RecoveryStatus != "success" || report.Results[1].FailureCode != FailureConnection {
t.Fatalf("部分目标失败必须恢复已推送目标:%+v,%v", report, err)
}
assertVerifiedLocks(t, client, first, false)
}
func TestVerifiedDeploymentRequiresVerifierAndDoesNotHideCleanupFailure(t *testing.T) {
client, server := newMemoryClient(), testServer("cleanup")
original := seedVerifiedFiles(client, server)
service := NewService(fixedClientFactory{client})
report, err := service.DeployVerified(t.Context(), []Target{{Server: server}}, testArtifact(t), Verification{})
if err == nil || report.Status != "failed" || len(client.commands) != 0 {
t.Fatal("缺少校验器不得推送")
}
assertVerifiedFiles(t, client, original)
client.removeError = ".scert-backup-"
report, err = service.DeployVerified(t.Context(), []Target{{Server: server}}, testArtifact(t), Verification{Verify: func(context.Context) error { return nil }})
if err == nil || report.Status != "failed" || report.VerificationStatus != "success" || report.Results[0].FailureCode != FailureCleanup {
t.Fatalf("校验成功不能掩盖清理失败:%+v,%v", report, err)
}
assertVerifiedLocks(t, client, server, true)
}