保存不确定部署处置重新核查实现及测试检查点
This commit is contained in:
1 parent
7fa5e9b3d2
commit
f9cd19f631
29 files changed
+907
-49
No files matched your search
@@ -43,6 +43,11 @@ func (r *SQLDeploymentTransactionRepository) FinishRecoveryDisposition(ctx conte
|
||||
if err := tx.QueryRowContext(ctx, r.query(`SELECT event_type FROM recovery_events WHERE disposition_id = ? AND event_type = 'started'`), id).Scan(&start); err != nil {
|
||||
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||
}
|
||||
if item.Intent.SourceDispositionID != "" {
|
||||
if err := r.validateReconciliationSource(ctx, tx, item.Intent); err != nil {
|
||||
return deployment.RecoveryDisposition{}, err
|
||||
}
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if err := r.finishRecoveryOriginal(ctx, tx, item.Intent, resultCode, now); err != nil {
|
||||
return deployment.RecoveryDisposition{}, err
|
||||
@@ -58,7 +63,13 @@ func (r *SQLDeploymentTransactionRepository) FinishRecoveryDisposition(ctx conte
|
||||
if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_events (disposition_id, event_type, result_code, created_at) VALUES (?, 'finished', ?, ?)`), id, resultCode, now); err != nil {
|
||||
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||
}
|
||||
if resultCode != deployment.RecoveryResultUncertain {
|
||||
if item.Intent.SourceDispositionID != "" && (resultCode == deployment.RecoveryResultRejected || resultCode == deployment.RecoveryResultUncertain) {
|
||||
// 只读核查未通过时归还原占用,保留两次审计;绝不解锁原事务。
|
||||
result, err := tx.ExecContext(ctx, r.query(`UPDATE recovery_claims SET disposition_id = ?, created_at = ? WHERE transaction_id = ? AND disposition_id = ?`), item.Intent.SourceDispositionID, now, item.Intent.TransactionID, id)
|
||||
if err != nil || !oneRecoveryRow(result) {
|
||||
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||
}
|
||||
} else if resultCode != deployment.RecoveryResultUncertain {
|
||||
result, err := tx.ExecContext(ctx, r.query(`DELETE FROM recovery_claims WHERE transaction_id = ? AND disposition_id = ?`), item.Intent.TransactionID, id)
|
||||
if err != nil || !oneRecoveryRow(result) {
|
||||
return deployment.RecoveryDisposition{}, dispositionError(ctx)
|
||||
@@ -96,6 +107,9 @@ func (r *SQLDeploymentTransactionRepository) finishRecoveryOriginal(ctx context.
|
||||
(record.Status != deployment.TransactionStatusPending && record.Status != deployment.TransactionStatusRecoveryRequired) {
|
||||
return deployment.ErrRecoveryDispositionConflict
|
||||
}
|
||||
if intent.SourceDispositionID != "" && !intent.MatchesRecord(record) {
|
||||
return deployment.ErrRecoveryReviewChanged
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, r.query(`UPDATE deployment_transactions SET status = ?, phase = ?, message = ?, updated_at = ?
|
||||
WHERE id = ? AND phase = ? AND status IN ('pending', 'recovery_required')`), status, status, message, now, intent.TransactionID, phase)
|
||||
if err != nil || !oneRecoveryRow(result) {
|
||||
|
||||
@@ -12,13 +12,13 @@ import (
|
||||
var _ deployment.RecoveryDispositionJournal = (*SQLDeploymentTransactionRepository)(nil)
|
||||
|
||||
const recoveryDispositionColumns = `id, transaction_id, server_id, operator_id, action, request_key,
|
||||
record_digest, server_digest, observation_digest, status, result_code, created_at, updated_at`
|
||||
record_digest, server_digest, observation_digest, status, result_code, created_at, updated_at, source_disposition_id`
|
||||
|
||||
func scanRecoveryDisposition(row deploymentTransactionScanner) (deployment.RecoveryDisposition, error) {
|
||||
var item deployment.RecoveryDisposition
|
||||
i := &item.Intent
|
||||
err := row.Scan(&i.ID, &i.TransactionID, &i.ServerID, &i.Operator, &i.Action, &i.RequestKey,
|
||||
&i.RecordDigest, &i.ServerDigest, &i.ObservationDigest, &item.Status, &item.ResultCode, &item.CreatedAt, &item.UpdatedAt)
|
||||
&i.RecordDigest, &i.ServerDigest, &i.ObservationDigest, &item.Status, &item.ResultCode, &item.CreatedAt, &item.UpdatedAt, &i.SourceDispositionID)
|
||||
return item, err
|
||||
}
|
||||
|
||||
@@ -75,9 +75,9 @@ func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx contex
|
||||
defer tx.Rollback()
|
||||
now := time.Now().UTC()
|
||||
item := deployment.RecoveryDisposition{Intent: intent, Status: "running", CreatedAt: now, UpdatedAt: now}
|
||||
_, err = tx.ExecContext(ctx, r.query(`INSERT INTO recovery_dispositions (`+recoveryDispositionColumns+`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`),
|
||||
_, err = tx.ExecContext(ctx, r.query(`INSERT INTO recovery_dispositions (`+recoveryDispositionColumns+`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`),
|
||||
intent.ID, intent.TransactionID, intent.ServerID, intent.Operator, intent.Action, intent.RequestKey,
|
||||
intent.RecordDigest, intent.ServerDigest, intent.ObservationDigest, item.Status, "", now, now)
|
||||
intent.RecordDigest, intent.ServerDigest, intent.ObservationDigest, item.Status, "", now, now, intent.SourceDispositionID)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
if existing, readErr := r.findRecoveryRequest(ctx, intent.RequestKey); readErr == nil {
|
||||
@@ -85,7 +85,7 @@ func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx contex
|
||||
}
|
||||
return deployment.RecoveryDisposition{}, false, dispositionError(ctx)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_claims (transaction_id, disposition_id, created_at) VALUES (?, ?, ?)`), intent.TransactionID, intent.ID, now); err != nil {
|
||||
if err := r.claimRecoveryDisposition(ctx, tx, intent, now); err != nil {
|
||||
_ = tx.Rollback()
|
||||
if claimed, readErr := r.HasRecoveryClaim(ctx, intent.TransactionID); readErr == nil && claimed {
|
||||
return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionBusy
|
||||
@@ -109,6 +109,39 @@ func (r *SQLDeploymentTransactionRepository) BeginRecoveryDisposition(ctx contex
|
||||
return item, true, nil
|
||||
}
|
||||
|
||||
func (r *SQLDeploymentTransactionRepository) claimRecoveryDisposition(ctx context.Context, tx *sql.Tx, intent deployment.RecoveryDispositionIntent, now time.Time) error {
|
||||
if intent.SourceDispositionID == "" {
|
||||
_, err := tx.ExecContext(ctx, r.query(`INSERT INTO recovery_claims (transaction_id, disposition_id, created_at) VALUES (?, ?, ?)`), intent.TransactionID, intent.ID, now)
|
||||
return err
|
||||
}
|
||||
if err := r.validateReconciliationSource(ctx, tx, intent); err != nil {
|
||||
return err
|
||||
}
|
||||
// 只从指定的已结束处置接管占用;运行中、其他处置或丢失的占用均不接管。
|
||||
result, err := tx.ExecContext(ctx, r.query(`UPDATE recovery_claims SET disposition_id = ?, created_at = ? WHERE transaction_id = ? AND disposition_id = ?`), intent.ID, now, intent.TransactionID, intent.SourceDispositionID)
|
||||
if err != nil || !oneRecoveryRow(result) {
|
||||
return deployment.ErrRecoveryDispositionBusy
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *SQLDeploymentTransactionRepository) validateReconciliationSource(ctx context.Context, tx *sql.Tx, intent deployment.RecoveryDispositionIntent) error {
|
||||
query := `SELECT ` + recoveryDispositionColumns + ` FROM recovery_dispositions WHERE id = ?`
|
||||
if r.driver != "sqlite" {
|
||||
query += " FOR UPDATE"
|
||||
}
|
||||
source, err := scanRecoveryDisposition(tx.QueryRowContext(ctx, r.query(query), intent.SourceDispositionID))
|
||||
if err != nil || !intent.Reconciles(source) {
|
||||
return deployment.ErrRecoveryDispositionConflict
|
||||
}
|
||||
var count int
|
||||
if err := tx.QueryRowContext(ctx, r.query(`SELECT COUNT(*) FROM recovery_events WHERE disposition_id = ? AND
|
||||
((event_type = 'started' AND result_code = '') OR (event_type = 'finished' AND result_code = 'uncertain'))`), intent.SourceDispositionID).Scan(&count); err != nil || count != 2 {
|
||||
return deployment.ErrRecoveryDispositionConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func recoveryRetry(existing deployment.RecoveryDisposition, intent deployment.RecoveryDispositionIntent) (deployment.RecoveryDisposition, bool, error) {
|
||||
if !existing.Intent.SameRequest(intent) {
|
||||
return deployment.RecoveryDisposition{}, false, deployment.ErrRecoveryDispositionConflict
|
||||
|
||||
@@ -68,6 +68,16 @@ func openRecoveryExternal(t *testing.T, cfg config.DatabaseConfig) *sql.DB {
|
||||
// 只替换远端文件实现,准备、确认、执行、迁移、占用和审计均走实际产品代码。
|
||||
func runExternalRecovery(t *testing.T, fixture recoveryExternalFixture) {
|
||||
t.Helper()
|
||||
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
|
||||
t.Run("reconciliation/"+string(action), func(t *testing.T) {
|
||||
cfg := fixture.database(t)
|
||||
db := openRecoveryExternal(t, cfg)
|
||||
if err := database.Migrate(t.Context(), db, cfg.Driver); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runReconciliationSuccess(t, NewSQLDeploymentTransactionRepository(db, cfg.Driver), db, action)
|
||||
})
|
||||
}
|
||||
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionRecord, deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
|
||||
for _, stage := range []string{"none", "started", "finished"} {
|
||||
t.Run(string(action)+"/"+stage, func(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"maps"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||
)
|
||||
|
||||
// 远端文件是测试替身;事务、关联、占用交接与审计均经过真实数据库和产品执行器。
|
||||
type reconciliationClient struct {
|
||||
*executorIntegrationClient
|
||||
failClosed, readOnly bool
|
||||
connections, writes int
|
||||
}
|
||||
|
||||
func (c *reconciliationClient) Connect(context.Context, deployment.Server) (deployment.Client, error) {
|
||||
c.connections++
|
||||
return c, nil
|
||||
}
|
||||
func (c *reconciliationClient) Close() error {
|
||||
if c.failClosed {
|
||||
if _, exists := c.files["/cert.pem.scert-lock"]; !exists {
|
||||
return errors.New("注入动作完成后的连接关闭失败")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (c *reconciliationClient) writable() error {
|
||||
c.writes++
|
||||
if c.readOnly {
|
||||
return errors.New("重新核查不得执行远端写操作")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (c *reconciliationClient) Remove(ctx context.Context, path string) error {
|
||||
if err := c.writable(); err != nil {
|
||||
return err
|
||||
}
|
||||
return c.executorIntegrationClient.Remove(ctx, path)
|
||||
}
|
||||
func (c *reconciliationClient) Rename(ctx context.Context, from, to string) error {
|
||||
if err := c.writable(); err != nil {
|
||||
return err
|
||||
}
|
||||
return c.executorIntegrationClient.Rename(ctx, from, to)
|
||||
}
|
||||
func (c *reconciliationClient) Unlock(ctx context.Context, path string) error {
|
||||
if err := c.writable(); err != nil {
|
||||
return err
|
||||
}
|
||||
return c.executorIntegrationClient.Unlock(ctx, path)
|
||||
}
|
||||
func (c *reconciliationClient) Run(ctx context.Context, command []string) (deployment.CommandResult, error) {
|
||||
if err := c.writable(); err != nil {
|
||||
return deployment.CommandResult{}, err
|
||||
}
|
||||
return c.executorIntegrationClient.Run(ctx, command)
|
||||
}
|
||||
|
||||
func seedUncertainReconciliation(t *testing.T, repo *SQLDeploymentTransactionRepository, db *sql.DB, action deployment.RecoveryAction) (*deployment.Service, *reconciliationClient, deployment.Server, deployment.TransactionRecord, deployment.RecoveryDisposition) {
|
||||
t.Helper()
|
||||
_, _, _, remote, server, record := seedRecoveryExecutor(t, repo, db)
|
||||
client := &reconciliationClient{executorIntegrationClient: remote, failClosed: true}
|
||||
service := deployment.NewServiceWithJournal(client, repo)
|
||||
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error { return nil }
|
||||
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
|
||||
confirmation, token, err := service.PrepareRecoveryConfirmation(t.Context(), record.ID, "unix:0", action, resolve)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
|
||||
if !errors.Is(err, deployment.ErrRecoveryActionUncertain) || source.Status != deployment.RecoveryResultUncertain {
|
||||
t.Fatalf("未形成真实执行器不确定结果:%+v %v", source, err)
|
||||
}
|
||||
client.failClosed, client.readOnly = false, true
|
||||
client.writes = 0
|
||||
return service, client, server, record, source
|
||||
}
|
||||
|
||||
func assertReconciliationSource(t *testing.T, repo *SQLDeploymentTransactionRepository, source deployment.RecoveryDisposition) {
|
||||
t.Helper()
|
||||
stored, err := repo.GetRecoveryDisposition(t.Context(), source.Intent.ID)
|
||||
if err != nil || !reflect.DeepEqual(stored, source) {
|
||||
t.Fatalf("重新核查改写了原不确定历史:%+v %v", stored, err)
|
||||
}
|
||||
}
|
||||
|
||||
func runReconciliationSuccess(t *testing.T, repo *SQLDeploymentTransactionRepository, db *sql.DB, action deployment.RecoveryAction) {
|
||||
t.Helper()
|
||||
service, client, server, record, source := seedUncertainReconciliation(t, repo, db, action)
|
||||
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
|
||||
files := maps.Clone(client.files)
|
||||
verified := false
|
||||
service.VerifyRecoveryTarget = func(_ context.Context, _ deployment.Server, _ deployment.RecoveryBaseline, committed bool) error {
|
||||
if committed != (action == deployment.RecoveryActionCommit) {
|
||||
t.Fatal("重新核查擅自切换方向")
|
||||
}
|
||||
verified = true
|
||||
return nil
|
||||
}
|
||||
confirmation, token, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve)
|
||||
if err != nil || confirmation.Summary().Validate() != nil {
|
||||
t.Fatal("准备重新核查失败", err)
|
||||
}
|
||||
other, otherToken, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
|
||||
if err != nil || result.Status != "succeeded" || result.Intent.SourceDispositionID != source.Intent.ID || !verified {
|
||||
t.Fatalf("重新核查失败:%+v %v", result, err)
|
||||
}
|
||||
if client.writes != 0 || !reflect.DeepEqual(files, client.files) {
|
||||
t.Fatal("只读重新核查修改了远端")
|
||||
}
|
||||
assertReconciliationSource(t, repo, source)
|
||||
recoveryCount(t, db, "recovery_dispositions", 2)
|
||||
recoveryCount(t, db, "recovery_events", 4)
|
||||
recoveryCount(t, db, "recovery_claims", 0)
|
||||
pending, err := repo.ListPendingDeploymentTransactions(t.Context())
|
||||
if err != nil || len(pending) != 0 {
|
||||
t.Fatal("原事务未原子结束", err)
|
||||
}
|
||||
view, err := service.GetRecoveryDisposition(t.Context(), result.Intent.ID)
|
||||
if err != nil || view.SourceDispositionID != source.Intent.ID || view.Validate() != nil {
|
||||
t.Fatal("查询缺少可信关联", err)
|
||||
}
|
||||
connections := client.connections
|
||||
if _, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve); !errors.Is(err, deployment.ErrRecoveryConfirmation) {
|
||||
t.Fatal("重新核查确认被重放", err)
|
||||
}
|
||||
if _, err := service.ExecuteRecoveryDisposition(t.Context(), other, "unix:0", otherToken, resolve); err == nil {
|
||||
t.Fatal("其他旧确认再次接管已完成的事务")
|
||||
}
|
||||
if client.connections != connections {
|
||||
t.Fatal("完成后仍使用旧确认连接远端")
|
||||
}
|
||||
if record.ID != result.Intent.TransactionID {
|
||||
t.Fatal("结束了其他事务")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryReconciliationSQLiteSuccess(t *testing.T) {
|
||||
for _, action := range []deployment.RecoveryAction{deployment.RecoveryActionCommit, deployment.RecoveryActionRollback} {
|
||||
t.Run(string(action), func(t *testing.T) {
|
||||
_, db := newTaskRepositoryTest(t)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
runReconciliationSuccess(t, NewSQLDeploymentTransactionRepository(db), db, action)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryReconciliationSQLiteRejectsUnsafeCompletion(t *testing.T) {
|
||||
for _, failure := range []string{"file", "backup", "temporary", "lock", "https", "during-https", "close", "changed-after-prepare", "started-audit", "finished-audit", "identity", "phase", "running", "direction"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
_, db := newTaskRepositoryTest(t)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
repo := NewSQLDeploymentTransactionRepository(db)
|
||||
service, client, server, record, source := seedUncertainReconciliation(t, repo, db, deployment.RecoveryActionCommit)
|
||||
resolve := func(context.Context, string) (deployment.Server, error) { return server, nil }
|
||||
switch failure {
|
||||
case "file":
|
||||
client.files[record.Paths[0]] = "tampered"
|
||||
case "backup":
|
||||
client.files[record.Backups[0].Saved] = "old"
|
||||
case "temporary":
|
||||
client.files[record.Paths[0]+".scert-tmp-"+record.ID] = "unexpected"
|
||||
case "lock":
|
||||
client.files[record.LockPaths[0]] = ""
|
||||
case "https":
|
||||
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error {
|
||||
return errors.New("严格 HTTPS 未通过")
|
||||
}
|
||||
case "during-https":
|
||||
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error {
|
||||
client.files[record.Paths[0]] = "changed"
|
||||
return nil
|
||||
}
|
||||
case "identity":
|
||||
server.PasswordRef = "secret://ssh/password/changed"
|
||||
case "phase":
|
||||
if _, err := db.ExecContext(t.Context(), `UPDATE deployment_transactions SET phase = 'reload_attempted'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "running":
|
||||
if _, err := db.ExecContext(t.Context(), `UPDATE recovery_dispositions SET status = 'running', result_code = ''`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "direction":
|
||||
if _, err := db.ExecContext(t.Context(), `UPDATE recovery_dispositions SET action = 'restore_baseline'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
confirmation, token, err := service.PrepareRecoveryReconciliation(t.Context(), source.Intent.ID, "unix:0", resolve)
|
||||
if failure == "identity" || failure == "phase" || failure == "running" || failure == "direction" {
|
||||
if err == nil {
|
||||
t.Fatal("危险来源允许重新核查")
|
||||
}
|
||||
recoveryCount(t, db, "recovery_claims", 1)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal("准备失败", err)
|
||||
}
|
||||
if failure == "close" {
|
||||
service.VerifyRecoveryTarget = func(context.Context, deployment.Server, deployment.RecoveryBaseline, bool) error {
|
||||
client.failClosed = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if failure == "changed-after-prepare" {
|
||||
client.files[record.Paths[1]] = "changed"
|
||||
}
|
||||
if failure == "started-audit" || failure == "finished-audit" {
|
||||
stage := "started"
|
||||
if failure == "finished-audit" {
|
||||
stage = "finished"
|
||||
}
|
||||
if _, err := db.ExecContext(t.Context(), `CREATE TRIGGER reject_recheck BEFORE INSERT ON recovery_events WHEN NEW.event_type = '`+stage+`' BEGIN SELECT RAISE(ABORT, 'injected'); END`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
result, err := service.ExecuteRecoveryDisposition(t.Context(), confirmation, "unix:0", token, resolve)
|
||||
if err == nil || result.Status == "succeeded" || client.writes != 0 {
|
||||
t.Fatalf("危险收尾未拒绝或执行了写操作:%+v %v", result, err)
|
||||
}
|
||||
assertReconciliationSource(t, repo, source)
|
||||
var owner string
|
||||
if err := db.QueryRowContext(t.Context(), `SELECT disposition_id FROM recovery_claims WHERE transaction_id = ?`, record.ID).Scan(&owner); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expected := source.Intent.ID
|
||||
if failure == "finished-audit" {
|
||||
expected = confirmation.DispositionID()
|
||||
}
|
||||
if owner != expected {
|
||||
t.Fatal("失败后占用归属不正确", owner, expected)
|
||||
}
|
||||
pending, err := repo.ListPendingDeploymentTransactions(t.Context())
|
||||
if err != nil || len(pending) != 1 || pending[0].Status != deployment.TransactionStatusRecoveryRequired {
|
||||
t.Fatal("失败改变了原事务", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user