修复MySQL部署迁移中断重入并更新实库验收进度

This commit is contained in:
xuans committed 2026-10-01 17:48:18 +08:00
1 parent dbcdf05b3e
commit c6982ab827
7 files changed
+448

No files matched your search

+3
View File
@@ -84,6 +84,9 @@ func applyMigration(ctx context.Context, db *sql.DB, driver string, version int)
return applySQLiteTaskMigration(ctx, db, statements, version)
}
if driver == "mysql" {
if version >= 15 && version <= 17 {
return applyMySQLDeploymentMigration(ctx, db, version, statements)
}
if version == 14 {
if err := installMySQLProjectMembers(ctx, db); err != nil {
return err
@@ -0,0 +1,215 @@
package database
import (
"context"
"database/sql"
"fmt"
"reflect"
"regexp"
"sort"
"strings"
)
// MySQL 的建表语句会单独提交,恢复时必须校验结构,不能直接吞掉同名表错误。
func applyMySQLDeploymentMigration(ctx context.Context, db *sql.DB, version int, statements []string) error {
if len(statements) == 0 {
return fmt.Errorf("部署迁移缺少建表语句:%d", version)
}
shape, err := parseMySQLDeploymentShape(statements[0])
if err != nil {
return err
}
create := strings.Replace(statements[0], "CREATE TABLE ", "CREATE TABLE IF NOT EXISTS ", 1)
if _, err := db.ExecContext(ctx, create); err != nil {
return fmt.Errorf("创建部署迁移表失败:%w", err)
}
if err := validateMySQLDeploymentShape(ctx, db, shape); err != nil {
return err
}
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if version == 15 {
if err := backfillMySQLTaskIdempotency(ctx, tx); err != nil {
return err
}
}
if _, err := tx.ExecContext(ctx, "INSERT INTO schema_migrations (version) VALUES (?)", version); err != nil {
return fmt.Errorf("记录部署迁移版本失败:%w", err)
}
return tx.Commit()
}
type mysqlDeploymentColumn struct {
Name string
Type string
Default string
HasDefault bool
}
type mysqlDeploymentShape struct {
Table string
Columns []mysqlDeploymentColumn
Indexes []string
}
var (
mysqlDeploymentTablePattern = regexp.MustCompile(`^CREATE TABLE ([a-z_]+) \($`)
mysqlDeploymentColumnPattern = regexp.MustCompile(`^([a-z_]+) (VARCHAR\([0-9]+\)|INT|TEXT|DATETIME) NOT NULL(?: DEFAULT (CURRENT_TIMESTAMP|0))?( PRIMARY KEY)?$`)
mysqlDeploymentIndexPattern = regexp.MustCompile(`^(PRIMARY KEY|UNIQUE KEY ([a-z_]+)|INDEX ([a-z_]+)) \(([a-z_, ]+)\)$`)
mysqlIntegerWidthPattern = regexp.MustCompile(`^int\([0-9]+\)$`)
)
// 只解析这三份内嵌迁移使用的语法;出现未知定义时拒绝迁移,避免漏检结构。
func parseMySQLDeploymentShape(statement string) (mysqlDeploymentShape, error) {
var shape mysqlDeploymentShape
lines := strings.Split(strings.TrimSpace(statement), "\n")
if len(lines) < 3 {
return shape, fmt.Errorf("部署迁移表定义不完整")
}
match := mysqlDeploymentTablePattern.FindStringSubmatch(strings.TrimSpace(lines[0]))
if len(match) != 2 || strings.TrimSpace(lines[len(lines)-1]) != ")" {
return shape, fmt.Errorf("部署迁移表定义不受支持")
}
shape.Table = match[1]
for _, raw := range lines[1 : len(lines)-1] {
line := strings.TrimSuffix(strings.TrimSpace(raw), ",")
if parts := mysqlDeploymentColumnPattern.FindStringSubmatch(line); len(parts) == 5 {
shape.Columns = append(shape.Columns, mysqlDeploymentColumn{parts[1], strings.ToLower(parts[2]), strings.ToLower(parts[3]), parts[3] != ""})
if parts[4] != "" {
shape.Indexes = append(shape.Indexes, "PRIMARY:0:1:"+parts[1])
}
continue
}
parts := mysqlDeploymentIndexPattern.FindStringSubmatch(line)
if len(parts) != 5 {
return shape, fmt.Errorf("部署迁移表包含不受支持的定义:%s", shape.Table)
}
name, nonUnique := "PRIMARY", 0
if parts[2] != "" {
name = parts[2]
} else if parts[3] != "" {
name, nonUnique = parts[3], 1
}
for index, column := range strings.Split(parts[4], ",") {
shape.Indexes = append(shape.Indexes, fmt.Sprintf("%s:%d:%d:%s", name, nonUnique, index+1, strings.TrimSpace(column)))
}
}
sort.Strings(shape.Indexes)
return shape, nil
}
func validateMySQLDeploymentShape(ctx context.Context, db *sql.DB, expected mysqlDeploymentShape) error {
var tableCollation, schemaCollation string
if err := db.QueryRowContext(ctx, `SELECT t.table_collation, s.default_collation_name
FROM information_schema.tables t JOIN information_schema.schemata s ON t.table_schema = s.schema_name
WHERE t.table_schema = DATABASE() AND t.table_name = ?`, expected.Table).Scan(&tableCollation, &schemaCollation); err != nil {
return err
}
if tableCollation != schemaCollation {
return fmt.Errorf("部署迁移表排序规则与数据库默认定义不匹配:%s", expected.Table)
}
var engine, tableType string
if err := db.QueryRowContext(ctx, `SELECT ENGINE, TABLE_TYPE FROM information_schema.tables
WHERE table_schema = DATABASE() AND table_name = ?`, expected.Table).Scan(&engine, &tableType); err != nil {
return fmt.Errorf("读取部署迁移表状态失败:%w", err)
}
if !strings.EqualFold(engine, "InnoDB") || tableType != "BASE TABLE" {
return fmt.Errorf("部署迁移表必须是 InnoDB 基础表:%s", expected.Table)
}
rows, err := db.QueryContext(ctx, `SELECT column_name, column_type, column_default, is_nullable, extra, collation_name
FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = ? ORDER BY ordinal_position`, expected.Table)
if err != nil {
return err
}
var actual []mysqlDeploymentColumn
for rows.Next() {
var column mysqlDeploymentColumn
var defaultValue sql.NullString
var columnCollation sql.NullString
var nullable, extra string
if err := rows.Scan(&column.Name, &column.Type, &defaultValue, &nullable, &extra, &columnCollation); err != nil {
rows.Close()
return err
}
column.Type = strings.ToLower(column.Type)
if mysqlIntegerWidthPattern.MatchString(column.Type) {
column.Type = "int"
}
column.Default = strings.TrimSuffix(strings.ToLower(defaultValue.String), "()")
column.HasDefault = defaultValue.Valid
if columnCollation.Valid && columnCollation.String != tableCollation {
rows.Close()
return fmt.Errorf("部署迁移表字段排序规则不匹配:%s.%s", expected.Table, column.Name)
}
if nullable != "NO" || (extra != "" && !(strings.EqualFold(extra, "DEFAULT_GENERATED") && column.Default == "current_timestamp")) {
rows.Close()
return fmt.Errorf("部署迁移表字段属性不匹配:%s.%s", expected.Table, column.Name)
}
actual = append(actual, column)
}
err = rows.Err()
rows.Close()
if err != nil {
return err
}
if !reflect.DeepEqual(actual, expected.Columns) {
return fmt.Errorf("部署迁移表字段定义不匹配:%s,拒绝跳过或覆盖现有数据", expected.Table)
}
rows, err = db.QueryContext(ctx, `SELECT index_name, non_unique, seq_in_index, column_name, sub_part, index_type, collation
FROM information_schema.statistics WHERE table_schema = DATABASE() AND table_name = ?`, expected.Table)
if err != nil {
return err
}
var indexes []string
for rows.Next() {
var name, column, indexType, collation string
var nonUnique, position int
var prefix sql.NullInt64
if err := rows.Scan(&name, &nonUnique, &position, &column, &prefix, &indexType, &collation); err != nil {
rows.Close()
return err
}
if prefix.Valid || indexType != "BTREE" || collation != "A" {
rows.Close()
return fmt.Errorf("部署迁移表索引属性不匹配:%s.%s", expected.Table, name)
}
indexes = append(indexes, fmt.Sprintf("%s:%d:%d:%s", name, nonUnique, position, column))
}
err = rows.Err()
rows.Close()
if err != nil {
return err
}
sort.Strings(indexes)
if !reflect.DeepEqual(indexes, expected.Indexes) {
return fmt.Errorf("部署迁移表索引定义不匹配:%s,拒绝跳过或覆盖现有数据", expected.Table)
}
return nil
}
// 回填与版本标记同事务提交;已落盘的历史回填只有内容完全一致才可复用。
func backfillMySQLTaskIdempotency(ctx context.Context, tx *sql.Tx) error {
const expected = `SELECT domain_id, idempotency_key, MIN(id) AS task_id, MIN(created_at) AS created_at
FROM tasks WHERE idempotency_key <> '' GROUP BY domain_id, idempotency_key`
var conflicts int
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM task_idempotency AS actual
LEFT JOIN (`+expected+`) AS expected ON actual.domain_id = expected.domain_id AND actual.idempotency_key = expected.idempotency_key
WHERE expected.task_id IS NULL OR BINARY actual.task_id <> BINARY expected.task_id OR actual.created_at <> expected.created_at
OR BINARY actual.domain_id <> BINARY expected.domain_id OR BINARY actual.idempotency_key <> BINARY expected.idempotency_key`).Scan(&conflicts); err != nil {
return fmt.Errorf("检查任务幂等历史回填失败:%w", err)
}
if conflicts != 0 {
return fmt.Errorf("任务幂等历史回填与已有数据冲突,拒绝覆盖,需核查后重试")
}
_, err := tx.ExecContext(ctx, `INSERT INTO task_idempotency (domain_id, idempotency_key, task_id, created_at)
SELECT expected.domain_id, expected.idempotency_key, expected.task_id, expected.created_at FROM (`+expected+`) AS expected
LEFT JOIN task_idempotency AS actual ON actual.domain_id = expected.domain_id AND actual.idempotency_key = expected.idempotency_key
WHERE actual.domain_id IS NULL`)
if err != nil {
return fmt.Errorf("回填任务幂等历史失败:%w", err)
}
return nil
}
@@ -0,0 +1,157 @@
package database
import (
"database/sql"
"fmt"
"strings"
"testing"
)
func TestMySQLDeploymentShapeDefinitions(t *testing.T) {
for version := 15; version <= 17; version++ {
statements := mysqlDeploymentStatements(t, version)
shape, err := parseMySQLDeploymentShape(statements[0])
if err != nil || len(shape.Columns) == 0 || len(shape.Indexes) == 0 {
t.Fatalf("内嵌迁移定义必须完整解析:%d:%v", version, err)
}
for _, invalid := range []string{
strings.Replace(statements[0], "NOT NULL", "NULL", 1),
strings.Replace(statements[0], "VARCHAR", "CHAR", 1),
strings.Replace(statements[0], "CREATE TABLE", "CREATE VIEW", 1),
} {
if _, err := parseMySQLDeploymentShape(invalid); err == nil {
t.Fatal("未知迁移定义不能被静默忽略")
}
}
}
}
func TestMySQLRealDeploymentMigrationRecovery(t *testing.T) {
f := startRealMySQL(t)
baseline := f.database(t)
if err := Migrate(t.Context(), baseline.DB, "mysql"); err != nil {
t.Fatal(err)
}
expectedSchema := mysqlSchemaSnapshot(t, baseline.DB)
for _, boundary := range []string{"before_create", "after_create", "partial_backfill", "after_backfill"} {
t.Run(boundary, func(t *testing.T) {
db := f.database(t)
mysqlPrepareVersion(t, db.DB, 14)
seedMySQLIdempotencyTasks(t, db.DB)
statements := mysqlDeploymentStatements(t, 15)
if boundary != "before_create" {
mysqlDeploymentExec(t, db.DB, statements[0])
}
switch boundary {
case "partial_backfill":
mysqlDeploymentExec(t, db.DB, `INSERT INTO task_idempotency VALUES ('domain-a','request-a','task-a','2026-01-01 00:00:00')`)
case "after_backfill":
mysqlDeploymentExec(t, db.DB, statements[1])
}
mysqlAssertUpgrade(t, db.DB, expectedSchema)
var count int
if err := db.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM task_idempotency").Scan(&count); err != nil || count != 3 {
t.Fatalf("幂等回填必须去重、跨域隔离并排除空键:%d:%v", count, err)
}
var task, created string
if err := db.DB.QueryRowContext(t.Context(), `SELECT task_id, DATE_FORMAT(created_at, '%Y-%m-%d %H:%i:%s') FROM task_idempotency WHERE domain_id='domain-a' AND idempotency_key='request-a'`).Scan(&task, &created); err != nil || task != "task-a" || created != "2026-01-01 00:00:00" {
t.Fatalf("历史代表任务及最早创建时间必须保留:%s:%s:%v", task, created, err)
}
})
}
for _, invalid := range []struct {
name string
statement string
}{
{"wrong_task", `INSERT INTO task_idempotency VALUES ('domain-a','request-a','task-z','2026-01-01 00:00:00')`},
{"wrong_task_case", `INSERT INTO task_idempotency VALUES ('domain-a','request-a','TASK-A','2026-01-01 00:00:00')`},
{"wrong_time", `INSERT INTO task_idempotency VALUES ('domain-a','request-a','task-a','2026-01-02 00:00:00')`},
{"orphan_mapping", `INSERT INTO task_idempotency VALUES ('unknown','unknown','unknown','2026-01-01 00:00:00')`},
} {
t.Run(invalid.name, func(t *testing.T) {
db := f.database(t)
mysqlPrepareVersion(t, db.DB, 14)
seedMySQLIdempotencyTasks(t, db.DB)
mysqlDeploymentExec(t, db.DB, mysqlDeploymentStatements(t, 15)[0])
mysqlDeploymentExec(t, db.DB, invalid.statement)
var before string
query := `SELECT CONCAT(domain_id, ':', idempotency_key, ':', task_id, ':', created_at) FROM task_idempotency`
if err := db.DB.QueryRowContext(t.Context(), query).Scan(&before); err != nil {
t.Fatal(err)
}
assertMySQLDeploymentMigrationRejected(t, db.DB, 14)
var after string
if err := db.DB.QueryRowContext(t.Context(), query).Scan(&after); err != nil || after != before {
t.Fatalf("拒绝恢复不得修改冲突数据:%v", err)
}
})
}
for _, invalid := range []struct {
name string
version int
statement string
}{
{"wrong_column", 15, "ALTER TABLE task_idempotency MODIFY COLUMN task_id VARCHAR(64) NOT NULL"},
{"empty_string_default", 15, "ALTER TABLE task_idempotency ALTER COLUMN task_id SET DEFAULT ''"},
{"wrong_collation", 15, "ALTER TABLE task_idempotency MODIFY COLUMN task_id VARCHAR(36) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL"},
{"nullable_column", 15, "ALTER TABLE task_idempotency MODIFY COLUMN task_id VARCHAR(36) NULL"},
{"missing_unique", 15, "ALTER TABLE task_idempotency DROP INDEX uq_task_idempotency_task"},
{"wrong_default", 16, "ALTER TABLE task_deployment_results ALTER COLUMN attempt_number SET DEFAULT 7"},
{"missing_index", 16, "ALTER TABLE task_deployment_results DROP INDEX idx_task_deployment_results_task_id"},
{"wrong_engine", 17, "ALTER TABLE deployment_transactions ENGINE=MyISAM"},
{"extra_column", 17, "ALTER TABLE deployment_transactions ADD COLUMN unexpected INT NOT NULL DEFAULT 0"},
{"wrong_index_order", 17, "ALTER TABLE deployment_transactions DROP INDEX idx_deployment_transactions_status, ADD INDEX idx_deployment_transactions_status (updated_at, status)"},
} {
t.Run(invalid.name, func(t *testing.T) {
db := f.database(t)
mysqlPrepareVersion(t, db.DB, invalid.version-1)
mysqlDeploymentExec(t, db.DB, mysqlDeploymentStatements(t, invalid.version)[0])
mysqlDeploymentExec(t, db.DB, invalid.statement)
before := fmt.Sprint(mysqlSchemaSnapshot(t, db.DB))
assertMySQLDeploymentMigrationRejected(t, db.DB, invalid.version-1)
if after := fmt.Sprint(mysqlSchemaSnapshot(t, db.DB)); after != before {
t.Fatal("拒绝恢复不得擅自修复或覆盖已有结构")
}
})
}
}
func mysqlDeploymentStatements(t *testing.T, version int) []string {
t.Helper()
content, err := migrationFiles.ReadFile(fmt.Sprintf("migrations/mysql/%04d_%s.sql", version, migrationName(version)))
if err != nil {
t.Fatal(err)
}
return migrationStatements(content)
}
func mysqlDeploymentExec(t *testing.T, db *sql.DB, statement string) {
t.Helper()
if _, err := db.ExecContext(t.Context(), statement); err != nil {
t.Fatal(err)
}
}
func seedMySQLIdempotencyTasks(t *testing.T, db *sql.DB) {
t.Helper()
mysqlDeploymentExec(t, db, `INSERT INTO tasks (id, domain_id, task_type, status, message, idempotency_key, created_at) VALUES
('task-z','domain-a','deploy','failed','','request-a','2026-01-01 00:00:00'),
('task-a','domain-a','deploy','success','','request-a','2026-01-02 00:00:00'),
('task-b','domain-a','deploy','success','','request-b','2026-01-03 00:00:00'),
('task-c','domain-b','deploy','success','','request-a','2026-01-04 00:00:00'),
('task-empty','domain-a','deploy','success','','','2026-01-05 00:00:00')`)
}
func assertMySQLDeploymentMigrationRejected(t *testing.T, db *sql.DB, expectedVersion int) {
t.Helper()
for repeat := 0; repeat < 2; repeat++ {
if err := Migrate(t.Context(), db, "mysql"); err == nil {
t.Fatal("结构或历史冲突时必须拒绝迁移")
}
version, err := currentMigrationVersion(t.Context(), db)
if err != nil || version != expectedVersion {
t.Fatalf("拒绝恢复不得提前写入版本:%d:%v", version, err)
}
mysqlAssertPreserved(t, db)
}
}