修复MySQL特殊列备份并补充官方工具互操作测试
保留并验证四个待提交源码文件：显式导出可写列以保留不可见列，跳过生成列并保留仅生成列表的行数；修复扫描越界，以实际结构元数据校验恢复结果。MySQL 5.7.44与8.0.45专项回归、后端常规测试、静态检查及模块校验通过；不代表完整生产验收。
This commit is contained in:
parent
e9bc156bed
commit
f260b221a9
@ -157,12 +157,22 @@ func dumpMySQLTable(ctx context.Context, db *sql.Tx, output *os.File, tableName
|
||||
return fmt.Errorf("写入 MySQL 表结构失败（%s）：%w", tableName, err)
|
||||
}
|
||||
|
||||
rows, err := db.QueryContext(ctx, "SELECT * FROM "+quotedTable)
|
||||
columns, err := mysqlInsertableColumns(ctx, db, tableName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 显式投影可写列：SELECT * 会遗漏不可见列，并错误地包含生成列的计算结果。
|
||||
projection := quoteMySQLColumns(columns)
|
||||
if len(columns) == 0 {
|
||||
// 只有生成列的表仍有行数，使用占位读取并导出空列 INSERT 保留每一行。
|
||||
projection = "1"
|
||||
}
|
||||
rows, err := db.QueryContext(ctx, "SELECT "+projection+" FROM "+quotedTable)
|
||||
if err != nil {
|
||||
return fmt.Errorf("读取 MySQL 表数据失败（%s）：%w", tableName, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
columns, err := rows.Columns()
|
||||
selectedColumns, err := rows.Columns()
|
||||
if err != nil {
|
||||
return fmt.Errorf("读取 MySQL 表字段失败（%s）：%w", tableName, err)
|
||||
}
|
||||
@ -171,17 +181,17 @@ func dumpMySQLTable(ctx context.Context, db *sql.Tx, output *os.File, tableName
|
||||
return fmt.Errorf("读取 MySQL 表字段类型失败（%s）：%w", tableName, err)
|
||||
}
|
||||
for rows.Next() {
|
||||
values := make([]any, len(columns))
|
||||
pointers := make([]any, len(columns))
|
||||
values := make([]any, len(selectedColumns))
|
||||
pointers := make([]any, len(values))
|
||||
for index := range values {
|
||||
pointers[index] = &values[index]
|
||||
}
|
||||
if err := rows.Scan(pointers...); err != nil {
|
||||
return fmt.Errorf("读取 MySQL 表行失败（%s）：%w", tableName, err)
|
||||
}
|
||||
literals := make([]string, len(values))
|
||||
for index, value := range values {
|
||||
literals[index] = mysqlColumnSQLLiteral(value, columnTypes[index].DatabaseTypeName())
|
||||
literals := make([]string, len(columns))
|
||||
for index := range columns {
|
||||
literals[index] = mysqlColumnSQLLiteral(values[index], columnTypes[index].DatabaseTypeName())
|
||||
}
|
||||
if _, err := fmt.Fprintf(output, "INSERT INTO %s (%s) VALUES (%s);\n", quotedTable, quoteMySQLColumns(columns), strings.Join(literals, ", ")); err != nil {
|
||||
return fmt.Errorf("写入 MySQL 表数据失败（%s）：%w", tableName, err)
|
||||
@ -194,6 +204,26 @@ func dumpMySQLTable(ctx context.Context, db *sql.Tx, output *os.File, tableName
|
||||
return err
|
||||
}
|
||||
|
||||
func mysqlInsertableColumns(ctx context.Context, db *sql.Tx, tableName string) ([]string, error) {
|
||||
rows, err := db.QueryContext(ctx, "SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COALESCE(GENERATION_EXPRESSION, '') = '' ORDER BY ORDINAL_POSITION", tableName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取 MySQL 可写字段失败（%s）：%w", tableName, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var columns []string
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err != nil {
|
||||
return nil, fmt.Errorf("读取 MySQL 可写字段名失败（%s）：%w", tableName, err)
|
||||
}
|
||||
columns = append(columns, name)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("遍历 MySQL 可写字段失败（%s）：%w", tableName, err)
|
||||
}
|
||||
return columns, nil
|
||||
}
|
||||
|
||||
func quoteMySQLIdentifier(value string) string {
|
||||
return "`" + strings.ReplaceAll(value, "`", "``") + "`"
|
||||
}
|
||||
|
||||
174
backend/internal/cli/mysql_interop_integration_test.go
Normal file
174
backend/internal/cli/mysql_interop_integration_test.go
Normal file
@ -0,0 +1,174 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/config"
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/database"
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/testsupport/mysqltest"
|
||||
)
|
||||
|
||||
func TestMySQLRealOfficialInteroperability(t *testing.T) {
|
||||
server := mysqltest.New(t)
|
||||
fixtures := []struct {
|
||||
name string
|
||||
create string
|
||||
insert string
|
||||
query string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "basic_values",
|
||||
create: "CREATE TABLE interop (id INT PRIMARY KEY, label TEXT, payload BLOB, amount DECIMAL(20,6), optional_value TEXT NULL) ENGINE=InnoDB",
|
||||
insert: "INSERT INTO interop VALUES (1, '证书;单引号''与反斜线\\\\', X'00013B5CFF', 123456789.123456, NULL)",
|
||||
query: "SELECT CONCAT(label, '|', HEX(payload), '|', CAST(amount AS CHAR), '|', optional_value IS NULL) FROM interop WHERE id = 1",
|
||||
want: "证书;单引号'与反斜线\\|00013B5CFF|123456789.123456|1",
|
||||
},
|
||||
{
|
||||
name: "generated_columns",
|
||||
create: "CREATE TABLE interop (id INT PRIMARY KEY, base_value DECIMAL(20,6), stored_value DECIMAL(21,6) GENERATED ALWAYS AS (base_value * 2) STORED, virtual_value VARCHAR(80) GENERATED ALWAYS AS (CONCAT('证书;', id)) VIRTUAL, label TEXT) ENGINE=InnoDB",
|
||||
insert: "INSERT INTO interop (id, base_value, label) VALUES (7, 1234.123456, '保留原值')",
|
||||
query: "SELECT CONCAT(CAST(base_value AS CHAR), '|', CAST(stored_value AS CHAR), '|', virtual_value, '|', label) FROM interop WHERE id = 7",
|
||||
want: "1234.123456|2468.246912|证书;7|保留原值",
|
||||
},
|
||||
{
|
||||
name: "only_generated_columns",
|
||||
create: "CREATE TABLE interop (stored_value INT GENERATED ALWAYS AS (42) STORED) ENGINE=InnoDB",
|
||||
insert: "INSERT INTO interop () VALUES (), (), ()",
|
||||
query: "SELECT CONCAT(COUNT(*), '|', SUM(stored_value)) FROM interop",
|
||||
want: "3|126",
|
||||
},
|
||||
}
|
||||
if server.IsMySQL8() {
|
||||
fixtures = append(fixtures, struct {
|
||||
name string
|
||||
create string
|
||||
insert string
|
||||
query string
|
||||
want string
|
||||
}{
|
||||
name: "invisible_column",
|
||||
create: "CREATE TABLE interop (id INT PRIMARY KEY, hidden_value VARCHAR(80) INVISIBLE DEFAULT '默认隐藏值', label TEXT) ENGINE=InnoDB",
|
||||
insert: "INSERT INTO interop (id, hidden_value, label) VALUES (1, '隐藏列实际值', '可见列')",
|
||||
query: "SELECT CONCAT(hidden_value, '|', label) FROM interop WHERE id = 1",
|
||||
want: "隐藏列实际值|可见列",
|
||||
})
|
||||
}
|
||||
for _, fixture := range fixtures {
|
||||
for _, direction := range []string{"scert_to_official", "official_to_scert"} {
|
||||
t.Run(fixture.name+"/"+direction, func(t *testing.T) {
|
||||
source := server.Database(t)
|
||||
db, err := database.Open(t.Context(), source)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
for _, statement := range []string{fixture.create, fixture.insert} {
|
||||
if _, err := db.DB.ExecContext(t.Context(), statement); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
checkMySQLInteropValue(t, source, fixture.query, fixture.want)
|
||||
definition := readMySQLInteropDefinition(t, db.DB)
|
||||
output := filepath.Join(server.Root(), fixture.name+"-"+direction+".sql")
|
||||
target := server.Database(t)
|
||||
if direction == "scert_to_official" {
|
||||
if err := backupMySQLDatabase(t.Context(), source, output); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
server.Import(t, target, output)
|
||||
} else {
|
||||
server.Dump(t, source, output)
|
||||
if err := restoreMySQL(t.Context(), target, output); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
checkMySQLInteropValue(t, target, fixture.query, fixture.want)
|
||||
verify, err := database.Open(t.Context(), target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer verify.Close()
|
||||
restoredDefinition := readMySQLInteropDefinition(t, verify.DB)
|
||||
if restoredDefinition != definition {
|
||||
t.Fatalf("互操作恢复后表结构不一致：\n源表：%s\n目标表：%s", definition, restoredDefinition)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readMySQLInteropDefinition(t *testing.T, db *sql.DB) string {
|
||||
t.Helper()
|
||||
queries := []string{
|
||||
`SELECT TABLE_TYPE, ENGINE, TABLE_COLLATION, ROW_FORMAT, CREATE_OPTIONS
|
||||
FROM information_schema.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'interop'`,
|
||||
`SELECT COLUMN_NAME, ORDINAL_POSITION, COLUMN_DEFAULT, IS_NULLABLE,
|
||||
COLUMN_TYPE, CHARACTER_SET_NAME, COLLATION_NAME, EXTRA,
|
||||
GENERATION_EXPRESSION, COLUMN_COMMENT
|
||||
FROM information_schema.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'interop'
|
||||
ORDER BY ORDINAL_POSITION`,
|
||||
`SELECT INDEX_NAME, NON_UNIQUE, SEQ_IN_INDEX, COLUMN_NAME,
|
||||
COLLATION, SUB_PART, PACKED, NULLABLE, INDEX_TYPE, COMMENT, INDEX_COMMENT
|
||||
FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'interop'
|
||||
ORDER BY INDEX_NAME, SEQ_IN_INDEX`,
|
||||
}
|
||||
var metadata [][][]sql.NullString
|
||||
for _, query := range queries {
|
||||
rows, err := db.QueryContext(t.Context(), query)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
columns, err := rows.Columns()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var section [][]sql.NullString
|
||||
for rows.Next() {
|
||||
values := make([]sql.NullString, len(columns))
|
||||
pointers := make([]any, len(values))
|
||||
for index := range values {
|
||||
pointers[index] = &values[index]
|
||||
}
|
||||
if err := rows.Scan(pointers...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
section = append(section, values)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
metadata = append(metadata, section)
|
||||
}
|
||||
encoded, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(encoded)
|
||||
}
|
||||
|
||||
func checkMySQLInteropValue(t *testing.T, cfg config.DatabaseConfig, query, want string) {
|
||||
t.Helper()
|
||||
db, err := database.Open(t.Context(), cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
var value string
|
||||
if err := db.DB.QueryRowContext(t.Context(), query).Scan(&value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value != want {
|
||||
t.Fatalf("官方工具互操作后数据不一致：得到 %q，期望 %q", value, want)
|
||||
}
|
||||
}
|
||||
@ -126,7 +126,7 @@ func waitForMySQLBackupLock(t *testing.T, ctx context.Context, admin *sql.DB, na
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
var connectionID uint64
|
||||
err := admin.QueryRowContext(ctx, "SELECT ID FROM information_schema.PROCESSLIST WHERE DB = ? AND STATE LIKE '%lock%' AND (INFO = 'SHOW CREATE TABLE `b_snapshot`' OR INFO = 'SELECT * FROM `b_snapshot`')", name).Scan(&connectionID)
|
||||
err := admin.QueryRowContext(ctx, "SELECT ID FROM information_schema.PROCESSLIST WHERE DB = ? AND STATE LIKE '%lock%' AND (INFO = 'SHOW CREATE TABLE `b_snapshot`' OR INFO LIKE 'SELECT % FROM `b_snapshot`')", name).Scan(&connectionID)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
106
backend/internal/testsupport/mysqltest/tools.go
Normal file
106
backend/internal/testsupport/mysqltest/tools.go
Normal file
@ -0,0 +1,106 @@
|
||||
package mysqltest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/config"
|
||||
)
|
||||
|
||||
// 官方工具仅连接本夹具创建的隔离实例，密码只传入子进程环境，不进入命令行或文件。
|
||||
func (f *Server) Dump(t *testing.T, cfg config.DatabaseConfig, output string) {
|
||||
t.Helper()
|
||||
file, err := os.OpenFile(output, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
f.runTool(t, "mysqldump", cfg, []string{
|
||||
"--single-transaction", "--no-tablespaces", "--set-gtid-purged=OFF",
|
||||
"--skip-add-locks", "--hex-blob", "--complete-insert", "--skip-comments", "--skip-extended-insert",
|
||||
}, nil, file)
|
||||
if err := file.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Server) Import(t *testing.T, cfg config.DatabaseConfig, input string) {
|
||||
t.Helper()
|
||||
file, err := os.Open(input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
f.runTool(t, "mysql", cfg, []string{
|
||||
"--binary-mode", "--batch", "--skip-reconnect", "--local-infile=0",
|
||||
}, file, io.Discard)
|
||||
}
|
||||
|
||||
func (f *Server) IsMySQL8() bool { return strings.Contains(f.version, "Ver 8.0.") }
|
||||
|
||||
func (f *Server) runTool(t *testing.T, name string, cfg config.DatabaseConfig, args []string, input io.Reader, output io.Writer) {
|
||||
t.Helper()
|
||||
parsed, err := mysql.ParseDSN(cfg.DSN)
|
||||
if err != nil || cfg.Driver != "mysql" {
|
||||
t.Fatal("官方工具测试连接无效")
|
||||
}
|
||||
if parsed.Net != "tcp" || parsed.Addr != "127.0.0.1:"+f.port || parsed.TLSConfig != f.tlsName ||
|
||||
parsed.Passwd != f.password || !strings.HasPrefix(parsed.DBName, "scert_test_") || !strings.HasPrefix(parsed.User, "scert_user_") {
|
||||
t.Fatal("官方工具只允许连接本次隔离实例的测试数据库")
|
||||
}
|
||||
binary := filepath.Join(filepath.Dir(f.binary), name)
|
||||
if runtime.GOOS == "windows" {
|
||||
binary += ".exe"
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
version, err := exec.CommandContext(ctx, binary, "--no-defaults", "--version").Output()
|
||||
if err != nil {
|
||||
t.Fatal("无法读取官方工具版本，不能跳过互操作验证")
|
||||
}
|
||||
expected := "5.7."
|
||||
if f.IsMySQL8() {
|
||||
expected = "8.0."
|
||||
}
|
||||
if !strings.Contains(string(version), expected) {
|
||||
t.Fatal("官方工具与本次测试服务端版本系列不一致")
|
||||
}
|
||||
t.Logf("官方工具：%s", strings.TrimSpace(string(version)))
|
||||
base := []string{
|
||||
"--no-defaults", "--protocol=TCP", "--host=127.0.0.1", "--port=" + f.port,
|
||||
"--user=" + parsed.User, "--ssl-mode=VERIFY_IDENTITY", "--ssl-ca=" + filepath.Join(f.root, "server.crt"),
|
||||
"--default-character-set=utf8mb4",
|
||||
}
|
||||
command := exec.CommandContext(ctx, binary, append(append(base, args...), parsed.DBName)...)
|
||||
command.Env = mysqlToolTestEnvironment(f.root, parsed.Passwd)
|
||||
command.Stdin, command.Stdout = input, output
|
||||
// 错误可能含数据库内容；只报告退出状态，不透传工具的原始错误输出。
|
||||
command.Stderr = io.Discard
|
||||
command.WaitDelay = 2 * time.Second
|
||||
if err := command.Run(); err != nil {
|
||||
t.Fatalf("官方 %s 执行失败：%v；原始输出已隐藏", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func mysqlToolTestEnvironment(root, password string) []string {
|
||||
env := make([]string, 0, len(os.Environ())+4)
|
||||
for _, entry := range os.Environ() {
|
||||
key, _, _ := strings.Cut(entry, "=")
|
||||
key = strings.ToUpper(key)
|
||||
if strings.HasPrefix(key, "MYSQL") || key == "LANG" || key == "LC_ALL" {
|
||||
continue
|
||||
}
|
||||
env = append(env, entry)
|
||||
}
|
||||
// --no-defaults 仍可能读取登录文件；指定隔离目录中不存在的文件，避免读取用户凭据。
|
||||
return append(env, "MYSQL_PWD="+password, "MYSQL_TEST_LOGIN_FILE="+filepath.Join(root, "no-login.cnf"), "LANG=C", "LC_ALL=C")
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user