From f260b221a9b0d7038dd39369118589f9171bea80 Mon Sep 17 00:00:00 2001 From: xuans Date: Sun, 27 Sep 2026 20:18:43 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8DMySQL=E7=89=B9=E6=AE=8A?= =?UTF-8?q?=E5=88=97=E5=A4=87=E4=BB=BD=E5=B9=B6=E8=A1=A5=E5=85=85=E5=AE=98?= =?UTF-8?q?=E6=96=B9=E5=B7=A5=E5=85=B7=E4=BA=92=E6=93=8D=E4=BD=9C=E6=B5=8B?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 保留并验证四个待提交源码文件：显式导出可写列以保留不可见列，跳过生成列并保留仅生成列表的行数；修复扫描越界，以实际结构元数据校验恢复结果。MySQL 5.7.44与8.0.45专项回归、后端常规测试、静态检查及模块校验通过；不代表完整生产验收。 --- backend/internal/cli/mysql_backup.go | 44 ++++- .../cli/mysql_interop_integration_test.go | 174 ++++++++++++++++++ .../cli/mysql_snapshot_integration_test.go | 2 +- .../internal/testsupport/mysqltest/tools.go | 106 +++++++++++ 4 files changed, 318 insertions(+), 8 deletions(-) create mode 100644 backend/internal/cli/mysql_interop_integration_test.go create mode 100644 backend/internal/testsupport/mysqltest/tools.go diff --git a/backend/internal/cli/mysql_backup.go b/backend/internal/cli/mysql_backup.go index 8bdd7cf..20299e6 100644 --- a/backend/internal/cli/mysql_backup.go +++ b/backend/internal/cli/mysql_backup.go @@ -157,12 +157,22 @@ func dumpMySQLTable(ctx context.Context, db *sql.Tx, output *os.File, tableName return fmt.Errorf("写入 MySQL 表结构失败（%s）：%w", tableName, err) } - rows, err := db.QueryContext(ctx, "SELECT * FROM "+quotedTable) + columns, err := mysqlInsertableColumns(ctx, db, tableName) + if err != nil { + return err + } + // 显式投影可写列：SELECT * 会遗漏不可见列，并错误地包含生成列的计算结果。 + projection := quoteMySQLColumns(columns) + if len(columns) == 0 { + // 只有生成列的表仍有行数，使用占位读取并导出空列 INSERT 保留每一行。 + projection = "1" + } + rows, err := db.QueryContext(ctx, "SELECT "+projection+" FROM "+quotedTable) if err != nil { return fmt.Errorf("读取 MySQL 表数据失败（%s）：%w", tableName, err) } defer rows.Close() - columns, err := rows.Columns() + selectedColumns, err := rows.Columns() if err != nil { return fmt.Errorf("读取 MySQL 表字段失败（%s）：%w", tableName, err) } @@ -171,17 +181,17 @@ func dumpMySQLTable(ctx context.Context, db *sql.Tx, output *os.File, tableName return fmt.Errorf("读取 MySQL 表字段类型失败（%s）：%w", tableName, err) } for rows.Next() { - values := make([]any, len(columns)) - pointers := make([]any, len(columns)) + values := make([]any, len(selectedColumns)) + pointers := make([]any, len(values)) for index := range values { pointers[index] = &values[index] } if err := rows.Scan(pointers...); err != nil { return fmt.Errorf("读取 MySQL 表行失败（%s）：%w", tableName, err) } - literals := make([]string, len(values)) - for index, value := range values { - literals[index] = mysqlColumnSQLLiteral(value, columnTypes[index].DatabaseTypeName()) + literals := make([]string, len(columns)) + for index := range columns { + literals[index] = mysqlColumnSQLLiteral(values[index], columnTypes[index].DatabaseTypeName()) } if _, err := fmt.Fprintf(output, "INSERT INTO %s (%s) VALUES (%s);\n", quotedTable, quoteMySQLColumns(columns), strings.Join(literals, ", ")); err != nil { return fmt.Errorf("写入 MySQL 表数据失败（%s）：%w", tableName, err) @@ -194,6 +204,26 @@ func dumpMySQLTable(ctx context.Context, db *sql.Tx, output *os.File, tableName return err } +func mysqlInsertableColumns(ctx context.Context, db *sql.Tx, tableName string) ([]string, error) { + rows, err := db.QueryContext(ctx, "SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COALESCE(GENERATION_EXPRESSION, '') = '' ORDER BY ORDINAL_POSITION", tableName) + if err != nil { + return nil, fmt.Errorf("读取 MySQL 可写字段失败（%s）：%w", tableName, err) + } + defer rows.Close() + var columns []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("读取 MySQL 可写字段名失败（%s）：%w", tableName, err) + } + columns = append(columns, name) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("遍历 MySQL 可写字段失败（%s）：%w", tableName, err) + } + return columns, nil +} + func quoteMySQLIdentifier(value string) string { return "`" + strings.ReplaceAll(value, "`", "``") + "`" } diff --git a/backend/internal/cli/mysql_interop_integration_test.go b/backend/internal/cli/mysql_interop_integration_test.go new file mode 100644 index 0000000..0913edb --- /dev/null +++ b/backend/internal/cli/mysql_interop_integration_test.go @@ -0,0 +1,174 @@ +package cli + +import ( + "database/sql" + "encoding/json" + "path/filepath" + "testing" + + "git.pcux.cn/xuans/Scert/backend/internal/config" + "git.pcux.cn/xuans/Scert/backend/internal/database" + "git.pcux.cn/xuans/Scert/backend/internal/testsupport/mysqltest" +) + +func TestMySQLRealOfficialInteroperability(t *testing.T) { + server := mysqltest.New(t) + fixtures := []struct { + name string + create string + insert string + query string + want string + }{ + { + name: "basic_values", + create: "CREATE TABLE interop (id INT PRIMARY KEY, label TEXT, payload BLOB, amount DECIMAL(20,6), optional_value TEXT NULL) ENGINE=InnoDB", + insert: "INSERT INTO interop VALUES (1, '证书;单引号''与反斜线\\\\', X'00013B5CFF', 123456789.123456, NULL)", + query: "SELECT CONCAT(label, '|', HEX(payload), '|', CAST(amount AS CHAR), '|', optional_value IS NULL) FROM interop WHERE id = 1", + want: "证书;单引号'与反斜线\\|00013B5CFF|123456789.123456|1", + }, + { + name: "generated_columns", + create: "CREATE TABLE interop (id INT PRIMARY KEY, base_value DECIMAL(20,6), stored_value DECIMAL(21,6) GENERATED ALWAYS AS (base_value * 2) STORED, virtual_value VARCHAR(80) GENERATED ALWAYS AS (CONCAT('证书;', id)) VIRTUAL, label TEXT) ENGINE=InnoDB", + insert: "INSERT INTO interop (id, base_value, label) VALUES (7, 1234.123456, '保留原值')", + query: "SELECT CONCAT(CAST(base_value AS CHAR), '|', CAST(stored_value AS CHAR), '|', virtual_value, '|', label) FROM interop WHERE id = 7", + want: "1234.123456|2468.246912|证书;7|保留原值", + }, + { + name: "only_generated_columns", + create: "CREATE TABLE interop (stored_value INT GENERATED ALWAYS AS (42) STORED) ENGINE=InnoDB", + insert: "INSERT INTO interop () VALUES (), (), ()", + query: "SELECT CONCAT(COUNT(*), '|', SUM(stored_value)) FROM interop", + want: "3|126", + }, + } + if server.IsMySQL8() { + fixtures = append(fixtures, struct { + name string + create string + insert string + query string + want string + }{ + name: "invisible_column", + create: "CREATE TABLE interop (id INT PRIMARY KEY, hidden_value VARCHAR(80) INVISIBLE DEFAULT '默认隐藏值', label TEXT) ENGINE=InnoDB", + insert: "INSERT INTO interop (id, hidden_value, label) VALUES (1, '隐藏列实际值', '可见列')", + query: "SELECT CONCAT(hidden_value, '|', label) FROM interop WHERE id = 1", + want: "隐藏列实际值|可见列", + }) + } + for _, fixture := range fixtures { + for _, direction := range []string{"scert_to_official", "official_to_scert"} { + t.Run(fixture.name+"/"+direction, func(t *testing.T) { + source := server.Database(t) + db, err := database.Open(t.Context(), source) + if err != nil { + t.Fatal(err) + } + defer db.Close() + for _, statement := range []string{fixture.create, fixture.insert} { + if _, err := db.DB.ExecContext(t.Context(), statement); err != nil { + t.Fatal(err) + } + } + checkMySQLInteropValue(t, source, fixture.query, fixture.want) + definition := readMySQLInteropDefinition(t, db.DB) + output := filepath.Join(server.Root(), fixture.name+"-"+direction+".sql") + target := server.Database(t) + if direction == "scert_to_official" { + if err := backupMySQLDatabase(t.Context(), source, output); err != nil { + t.Fatal(err) + } + server.Import(t, target, output) + } else { + server.Dump(t, source, output) + if err := restoreMySQL(t.Context(), target, output); err != nil { + t.Fatal(err) + } + } + checkMySQLInteropValue(t, target, fixture.query, fixture.want) + verify, err := database.Open(t.Context(), target) + if err != nil { + t.Fatal(err) + } + defer verify.Close() + restoredDefinition := readMySQLInteropDefinition(t, verify.DB) + if restoredDefinition != definition { + t.Fatalf("互操作恢复后表结构不一致：\n源表：%s\n目标表：%s", definition, restoredDefinition) + } + }) + } + } +} + +func readMySQLInteropDefinition(t *testing.T, db *sql.DB) string { + t.Helper() + queries := []string{ + `SELECT TABLE_TYPE, ENGINE, TABLE_COLLATION, ROW_FORMAT, CREATE_OPTIONS + FROM information_schema.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'interop'`, + `SELECT COLUMN_NAME, ORDINAL_POSITION, COLUMN_DEFAULT, IS_NULLABLE, + COLUMN_TYPE, CHARACTER_SET_NAME, COLLATION_NAME, EXTRA, + GENERATION_EXPRESSION, COLUMN_COMMENT + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'interop' + ORDER BY ORDINAL_POSITION`, + `SELECT INDEX_NAME, NON_UNIQUE, SEQ_IN_INDEX, COLUMN_NAME, + COLLATION, SUB_PART, PACKED, NULLABLE, INDEX_TYPE, COMMENT, INDEX_COMMENT + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'interop' + ORDER BY INDEX_NAME, SEQ_IN_INDEX`, + } + var metadata [][][]sql.NullString + for _, query := range queries { + rows, err := db.QueryContext(t.Context(), query) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + columns, err := rows.Columns() + if err != nil { + t.Fatal(err) + } + var section [][]sql.NullString + for rows.Next() { + values := make([]sql.NullString, len(columns)) + pointers := make([]any, len(values)) + for index := range values { + pointers[index] = &values[index] + } + if err := rows.Scan(pointers...); err != nil { + t.Fatal(err) + } + section = append(section, values) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if err := rows.Close(); err != nil { + t.Fatal(err) + } + metadata = append(metadata, section) + } + encoded, err := json.Marshal(metadata) + if err != nil { + t.Fatal(err) + } + return string(encoded) +} + +func checkMySQLInteropValue(t *testing.T, cfg config.DatabaseConfig, query, want string) { + t.Helper() + db, err := database.Open(t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var value string + if err := db.DB.QueryRowContext(t.Context(), query).Scan(&value); err != nil { + t.Fatal(err) + } + if value != want { + t.Fatalf("官方工具互操作后数据不一致：得到 %q，期望 %q", value, want) + } +} diff --git a/backend/internal/cli/mysql_snapshot_integration_test.go b/backend/internal/cli/mysql_snapshot_integration_test.go index 8d4a2ac..f06176e 100644 --- a/backend/internal/cli/mysql_snapshot_integration_test.go +++ b/backend/internal/cli/mysql_snapshot_integration_test.go @@ -126,7 +126,7 @@ func waitForMySQLBackupLock(t *testing.T, ctx context.Context, admin *sql.DB, na defer ticker.Stop() for { var connectionID uint64 - err := admin.QueryRowContext(ctx, "SELECT ID FROM information_schema.PROCESSLIST WHERE DB = ? AND STATE LIKE '%lock%' AND (INFO = 'SHOW CREATE TABLE `b_snapshot`' OR INFO = 'SELECT * FROM `b_snapshot`')", name).Scan(&connectionID) + err := admin.QueryRowContext(ctx, "SELECT ID FROM information_schema.PROCESSLIST WHERE DB = ? AND STATE LIKE '%lock%' AND (INFO = 'SHOW CREATE TABLE `b_snapshot`' OR INFO LIKE 'SELECT % FROM `b_snapshot`')", name).Scan(&connectionID) if err != nil && !errors.Is(err, sql.ErrNoRows) { t.Fatal(err) } diff --git a/backend/internal/testsupport/mysqltest/tools.go b/backend/internal/testsupport/mysqltest/tools.go new file mode 100644 index 0000000..90c2af2 --- /dev/null +++ b/backend/internal/testsupport/mysqltest/tools.go @@ -0,0 +1,106 @@ +package mysqltest + +import ( + "context" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/go-sql-driver/mysql" + + "git.pcux.cn/xuans/Scert/backend/internal/config" +) + +// 官方工具仅连接本夹具创建的隔离实例，密码只传入子进程环境，不进入命令行或文件。 +func (f *Server) Dump(t *testing.T, cfg config.DatabaseConfig, output string) { + t.Helper() + file, err := os.OpenFile(output, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + defer file.Close() + f.runTool(t, "mysqldump", cfg, []string{ + "--single-transaction", "--no-tablespaces", "--set-gtid-purged=OFF", + "--skip-add-locks", "--hex-blob", "--complete-insert", "--skip-comments", "--skip-extended-insert", + }, nil, file) + if err := file.Close(); err != nil { + t.Fatal(err) + } +} + +func (f *Server) Import(t *testing.T, cfg config.DatabaseConfig, input string) { + t.Helper() + file, err := os.Open(input) + if err != nil { + t.Fatal(err) + } + defer file.Close() + f.runTool(t, "mysql", cfg, []string{ + "--binary-mode", "--batch", "--skip-reconnect", "--local-infile=0", + }, file, io.Discard) +} + +func (f *Server) IsMySQL8() bool { return strings.Contains(f.version, "Ver 8.0.") } + +func (f *Server) runTool(t *testing.T, name string, cfg config.DatabaseConfig, args []string, input io.Reader, output io.Writer) { + t.Helper() + parsed, err := mysql.ParseDSN(cfg.DSN) + if err != nil || cfg.Driver != "mysql" { + t.Fatal("官方工具测试连接无效") + } + if parsed.Net != "tcp" || parsed.Addr != "127.0.0.1:"+f.port || parsed.TLSConfig != f.tlsName || + parsed.Passwd != f.password || !strings.HasPrefix(parsed.DBName, "scert_test_") || !strings.HasPrefix(parsed.User, "scert_user_") { + t.Fatal("官方工具只允许连接本次隔离实例的测试数据库") + } + binary := filepath.Join(filepath.Dir(f.binary), name) + if runtime.GOOS == "windows" { + binary += ".exe" + } + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + version, err := exec.CommandContext(ctx, binary, "--no-defaults", "--version").Output() + if err != nil { + t.Fatal("无法读取官方工具版本，不能跳过互操作验证") + } + expected := "5.7." + if f.IsMySQL8() { + expected = "8.0." + } + if !strings.Contains(string(version), expected) { + t.Fatal("官方工具与本次测试服务端版本系列不一致") + } + t.Logf("官方工具：%s", strings.TrimSpace(string(version))) + base := []string{ + "--no-defaults", "--protocol=TCP", "--host=127.0.0.1", "--port=" + f.port, + "--user=" + parsed.User, "--ssl-mode=VERIFY_IDENTITY", "--ssl-ca=" + filepath.Join(f.root, "server.crt"), + "--default-character-set=utf8mb4", + } + command := exec.CommandContext(ctx, binary, append(append(base, args...), parsed.DBName)...) + command.Env = mysqlToolTestEnvironment(f.root, parsed.Passwd) + command.Stdin, command.Stdout = input, output + // 错误可能含数据库内容；只报告退出状态，不透传工具的原始错误输出。 + command.Stderr = io.Discard + command.WaitDelay = 2 * time.Second + if err := command.Run(); err != nil { + t.Fatalf("官方 %s 执行失败：%v；原始输出已隐藏", name, err) + } +} + +func mysqlToolTestEnvironment(root, password string) []string { + env := make([]string, 0, len(os.Environ())+4) + for _, entry := range os.Environ() { + key, _, _ := strings.Cut(entry, "=") + key = strings.ToUpper(key) + if strings.HasPrefix(key, "MYSQL") || key == "LANG" || key == "LC_ALL" { + continue + } + env = append(env, entry) + } + // --no-defaults 仍可能读取登录文件；指定隔离目录中不存在的文件，避免读取用户凭据。 + return append(env, "MYSQL_PWD="+password, "MYSQL_TEST_LOGIN_FILE="+filepath.Join(root, "no-login.cnf"), "LANG=C", "LC_ALL=C") +}