diff --git a/backend/config/database_migration.go b/backend/config/database_migration.go index 8d97efad..bb49c5aa 100644 --- a/backend/config/database_migration.go +++ b/backend/config/database_migration.go @@ -3,14 +3,14 @@ package config import ( "context" "errors" + "strings" "time" ) var ErrDatabaseMigrationTarget = errors.New("database migration target is invalid") -// SwitchDatabase commits only the database name or SQLite local reference. -// It deliberately does not open a connection or restart the service; the -// caller must complete the physical migration before invoking this method. +// SwitchDatabase 提交已完成复制与校验的目标连接,保留旧库并标记重启。 +// 普通配置编辑不能绕过本迁移入口切换数据库类型。 func (m *ConfigManager) SwitchDatabase(ctx context.Context, expectedGeneration uint64, target DatabaseConfig, actor AuditActor) error { if err := contextError(ctx); err != nil { return err @@ -42,14 +42,7 @@ func (m *ConfigManager) SwitchDatabase(ctx context.Context, expectedGeneration u if err := ValidateDatabaseConfig(target); err != nil { return ErrDatabaseMigrationTarget } - if !sameDatabaseConnection(current.Root.Database, target) { - return ErrDatabaseMigrationTarget - } - if target.Type == DatabaseTypeSQLite { - if target.LocalRef == current.Root.Database.LocalRef { - return ErrDatabaseMigrationTarget - } - } else if target.Name == current.Root.Database.Name { + if SameDatabaseLocation(current.Root.Database, target) { return ErrDatabaseMigrationTarget } @@ -95,6 +88,9 @@ func (m *ConfigManager) SwitchDatabase(ctx context.Context, expectedGeneration u if target.Type == DatabaseTypeSQLite { changedField = "root.database.localRef" } + if !sameDatabaseConnection(current.Root.Database, target) { + changedField = "root.database" + } return m.writeAudit(ctx, ConfigAuditEvent{ Operation: ConfigOperationMigration, Section: SectionRoot, @@ -109,6 +105,20 @@ func (m *ConfigManager) SwitchDatabase(ctx context.Context, expectedGeneration u }) } +// SameDatabaseLocation 不把用户名或密码不同误认为不同数据库。 +func SameDatabaseLocation(current, target DatabaseConfig) bool { + current = NormalizeDatabaseConfig(current) + target = NormalizeDatabaseConfig(target) + if current.Type != target.Type { + return false + } + if current.Type == DatabaseTypeSQLite { + return strings.EqualFold(current.LocalRef, target.LocalRef) + } + return strings.EqualFold(current.Host, target.Host) && current.Port == target.Port && + strings.EqualFold(current.Name, target.Name) +} + func sameDatabaseConnection(current, target DatabaseConfig) bool { if current.Type != target.Type || current.Host != target.Host || diff --git a/backend/config/database_migration_test.go b/backend/config/database_migration_test.go index 1f1ff535..38e3e019 100644 --- a/backend/config/database_migration_test.go +++ b/backend/config/database_migration_test.go @@ -2,8 +2,11 @@ package config import ( "context" + "encoding/json" "errors" + "strings" "testing" + "time" ) func TestConfigManagerSwitchDatabaseCommitsSQLiteReference(t *testing.T) { @@ -36,6 +39,43 @@ func TestConfigManagerSwitchDatabaseCommitsSQLiteReference(t *testing.T) { } } +func TestConfigManagerSwitchAcrossTypesPersistsAndRedactsAudit(t *testing.T) { + for _, kind := range []DatabaseType{DatabaseTypeMySQL, DatabaseTypePostgreSQL} { + t.Run(string(kind), func(t *testing.T) { + root := t.TempDir() + writeRuntimeFixture(t, root, validRuntimeConfig()) + options := ConfigLoaderOptions{Validation: RuntimeValidationOptions{HasSuperAdmin: true, Production: true}} + manager, err := NewConfigManager(root, options) + if err != nil { + t.Fatal(err) + } + secret, err := ApplySecretChange(manager.vault, StoredSecret{}, SecretChange{Operation: SecretReplace, Value: &SecretInput{Plaintext: "migration-test-secret"}}, SecretPolicy{Required: true, MaxLength: 4096}, time.Now().UTC()) + if err != nil { + t.Fatal(err) + } + target := NormalizeDatabaseConfig(DatabaseConfig{Type: kind, Host: "127.0.0.1", Username: "review", Name: "smail_next", Password: secret}) + if err := manager.SwitchDatabase(context.Background(), 1, target, authorizedConfigActor()); err != nil { + t.Fatal(err) + } + reloaded, err := NewConfigManager(root, options) + if err != nil { + t.Fatal(err) + } + if got := reloaded.Snapshot(); got.Generation != 2 || got.Root.Database != target { + t.Fatal("target configuration was not persisted") + } + events, err := manager.security.Audit.(*FileAuditWriter).Query(context.Background(), ConfigAuditQuery{Operation: ConfigOperationMigration, Limit: 10}) + if err != nil || len(events) != 1 { + t.Fatal("migration audit missing") + } + data, _ := json.Marshal(events) + if strings.Contains(string(data), "migration-test-secret") || strings.Contains(string(data), secret.Ciphertext) || events[0].ChangedFields[0] != "root.database" { + t.Fatal("migration audit disclosed secret or omitted connection change") + } + }) + } +} + func TestConfigManagerSwitchDatabaseRejectsImmutableChangesAndStaleGeneration(t *testing.T) { root := t.TempDir() writeRuntimeFixture(t, root, validRuntimeConfig()) diff --git a/backend/config/partitioned_compatibility.go b/backend/config/partitioned_compatibility.go index cb981689..93d90247 100644 --- a/backend/config/partitioned_compatibility.go +++ b/backend/config/partitioned_compatibility.go @@ -101,9 +101,12 @@ func compatibilityDatabase(database DatabaseConfig, vault *credential.Vault) (st settings.User, settings.Passwd = database.Username, password settings.Net, settings.Addr, settings.DBName = "tcp", address, database.Name settings.ParseTime = true + // 与迁移连接保持相同时区,避免重启后 TIMESTAMP 会话到期时间偏移。 + settings.Params = map[string]string{"charset": "utf8mb4", "time_zone": "'+00:00'"} return DBTypeMySQL, settings.FormatDSN(), nil case DatabaseTypePostgreSQL: uri := url.URL{Scheme: "postgres", User: url.UserPassword(database.Username, password), Host: address, Path: "/" + database.Name} + uri.RawQuery = "sslmode=require&timezone=UTC&search_path=public" return DBTypePostgres, uri.String(), nil default: return "", "", fmt.Errorf("不支持的数据库类型") diff --git a/backend/config/partitioned_compatibility_test.go b/backend/config/partitioned_compatibility_test.go index 00e4f59c..2bea4091 100644 --- a/backend/config/partitioned_compatibility_test.go +++ b/backend/config/partitioned_compatibility_test.go @@ -2,11 +2,48 @@ package config import ( "bytes" + "net/url" "os" "path/filepath" + "strings" "testing" + "time" + + "git.pcux.cn/xuans/Smail/utils/credential" + "github.com/go-sql-driver/mysql" ) +func TestPartitionedDatabaseConnectionPreservesMigrationTimezone(t *testing.T) { + vault, err := credential.NewVault(t.TempDir()) + if err != nil { + t.Fatal(err) + } + secret, err := ApplySecretChange(vault, StoredSecret{}, SecretChange{Operation: SecretReplace, Value: &SecretInput{Plaintext: "test-password"}}, SecretPolicy{Required: true, MaxLength: 4096}, time.Now().UTC()) + if err != nil { + t.Fatal(err) + } + for _, kind := range []DatabaseType{DatabaseTypeMySQL, DatabaseTypePostgreSQL} { + _, dsn, err := compatibilityDatabase(DatabaseConfig{Type: kind, Host: "::1", Port: 3306, Username: "test", Name: "test", Password: secret}, vault) + if err != nil { + t.Fatal("runtime connection failed") + } + if kind == DatabaseTypeMySQL { + settings, err := mysql.ParseDSN(dsn) + _, rawQuery, _ := strings.Cut(dsn, "?") + query, queryErr := url.ParseQuery(rawQuery) + // charset 会被驱动解析为内部选项,不再保留在 Params 中。 + if err != nil || queryErr != nil || !settings.ParseTime || settings.Params["time_zone"] != "'+00:00'" || query.Get("charset") != "utf8mb4" { + t.Fatal("MySQL runtime timezone differs from migration") + } + } else { + uri, err := url.Parse(dsn) + if err != nil || uri.Query().Get("timezone") != "UTC" || uri.Query().Get("search_path") != "public" || uri.Query().Get("sslmode") != "require" { + t.Fatal("PostgreSQL runtime options differ from migration") + } + } + } +} + func TestPartitionedCompatibilityStartupAndWriteProtection(t *testing.T) { root := t.TempDir() oldRoot, oldArgs, oldDevelopment := ROOT_PATH, os.Args, DevelopmentMode diff --git a/backend/controllers/config_error.go b/backend/controllers/config_error.go index a0ea56b3..4f8f1c00 100644 --- a/backend/controllers/config_error.go +++ b/backend/controllers/config_error.go @@ -49,6 +49,9 @@ func configErrorDetails(err error) (string, int, string, map[string]string) { code = string(denial.Code) status = http.StatusForbidden message = "无权访问或请求不满足安全条件" + if denial.Code == access.CodeRecentAuthRequired { + message = "此操作需要最近十分钟内完成登录,请重新登录后再试" + } case errors.Is(err, config.ErrManagerStorage): code = "CONFIG_MANAGER_UNAVAILABLE" status = http.StatusServiceUnavailable diff --git a/backend/controllers/database_migration.go b/backend/controllers/database_migration.go index 7b9d3251..dc263dd4 100644 --- a/backend/controllers/database_migration.go +++ b/backend/controllers/database_migration.go @@ -3,6 +3,7 @@ package controllers import ( "net/http" "strings" + "time" "git.pcux.cn/xuans/Smail/config" "git.pcux.cn/xuans/Smail/dto/response" @@ -12,8 +13,9 @@ import ( ) type databaseMigrationPreviewRequest struct { - TargetName string `json:"targetName"` - ExpectedGeneration uint64 `json:"expectedGeneration"` + TargetName string `json:"targetName"` + Target *dbmigration.TargetInput `json:"target,omitempty"` + ExpectedGeneration uint64 `json:"expectedGeneration"` } type databaseMigrationExecuteRequest struct { @@ -21,7 +23,7 @@ type databaseMigrationExecuteRequest struct { Token string `json:"token"` } -// PreviewDatabaseMigration 校验只改数据库名称的迁移计划,不连接或修改目标数据库。 +// PreviewDatabaseMigration 验证三种数据库的结构化目标连接,不创建或修改目标库。 func PreviewDatabaseMigration(ctx *appcontext.Context, w http.ResponseWriter, req *http.Request) { if err := authorizeCapability(ctx, access.CapabilityDatabaseMigrate, "", false); err != nil { writeConfigAccessError(w, err) @@ -42,11 +44,27 @@ func PreviewDatabaseMigration(ctx *appcontext.Context, w http.ResponseWriter, re return } status := manager.Status() - if input.ExpectedGeneration != status.ActiveGeneration { + if input.ExpectedGeneration != status.ActiveGeneration || len(status.PendingRestart) > 0 { writeConfigError(w, config.ErrGenerationConflict) return } - record, token, err := service.Preview(req.Context(), manager.Snapshot().Root.Database, status.ActiveGeneration, input.TargetName) + var record dbmigration.Record + var token string + var err error + if input.Target != nil { + if input.TargetName != "" { + writeConfigError(w, dbmigration.ErrInvalidRequest) + return + } + target, prepareErr := input.Target.Prepare(config.ROOT_PATH) + if prepareErr != nil { + writeConfigError(w, prepareErr) + return + } + record, token, err = service.PreviewTarget(req.Context(), manager.Snapshot().Root.Database, status.ActiveGeneration, target) + } else { + record, token, err = service.Preview(req.Context(), manager.Snapshot().Root.Database, status.ActiveGeneration, input.TargetName) + } if err != nil { writeConfigError(w, err) return @@ -69,6 +87,8 @@ func ExecuteDatabaseMigration(ctx *appcontext.Context, w http.ResponseWriter, re writeConfigError(w, dbmigration.ErrInvalidRequest) return } + // 迁移最长十分钟,不能沿用普通接口的九十秒响应写入期限。 + _ = http.NewResponseController(w).SetWriteDeadline(time.Now().Add(11 * time.Minute)) record, err := dbmigration.Current().ExecuteAuthorized(req.Context(), input.ID, input.Token, configActor(ctx, req)) if err != nil { writeConfigError(w, err) diff --git a/backend/controllers/database_migration_test.go b/backend/controllers/database_migration_test.go new file mode 100644 index 00000000..6193f36e --- /dev/null +++ b/backend/controllers/database_migration_test.go @@ -0,0 +1,95 @@ +package controllers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "git.pcux.cn/xuans/Smail/config" + "git.pcux.cn/xuans/Smail/services/access" + "git.pcux.cn/xuans/Smail/services/dbmigration" + appcontext "git.pcux.cn/xuans/Smail/utils/context" +) + +// 这里只验证控制器契约;真实数据复制由迁移服务集成测试覆盖。 +type migrationContractRunner struct{} + +func (migrationContractRunner) Run(context.Context, dbmigration.Plan, func(dbmigration.Status) error) error { + return nil +} + +func TestDatabaseMigrationPreviewContract(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "config"), 0700); err != nil { + t.Fatal(err) + } + legacy := []byte(`{"domain":"mail.example.com","webDomain":"mail.example.com","dbType":"sqlite","dbDSN":"./config/source.db","isInit":true,"httpPort":80,"httpsPort":443,"httpsEnabled":1}`) + if err := os.WriteFile(filepath.Join(root, "config", "config.json"), legacy, 0600); err != nil { + t.Fatal(err) + } + validation := config.RuntimeValidationOptions{HasSuperAdmin: true, Production: true} + if _, err := config.MigrateLegacyConfig(root, validation); err != nil { + t.Fatal(err) + } + manager, err := config.NewConfigManager(root, config.ConfigLoaderOptions{Validation: validation}) + if err != nil { + t.Fatal(err) + } + previousRoot, previousManager, previousService := config.ROOT_PATH, config.CurrentManager(), dbmigration.Current() + config.ROOT_PATH = root + config.SetRuntimeManager(manager) + dbmigration.SetCurrent(dbmigration.NewService(migrationContractRunner{})) + t.Cleanup(func() { + config.ROOT_PATH = previousRoot + config.SetRuntimeManager(previousManager) + dbmigration.SetCurrent(previousService) + }) + admin := &appcontext.Context{UserID: 1, Role: access.RoleSuperAdmin, RecentAuth: true} + for _, test := range []struct { + name, body string + ctx *appcontext.Context + code int + }{ + {"sqlite", `{"target":{"type":"sqlite","localRef":"next.db"},"expectedGeneration":1}`, admin, 200}, + {"mysql", `{"target":{"type":"mysql","host":"localhost","username":"review","name":"next","password":{"plaintext":"private-test-value"}},"expectedGeneration":1}`, admin, 200}, + {"postgresql", `{"target":{"type":"postgresql","host":"localhost","username":"review","name":"next","password":{"plaintext":"private-test-value"}},"expectedGeneration":1}`, admin, 200}, + {"legacy", `{"targetName":"next.db","expectedGeneration":1}`, admin, 200}, + {"both targets", `{"targetName":"next.db","target":{"type":"sqlite","localRef":"next.db"},"expectedGeneration":1}`, admin, 422}, + {"raw dsn", `{"target":{"type":"mysql","dsn":"forbidden"},"expectedGeneration":1}`, admin, 422}, + {"unsafe path", `{"target":{"type":"sqlite","localRef":"../next.db"},"expectedGeneration":1}`, admin, 422}, + {"sqlite password", `{"target":{"type":"sqlite","localRef":"next.db","password":{"plaintext":"private-test-value"}},"expectedGeneration":1}`, admin, 422}, + {"missing password", `{"target":{"type":"mysql","host":"localhost","username":"review","name":"next"},"expectedGeneration":1}`, admin, 422}, + {"generation conflict", `{"targetName":"next.db","expectedGeneration":9}`, admin, 409}, + {"no recent auth", `{"targetName":"next.db","expectedGeneration":1}`, &appcontext.Context{UserID: 1, Role: access.RoleSuperAdmin}, 403}, + {"ordinary admin", `{"targetName":"next.db","expectedGeneration":1}`, &appcontext.Context{UserID: 2, Role: access.RoleAdmin, RecentAuth: true}, 403}, + {"ordinary user", `{"targetName":"next.db","expectedGeneration":1}`, &appcontext.Context{UserID: 3, Role: access.RoleUser, RecentAuth: true}, 403}, + } { + t.Run(test.name, func(t *testing.T) { + recorder := httptest.NewRecorder() + PreviewDatabaseMigration(test.ctx, recorder, httptest.NewRequest(http.MethodPost, "/api/v1/database/migrations/preview", strings.NewReader(test.body))) + if recorder.Code != test.code { + t.Fatalf("unexpected status: %d", recorder.Code) + } + if strings.Contains(recorder.Body.String(), "private-test-value") || strings.Contains(recorder.Body.String(), "ciphertext") { + t.Fatal("migration response disclosed secret") + } + if test.code == 200 { + var payload struct { + ErrorNo int `json:"errorNo"` + Data struct { + Migration dbmigration.Record `json:"migration"` + Token string `json:"token"` + } `json:"data"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil || payload.ErrorNo != 0 || payload.Data.Token == "" || payload.Data.Migration.Status != dbmigration.StatusPreviewed { + t.Fatal("invalid preview response") + } + } + }) + } +} diff --git a/backend/listen/http_server/http_server.go b/backend/listen/http_server/http_server.go index 1d70c2c4..1fb27455 100644 --- a/backend/listen/http_server/http_server.go +++ b/backend/listen/http_server/http_server.go @@ -12,6 +12,7 @@ import ( "git.pcux.cn/xuans/Smail/config" "git.pcux.cn/xuans/Smail/controllers" "git.pcux.cn/xuans/Smail/controllers/email" + "git.pcux.cn/xuans/Smail/services/dbmigration" "git.pcux.cn/xuans/Smail/session" "git.pcux.cn/xuans/Smail/utils/context" log "github.com/sirupsen/logrus" @@ -290,7 +291,7 @@ func newHTTPHandler(cfg config.HTTPConfig) (http.Handler, error) { if session.Instance == nil { return nil, fmt.Errorf("session manager is unavailable") } - handler = session.Instance.LoadAndSave(mux) + handler = dbmigration.GuardHTTP(session.Instance.LoadAndSave(mux)) } else { mux.HandleFunc("/.well-known/", controllers.AcmeChallenge) mux.HandleFunc("/api/ping", controllers.Ping) diff --git a/backend/listen/http_server/https_server.go b/backend/listen/http_server/https_server.go index cc216447..e7a338ff 100644 --- a/backend/listen/http_server/https_server.go +++ b/backend/listen/http_server/https_server.go @@ -19,6 +19,7 @@ import ( "git.pcux.cn/xuans/Smail/i18n" "git.pcux.cn/xuans/Smail/models" "git.pcux.cn/xuans/Smail/services/access" + "git.pcux.cn/xuans/Smail/services/dbmigration" "git.pcux.cn/xuans/Smail/session" "git.pcux.cn/xuans/Smail/utils/context" "git.pcux.cn/xuans/Smail/utils/csrf" @@ -73,7 +74,7 @@ func HttpsStart() error { httpsServer = &http.Server{ Addr: fmt.Sprintf(":%d", httpsPort), - Handler: securityHeaders(session.Instance.LoadAndSave(mux)), + Handler: securityHeaders(dbmigration.GuardHTTP(session.Instance.LoadAndSave(mux))), ReadTimeout: time.Second * 90, ReadHeaderTimeout: time.Second * 10, WriteTimeout: time.Second * 90, @@ -160,6 +161,7 @@ func contextIterceptor(h controllers.HandlerFunc) http.HandlerFunc { ctx.IsAdmin = ctx.Role == access.RoleSuperAdmin || ctx.Role == access.RoleAdmin ctx.MustChangePassword = currentUser.MustChangePassword ctx.SecuritySetupAcknowledged = currentUser.SecuritySetupAcknowledged + ctx.RecentAuth = identity.HasRecentAuthentication(time.Now().UTC()) controllers.LoadSecurityContext(ctx) } } diff --git a/backend/listen/http_server/security_setup_browser_test.go b/backend/listen/http_server/security_setup_browser_test.go index c7d11a28..ccfc90c7 100644 --- a/backend/listen/http_server/security_setup_browser_test.go +++ b/backend/listen/http_server/security_setup_browser_test.go @@ -1,6 +1,8 @@ package http_server import ( + "context" + "encoding/json" "net" "net/http" "net/http/httptest" @@ -17,6 +19,7 @@ import ( "git.pcux.cn/xuans/Smail/services/access" "git.pcux.cn/xuans/Smail/services/auth" "git.pcux.cn/xuans/Smail/services/credentials" + "git.pcux.cn/xuans/Smail/services/dbmigration" "git.pcux.cn/xuans/Smail/services/externalemail" "git.pcux.cn/xuans/Smail/session" ) @@ -35,7 +38,14 @@ func TestSecuritySetupBrowserFixture(t *testing.T) { if err := os.MkdirAll(root, 0700); err != nil { t.Fatal(err) } + migrationEnabled := os.Getenv("SMAIL_BROWSER_MIGRATION") == "1" databasePath := filepath.Join(root, "browser.db") + if migrationEnabled { + if err := os.MkdirAll(filepath.Join(root, "config"), 0700); err != nil { + t.Fatal(err) + } + databasePath = filepath.Join(root, "config", "browser.db") + } if _, err := os.Stat(databasePath); !os.IsNotExist(err) { t.Fatal("fixture requires a fresh database path") } @@ -47,11 +57,53 @@ func TestSecuritySetupBrowserFixture(t *testing.T) { IsInit: true, SetupState: config.SetupStateReady, HttpsPort: 15443, } config.SetInitState(true) + if migrationEnabled { + legacy, err := json.Marshal(map[string]any{ + "domain": "localhost", "webDomain": "localhost", "accessHosts": []string{"127.0.0.1"}, + "dbType": "sqlite", "dbDSN": databasePath, "isInit": true, + "httpPort": 15080, "httpsPort": 15443, "httpsEnabled": 1, + }) + if err != nil { + t.Fatal(err) + } + configPath := filepath.Join(root, "config", "config.json") + file, err := os.OpenFile(configPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + t.Fatal(err) + } + _, writeErr := file.Write(legacy) + closeErr := file.Close() + if writeErr != nil || closeErr != nil { + t.Fatal("isolated configuration could not be saved") + } + validation := config.RuntimeValidationOptions{HasSuperAdmin: true, Production: true} + if _, err := config.MigrateLegacyConfig(root, validation); err != nil { + t.Fatal(err) + } + manager, err := config.NewConfigManager(root, config.ConfigLoaderOptions{Validation: validation}) + if err != nil { + t.Fatal(err) + } + config.SetRuntimeManager(manager) + factory, err := dbmigration.NewRuntimeSQLConnectionFactory(root) + if err != nil { + t.Fatal(err) + } + service := dbmigration.NewService(dbmigration.NewPortableDatabaseExecutor(root, factory, + func(ctx context.Context, generation uint64, target config.DatabaseConfig, actor config.AuditActor) error { + return manager.SwitchDatabase(ctx, generation, target, actor) + })) + dbmigration.SetCurrent(service) + defer service.Close() + } if err := db.Init("browser-fixture"); err != nil { t.Fatal(err) } defer db.Instance.Close() user := models.User{Account: "xuans", Name: "Test", SessionVersion: 1, SecurityVersion: 1} + if migrationEnabled { + user.SecuritySetupAcknowledged = 1 + } user.ApplyRole(access.RoleSuperAdmin) if err := auth.SetPassword(&user, password); err != nil { t.Fatal(err) @@ -75,6 +127,7 @@ func TestSecuritySetupBrowserFixture(t *testing.T) { if err := session.Init(); err != nil { t.Fatal(err) } + defer session.Stop() mux := http.NewServeMux() if err := router(mux); err != nil { t.Fatal(err) @@ -92,7 +145,7 @@ func TestSecuritySetupBrowserFixture(t *testing.T) { if err != nil { t.Fatal(err) } - server := httptest.NewUnstartedServer(securityHeaders(session.Instance.LoadAndSave(handler))) + server := httptest.NewUnstartedServer(securityHeaders(dbmigration.GuardHTTP(session.Instance.LoadAndSave(handler)))) _ = server.Listener.Close() server.Listener = listener server.StartTLS() diff --git a/backend/listen/http_server/session_validation_test.go b/backend/listen/http_server/session_validation_test.go index 2011432c..bc20b33d 100644 --- a/backend/listen/http_server/session_validation_test.go +++ b/backend/listen/http_server/session_validation_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "path/filepath" "testing" + "time" "git.pcux.cn/xuans/Smail/config" "git.pcux.cn/xuans/Smail/controllers" @@ -49,7 +50,7 @@ func TestContextInterceptorValidatesCurrentUserState(t *testing.T) { session.Instance = manager called := false handler := contextIterceptor(controllers.HandlerFunc(func(ctx *appcontext.Context, w http.ResponseWriter, r *http.Request) { - called = ctx.UserID == user.ID && ctx.UserAccount == user.Account && ctx.IsAdmin + called = ctx.UserID == user.ID && ctx.UserAccount == user.Account && ctx.IsAdmin && ctx.RecentAuth response.NewSuccessResponse("ok").FPrint(w) })) request := httptest.NewRequest(http.MethodGet, "/api/user/info", nil).WithContext(loaded) @@ -60,6 +61,37 @@ func TestContextInterceptorValidatesCurrentUserState(t *testing.T) { } }) + t.Run("refresh does not elevate or renew authentication", func(t *testing.T) { + for _, authenticatedAt := range []int64{0, time.Now().Add(-11 * time.Minute).Unix(), time.Now().Add(time.Hour).Unix()} { + manager, loaded := sessionContext(t, &user) + session.Instance = manager + identity, err := session.GetUser(loaded) + if err != nil { + t.Fatal(err) + } + identity.AuthenticatedAt = authenticatedAt + encoded, err := json.Marshal(identity) + if err != nil { + t.Fatal(err) + } + manager.Put(loaded, "user", string(encoded)) + called := false + handler := contextIterceptor(controllers.HandlerFunc(func(ctx *appcontext.Context, w http.ResponseWriter, r *http.Request) { + called = true + if ctx.UserID != user.ID || ctx.RecentAuth { + t.Fatalf("refresh changed authentication authority: user=%d recent=%v", ctx.UserID, ctx.RecentAuth) + } + })) + for range 2 { + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/api/v1/user/info", nil).WithContext(loaded)) + } + stored, err := session.GetUser(loaded) + if err != nil || !called || stored.AuthenticatedAt != authenticatedAt { + t.Fatalf("refresh renewed authentication: called=%v identity=%+v err=%v", called, stored, err) + } + } + }) + t.Run("disabled identity", func(t *testing.T) { manager, loaded := sessionContext(t, &user) session.Instance = manager diff --git a/backend/res_init/init.go b/backend/res_init/init.go index eb736b35..ca3c53b0 100644 --- a/backend/res_init/init.go +++ b/backend/res_init/init.go @@ -82,18 +82,13 @@ func startConfigManager() error { authorizedSwitch := func(ctx context.Context, expectedGeneration uint64, target config.DatabaseConfig, actor config.AuditActor) error { return manager.SwitchDatabase(ctx, expectedGeneration, target, actor) } - sqliteExecutor := dbmigration.NewAuthorizedSQLiteExecutor(config.ROOT_PATH, authorizedSwitch) - var structuredExecutor dbmigration.Runner - if manager.Snapshot().Root.Database.Type != config.DatabaseTypeSQLite { - factory, factoryErr := dbmigration.NewRuntimeSQLConnectionFactory(config.ROOT_PATH) - if factoryErr != nil { - return fmt.Errorf("initialize structured database migration: %w", factoryErr) - } - structuredExecutor = dbmigration.NewSQLExecutor(dbmigration.NewSQLCopier(factory), authorizedSwitch) + factory, factoryErr := dbmigration.NewRuntimeSQLConnectionFactory(config.ROOT_PATH) + if factoryErr != nil { + return fmt.Errorf("initialize database migration: %w", factoryErr) } dbmigration.SetCurrent( dbmigration.NewService( - dbmigration.NewDatabaseExecutor(sqliteExecutor, structuredExecutor), + dbmigration.NewPortableDatabaseExecutor(config.ROOT_PATH, factory, authorizedSwitch), ), ) @@ -101,6 +96,7 @@ func startConfigManager() error { } func stopConfigManager() { + dbmigration.Current().Close() config.SetRuntimeManager(nil) dbmigration.SetCurrent(nil) } @@ -231,11 +227,12 @@ func Init(serverVersion string) error { return err case <-signal.RestartChan: log.Infof("Server Restart!") - stopServices() + if err := stopServices(); err != nil { + return err + } case <-signal.StopChan: log.Infof("Server Stop!") - stopServices() - return nil + return stopServices() } log.Infof("Server Stop Success!") time.Sleep(5 * time.Second) @@ -252,9 +249,9 @@ func startService(serviceErrors chan<- error, name string, service func() error) }() } -func stopServices() { +func stopServices() error { restart.SetCurrent(nil) - stopConfigManager() + session.Stop() stopAdminResetControl() stopVerificationRuntime() smtp_server.Stop() @@ -263,6 +260,17 @@ func stopServices() { pop3_server.Stop() imap_server.Stop() hooks.Stop() + // 关闭连接池后还要等待借出的连接归还,不能提前解除迁移源锁。 + if db.Instance != nil { + _ = db.Instance.Close() + ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second) + defer cancel() + if err := dbmigration.WaitForDatabaseDrain(ctx, db.Instance.DB().DB); err != nil { + return fmt.Errorf("database operations did not drain; migration lock retained: %w", err) + } + } + stopConfigManager() + return nil } func startVerificationRuntime() error { diff --git a/backend/services/dbmigration/maintenance.go b/backend/services/dbmigration/maintenance.go new file mode 100644 index 00000000..26776e14 --- /dev/null +++ b/backend/services/dbmigration/maintenance.go @@ -0,0 +1,125 @@ +package dbmigration + +import ( + "context" + "database/sql" + "net/http" + "strings" + "time" +) + +// WritePaused 表示迁移已冻结源数据库,只有读取状态与重启可以继续。 +func (s *Service) WritePaused() bool { + if s == nil { + return false + } + executor, ok := s.runner.(*DatabaseExecutor) + return ok && executor.portable != nil && executor.portable.paused.Load() +} + +// GuardHTTP 等待在途 HTTP 请求及会话提交结束,再独占执行迁移。 +// 放在会话中间件外层,避免迁移请求返回时才开始排空旧会话写入。 +func GuardHTTP(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + service := Current() + if service == nil { + next.ServeHTTP(w, r) + return + } + service.requests.RLock() + lease := &requestLease{service: service} + defer lease.release() + r = r.WithContext(context.WithValue(r.Context(), requestLeaseKey{}, lease)) + if service.WritePaused() && !maintenanceRequest(r) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Retry-After", "30") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"errorNo":500,"errorMsg":"数据库迁移已暂停业务操作,请立即重启后端","data":"DATABASE_RESTART_REQUIRED"}`)) + return + } + next.ServeHTTP(w, r) + }) +} + +type requestLeaseKey struct{} + +type requestLease struct { + service *Service + exclusive bool +} + +func (lease *requestLease) release() { + if lease.exclusive { + lease.service.requests.Unlock() + } else { + lease.service.requests.RUnlock() + } +} + +// 只有权限、请求体与一次性票据均已验证,才排空请求并独占迁移。 +// 独占状态由外层中间件持有到会话响应提交完成,不能在控制器返回前释放。 +func (s *Service) drainRequests(ctx context.Context) func() { + if lease, ok := ctx.Value(requestLeaseKey{}).(*requestLease); ok && lease.service == s { + lease.service.requests.RUnlock() + lease.service.requests.Lock() + lease.exclusive = true + return func() {} + } + s.requests.Lock() + return s.requests.Unlock +} + +func maintenanceRequest(r *http.Request) bool { + path := r.URL.Path + if r.Method == http.MethodGet { + if !strings.HasPrefix(path, "/api/") && !strings.HasPrefix(path, "/attachments/") { + return true + } + return path == "/api/ping" || path == "/api/v1/ping" || + path == "/api/v1/config/status" || path == "/api/v1/config/base" || + (strings.HasPrefix(path, "/api/v1/database/migrations/") && path != "/api/v1/database/migrations/preview") + } + return r.Method == http.MethodPost && (path == "/api/user/info" || path == "/api/v1/user/info" || + path == "/api/v1/system/restart" || path == "/api/v1/system/restart/preview") +} + +// WhileWritable 用于会话后台清理,与 HTTP 迁移执行共享排空边界。 +func WhileWritable(action func() error) error { + service := Current() + if service == nil { + return action() + } + service.requests.RLock() + defer service.requests.RUnlock() + if service.WritePaused() { + return nil + } + return action() +} + +func (s *Service) Close() { + if s == nil { + return + } + s.requests.Lock() + defer s.requests.Unlock() + if executor, ok := s.runner.(*DatabaseExecutor); ok && executor.portable != nil { + executor.portable.Close() + } +} + +// WaitForDatabaseDrain 在关闭连接池后确认借出的连接全部归还。 +// 等待失败时不能释放迁移源锁或继续内部重启,应终止进程并由外部重新启动。 +func WaitForDatabaseDrain(ctx context.Context, database *sql.DB) error { + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() + for database != nil && database.Stats().InUse > 0 { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } + return nil +} diff --git a/backend/services/dbmigration/maintenance_test.go b/backend/services/dbmigration/maintenance_test.go new file mode 100644 index 00000000..553aeee7 --- /dev/null +++ b/backend/services/dbmigration/maintenance_test.go @@ -0,0 +1,102 @@ +package dbmigration + +import ( + "context" + "database/sql" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestDatabaseDrainWaitsForBorrowedConnectionAfterPoolClose(t *testing.T) { + database, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + defer database.Close() + connection, err := database.Conn(context.Background()) + if err != nil { + t.Fatal(err) + } + defer connection.Close() + if err := database.Close(); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + if err := WaitForDatabaseDrain(ctx, database); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("borrowed connection was not protected: %v", err) + } + if err := connection.Close(); err != nil { + t.Fatal(err) + } + if err := WaitForDatabaseDrain(context.Background(), database); err != nil { + t.Fatal(err) + } +} + +func TestMaintenanceGuardDrainsRequestsAndKeepsRestartRoute(t *testing.T) { + executor := &PortableExecutor{} + service := NewService(&DatabaseExecutor{portable: executor}) + previous := Current() + SetCurrent(service) + t.Cleanup(func() { SetCurrent(previous) }) + entered, release, migrated := make(chan struct{}), make(chan struct{}), make(chan struct{}) + handler := GuardHTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/write": + close(entered) + <-release + case "/api/v1/database/migrations": + release := service.drainRequests(r.Context()) + defer release() + executor.paused.Store(true) + close(migrated) + } + w.WriteHeader(http.StatusOK) + })) + done := make(chan struct{}) + go func() { + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/write", nil)) + close(done) + }() + <-entered + migrationDone := make(chan struct{}) + go func() { + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/api/v1/database/migrations", nil)) + close(migrationDone) + }() + select { + case <-migrated: + t.Error("migration did not drain request") + case <-time.After(20 * time.Millisecond): + } + close(release) + <-done + <-migrationDone + for _, test := range []struct { + method, path string + code int + }{ + {http.MethodPost, "/api/v1/email/send", 503}, + {http.MethodPost, "/api/v1/database/migrations", 503}, + {http.MethodPut, "/api/v1/config/base", 503}, + {http.MethodGet, "/api/v1/config/status", 200}, + {http.MethodGet, "/api/v1/config/base", 200}, + {http.MethodPost, "/api/v1/user/info", 200}, + {http.MethodPost, "/api/v1/system/restart", 200}, + {http.MethodPost, "/api/v1/system/restart/preview", 200}, + } { + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(test.method, test.path, nil)) + if recorder.Code != test.code { + t.Errorf("%s: %d", test.path, recorder.Code) + } + } + called := false + if err := WhileWritable(func() error { called = true; return nil }); err != nil || called { + t.Fatal("cleanup wrote to frozen source") + } +} diff --git a/backend/services/dbmigration/portable_connection_test.go b/backend/services/dbmigration/portable_connection_test.go new file mode 100644 index 00000000..8b9f0e15 --- /dev/null +++ b/backend/services/dbmigration/portable_connection_test.go @@ -0,0 +1,34 @@ +package dbmigration + +import ( + "net/url" + "testing" + + "git.pcux.cn/xuans/Smail/config" + "github.com/lib/pq" +) + +func TestPortablePostgresUsesSupportedRuntimeTLSMode(t *testing.T) { + for _, name := range []string{"", "smail_next"} { + driver, dsn, err := runtimeSQLDSN(config.DatabaseConfig{ + Type: config.DatabaseTypePostgreSQL, Host: "::1", Port: 5432, Username: "review", + }, "test-only-password", name) + if err != nil || driver != "postgres" { + t.Fatal("PostgreSQL connection was not built") + } + parsed, err := url.Parse(dsn) + if err != nil { + t.Fatal(err) + } + query := parsed.Query() + if query.Get("sslmode") != "require" || query.Get("timezone") != "UTC" || query.Get("search_path") != "public" { + t.Fatal("migration connection differs from supported runtime TLS or time settings") + } + if _, err := pq.NewConnector(dsn); err != nil { + t.Fatal("driver rejected generated connection parameters") + } + if name == "" && parsed.Path != "/postgres" { + t.Fatal("maintenance connection did not select postgres database") + } + } +} diff --git a/backend/services/dbmigration/portable_data.go b/backend/services/dbmigration/portable_data.go new file mode 100644 index 00000000..109b7054 --- /dev/null +++ b/backend/services/dbmigration/portable_data.go @@ -0,0 +1,329 @@ +package dbmigration + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/binary" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" + + "git.pcux.cn/xuans/Smail/config" + "git.pcux.cn/xuans/Smail/db/migration" +) + +type portableQuery interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) +} + +func portableTables(ctx context.Context, database portableQuery, kind config.DatabaseType) ([]string, error) { + query := "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name" + if kind == config.DatabaseTypeMySQL { + query = "SELECT table_name FROM information_schema.tables WHERE table_schema=DATABASE() AND table_type='BASE TABLE' ORDER BY table_name" + } + if kind == config.DatabaseTypePostgreSQL { + query = "SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE' ORDER BY table_name" + } + rows, err := database.QueryContext(ctx, query) + if err != nil { + return nil, ErrSQLConnectionUnavailable + } + defer rows.Close() + var tables []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, ErrSQLUnsupportedObject + } + tables = append(tables, name) + } + if rows.Err() != nil || len(tables) == 0 { + return nil, ErrSQLUnsupportedObject + } + return tables, nil +} + +func portableColumns(ctx context.Context, database portableQuery, kind config.DatabaseType, table string) ([]string, map[string]string, error) { + rows, err := database.QueryContext(ctx, "SELECT * FROM "+quoteDatabaseIdentifier(kind, table)+" WHERE 1=0") + if err != nil { + return nil, nil, ErrSQLUnsupportedObject + } + defer rows.Close() + columns, err := rows.Columns() + if err != nil { + return nil, nil, err + } + types, err := rows.ColumnTypes() + if err != nil { + return nil, nil, err + } + result := make(map[string]string, len(types)) + for _, column := range types { + result[column.Name()] = strings.ToUpper(column.DatabaseTypeName()) + } + sort.Strings(columns) + return columns, result, nil +} + +func validatePortableSchema(ctx context.Context, source portableQuery, target *sql.DB, plan Plan, tables []string) error { + targetTables, err := portableTables(ctx, target, plan.Target.Type) + if err != nil || !sameStringSet(tables, targetTables) { + return ErrSQLUnsupportedObject + } + rows, err := source.QueryContext(ctx, "SELECT version, checksum FROM schema_migration ORDER BY version") + if err != nil { + return ErrSQLUnsupportedObject + } + defer rows.Close() + definitions := migration.DefaultMigrations() + index := 0 + for rows.Next() { + var version int64 + var checksum string + if err := rows.Scan(&version, &checksum); err != nil || index >= len(definitions) || definitions[index].Version != version || !definitions[index].AcceptsChecksum(checksum) { + return ErrSQLUnsupportedObject + } + index++ + } + if rows.Err() != nil || index != len(definitions) { + return ErrSQLUnsupportedObject + } + return nil +} + +func copyPortableTables(ctx context.Context, source *sql.Conn, target *sql.DB, plan Plan, tables []string) error { + if err := validatePortableSchema(ctx, source, target, plan, tables); err != nil { + return err + } + transaction, err := target.BeginTx(ctx, nil) + if err != nil { + return ErrSQLConnectionUnavailable + } + defer transaction.Rollback() + for _, table := range tables { + if table == "schema_migration" { + continue + } + columns, _, err := portableColumns(ctx, source, plan.Source.Type, table) + if err != nil { + return err + } + targetColumns, types, err := portableColumns(ctx, transaction, plan.Target.Type, table) + if err != nil || !sameStringSet(columns, targetColumns) { + return ErrSQLUnsupportedObject + } + query := "INSERT INTO " + quoteDatabaseIdentifier(plan.Target.Type, table) + " (" + portableColumnList(plan.Target.Type, columns) + ") VALUES (" + placeholders := make([]string, len(columns)) + for i := range columns { + placeholders[i] = "?" + if plan.Target.Type == config.DatabaseTypePostgreSQL { + placeholders[i] = fmt.Sprintf("$%d", i+1) + } + } + statement, err := transaction.PrepareContext(ctx, query+strings.Join(placeholders, ",")+")") + if err != nil { + return ErrSQLUnsupportedObject + } + err = visitPortableRows(ctx, source, plan.Source.Type, table, columns, func(values []any) error { + for i, value := range values { + converted, err := convertPortableValue(value, table, columns[i], types[columns[i]], plan.Target.Type) + if err != nil { + return err + } + values[i] = converted + } + if _, err := statement.ExecContext(ctx, values...); err != nil { + return ErrSQLDataVerification + } + return nil + }) + _ = statement.Close() + if err != nil { + return err + } + if plan.Target.Type == config.DatabaseTypePostgreSQL && types["id"] != "" { + // 显式复制主键后必须推进序列,否则重启后的新增记录会冲突。 + quoted := quotePostgreSQLIdentifier(table) + query := "SELECT setval(pg_get_serial_sequence($1, 'id'), COALESCE(MAX(id), 1), MAX(id) IS NOT NULL) FROM " + quoted + if _, err := transaction.ExecContext(ctx, query, quoted); err != nil { + return ErrSQLDataVerification + } + } + } + if err := transaction.Commit(); err != nil { + return ErrSQLDataVerification + } + return nil +} + +func portableColumnList(kind config.DatabaseType, columns []string) string { + result := make([]string, len(columns)) + for i, column := range columns { + result[i] = quoteDatabaseIdentifier(kind, column) + } + return strings.Join(result, ",") +} + +func visitPortableRows(ctx context.Context, database portableQuery, kind config.DatabaseType, table string, columns []string, visit func([]any) error) error { + rows, err := database.QueryContext(ctx, "SELECT "+portableColumnList(kind, columns)+" FROM "+quoteDatabaseIdentifier(kind, table)) + if err != nil { + return ErrSQLConnectionUnavailable + } + defer rows.Close() + for rows.Next() { + values, pointers := make([]any, len(columns)), make([]any, len(columns)) + for i := range values { + pointers[i] = &values[i] + } + if err := rows.Scan(pointers...); err != nil { + return ErrSQLDataVerification + } + if err := visit(values); err != nil { + return err + } + } + if rows.Err() != nil { + return ErrSQLDataVerification + } + return nil +} + +func convertPortableValue(value any, table, column, targetType string, target config.DatabaseType) (any, error) { + if value == nil { + return nil, nil + } + if table == "sessions" && column == "expiry" { + var instant time.Time + switch typed := value.(type) { + case time.Time: + instant = typed + case float64: + // sqlite3store 实际保存儒略日,而非 Unix 秒。 + seconds, fraction := math.Modf((typed - 2440587.5) * 86400) + instant = time.Unix(int64(seconds), int64(fraction*1e9)).UTC() + case int64: + return nil, ErrSQLDataVerification + default: + return nil, ErrSQLDataVerification + } + if target == config.DatabaseTypeSQLite { + if original, ok := value.(float64); ok { + return original, nil + } + return 2440587.5 + (float64(instant.Unix())+float64(instant.Nanosecond())/1e9)/86400, nil + } + return instant.UTC().Truncate(time.Microsecond), nil + } + if bytes, ok := value.([]byte); ok && !strings.Contains(targetType, "BLOB") && targetType != "BYTEA" { + value = string(bytes) + } + if targetType == "BOOL" || targetType == "BOOLEAN" { + switch typed := value.(type) { + case bool: + return typed, nil + case int64: + if typed == 0 || typed == 1 { + return typed == 1, nil + } + default: + return nil, ErrSQLDataVerification + } + return nil, ErrSQLDataVerification + } + if boolean, ok := value.(bool); ok { + if boolean { + return int64(1), nil + } + return int64(0), nil + } + if instant, ok := value.(time.Time); ok { + return instant.UTC().Truncate(time.Microsecond), nil + } + if text, ok := value.(string); ok && (strings.Contains(targetType, "TIME") || targetType == "DATE") { + for _, layout := range []string{time.RFC3339Nano, "2006-01-02 15:04:05.999999999-07:00", "2006-01-02 15:04:05.999999999 -0700 MST", "2006-01-02 15:04:05.999999999", "2006-01-02"} { + if instant, err := time.Parse(layout, text); err == nil { + return instant.UTC().Truncate(time.Microsecond), nil + } + } + return nil, ErrSQLDataVerification + } + return value, nil +} + +type portableDigest struct { + Count uint64 + Sum [32]byte +} + +func digestPortableTable(ctx context.Context, database portableQuery, kind config.DatabaseType, table string, columns []string, types map[string]string, target config.DatabaseType) (portableDigest, error) { + var digest portableDigest + err := visitPortableRows(ctx, database, kind, table, columns, func(values []any) error { + hash := sha256.New() + for i, value := range values { + value, err := convertPortableValue(value, table, columns[i], types[columns[i]], target) + if err != nil { + return err + } + var text string + switch typed := value.(type) { + case nil: + _, _ = hash.Write([]byte{0}) + continue + case time.Time: + text = typed.UTC().Format(time.RFC3339Nano) + case []byte: + text = string(typed) + case bool: + text = strconv.FormatBool(typed) + case float64: + // 儒略日单位为天,六位小数会掩盖约八十六毫秒的差异。 + text = strconv.FormatFloat(typed, 'g', -1, 64) + default: + text = fmt.Sprint(typed) + } + var length [8]byte + _, _ = hash.Write([]byte{1}) + binary.BigEndian.PutUint64(length[:], uint64(len(text))) + _, _ = hash.Write(length[:]) + _, _ = hash.Write([]byte(text)) + } + row := hash.Sum(nil) + carry := uint16(0) + for i := 31; i >= 0; i-- { + carry += uint16(digest.Sum[i]) + uint16(row[i]) + digest.Sum[i] = byte(carry) + carry >>= 8 + } + digest.Count++ + return nil + }) + return digest, err +} + +func verifyPortableTables(ctx context.Context, source *sql.Conn, target *sql.DB, plan Plan, tables []string) error { + for _, table := range tables { + if table == "schema_migration" { + continue + } + columns, types, err := portableColumns(ctx, target, plan.Target.Type, table) + if err != nil { + return err + } + before, err := digestPortableTable(ctx, source, plan.Source.Type, table, columns, types, plan.Target.Type) + if err != nil { + return err + } + after, err := digestPortableTable(ctx, target, plan.Target.Type, table, columns, types, plan.Target.Type) + if err != nil { + return err + } + if before != after { + return ErrSQLDataVerification + } + } + return nil +} diff --git a/backend/services/dbmigration/portable_executor.go b/backend/services/dbmigration/portable_executor.go new file mode 100644 index 00000000..dd21bfdf --- /dev/null +++ b/backend/services/dbmigration/portable_executor.go @@ -0,0 +1,282 @@ +package dbmigration + +import ( + "context" + "database/sql" + "errors" + "os" + "strings" + "sync" + "sync/atomic" + "time" + + "git.pcux.cn/xuans/Smail/config" + "git.pcux.cn/xuans/Smail/db/migration" +) + +// PortableExecutor 使用版本化目标结构复制业务数据,不拼接用户提供的 SQL。 +// 成功切换后保留源库写锁直到进程重启,避免旧连接继续接收邮件造成数据丢失。 +type PortableExecutor struct { + root string + open SQLConnectionFactory + switchConfig AuthorizedDatabaseSwitch + mu sync.Mutex + pending bool + paused atomic.Bool + releaseSource func() +} + +func NewPortableDatabaseExecutor(root string, open SQLConnectionFactory, switchConfig AuthorizedDatabaseSwitch) Runner { + return &DatabaseExecutor{portable: &PortableExecutor{root: root, open: open, switchConfig: switchConfig}} +} + +func (e *PortableExecutor) connect(ctx context.Context, database config.DatabaseConfig) (*sql.DB, error) { + if database.Type != config.DatabaseTypeSQLite { + if e.open == nil { + return nil, ErrSQLConnectionUnavailable + } + return e.open(ctx, database, database.Name) + } + path, err := NewSQLiteExecutor(e.root, nil).resolveReference(database.LocalRef) + if err != nil { + return nil, err + } + if err := checkRegularFile(path, false); err != nil { + return nil, ErrSourceDatabase + } + connection, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)") + if err != nil { + return nil, ErrSQLConnectionUnavailable + } + connection.SetMaxOpenConns(1) + return connection, nil +} + +func (e *PortableExecutor) Preview(ctx context.Context, plan Plan) error { + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + if _, err := buildTargetPlan(plan.sourceConfig, plan.targetConfig); err != nil { + return err + } + source, err := e.connect(ctx, plan.sourceConfig) + if err != nil { + return ErrSQLConnectionUnavailable + } + defer source.Close() + if err := source.PingContext(ctx); err != nil { + return ErrSQLConnectionUnavailable + } + return e.checkTarget(ctx, plan.targetConfig) +} + +func (e *PortableExecutor) checkTarget(ctx context.Context, target config.DatabaseConfig) error { + if target.Type == config.DatabaseTypeSQLite { + path, err := NewSQLiteExecutor(e.root, nil).resolveReference(target.LocalRef) + if err != nil { + return err + } + return checkRegularFile(path, true) + } + if e.open == nil { + return ErrSQLConnectionUnavailable + } + admin, err := e.open(ctx, target, "") + if err != nil { + return ErrSQLConnectionUnavailable + } + defer admin.Close() + query := "SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = ?" + if target.Type == config.DatabaseTypePostgreSQL { + query = "SELECT COUNT(*) FROM pg_database WHERE datname = $1" + } + var count int + if err := admin.QueryRowContext(ctx, query, target.Name).Scan(&count); err != nil { + return ErrSQLConnectionUnavailable + } + if count != 0 { + return ErrSQLTargetExists + } + return nil +} + +func (e *PortableExecutor) createTarget(ctx context.Context, target config.DatabaseConfig) (*sql.DB, error) { + if err := e.checkTarget(ctx, target); err != nil { + return nil, err + } + if target.Type == config.DatabaseTypeSQLite { + path, err := NewSQLiteExecutor(e.root, nil).resolveReference(target.LocalRef) + if err != nil { + return nil, err + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return nil, ErrTargetExists + } + if err := file.Close(); err != nil { + return nil, err + } + } else { + admin, err := e.open(ctx, target, "") + if err != nil { + return nil, ErrSQLConnectionUnavailable + } + defer admin.Close() + query := "CREATE DATABASE " + quoteDatabaseIdentifier(target.Type, target.Name) + if target.Type == config.DatabaseTypeMySQL { + query += " CHARACTER SET utf8mb4 COLLATE utf8mb4_bin" + } + if target.Type == config.DatabaseTypePostgreSQL { + query += " TEMPLATE template0 ENCODING 'UTF8'" + } + if _, err := admin.ExecContext(ctx, query); err != nil { + if isDatabaseAlreadyExistsError(err, target.Type) { + return nil, ErrSQLTargetExists + } + return nil, ErrSQLConnectionUnavailable + } + } + // 失败目标保留供检查,后续重试必须换用未占用的名称,绝不删除旧库。 + return e.connect(ctx, target) +} + +func (e *PortableExecutor) Run(ctx context.Context, plan Plan, update func(Status) error) error { + if !e.mu.TryLock() { + return ErrSQLSourceBusy + } + defer e.mu.Unlock() + if e.pending { + return ErrSQLSourceBusy + } + if e.switchConfig == nil || update == nil { + return ErrSwitchUnavailable + } + ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + if err := e.Preview(ctx, plan); err != nil { + return err + } + source, err := e.connect(ctx, plan.sourceConfig) + if err != nil { + return ErrSQLConnectionUnavailable + } + keepLock := false + defer func() { + if !keepLock { + _ = source.Close() + } + }() + connection, err := source.Conn(ctx) + if err != nil { + return ErrSQLConnectionUnavailable + } + defer func() { + if !keepLock { + _ = connection.Close() + } + }() + tables, err := portableTables(ctx, connection, plan.Source.Type) + if err != nil { + return err + } + if err := validatePortableObjects(ctx, connection, plan.Source.Type); err != nil { + return err + } + e.paused.Store(true) + defer func() { + if !keepLock { + e.paused.Store(false) + } + }() + release, err := lockPortableSource(ctx, connection, plan.Source.Type, tables) + if err != nil { + return err + } + defer func() { + if !keepLock { + release() + } + }() + target, err := e.createTarget(ctx, plan.targetConfig) + if err != nil { + return err + } + defer target.Close() + dialect := migration.Dialect(plan.Target.Type) + if plan.Target.Type == config.DatabaseTypePostgreSQL { + dialect = migration.DialectPostgres + } + if _, err := (migration.Runner{DB: target, Dialect: dialect, Migrations: migration.DefaultMigrations()}).Run(ctx); err != nil { + return ErrSQLUnsupportedObject + } + if err := copyPortableTables(ctx, connection, target, plan, tables); err != nil { + return err + } + if err := update(StatusVerifying); err != nil { + return err + } + if err := verifyPortableTables(ctx, connection, target, plan, tables); err != nil { + return err + } + if err := update(StatusSwitching); err != nil { + return err + } + err = e.switchConfig(ctx, plan.expectedGeneration, plan.targetConfig, plan.actor) + if err != nil && isKnownSwitchRejection(err) { + return err + } + // 切换结果未知也保留写锁和目标,避免在不确定状态下继续写入旧库。 + keepLock = true + e.pending = true + e.releaseSource = func() { release(); _ = connection.Close(); _ = source.Close() } + if err != nil { + return errors.Join(ErrSwitchOutcomeUnknown, err) + } + return nil +} + +// Close 只能在请求和邮件监听器停止后调用,不能提前解除旧库保护。 +func (e *PortableExecutor) Close() { + e.mu.Lock() + defer e.mu.Unlock() + if e.releaseSource != nil { + e.releaseSource() + e.releaseSource = nil + } +} + +func quoteDatabaseIdentifier(kind config.DatabaseType, name string) string { + if kind == config.DatabaseTypeMySQL { + return quoteMySQLIdentifier(name) + } + return quotePostgreSQLIdentifier(name) +} + +func lockPortableSource(ctx context.Context, connection *sql.Conn, kind config.DatabaseType, tables []string) (func(), error) { + query, unlock := "BEGIN IMMEDIATE", "ROLLBACK" + if kind == config.DatabaseTypeMySQL { + parts := make([]string, len(tables)) + for i, table := range tables { + parts[i] = quoteMySQLIdentifier(table) + " READ" + } + query, unlock = "LOCK TABLES "+strings.Join(parts, ","), "UNLOCK TABLES" + } else if kind == config.DatabaseTypePostgreSQL { + if _, err := connection.ExecContext(ctx, "BEGIN"); err != nil { + return nil, ErrSQLSourceBusy + } + parts := make([]string, len(tables)) + for i, table := range tables { + parts[i] = quotePostgreSQLIdentifier(table) + } + query = "LOCK TABLE " + strings.Join(parts, ",") + " IN SHARE MODE NOWAIT" + } + release := func() { + cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, _ = connection.ExecContext(cleanup, unlock) + } + if _, err := connection.ExecContext(ctx, query); err != nil { + release() + return nil, ErrSQLSourceBusy + } + return release, nil +} diff --git a/backend/services/dbmigration/portable_integration_test.go b/backend/services/dbmigration/portable_integration_test.go new file mode 100644 index 00000000..05e3c8d6 --- /dev/null +++ b/backend/services/dbmigration/portable_integration_test.go @@ -0,0 +1,104 @@ +package dbmigration + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" + + "git.pcux.cn/xuans/Smail/config" +) + +// 集成测试只创建随机命名的新库,不修改现有业务库,也不自动删除测试库。 +// 必须显式提供专用测试服务连接;缺少连接时跳过,不冒充实库验收成功。 +func TestPortableSQLIntegration(t *testing.T) { + for _, item := range []struct { + kind config.DatabaseType + env string + }{ + {config.DatabaseTypeMySQL, "SMAIL_TEST_MIGRATION_MYSQL"}, + {config.DatabaseTypePostgreSQL, "SMAIL_TEST_MIGRATION_POSTGRES"}, + } { + t.Run(string(item.kind), func(t *testing.T) { + input := portableIntegrationInput(t, item.env, item.kind) + runPortableIntegrationChain(t, []TargetInput{input, input}) + }) + } + t.Run("mysql_postgresql", func(t *testing.T) { + mysql := portableIntegrationInput(t, "SMAIL_TEST_MIGRATION_MYSQL", config.DatabaseTypeMySQL) + postgres := portableIntegrationInput(t, "SMAIL_TEST_MIGRATION_POSTGRES", config.DatabaseTypePostgreSQL) + runPortableIntegrationChain(t, []TargetInput{mysql, postgres, mysql}) + }) +} + +func portableIntegrationInput(t *testing.T, name string, kind config.DatabaseType) TargetInput { + t.Helper() + raw := strings.TrimSpace(os.Getenv(name)) + if raw == "" { + t.Skip(name + " is not configured") + } + var input TargetInput + decoder := json.NewDecoder(strings.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&input); err != nil || input.Type != kind || input.Name != "" || input.LocalRef != "" { + t.Fatal("integration connection must specify the expected type without a database name or local path") + } + return input +} + +func runPortableIntegrationChain(t *testing.T, inputs []TargetInput) { + t.Helper() + root, source := portableFixture(t) + open, err := NewRuntimeSQLConnectionFactory(root) + if err != nil { + t.Fatal("integration connection factory unavailable") + } + inputs = append(inputs, TargetInput{Type: config.DatabaseTypeSQLite, LocalRef: "roundtrip.db"}) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + for index, input := range inputs { + if input.Type != config.DatabaseTypeSQLite { + input.Name, err = randomID("smail_migration_test_") + if err != nil { + t.Fatal("integration name generation failed") + } + } + target, err := input.Prepare(root) + if err != nil { + t.Fatal("integration connection configuration is invalid") + } + plan, err := buildTargetPlan(source, target) + if err != nil { + t.Fatal("integration plan validation failed") + } + switched := false + executor := &PortableExecutor{root: root, open: open, switchConfig: func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error { + switched = true + return nil + }} + // 路径只记录随机测试库名,不记录连接账号、密码或底层驱动错误。 + t.Logf("hop %d: %s -> %s, retained database: %s%s", index+1, source.Type, target.Type, target.Name, target.LocalRef) + if err := executor.Run(ctx, plan, func(Status) error { return nil }); err != nil { + executor.Close() + t.Fatalf("migration failed: %s", migrationErrorCode(err)) + } + executor.Close() + if !switched { + t.Fatal("configuration switch was not reached") + } + connection, err := executor.connect(ctx, target) + if err != nil { + t.Fatal("migrated target could not be reopened") + } + var subject string + queryErr := connection.QueryRowContext(ctx, "SELECT subject FROM email WHERE id=82").Scan(&subject) + _, insertErr := connection.ExecContext(ctx, "INSERT INTO "+quoteDatabaseIdentifier(target.Type, "user")+" (account,name,password) VALUES ('next_hop_"+string(rune('a'+index))+"','sequence','test-only')") + _ = connection.Close() + if queryErr != nil || subject != "中文主题" || insertErr != nil { + t.Fatal("target content or identity sequence was not preserved") + } + source = target + } +} diff --git a/backend/services/dbmigration/portable_schema.go b/backend/services/dbmigration/portable_schema.go new file mode 100644 index 00000000..0475835d --- /dev/null +++ b/backend/services/dbmigration/portable_schema.go @@ -0,0 +1,43 @@ +package dbmigration + +import ( + "context" + + "git.pcux.cn/xuans/Smail/config" +) + +// 仅迁移 Smail 版本化业务表;不能静默忽略自定义对象。 +func validatePortableObjects(ctx context.Context, database portableQuery, kind config.DatabaseType) error { + queries := []string{"SELECT COUNT(*) FROM sqlite_master WHERE type IN ('view','trigger')"} + switch kind { + case config.DatabaseTypeMySQL: + queries = []string{ + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=DATABASE() AND (table_type <> 'BASE TABLE' OR engine <> 'InnoDB')", + "SELECT COUNT(*) FROM information_schema.triggers WHERE trigger_schema=DATABASE()", + "SELECT COUNT(*) FROM information_schema.events WHERE event_schema=DATABASE()", + "SELECT COUNT(*) FROM information_schema.routines WHERE routine_schema=DATABASE()", + "SELECT COUNT(*) FROM information_schema.table_constraints WHERE constraint_schema=DATABASE() AND constraint_type='FOREIGN KEY'", + } + case config.DatabaseTypePostgreSQL: + queries = []string{ + "SELECT COUNT(*) FROM pg_namespace WHERE nspname NOT IN ('public','information_schema') AND nspname NOT LIKE 'pg_%'", + "SELECT COUNT(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind NOT IN ('r','i','S')", + "SELECT COUNT(*) FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND NOT t.tgisinternal", + "SELECT COUNT(*) FROM pg_proc p JOIN pg_namespace n ON n.oid=p.pronamespace WHERE n.nspname='public'", + "SELECT COUNT(*) FROM pg_constraint c JOIN pg_namespace n ON n.oid=c.connamespace WHERE n.nspname='public' AND c.contype IN ('f','c','x')", + } + } + for _, query := range queries { + rows, err := database.QueryContext(ctx, query) + if err != nil { + return ErrSQLUnsupportedObject + } + var count int + valid := rows.Next() && rows.Scan(&count) == nil && count == 0 && rows.Err() == nil + _ = rows.Close() + if !valid { + return ErrSQLUnsupportedObject + } + } + return nil +} diff --git a/backend/services/dbmigration/portable_test.go b/backend/services/dbmigration/portable_test.go new file mode 100644 index 00000000..8dac0918 --- /dev/null +++ b/backend/services/dbmigration/portable_test.go @@ -0,0 +1,211 @@ +package dbmigration + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.pcux.cn/xuans/Smail/config" + "git.pcux.cn/xuans/Smail/db/migration" + "git.pcux.cn/xuans/Smail/utils/credential" +) + +func portableFixture(t *testing.T) (string, config.DatabaseConfig) { + t.Helper() + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "config"), 0700); err != nil { + t.Fatal(err) + } + database, err := sql.Open("sqlite", filepath.Join(root, "config", "source.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + if _, err := (migration.Runner{DB: database, Dialect: migration.DialectSQLite, Migrations: migration.DefaultMigrations()}).Run(context.Background()); err != nil { + t.Fatal(err) + } + for _, query := range []string{ + `INSERT INTO "user" (id,account,name,password) VALUES (40,'review','测试','hash')`, + `INSERT INTO email (id,subject,text,create_time) VALUES (82,'中文主题','正文😀','2026-09-05 12:30:00.123456')`, + `INSERT INTO sessions (token,data,expiry) VALUES ('token',x'0001ff',julianday('2026-09-12 12:00:00'))`, + `INSERT INTO plugin_user_setting (plugin_id,user_id,schema_version,settings_ciphertext,settings_hash,enabled,created_at,updated_at) VALUES ('test',40,1,'cipher','hash',1,'2026-09-05 12:00:00','2026-09-05 12:00:00')`, + } { + if _, err := database.Exec(query); err != nil { + t.Fatal(err) + } + } + return root, config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "source.db"} +} + +func TestPortableSQLiteCopyPreservesDataAndLocksUntilRestart(t *testing.T) { + root, source := portableFixture(t) + var switched bool + executor := &PortableExecutor{root: root, switchConfig: func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error { + switched = true + return nil + }} + plan, err := buildTargetPlan(source, config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "target.db"}) + if err != nil { + t.Fatal(err) + } + if err := executor.Preview(context.Background(), plan); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(root, "config", "target.db")); !os.IsNotExist(err) { + t.Fatal("preview created target") + } + if err := executor.Run(context.Background(), plan, func(Status) error { return nil }); err != nil { + t.Fatal(err) + } + defer executor.releaseSource() + if !switched || !executor.pending { + t.Fatal("switch or restart state missing") + } + target, err := executor.connect(context.Background(), plan.targetConfig) + if err != nil { + t.Fatal(err) + } + defer target.Close() + var subject string + if err := target.QueryRow("SELECT subject FROM email WHERE id=82").Scan(&subject); err != nil || subject != "中文主题" { + t.Fatal("content not preserved") + } + var expiry float64 + if err := target.QueryRow("SELECT expiry FROM sessions").Scan(&expiry); err != nil || expiry < 2400000 || expiry > 2500000 { + t.Fatal("session Julian date not preserved") + } + if _, err := target.Exec(`INSERT INTO "user" (account,name,password) VALUES ('next','新增','hash')`); err != nil { + t.Fatal(err) + } + var next int + if err := target.QueryRow(`SELECT id FROM "user" WHERE account='next'`).Scan(&next); err != nil || next <= 40 { + t.Fatal("identity sequence not advanced") + } + old, err := sql.Open("sqlite", filepath.Join(root, "config", "source.db")+"?_pragma=busy_timeout(1)") + if err != nil { + t.Fatal(err) + } + defer old.Close() + if _, err := old.Exec(`UPDATE "user" SET name='bad'`); err == nil { + t.Fatal("old source accepted writes after switch") + } +} + +func TestPortableSwitchFailurePreservesSourceAndTarget(t *testing.T) { + root, source := portableFixture(t) + executor := &PortableExecutor{root: root, switchConfig: func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error { + return config.ErrGenerationConflict + }} + plan, err := buildTargetPlan(source, config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "target.db"}) + if err != nil { + t.Fatal(err) + } + if err := executor.Run(context.Background(), plan, func(Status) error { return nil }); !errors.Is(err, config.ErrGenerationConflict) { + t.Fatalf("switch failure: %v", err) + } + if executor.pending { + t.Fatal("rejected switch kept write lock") + } + for _, name := range []string{"source.db", "target.db"} { + if _, err := os.Stat(filepath.Join(root, "config", name)); err != nil { + t.Fatal("database removed") + } + } + if err := executor.Preview(context.Background(), plan); !errors.Is(err, ErrTargetExists) { + t.Fatal("existing target accepted") + } +} + +func TestTargetSupportsAllDatabasePairsWithoutSecretDisclosure(t *testing.T) { + root := t.TempDir() + inputs := []TargetInput{ + {Type: config.DatabaseTypeSQLite, LocalRef: "next.db"}, + {Type: config.DatabaseTypeMySQL, Host: "localhost", Username: "test", Name: "next", Password: &config.SecretInput{Plaintext: "private-test-value"}}, + {Type: config.DatabaseTypePostgreSQL, Host: "localhost", Username: "test", Name: "next", Password: &config.SecretInput{Plaintext: "private-test-value"}}, + } + for _, input := range inputs { + target, err := input.Prepare(root) + if err != nil { + t.Fatal(err) + } + for _, sourceInput := range inputs { + sourceInput.Name = "" + if sourceInput.Type == config.DatabaseTypeSQLite { + sourceInput.LocalRef = "source.db" + } else { + sourceInput.Name = "source" + } + source, err := sourceInput.Prepare(root) + if err != nil { + t.Fatal(err) + } + service := NewService(testRunner{}) + record, _, err := service.PreviewTarget(context.Background(), source, 1, target) + if err != nil { + t.Fatal(err) + } + data, _ := json.Marshal(record) + if strings.Contains(string(data), "password") || strings.Contains(string(data), "private-test-value") || strings.Contains(string(data), "ciphertext") { + t.Fatal("preview leaked password") + } + } + if target.Type != config.DatabaseTypeSQLite { + vault, _ := credential.NewVault(root) + value, err := config.OpenStoredSecret(vault, target.Password) + if err != nil || value != "private-test-value" { + t.Fatal("target secret did not roundtrip") + } + } + } +} + +func TestPortableConversions(t *testing.T) { + now := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) + julian, err := convertPortableValue(now, "sessions", "expiry", "REAL", config.DatabaseTypeSQLite) + if err != nil { + t.Fatal(err) + } + restored, err := convertPortableValue(julian, "sessions", "expiry", "TIMESTAMPTZ", config.DatabaseTypePostgreSQL) + if err != nil || restored.(time.Time).Sub(now).Abs() > time.Millisecond { + t.Fatal("session expiry did not roundtrip") + } + boolean, err := convertPortableValue(int64(1), "plugin_user_setting", "enabled", "BOOL", config.DatabaseTypePostgreSQL) + if err != nil || boolean != true { + t.Fatal("boolean conversion failed") + } + if _, err := convertPortableValue(int64(2), "plugin_user_setting", "enabled", "BOOL", config.DatabaseTypePostgreSQL); err == nil { + t.Fatal("invalid boolean accepted") + } +} + +func TestPortableRejectsCustomObjectsBeforeCreatingTarget(t *testing.T) { + root, source := portableFixture(t) + executor := &PortableExecutor{root: root, switchConfig: func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error { + t.Error("unsupported schema switched configuration") + return nil + }} + database, err := executor.connect(context.Background(), source) + if err != nil { + t.Fatal(err) + } + if _, err := database.Exec(`CREATE VIEW custom_view AS SELECT id FROM "user"`); err != nil { + t.Fatal(err) + } + _ = database.Close() + plan, err := buildTargetPlan(source, config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "target.db"}) + if err != nil { + t.Fatal(err) + } + if err := executor.Run(context.Background(), plan, func(Status) error { return nil }); !errors.Is(err, ErrSQLUnsupportedObject) { + t.Fatalf("custom view accepted: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "config", "target.db")); !os.IsNotExist(err) { + t.Fatal("unsupported source created target") + } +} diff --git a/backend/services/dbmigration/service.go b/backend/services/dbmigration/service.go index e6bebdd9..ab66b6e7 100644 --- a/backend/services/dbmigration/service.go +++ b/backend/services/dbmigration/service.go @@ -81,10 +81,11 @@ type Runner interface { } type Service struct { - mu sync.RWMutex - records map[string]*record - runner Runner - now func() time.Time + requests sync.RWMutex + mu sync.RWMutex + records map[string]*record + runner Runner + now func() time.Time } type record struct { @@ -117,6 +118,15 @@ func Current() *Service { } func (s *Service) Preview(ctx context.Context, source config.DatabaseConfig, currentGeneration uint64, targetName string) (Record, string, error) { + plan, err := buildPlan(source, strings.TrimSpace(targetName)) + if err != nil { + return Record{}, "", err + } + return s.PreviewTarget(ctx, source, currentGeneration, plan.targetConfig) +} + +// PreviewTarget 固定完整目标连接;返回记录不包含密码或密文。 +func (s *Service) PreviewTarget(ctx context.Context, source config.DatabaseConfig, currentGeneration uint64, target config.DatabaseConfig) (Record, string, error) { if err := contextError(ctx); err != nil { return Record{}, "", err } @@ -126,14 +136,17 @@ func (s *Service) Preview(ctx context.Context, source config.DatabaseConfig, cur if err := config.ValidateDatabaseConfig(source); err != nil { return Record{}, "", ErrSourceDatabase } - targetName = strings.TrimSpace(targetName) - if (source.Type == config.DatabaseTypeSQLite && !sqliteTargetNamePattern.MatchString(targetName)) || (source.Type != config.DatabaseTypeSQLite && !targetNamePattern.MatchString(targetName)) { - return Record{}, "", ErrInvalidTargetName - } - plan, err := buildPlan(source, targetName) + plan, err := buildTargetPlan(source, target) if err != nil { return Record{}, "", err } + if checker, ok := s.runner.(interface { + Preview(context.Context, Plan) error + }); ok { + if err := checker.Preview(ctx, plan); err != nil { + return Record{}, "", err + } + } id, err := randomID("dbmig-") if err != nil { return Record{}, "", ErrInvalidRequest @@ -189,8 +202,11 @@ func (s *Service) execute(ctx context.Context, id, token string, actor config.Au item.Status = StatusFailed item.ErrorCode = "PREVIEW_EXPIRED" item.UpdatedAt = now + item.sourceConfig = config.DatabaseConfig{} + item.targetConfig = config.DatabaseConfig{} + snapshot := cloneRecord(item.Record) s.mu.Unlock() - return cloneRecord(item.Record), ErrPreviewExpired + return snapshot, ErrPreviewExpired } hash := sha256.Sum256([]byte(token)) if subtle.ConstantTimeCompare(hash[:], item.tokenHash[:]) != 1 { @@ -201,8 +217,12 @@ func (s *Service) execute(ctx context.Context, id, token string, actor config.Au item.Status = StatusCreated item.UpdatedAt = now plan := Plan{Source: item.Source, Target: item.Target, sourceConfig: item.sourceConfig, targetConfig: item.targetConfig, expectedGeneration: item.CurrentGeneration, actor: actor} + item.sourceConfig = config.DatabaseConfig{} + item.targetConfig = config.DatabaseConfig{} s.mu.Unlock() + releaseRequests := s.drainRequests(ctx) + defer releaseRequests() if err := s.setStatus(id, StatusCopying); err != nil { return s.fail(id, err) } diff --git a/backend/services/dbmigration/sql_executor.go b/backend/services/dbmigration/sql_executor.go index db66d420..6528dd78 100644 --- a/backend/services/dbmigration/sql_executor.go +++ b/backend/services/dbmigration/sql_executor.go @@ -44,6 +44,7 @@ type StructuredDatabaseCleanup interface { type DatabaseExecutor struct { sqlite Runner structured Runner + portable *PortableExecutor } func NewDatabaseExecutor(sqlite, structured Runner) Runner { @@ -54,6 +55,9 @@ func (e *DatabaseExecutor) Run(ctx context.Context, plan Plan, update func(Statu if e == nil { return ErrExecutorUnavailable } + if e.portable != nil { + return e.portable.Run(ctx, plan, update) + } if plan.sourceConfig.Type == config.DatabaseTypeSQLite { if e.sqlite == nil { return ErrExecutorUnavailable @@ -883,13 +887,17 @@ func runtimeSQLDSN(database config.DatabaseConfig, password, databaseName string switch database.Type { case config.DatabaseTypeMySQL: settings := mysql.Config{ - User: database.Username, - Passwd: password, - Net: "tcp", - Addr: address, - DBName: databaseName, - ParseTime: true, - Params: map[string]string{"charset": "utf8mb4"}, + User: database.Username, + Passwd: password, + Net: "tcp", + Addr: address, + DBName: databaseName, + ParseTime: true, + AllowNativePasswords: true, + Timeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + Params: map[string]string{"charset": "utf8mb4", "time_zone": "'+00:00'", "sql_mode": "'STRICT_ALL_TABLES,NO_ZERO_DATE,NO_ZERO_IN_DATE'"}, } return "mysql", settings.FormatDSN(), nil case config.DatabaseTypePostgreSQL: @@ -898,11 +906,12 @@ func runtimeSQLDSN(database config.DatabaseConfig, password, databaseName string maintenanceDatabase = "postgres" } databaseURL := url.URL{ - Scheme: "postgres", - User: url.UserPassword(database.Username, password), - Host: address, - Path: "/" + maintenanceDatabase, - RawQuery: "sslmode=prefer", + Scheme: "postgres", + User: url.UserPassword(database.Username, password), + Host: address, + Path: "/" + maintenanceDatabase, + // 与分区配置运行连接的默认模式一致;lib/pq 不支持 prefer。 + RawQuery: "sslmode=require&connect_timeout=10&timezone=UTC&search_path=public", } return "postgres", databaseURL.String(), nil default: diff --git a/backend/services/dbmigration/target.go b/backend/services/dbmigration/target.go new file mode 100644 index 00000000..a7c52611 --- /dev/null +++ b/backend/services/dbmigration/target.go @@ -0,0 +1,85 @@ +package dbmigration + +import ( + "context" + "strings" + "time" + + "git.pcux.cn/xuans/Smail/config" + "git.pcux.cn/xuans/Smail/utils/credential" +) + +// TargetInput 只接受结构化连接字段,不接受任意 DSN、SQL 或密码密文。 +type TargetInput struct { + Type config.DatabaseType `json:"type"` + Host string `json:"host,omitempty"` + Port uint16 `json:"port,omitempty"` + Username string `json:"username,omitempty"` + Name string `json:"name,omitempty"` + LocalRef string `json:"localRef,omitempty"` + Password *config.SecretInput `json:"password,omitempty"` +} + +func (input TargetInput) Prepare(root string) (config.DatabaseConfig, error) { + target := config.NormalizeDatabaseConfig(config.DatabaseConfig{ + Type: input.Type, Host: strings.TrimSpace(input.Host), Port: input.Port, + Username: strings.TrimSpace(input.Username), Name: strings.TrimSpace(input.Name), + LocalRef: strings.TrimSpace(input.LocalRef), + }) + if target.Type == config.DatabaseTypeSQLite { + if input.Password != nil { + return config.DatabaseConfig{}, ErrInvalidRequest + } + } else if target.Type.Valid() { + vault, err := credential.NewVault(root) + if err != nil { + return config.DatabaseConfig{}, config.ErrSecretUnavailable + } + secret, err := config.ApplySecretChange(vault, config.StoredSecret{}, config.SecretChange{ + Operation: config.SecretReplace, Value: input.Password, + }, config.SecretPolicy{Required: true, MaxLength: 4096}, time.Now().UTC()) + if err != nil { + return config.DatabaseConfig{}, ErrInvalidRequest + } + target.Password = secret + } + if err := config.ValidateDatabaseConfig(target); err != nil { + return config.DatabaseConfig{}, ErrInvalidRequest + } + return target, nil +} + +func databaseIdentity(database config.DatabaseConfig) DatabaseIdentity { + return DatabaseIdentity{Type: database.Type, Host: database.Host, Port: database.Port, + Username: database.Username, Name: database.Name, LocalRef: database.LocalRef} +} + +func buildTargetPlan(source, target config.DatabaseConfig) (Plan, error) { + source = config.NormalizeDatabaseConfig(source) + target = config.NormalizeDatabaseConfig(target) + if err := config.ValidateDatabaseConfig(source); err != nil { + return Plan{}, ErrSourceDatabase + } + if err := config.ValidateDatabaseConfig(target); err != nil { + return Plan{}, ErrInvalidRequest + } + if target.Type == config.DatabaseTypeSQLite { + if !sqliteTargetNamePattern.MatchString(target.LocalRef) { + return Plan{}, ErrInvalidTargetName + } + } else if !targetNamePattern.MatchString(target.Name) { + return Plan{}, ErrInvalidTargetName + } + if config.SameDatabaseLocation(source, target) { + return Plan{}, ErrInvalidTargetName + } + return Plan{Source: databaseIdentity(source), Target: databaseIdentity(target), sourceConfig: source, targetConfig: target}, nil +} + +// Preview 不创建目标库,只验证源库可读、目标服务可连接且名称未占用。 +func (e *DatabaseExecutor) Preview(ctx context.Context, plan Plan) error { + if e.portable != nil { + return e.portable.Preview(ctx, plan) + } + return nil +} diff --git a/backend/session/init.go b/backend/session/init.go index bc7d8613..d127ae89 100644 --- a/backend/session/init.go +++ b/backend/session/init.go @@ -5,6 +5,7 @@ import ( "git.pcux.cn/xuans/Smail/config" "git.pcux.cn/xuans/Smail/db" + "git.pcux.cn/xuans/Smail/services/dbmigration" "github.com/alexedwards/scs/mysqlstore" "github.com/alexedwards/scs/postgresstore" "github.com/alexedwards/scs/sqlite3store" @@ -32,14 +33,18 @@ func Init() error { switch cfg.DbType { case config.DBTypeMySQL: - Instance.Store = mysqlstore.New(db.Instance.DB().DB) + Instance.Store = mysqlstore.NewWithCleanupInterval(db.Instance.DB().DB, 0) case config.DBTypeSQLite: - Instance.Store = sqlite3store.New(db.Instance.DB().DB) + Instance.Store = sqlite3store.NewWithCleanupInterval(db.Instance.DB().DB, 0) case config.DBTypePostgres: - Instance.Store = postgresstore.New(db.Instance.DB().DB) + Instance.Store = postgresstore.NewWithCleanupInterval(db.Instance.DB().DB, 0) default: return fmt.Errorf("unsupported database type: %s", cfg.DbType) } + Instance.Store = &migrationStore{Store: Instance.Store, codec: Instance.Codec, paused: func() bool { + return dbmigration.Current().WritePaused() + }} + startCleanup(db.Instance.DB().DB, cfg.DbType == config.DBTypeSQLite) return nil } diff --git a/backend/session/migration_store.go b/backend/session/migration_store.go new file mode 100644 index 00000000..ab87a85f --- /dev/null +++ b/backend/session/migration_store.go @@ -0,0 +1,103 @@ +package session + +import ( + "context" + "database/sql" + "errors" + "reflect" + "time" + + "git.pcux.cn/xuans/Smail/services/dbmigration" + "github.com/alexedwards/scs/v2" + log "github.com/sirupsen/logrus" +) + +var errMigrationSessionWrite = errors.New("session changes are paused during database migration") + +// migrationStore 只跳过冻结期间未改变内容的会话续期。 +// 不能新增会话、修改权限、延长有效期或伪称已保存安全状态变更。 +type migrationStore struct { + scs.Store + codec scs.Codec + paused func() bool +} + +func (s *migrationStore) Commit(token string, data []byte, expiry time.Time) error { + if !s.paused() { + return s.Store.Commit(token, data, expiry) + } + previous, found, err := s.Store.Find(token) + if err != nil { + return err + } + if !found { + return errMigrationSessionWrite + } + beforeDeadline, before, err := s.codec.Decode(previous) + if err != nil { + return err + } + afterDeadline, after, err := s.codec.Decode(data) + if err != nil { + return err + } + if !beforeDeadline.Equal(afterDeadline) || !reflect.DeepEqual(before, after) { + return errMigrationSessionWrite + } + return nil +} + +func (s *migrationStore) Delete(token string) error { + if s.paused() { + return errMigrationSessionWrite + } + return s.Store.Delete(token) +} + +func (s *migrationStore) All() (map[string][]byte, error) { + store, ok := s.Store.(scs.IterableStore) + if !ok { + return nil, errors.New("session store does not support iteration") + } + return store.All() +} + +var stopCleanup context.CancelFunc + +// Stop 在内部重启时停止旧会话清理器,防止它继续访问旧库。 +func Stop() { + if stopCleanup != nil { + stopCleanup() + stopCleanup = nil + } +} + +func startCleanup(database *sql.DB, sqlite bool) { + Stop() + ctx, cancel := context.WithCancel(context.Background()) + stopCleanup = cancel + go func() { + ticker := time.NewTicker(5 * time.Minute) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + err := dbmigration.WhileWritable(func() error { + operation, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + query := "DELETE FROM sessions WHERE expiry < CURRENT_TIMESTAMP" + if sqlite { + query = "DELETE FROM sessions WHERE expiry < julianday('now')" + } + _, err := database.ExecContext(operation, query) + return err + }) + if err != nil && ctx.Err() == nil { + log.Warn("清理过期会话失败") + } + } + } + }() +} diff --git a/backend/session/migration_store_test.go b/backend/session/migration_store_test.go new file mode 100644 index 00000000..abad6126 --- /dev/null +++ b/backend/session/migration_store_test.go @@ -0,0 +1,93 @@ +package session + +import ( + "context" + "database/sql" + "errors" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/alexedwards/scs/sqlite3store" + "github.com/alexedwards/scs/v2" + _ "modernc.org/sqlite" +) + +func TestMigrationSessionResponseDoesNotWriteLockedDatabase(t *testing.T) { + path := filepath.Join(t.TempDir(), "session.db") + database, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(1)") + if err != nil { + t.Fatal(err) + } + defer database.Close() + if _, err := database.Exec("CREATE TABLE sessions (token TEXT PRIMARY KEY, data BLOB NOT NULL, expiry REAL NOT NULL)"); err != nil { + t.Fatal(err) + } + manager := scs.New() + manager.IdleTimeout = time.Hour + paused := false + store := &migrationStore{Store: sqlite3store.NewWithCleanupInterval(database, 0), codec: manager.Codec, paused: func() bool { return paused }} + manager.Store = store + ctx, err := manager.Load(context.Background(), "") + if err != nil { + t.Fatal(err) + } + manager.Put(ctx, "role", "super_admin") + token, _, err := manager.Commit(ctx) + if err != nil { + t.Fatal(err) + } + var originalExpiry float64 + if err := database.QueryRow("SELECT expiry FROM sessions").Scan(&originalExpiry); err != nil { + t.Fatal(err) + } + lockDB, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + defer lockDB.Close() + lock, err := lockDB.Conn(context.Background()) + if err != nil { + t.Fatal(err) + } + defer lock.Close() + handler := manager.LoadAndSave(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if manager.GetString(r.Context(), "role") != "super_admin" { + t.Error("identity lost") + } + if !paused { + if _, err := lock.ExecContext(r.Context(), "BEGIN IMMEDIATE"); err != nil { + t.Error(err) + } + paused = true + } + _, _ = w.Write([]byte("ok")) + })) + defer lock.ExecContext(context.Background(), "ROLLBACK") + for range 2 { + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/api/v1/config/status", nil) + request.AddCookie(&http.Cookie{Name: manager.Cookie.Name, Value: token}) + handler.ServeHTTP(recorder, request) + if recorder.Code != 200 || recorder.Body.String() != "ok" { + t.Fatalf("session response: %d %s", recorder.Code, recorder.Body.String()) + } + } + ctx, err = manager.Load(context.Background(), token) + if err != nil { + t.Fatal(err) + } + manager.Put(ctx, "role", "admin") + if _, _, err := manager.Commit(ctx); !errors.Is(err, errMigrationSessionWrite) { + t.Fatalf("changed session accepted: %v", err) + } + if err := store.Delete(token); !errors.Is(err, errMigrationSessionWrite) { + t.Fatal("deletion accepted") + } + var expiry float64 + if err := database.QueryRow("SELECT expiry FROM sessions").Scan(&expiry); err != nil || expiry != originalExpiry { + t.Fatal("session expiry changed while frozen") + } +} diff --git a/backend/session/user.go b/backend/session/user.go index f6f17a86..e3f9362a 100644 --- a/backend/session/user.go +++ b/backend/session/user.go @@ -4,12 +4,14 @@ import ( "context" "encoding/json" "errors" + "time" "git.pcux.cn/xuans/Smail/models" "git.pcux.cn/xuans/Smail/services/access" ) const userSessionKey = "user" +const recentAuthenticationLifetime = 10 * time.Minute type UserIdentity struct { ID int `json:"id"` @@ -20,6 +22,17 @@ type UserIdentity struct { SessionVersion int `json:"session_version"` SecurityVersion int `json:"security_version"` MustChangePassword int `json:"must_change_password"` + AuthenticatedAt int64 `json:"authenticated_at,omitempty"` +} + +// HasRecentAuthentication 只信任完成登录时写入的服务端时间,刷新不会续期。 +// 旧会话缺少时间或时间来自未来时必须重新登录,不能自动获得高风险权限。 +func (identity UserIdentity) HasRecentAuthentication(now time.Time) bool { + if identity.ID <= 0 || identity.AuthenticatedAt <= 0 { + return false + } + authenticated := time.Unix(identity.AuthenticatedAt, 0) + return !now.Before(authenticated) && now.Before(authenticated.Add(recentAuthenticationLifetime)) } func PutUser(ctx context.Context, user *models.User) error { @@ -36,6 +49,7 @@ func PutUser(ctx context.Context, user *models.User) error { SessionVersion: user.SessionVersion, SecurityVersion: user.SecurityVersion, MustChangePassword: user.MustChangePassword, + AuthenticatedAt: time.Now().UTC().Unix(), }) if err != nil { return err diff --git a/backend/session/user_test.go b/backend/session/user_test.go index 1011dbec..9a81a0f1 100644 --- a/backend/session/user_test.go +++ b/backend/session/user_test.go @@ -3,6 +3,7 @@ package session import ( "context" "testing" + "time" "git.pcux.cn/xuans/Smail/models" "github.com/alexedwards/scs/v2" @@ -28,4 +29,30 @@ func TestUserIdentityRoundTrip(t *testing.T) { if identity.ID != user.ID || identity.Account != user.Account || identity.Name != user.Name || identity.IsAdmin != 1 || identity.SessionVersion != 4 || identity.SecurityVersion != 3 || identity.MustChangePassword != 1 { t.Fatalf("unexpected session identity: %+v", identity) } + if !identity.HasRecentAuthentication(time.Now().UTC()) { + t.Fatal("completed login did not record recent authentication") + } +} + +func TestRecentAuthenticationRejectsMissingFutureAndExpiredTimes(t *testing.T) { + now := time.Date(2026, 9, 6, 0, 0, 0, 0, time.UTC) + for _, test := range []struct { + name string + at int64 + want bool + }{ + {"legacy session", 0, false}, + {"future time", now.Add(time.Second).Unix(), false}, + {"fresh login", now.Unix(), true}, + {"before expiry", now.Add(-recentAuthenticationLifetime + time.Second).Unix(), true}, + {"exact expiry", now.Add(-recentAuthenticationLifetime).Unix(), false}, + {"expired login", now.Add(-time.Hour).Unix(), false}, + } { + t.Run(test.name, func(t *testing.T) { + identity := UserIdentity{ID: 1, AuthenticatedAt: test.at} + if identity.HasRecentAuthentication(now) != test.want { + t.Fatal("unexpected recent authentication state") + } + }) + } } diff --git a/frontend/src/views/components/ConfigSettings/ConfigForm/ConfigForm.js b/frontend/src/views/components/ConfigSettings/ConfigForm/ConfigForm.js index 5cdcb32c..e060a09e 100644 --- a/frontend/src/views/components/ConfigSettings/ConfigForm/ConfigForm.js +++ b/frontend/src/views/components/ConfigSettings/ConfigForm/ConfigForm.js @@ -41,7 +41,7 @@ const baseGroups = [ field('domains.accessHosts', '额外允许的访问主机', 'list', { hint: '按需填写域名或 IP 地址,不填写则保持默认访问范围。', placeholder: 'mail.example.com' }), field('logLevel', '日志级别', 'select', { options: options([['', '默认级别'], ['debug', '调试'], ['info', '信息'], ['warn', '警告'], ['error', '错误']]) }) ]), - group('database', '数据库连接', '数据库类型保持不变;修改数据库名称或文件请使用右侧“数据库迁移”,避免切换到空数据库。', [ + group('database', '数据库连接', '修改数据库类型、名称或文件请使用右侧“数据库迁移”,支持 SQLite、MySQL、PostgreSQL,避免切换到空数据库。', [ field('database.type', '数据库类型', 'select', { locked: true, options: options([['sqlite', 'SQLite(本地文件)'], ['mysql', 'MySQL'], ['postgresql', 'PostgreSQL']]) }), field('database.localRef', '数据库文件', 'text', { locked: true, show: config => !networkDatabase(config) }), field('database.name', '数据库名称', 'text', { locked: true, show: networkDatabase }), diff --git a/frontend/src/views/components/ConfigSettings/ConfigMaintenance/ConfigMaintenance.vue b/frontend/src/views/components/ConfigSettings/ConfigMaintenance/ConfigMaintenance.vue index a83412a0..151c51a2 100644 --- a/frontend/src/views/components/ConfigSettings/ConfigMaintenance/ConfigMaintenance.vue +++ b/frontend/src/views/components/ConfigSettings/ConfigMaintenance/ConfigMaintenance.vue @@ -12,7 +12,9 @@
-

系统先生成一次性预览票据,再复制并校验数据。主机、端口、用户名和密码保持不变,旧数据库不会自动删除。

+

支持 SQLite、MySQL、PostgreSQL 之间迁移。先验证目标连接,再复制并逐表校验数据;旧数据库不会删除。迁移期间暂停写入,成功后必须立即重启后端。

+ +
当前数据库类型 @@ -24,19 +26,43 @@
- + + + SQLite + MySQL + PostgreSQL + + +
+ + + + + + + + + + + + +
+ +

{{ migrationTarget.type === 'sqlite' ? '文件固定存放在运行目录 config 下,只填写新文件名,不填写路径。' : '请使用尚不存在的库名,系统执行时自动建库。新连接必须填写密码,不会复用源数据库密码。' }}

+

PostgreSQL 连接要求数据库服务启用 TLS 加密,与迁移后的运行连接保持一致。

- - 生成迁移预览 + + 验证连接并预览 执行迁移 @@ -52,7 +78,9 @@
迁移编号:{{ migrationRecord.id }} 源数据库:{{ migrationRecord.source?.name || migrationRecord.source?.localRef || '-' }} - 目标数据库:{{ migrationRecord.target?.name || migrationRecord.target?.localRef || '-' }} + 目标数据库:{{ databaseTypeLabel(migrationRecord.target?.type) }} · {{ migrationRecord.target?.name || migrationRecord.target?.localRef || '-' }} + 目标连接:{{ migrationRecord.target.host }}:{{ migrationRecord.target.port }} · {{ migrationRecord.target.username }} + 错误代码:{{ migrationRecord.errorCode }}
@@ -160,6 +188,7 @@ const { loading, availabilityError, working, status, refreshPage, beforeLeave, sectionError, canMigrate, databaseConfig, databaseTypeLabel, migrationTargetName, migrationPreview, migrationToken, migrationRecord, migrationStatusType, migrationStatusLabel, + migrationTarget, migrationReady, migrationError, invalidateMigration, changeMigrationType, previewDatabaseMigration, executeDatabaseMigration, loadMigrationStatus, canRestart, serviceLabel, restartPreview, previewRestart, requestRestart, canRollback, backupsLoading, backupsError, backups, rollbackGeneration, rollbackConfirmed, diff --git a/frontend/src/views/components/ConfigSettings/ConfigSettings.css b/frontend/src/views/components/ConfigSettings/ConfigSettings.css index 9610c6b1..7b1cbc98 100644 --- a/frontend/src/views/components/ConfigSettings/ConfigSettings.css +++ b/frontend/src/views/components/ConfigSettings/ConfigSettings.css @@ -471,8 +471,22 @@ overflow-wrap: anywhere; } .migration-form { + max-width: 880px; margin-top: 16px; } +.migration-connection-grid { + display: grid; + grid-template-columns: minmax(0, 2fr) minmax(0, 1fr); + gap: 0 20px; +} +.migration-connection-grid :deep(.el-input-number) { + width: 100%; +} +@container config-main (max-width: 620px) { + .migration-connection-grid { + grid-template-columns: minmax(0, 1fr); + } +} .migration-form :deep(.el-form-item) { margin-bottom: 0; } diff --git a/frontend/src/views/components/ConfigSettings/ConfigSettings.js b/frontend/src/views/components/ConfigSettings/ConfigSettings.js index fe3a0824..66af914f 100644 --- a/frontend/src/views/components/ConfigSettings/ConfigSettings.js +++ b/frontend/src/views/components/ConfigSettings/ConfigSettings.js @@ -13,6 +13,7 @@ const sections = [ ] const serviceLabels = { + database: '数据库服务', smtp: 'SMTP 服务', email: '邮箱服务', ssl: 'SSL 服务', @@ -68,9 +69,43 @@ export function usePage(pageKey = 'base') { const auditItems = ref([]) const databaseConfig = reactive({ type: '', name: '', localRef: '' }) const migrationTargetName = ref('') + const migrationTarget = reactive({ type: 'sqlite', host: '127.0.0.1', port: 3306, username: '', password: '' }) + const migrationError = ref('') + let migrationInput = '' const migrationPreview = ref(null) const migrationToken = ref('') const migrationRecord = ref(null) + const invalidateMigration = () => { + migrationPreview.value = null + migrationToken.value = '' + migrationInput = '' + migrationError.value = '' + } + const changeMigrationType = type => { + migrationTarget.type = type + migrationTarget.port = type === 'postgresql' ? 5432 : 3306 + migrationTarget.password = '' + migrationTargetName.value = '' + invalidateMigration() + } + const migrationPayload = () => migrationTarget.type === 'sqlite' + ? { type: 'sqlite', localRef: migrationTargetName.value.trim() } + : { + type: migrationTarget.type, + host: migrationTarget.host.trim(), + port: Number(migrationTarget.port), + username: migrationTarget.username.trim(), + name: migrationTargetName.value.trim(), + password: { plaintext: migrationTarget.password } + } + const migrationReady = computed(() => { + if (!migrationTargetName.value.trim() || status.pendingRestart?.length) return false + if (migrationTarget.type === 'sqlite') return /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(migrationTargetName.value.trim()) + return ['mysql', 'postgresql'].includes(migrationTarget.type) && + /^[A-Za-z][A-Za-z0-9_-]{0,62}$/.test(migrationTargetName.value.trim()) && + !!migrationTarget.host.trim() && !!migrationTarget.username.trim() && !!migrationTarget.password && + Number.isInteger(Number(migrationTarget.port)) && Number(migrationTarget.port) > 0 && Number(migrationTarget.port) <= 65535 + }) const restartPreview = ref(null) const hasChanges = computed(() => editorText.value !== loadedText.value || !!secretValue.value || !!savedGeneration.value) @@ -219,22 +254,27 @@ export function usePage(pageKey = 'base') { } const previewDatabaseMigration = async () => { - if (!canMigrate.value || working.value || sectionError.value || !migrationTargetName.value.trim()) return + if (!canMigrate.value || working.value || sectionError.value || !migrationReady.value) return working.value = 'migration-preview' migrationPreview.value = null migrationRecord.value = null migrationToken.value = '' + migrationError.value = '' try { + const target = migrationPayload() const response = await http.post('/api/v1/database/migrations/preview', { - targetName: migrationTargetName.value.trim(), + target, expectedGeneration: Number(status.activeGeneration) }) + if (disposed) return notifyResponseError(response, '数据库迁移预览失败') migrationPreview.value = response.data?.migration || null migrationRecord.value = migrationPreview.value migrationToken.value = response.data?.token || '' - ElMessage.success('数据库迁移预览已生成,请确认目标名称后执行') + migrationInput = JSON.stringify(target) + ElMessage.success('目标连接验证通过,请核对数据库类型和连接后执行') } catch (error) { + migrationError.value = error.message || '数据库迁移预览失败' ElMessage.error(error.message || '数据库迁移预览失败') } finally { working.value = '' @@ -257,10 +297,15 @@ export function usePage(pageKey = 'base') { const executeDatabaseMigration = async () => { if (!canMigrate.value || working.value || !migrationPreview.value?.id || !migrationToken.value) return + if (migrationInput !== JSON.stringify(migrationPayload())) { + invalidateMigration() + migrationError.value = '目标连接已修改,请重新生成迁移预览' + return + } working.value = 'migration-confirm' try { await ElMessageBox.confirm( - '迁移将复制数据库内容并切换配置,旧数据库会保留。确认继续吗?', + `将迁移到 ${databaseTypeLabel(migrationPreview.value.target?.type)}:${migrationPreview.value.target?.name || migrationPreview.value.target?.localRef || migrationTargetName.value.trim()}。迁移期间暂停写入,成功后必须立即重启后端。旧数据库及失败目标均保留。确认继续吗?`, '确认执行数据库迁移', { type: 'warning', confirmButtonText: '继续迁移', cancelButtonText: '取消' } ) @@ -268,19 +313,29 @@ export function usePage(pageKey = 'base') { const response = await http.post('/api/v1/database/migrations', { id: migrationPreview.value.id, token: migrationToken.value - }) + }, { timeout: 11 * 60 * 1000 }) notifyResponseError(response, '数据库迁移执行失败') migrationRecord.value = response.data || migrationPreview.value migrationToken.value = '' migrationPreview.value = null + migrationTarget.password = '' + migrationInput = '' await Promise.all([loadStatus(), loadSection()]) if (migrationRecord.value.status === 'succeeded') { - ElMessage.success('数据库迁移已完成,旧数据库仍然保留') + ElMessage.success('数据已复制校验并切换配置,请立即重启后端以恢复写入;旧数据库保留') } else { ElMessage.warning(`数据库迁移${migrationStatusLabel(migrationRecord.value.status)}`) } } catch (error) { - if (error !== 'cancel' && error !== 'close') ElMessage.error(error.message || '数据库迁移执行失败') + if (error !== 'cancel' && error !== 'close') { + migrationToken.value = '' + migrationPreview.value = null + migrationTarget.password = '' + migrationInput = '' + migrationError.value = error.message || '数据库迁移执行失败' + await loadMigrationStatus() + ElMessage.error(migrationError.value) + } } finally { working.value = '' } @@ -321,6 +376,8 @@ export function usePage(pageKey = 'base') { } } const loadAll = async () => { + invalidateMigration() + migrationTarget.password = '' loading.value = true availabilityError.value = '' try { @@ -539,6 +596,8 @@ export function usePage(pageKey = 'base') { savedInput = null confirmed.value = false migrationToken.value = '' + migrationTarget.password = '' + migrationInput = '' }) return { @@ -551,6 +610,7 @@ export function usePage(pageKey = 'base') { loadBackups, previewRollback, executeRollback, loadAudit, exportAudit, formatTime, serviceLabel, sectionLabel, operationLabel, databaseConfig, databaseTypeLabel, canMigrate, migrationTargetName, migrationPreview, migrationToken, migrationRecord, migrationStatusLabel, migrationStatusType, + migrationTarget, migrationReady, migrationError, invalidateMigration, changeMigrationType, previewDatabaseMigration, loadMigrationStatus, executeDatabaseMigration, canRestart, restartPreview, previewRestart, requestRestart, beforeLeave, refreshPage, hasChanges } diff --git a/test/README.md b/test/README.md index 9fe16984..c491612a 100644 --- a/test/README.md +++ b/test/README.md @@ -32,6 +32,22 @@ Set-Location test/tools/password-check go build -o ../../../out/test-tools/password-check.exe . ``` +## 三种数据库迁移验收 + +```sh +node test/run.mjs backend -count=1 -v ./services/dbmigration -run TestPortableSQLIntegration +``` + +网络数据库实测必须先配置专用测试服务的环境变量: + +- `SMAIL_TEST_MIGRATION_MYSQL`:结构化连接,`type` 为 `mysql`。 +- `SMAIL_TEST_MIGRATION_POSTGRES`:结构化连接,`type` 为 `postgresql`,服务必须支持 TLS。 +- 两个变量的内容均为 JSON,字段为 `type`、`host`、`port`、`username`、`password`,其中 `password` 是包含 `plaintext` 字段的对象;不填写 `name` 或 `localRef`,不把真实凭据写入仓库。 +- 使用拥有创建数据库权限的专用账号。测试仅创建 `smail_migration_test_` 前缀的随机新库,不访问现有业务库,不自动删除网络测试库。 +- 单个服务执行 SQLite → 网络库 → 同类型新库 → SQLite;同时提供两种服务时,额外执行 SQLite → MySQL → PostgreSQL → MySQL → SQLite。 +- 逐表校验内容,并检查中文主题、二进制会话、日期、布尔值和新增记录主键。配置切换使用测试回调,不代表完整部署重启验收。 +- 未提供连接时显示跳过,不能作为 MySQL/PostgreSQL 实库迁移通过的证据。 + ## Go 测试源码的必要例外 `backend/` 中的同包 `*_test.go` 保持原位,其中许多测试使用包内私有函数或相对路径。将其机械移动到独立目录会破坏 Go 测试语义,因此只集中运行入口,不改变业务包的可见性和 API。后续跨模块、浏览器和独立协议测试应放到本目录。 diff --git a/test/frontend/config-settings.test.mjs b/test/frontend/config-settings.test.mjs index 3c122db7..591289a5 100644 --- a/test/frontend/config-settings.test.mjs +++ b/test/frontend/config-settings.test.mjs @@ -190,10 +190,48 @@ test('迁移预览票据返回模板,迁移执行取消时不发送写入请 await page.executeDatabaseMigration() assert.deepEqual(requests, [{ path: '/api/v1/database/migrations/preview', - payload: { targetName: 'test-next.db', expectedGeneration: 3 } + payload: { target: { type: 'sqlite', localRef: 'test-next.db' }, expectedGeneration: 3 } }]) }) +test('三种数据库使用独立目标字段,类型切换清除密码及旧预览', async () => { + const requests = [] + const { page, dispose } = fixture(async () => ({ errorNo: 0, data: {} }), 'migration', { + http: { post: async (path, payload) => { + requests.push({ path, payload }) + return { errorNo: 0, data: { migration: { id: 'next', target: payload.target }, token: 'ticket' } } + } } + }) + for (const type of ['mysql', 'postgresql']) { + page.changeMigrationType(type) + assert.equal(page.migrationTarget.port, type === 'mysql' ? 3306 : 5432) + assert.equal(page.migrationReady.value, false) + page.migrationTargetName.value = 'smail_next' + Object.assign(page.migrationTarget, { host: ' db.local ', username: ' smail ', password: ' test-secret ' }) + assert.equal(page.migrationReady.value, true) + await page.previewDatabaseMigration() + assert.equal(requests.at(-1).payload.target.type, type) + assert.equal(requests.at(-1).payload.target.host, 'db.local') + assert.equal(requests.at(-1).payload.target.password.plaintext, ' test-secret ') + page.migrationTarget.host = 'other.local' + await page.executeDatabaseMigration() + assert.equal(page.migrationToken.value, '') + assert.match(page.migrationError.value, /重新生成/) + } + assert.equal(requests.length, 2) + page.changeMigrationType('sqlite') + assert.equal(page.migrationTarget.password, '') + assert.equal(page.migrationPreview.value, null) + page.migrationTargetName.value = '../bad.db' + assert.equal(page.migrationReady.value, false) + page.migrationTargetName.value = 'next.db' + page.status.pendingRestart = ['database'] + assert.equal(page.migrationReady.value, false) + page.migrationTarget.password = 'temporary' + dispose() + assert.equal(page.migrationTarget.password, '') +}) + test('独立页模板可编译,编辑器不再连接维护面板且权限绑定保留', () => { const require = createRequire(new URL('../../frontend/package.json', import.meta.url)) const { parse, compileScript, compileStyle } = require('@vue/compiler-sfc')