From 8bd6f1edfb66c9230f2a5e75d2167a96999ecc9b Mon Sep 17 00:00:00 2001 From: smail Date: Sun, 6 Sep 2026 06:34:10 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=BF=81=E7=A7=BB=E4=BA=8B?= =?UTF-8?q?=E5=8A=A1=E5=92=8C=E6=95=B0=E6=8D=AE=E5=BA=93=E5=88=87=E6=8D=A2?= =?UTF-8?q?=E8=BE=B9=E7=95=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/config/migrate.go | 31 ++++-- backend/config/transaction.go | 16 +-- backend/config/validator.go | 16 ++- backend/services/dbmigration/maintenance.go | 101 +++++++++++++++++- backend/services/dbmigration/portable_data.go | 77 +++++++++++++ .../services/dbmigration/portable_schema.go | 3 + backend/services/dbmigration/service.go | 9 +- .../services/dbmigration/sqlite_executor.go | 54 +++++++++- 8 files changed, 286 insertions(+), 21 deletions(-) diff --git a/backend/config/migrate.go b/backend/config/migrate.go index d728af33..a04ed37a 100644 --- a/backend/config/migrate.go +++ b/backend/config/migrate.go @@ -581,12 +581,8 @@ func legacyDatabaseConfig(root string, vault *credential.Vault, legacy legacyMig func legacyDatabasePlan(legacy legacyMigrationConfig) (DatabaseConfig, string, []string) { switch legacy.DbType { case "", DBTypeSQLite: - name := strings.TrimSpace(legacy.DbDSN) - if name == "" { - name = "smail.db" - } - name = filepath.Base(filepath.Clean(name)) - if !databaseLocalRefPattern.MatchString(name) { + name, ok := legacySQLiteLocalRef(legacy.DbDSN) + if !ok { return DatabaseConfig{}, "", []string{"dbDSN"} } return DatabaseConfig{Type: DatabaseTypeSQLite, LocalRef: name}, "", nil @@ -607,6 +603,29 @@ func legacyDatabasePlan(legacy legacyMigrationConfig) (DatabaseConfig, string, [ } } +func legacySQLiteLocalRef(value string) (string, bool) { + value = strings.TrimSpace(value) + if value == "" { + value = "smail.db" + } + if filepath.IsAbs(value) || isWindowsAbsolute(value) { + return "", false + } + value = filepath.ToSlash(value) + value = strings.TrimPrefix(value, "./") + value = strings.TrimPrefix(value, "config/") + if value == "" || strings.HasPrefix(value, "../") || value == ".." || strings.Contains(value, "/../") || strings.Contains(value, "\\") || strings.Contains(value, ":") { + return "", false + } + parts := strings.Split(value, "/") + for _, part := range parts { + if part == "" || part == "." || !databaseLocalRefPattern.MatchString(part) { + return "", false + } + } + return value, true +} + func parseLegacyMySQLDSN(value string) (DatabaseConfig, string, bool) { matches := legacyMySQLDSNPattern.FindStringSubmatch(value) if matches == nil { diff --git a/backend/config/transaction.go b/backend/config/transaction.go index 8c4b78a2..36550691 100644 --- a/backend/config/transaction.go +++ b/backend/config/transaction.go @@ -226,6 +226,14 @@ func (s *TransactionStore) Recover() error { if err := validateTransactionFiles(transactionDirectory, "original", manifest.OriginalFiles); err != nil { return err } + loader := NewConfigLoader(s.root, ConfigLoaderOptions{Validation: s.validation}) + runtime, loadErr := loader.Load() + if loadErr == nil && runtime.Generation >= manifest.ToGeneration { + if err := os.RemoveAll(transactionDirectory); err != nil { + return err + } + continue + } switch manifest.Phase { case TransactionPrepared, TransactionSectionsCommitted: if err := restoreTransactionOriginal(directory, transactionDirectory, manifest); err != nil { @@ -239,14 +247,6 @@ func (s *TransactionStore) Recover() error { return err } case TransactionRootCommitted, TransactionVerified: - loader := NewConfigLoader(s.root, ConfigLoaderOptions{Validation: s.validation}) - runtime, loadErr := loader.Load() - if loadErr == nil && runtime.Generation == manifest.ToGeneration { - if err := os.RemoveAll(transactionDirectory); err != nil { - return err - } - continue - } if err := restoreTransactionOriginal(directory, transactionDirectory, manifest); err != nil { return err } diff --git a/backend/config/validator.go b/backend/config/validator.go index fa62605c..0b6c58e0 100644 --- a/backend/config/validator.go +++ b/backend/config/validator.go @@ -75,7 +75,7 @@ func validateDatabase(database DatabaseConfig) error { database = NormalizeDatabaseConfig(database) switch database.Type { case DatabaseTypeSQLite: - if !databaseLocalRefPattern.MatchString(database.LocalRef) || + if !validSQLiteLocalRef(database.LocalRef) || filepath.IsAbs(database.LocalRef) || isWindowsAbsolute(database.LocalRef) || strings.ContainsAny(database.LocalRef, `/\\:`) || @@ -99,6 +99,20 @@ func validateDatabase(database DatabaseConfig) error { return nil } +func validSQLiteLocalRef(value string) bool { + value = filepath.ToSlash(value) + if value == "" || filepath.IsAbs(value) || isWindowsAbsolute(value) || strings.HasPrefix(value, "config/") || strings.ContainsAny(value, `\\:`) { + return false + } + parts := strings.Split(value, "/") + for _, part := range parts { + if part == "" || part == "." || part == ".." || !databaseLocalRefPattern.MatchString(part) { + return false + } + } + return true +} + func validateSMTP(smtp SMTPConfig) error { if !validLoginVerificationPolicy(smtp.LoginPolicy) { return ErrInvalidSMTP diff --git a/backend/services/dbmigration/maintenance.go b/backend/services/dbmigration/maintenance.go index 26776e14..568e312c 100644 --- a/backend/services/dbmigration/maintenance.go +++ b/backend/services/dbmigration/maintenance.go @@ -5,6 +5,7 @@ import ( "database/sql" "net/http" "strings" + "sync" "time" ) @@ -50,6 +51,9 @@ type requestLease struct { } func (lease *requestLease) release() { + if lease == nil || lease.service == nil { + return + } if lease.exclusive { lease.service.requests.Unlock() } else { @@ -62,11 +66,16 @@ func (lease *requestLease) release() { func (s *Service) drainRequests(ctx context.Context) func() { if lease, ok := ctx.Value(requestLeaseKey{}).(*requestLease); ok && lease.service == s { lease.service.requests.RUnlock() - lease.service.requests.Lock() + if !lease.service.requests.lockContext(ctx) { + lease.service = nil + return func() {} + } lease.exclusive = true return func() {} } - s.requests.Lock() + if !s.requests.lockContext(ctx) { + return func() {} + } return s.requests.Unlock } @@ -84,6 +93,94 @@ func maintenanceRequest(r *http.Request) bool { path == "/api/v1/system/restart" || path == "/api/v1/system/restart/preview") } +type requestGate struct { + mu sync.Mutex + readers int + writer bool + changed chan struct{} +} + +func (gate *requestGate) initialize() { + gate.mu.Lock() + if gate.changed == nil { + gate.changed = make(chan struct{}) + } + gate.mu.Unlock() +} + +func (gate *requestGate) RLock() { + gate.initialize() + for { + gate.mu.Lock() + if !gate.writer { + gate.readers++ + gate.mu.Unlock() + return + } + changed := gate.changed + gate.mu.Unlock() + <-changed + } +} + +func (gate *requestGate) RUnlock() { + gate.mu.Lock() + if gate.readers > 0 { + gate.readers-- + } + gate.notifyLocked() + gate.mu.Unlock() +} + +func (gate *requestGate) Lock() { + gate.initialize() + for { + gate.mu.Lock() + if !gate.writer && gate.readers == 0 { + gate.writer = true + gate.mu.Unlock() + return + } + changed := gate.changed + gate.mu.Unlock() + <-changed + } +} + +func (gate *requestGate) lockContext(ctx context.Context) bool { + gate.initialize() + if ctx == nil { + ctx = context.Background() + } + for { + gate.mu.Lock() + if !gate.writer && gate.readers == 0 { + gate.writer = true + gate.mu.Unlock() + return true + } + changed := gate.changed + gate.mu.Unlock() + select { + case <-ctx.Done(): + return false + case <-changed: + } + } +} + +func (gate *requestGate) Unlock() { + gate.mu.Lock() + gate.writer = false + gate.notifyLocked() + gate.mu.Unlock() +} + +func (gate *requestGate) notifyLocked() { + close(gate.changed) + gate.changed = make(chan struct{}) +} + // WhileWritable 用于会话后台清理，与 HTTP 迁移执行共享排空边界。 func WhileWritable(action func() error) error { service := Current() diff --git a/backend/services/dbmigration/portable_data.go b/backend/services/dbmigration/portable_data.go index 109b7054..16ff9f8f 100644 --- a/backend/services/dbmigration/portable_data.go +++ b/backend/services/dbmigration/portable_data.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "database/sql" "encoding/binary" + "errors" "fmt" "math" "sort" @@ -20,6 +21,34 @@ type portableQuery interface { QueryContext(context.Context, string, ...any) (*sql.Rows, error) } +type sqliteSequence struct { + Name string + Seq int64 +} + +func readSQLiteSequences(ctx context.Context, database portableQuery) ([]sqliteSequence, error) { + rows, err := database.QueryContext(ctx, "SELECT name, seq FROM sqlite_sequence") + if err != nil { + if strings.Contains(strings.ToLower(err.Error()), "no such table") { + return nil, nil + } + return nil, ErrSQLUnsupportedObject + } + defer rows.Close() + var sequences []sqliteSequence + for rows.Next() { + var item sqliteSequence + if err := rows.Scan(&item.Name, &item.Seq); err != nil || item.Seq < 0 { + return nil, ErrSQLUnsupportedObject + } + sequences = append(sequences, item) + } + if err := rows.Err(); err != nil { + return nil, ErrSQLUnsupportedObject + } + return sequences, nil +} + func portableTables(ctx context.Context, database portableQuery, kind config.DatabaseType) ([]string, error) { query := "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name" if kind == config.DatabaseTypeMySQL { @@ -104,6 +133,13 @@ func copyPortableTables(ctx context.Context, source *sql.Conn, target *sql.DB, p return ErrSQLConnectionUnavailable } defer transaction.Rollback() + var sequences []sqliteSequence + if plan.Source.Type == config.DatabaseTypeSQLite { + sequences, err = readSQLiteSequences(ctx, source) + if err != nil { + return err + } + } for _, table := range tables { if table == "schema_migration" { continue @@ -154,12 +190,53 @@ func copyPortableTables(ctx context.Context, source *sql.Conn, target *sql.DB, p } } } + if err := synchronizeSequences(ctx, transaction, plan, sequences); err != nil { + return err + } if err := transaction.Commit(); err != nil { return ErrSQLDataVerification } return nil } +func synchronizeSequences(ctx context.Context, transaction *sql.Tx, plan Plan, sequences []sqliteSequence) error { + for _, sequence := range sequences { + if sequence.Name == "schema_migration" { + continue + } + switch plan.Target.Type { + case config.DatabaseTypeSQLite: + var current int64 + err := transaction.QueryRowContext(ctx, "SELECT seq FROM sqlite_sequence WHERE name=?", sequence.Name).Scan(¤t) + if errors.Is(err, sql.ErrNoRows) { + if _, err := transaction.ExecContext(ctx, "INSERT INTO sqlite_sequence(name, seq) VALUES(?, ?)", sequence.Name, sequence.Seq); err != nil { + return ErrSQLDataVerification + } + continue + } + if err != nil { + return ErrSQLDataVerification + } + if current < sequence.Seq { + if _, err := transaction.ExecContext(ctx, "UPDATE sqlite_sequence SET seq=? WHERE name=?", sequence.Seq, sequence.Name); err != nil { + return ErrSQLDataVerification + } + } + case config.DatabaseTypeMySQL: + query := "ALTER TABLE " + quoteMySQLIdentifier(sequence.Name) + " AUTO_INCREMENT = " + strconv.FormatInt(sequence.Seq+1, 10) + if _, err := transaction.ExecContext(ctx, query); err != nil { + return ErrSQLDataVerification + } + case config.DatabaseTypePostgreSQL: + query := "SELECT setval(pg_get_serial_sequence($1, 'id'), GREATEST(COALESCE(MAX(id), 0), $2), true) FROM " + quotePostgreSQLIdentifier(sequence.Name) + if _, err := transaction.ExecContext(ctx, query, sequence.Name, sequence.Seq); err != nil { + return ErrSQLDataVerification + } + } + } + return nil +} + func portableColumnList(kind config.DatabaseType, columns []string) string { result := make([]string, len(columns)) for i, column := range columns { diff --git a/backend/services/dbmigration/portable_schema.go b/backend/services/dbmigration/portable_schema.go index 0475835d..93a0fed6 100644 --- a/backend/services/dbmigration/portable_schema.go +++ b/backend/services/dbmigration/portable_schema.go @@ -9,6 +9,9 @@ import ( // 仅迁移 Smail 版本化业务表；不能静默忽略自定义对象。 func validatePortableObjects(ctx context.Context, database portableQuery, kind config.DatabaseType) error { queries := []string{"SELECT COUNT(*) FROM sqlite_master WHERE type IN ('view','trigger')"} + if kind == config.DatabaseTypeSQLite { + queries = append(queries, "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND sql IS NOT NULL") + } switch kind { case config.DatabaseTypeMySQL: queries = []string{ diff --git a/backend/services/dbmigration/service.go b/backend/services/dbmigration/service.go index ab66b6e7..599a1315 100644 --- a/backend/services/dbmigration/service.go +++ b/backend/services/dbmigration/service.go @@ -81,7 +81,7 @@ type Runner interface { } type Service struct { - requests sync.RWMutex + requests requestGate mu sync.RWMutex records map[string]*record runner Runner @@ -102,7 +102,9 @@ var current struct { } func NewService(runner Runner) *Service { - return &Service{records: make(map[string]*record), runner: runner, now: func() time.Time { return time.Now().UTC() }} + service := &Service{records: make(map[string]*record), runner: runner, now: func() time.Time { return time.Now().UTC() }} + service.requests.initialize() + return service } func SetCurrent(service *Service) { @@ -223,6 +225,9 @@ func (s *Service) execute(ctx context.Context, id, token string, actor config.Au releaseRequests := s.drainRequests(ctx) defer releaseRequests() + if err := contextError(ctx); err != nil { + return s.fail(id, err) + } if err := s.setStatus(id, StatusCopying); err != nil { return s.fail(id, err) } diff --git a/backend/services/dbmigration/sqlite_executor.go b/backend/services/dbmigration/sqlite_executor.go index 9208071e..47bbc0e1 100644 --- a/backend/services/dbmigration/sqlite_executor.go +++ b/backend/services/dbmigration/sqlite_executor.go @@ -145,7 +145,7 @@ func (e *SQLiteExecutor) Run(ctx context.Context, plan Plan, update func(Status) } func (e *SQLiteExecutor) resolveReference(reference string) (string, error) { - if e == nil || strings.TrimSpace(e.root) == "" || !sqliteTargetNamePattern.MatchString(reference) { + if e == nil || strings.TrimSpace(e.root) == "" || !validSQLiteReference(reference) { return "", ErrInvalidTargetName } root, err := filepath.Abs(filepath.Clean(e.root)) @@ -159,12 +159,62 @@ func (e *SQLiteExecutor) resolveReference(reference string) (string, error) { } path := filepath.Join(configDirectory, reference) absolute, err := filepath.Abs(filepath.Clean(path)) - if err != nil || filepath.Dir(absolute) != configDirectory { + if err != nil || !pathWithinDirectory(configDirectory, absolute) { return "", ErrInvalidTargetName } + if realConfig, configErr := filepath.EvalSymlinks(configDirectory); configErr == nil { + if realTarget, targetErr := evalWithExistingAncestor(absolute); targetErr == nil && !pathWithinDirectory(realConfig, realTarget) { + return "", ErrInvalidTargetName + } + } return absolute, nil } +func validSQLiteReference(value string) bool { + value = filepath.ToSlash(strings.TrimSpace(value)) + if value == "" || filepath.IsAbs(value) || isWindowsAbsoluteReference(value) || strings.ContainsAny(value, `\\:`) { + return false + } + for _, part := range strings.Split(value, "/") { + if part == "" || part == "." || part == ".." || !sqliteTargetNamePattern.MatchString(part) { + return false + } + } + return true +} + +func isWindowsAbsoluteReference(value string) bool { + return strings.HasPrefix(value, "//") || (len(value) >= 3 && ((value[0] >= 'A' && value[0] <= 'Z') || (value[0] >= 'a' && value[0] <= 'z')) && value[1] == ':' && value[2] == '/') +} + +func pathWithinDirectory(directory, target string) bool { + relative, err := filepath.Rel(directory, target) + return err == nil && !filepath.IsAbs(relative) && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) +} + +func evalWithExistingAncestor(target string) (string, error) { + current := target + parts := make([]string, 0) + for { + resolved, err := filepath.EvalSymlinks(current) + if err == nil { + for index := len(parts) - 1; index >= 0; index-- { + resolved = filepath.Join(resolved, parts[index]) + } + return filepath.Abs(resolved) + } + if !os.IsNotExist(err) { + return "", err + } + parent := filepath.Dir(current) + if parent == current { + return "", err + } + parts = append(parts, filepath.Base(current)) + current = parent + } +} + func (e *SQLiteExecutor) temporaryPath(target string) (string, error) { for attempt := 0; attempt < 8; attempt++ { id, err := randomID(".migration-")