修复迁移事务和数据库切换边界
Some checks are pending
Security Checks / security (push) Waiting to run

This commit is contained in:
smail 2026-09-06 06:34:10 +08:00
parent 8ca7187721
commit 8bd6f1edfb
8 changed files with 286 additions and 21 deletions

View File

@ -581,12 +581,8 @@ func legacyDatabaseConfig(root string, vault *credential.Vault, legacy legacyMig
func legacyDatabasePlan(legacy legacyMigrationConfig) (DatabaseConfig, string, []string) {
switch legacy.DbType {
case "", DBTypeSQLite:
name := strings.TrimSpace(legacy.DbDSN)
if name == "" {
name = "smail.db"
}
name = filepath.Base(filepath.Clean(name))
if !databaseLocalRefPattern.MatchString(name) {
name, ok := legacySQLiteLocalRef(legacy.DbDSN)
if !ok {
return DatabaseConfig{}, "", []string{"dbDSN"}
}
return DatabaseConfig{Type: DatabaseTypeSQLite, LocalRef: name}, "", nil
@ -607,6 +603,29 @@ func legacyDatabasePlan(legacy legacyMigrationConfig) (DatabaseConfig, string, [
}
}
func legacySQLiteLocalRef(value string) (string, bool) {
value = strings.TrimSpace(value)
if value == "" {
value = "smail.db"
}
if filepath.IsAbs(value) || isWindowsAbsolute(value) {
return "", false
}
value = filepath.ToSlash(value)
value = strings.TrimPrefix(value, "./")
value = strings.TrimPrefix(value, "config/")
if value == "" || strings.HasPrefix(value, "../") || value == ".." || strings.Contains(value, "/../") || strings.Contains(value, "\\") || strings.Contains(value, ":") {
return "", false
}
parts := strings.Split(value, "/")
for _, part := range parts {
if part == "" || part == "." || !databaseLocalRefPattern.MatchString(part) {
return "", false
}
}
return value, true
}
func parseLegacyMySQLDSN(value string) (DatabaseConfig, string, bool) {
matches := legacyMySQLDSNPattern.FindStringSubmatch(value)
if matches == nil {

View File

@ -226,6 +226,14 @@ func (s *TransactionStore) Recover() error {
if err := validateTransactionFiles(transactionDirectory, "original", manifest.OriginalFiles); err != nil {
return err
}
loader := NewConfigLoader(s.root, ConfigLoaderOptions{Validation: s.validation})
runtime, loadErr := loader.Load()
if loadErr == nil && runtime.Generation >= manifest.ToGeneration {
if err := os.RemoveAll(transactionDirectory); err != nil {
return err
}
continue
}
switch manifest.Phase {
case TransactionPrepared, TransactionSectionsCommitted:
if err := restoreTransactionOriginal(directory, transactionDirectory, manifest); err != nil {
@ -239,14 +247,6 @@ func (s *TransactionStore) Recover() error {
return err
}
case TransactionRootCommitted, TransactionVerified:
loader := NewConfigLoader(s.root, ConfigLoaderOptions{Validation: s.validation})
runtime, loadErr := loader.Load()
if loadErr == nil && runtime.Generation == manifest.ToGeneration {
if err := os.RemoveAll(transactionDirectory); err != nil {
return err
}
continue
}
if err := restoreTransactionOriginal(directory, transactionDirectory, manifest); err != nil {
return err
}

View File

@ -75,7 +75,7 @@ func validateDatabase(database DatabaseConfig) error {
database = NormalizeDatabaseConfig(database)
switch database.Type {
case DatabaseTypeSQLite:
if !databaseLocalRefPattern.MatchString(database.LocalRef) ||
if !validSQLiteLocalRef(database.LocalRef) ||
filepath.IsAbs(database.LocalRef) ||
isWindowsAbsolute(database.LocalRef) ||
strings.ContainsAny(database.LocalRef, `/\\:`) ||
@ -99,6 +99,20 @@ func validateDatabase(database DatabaseConfig) error {
return nil
}
func validSQLiteLocalRef(value string) bool {
value = filepath.ToSlash(value)
if value == "" || filepath.IsAbs(value) || isWindowsAbsolute(value) || strings.HasPrefix(value, "config/") || strings.ContainsAny(value, `\\:`) {
return false
}
parts := strings.Split(value, "/")
for _, part := range parts {
if part == "" || part == "." || part == ".." || !databaseLocalRefPattern.MatchString(part) {
return false
}
}
return true
}
func validateSMTP(smtp SMTPConfig) error {
if !validLoginVerificationPolicy(smtp.LoginPolicy) {
return ErrInvalidSMTP

View File

@ -5,6 +5,7 @@ import (
"database/sql"
"net/http"
"strings"
"sync"
"time"
)
@ -50,6 +51,9 @@ type requestLease struct {
}
func (lease *requestLease) release() {
if lease == nil || lease.service == nil {
return
}
if lease.exclusive {
lease.service.requests.Unlock()
} else {
@ -62,11 +66,16 @@ func (lease *requestLease) release() {
func (s *Service) drainRequests(ctx context.Context) func() {
if lease, ok := ctx.Value(requestLeaseKey{}).(*requestLease); ok && lease.service == s {
lease.service.requests.RUnlock()
lease.service.requests.Lock()
if !lease.service.requests.lockContext(ctx) {
lease.service = nil
return func() {}
}
lease.exclusive = true
return func() {}
}
s.requests.Lock()
if !s.requests.lockContext(ctx) {
return func() {}
}
return s.requests.Unlock
}
@ -84,6 +93,94 @@ func maintenanceRequest(r *http.Request) bool {
path == "/api/v1/system/restart" || path == "/api/v1/system/restart/preview")
}
type requestGate struct {
mu sync.Mutex
readers int
writer bool
changed chan struct{}
}
func (gate *requestGate) initialize() {
gate.mu.Lock()
if gate.changed == nil {
gate.changed = make(chan struct{})
}
gate.mu.Unlock()
}
func (gate *requestGate) RLock() {
gate.initialize()
for {
gate.mu.Lock()
if !gate.writer {
gate.readers++
gate.mu.Unlock()
return
}
changed := gate.changed
gate.mu.Unlock()
<-changed
}
}
func (gate *requestGate) RUnlock() {
gate.mu.Lock()
if gate.readers > 0 {
gate.readers--
}
gate.notifyLocked()
gate.mu.Unlock()
}
func (gate *requestGate) Lock() {
gate.initialize()
for {
gate.mu.Lock()
if !gate.writer && gate.readers == 0 {
gate.writer = true
gate.mu.Unlock()
return
}
changed := gate.changed
gate.mu.Unlock()
<-changed
}
}
func (gate *requestGate) lockContext(ctx context.Context) bool {
gate.initialize()
if ctx == nil {
ctx = context.Background()
}
for {
gate.mu.Lock()
if !gate.writer && gate.readers == 0 {
gate.writer = true
gate.mu.Unlock()
return true
}
changed := gate.changed
gate.mu.Unlock()
select {
case <-ctx.Done():
return false
case <-changed:
}
}
}
func (gate *requestGate) Unlock() {
gate.mu.Lock()
gate.writer = false
gate.notifyLocked()
gate.mu.Unlock()
}
func (gate *requestGate) notifyLocked() {
close(gate.changed)
gate.changed = make(chan struct{})
}
// WhileWritable 用于会话后台清理,与 HTTP 迁移执行共享排空边界。
func WhileWritable(action func() error) error {
service := Current()

View File

@ -5,6 +5,7 @@ import (
"crypto/sha256"
"database/sql"
"encoding/binary"
"errors"
"fmt"
"math"
"sort"
@ -20,6 +21,34 @@ type portableQuery interface {
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
}
type sqliteSequence struct {
Name string
Seq int64
}
func readSQLiteSequences(ctx context.Context, database portableQuery) ([]sqliteSequence, error) {
rows, err := database.QueryContext(ctx, "SELECT name, seq FROM sqlite_sequence")
if err != nil {
if strings.Contains(strings.ToLower(err.Error()), "no such table") {
return nil, nil
}
return nil, ErrSQLUnsupportedObject
}
defer rows.Close()
var sequences []sqliteSequence
for rows.Next() {
var item sqliteSequence
if err := rows.Scan(&item.Name, &item.Seq); err != nil || item.Seq < 0 {
return nil, ErrSQLUnsupportedObject
}
sequences = append(sequences, item)
}
if err := rows.Err(); err != nil {
return nil, ErrSQLUnsupportedObject
}
return sequences, nil
}
func portableTables(ctx context.Context, database portableQuery, kind config.DatabaseType) ([]string, error) {
query := "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
if kind == config.DatabaseTypeMySQL {
@ -104,6 +133,13 @@ func copyPortableTables(ctx context.Context, source *sql.Conn, target *sql.DB, p
return ErrSQLConnectionUnavailable
}
defer transaction.Rollback()
var sequences []sqliteSequence
if plan.Source.Type == config.DatabaseTypeSQLite {
sequences, err = readSQLiteSequences(ctx, source)
if err != nil {
return err
}
}
for _, table := range tables {
if table == "schema_migration" {
continue
@ -154,12 +190,53 @@ func copyPortableTables(ctx context.Context, source *sql.Conn, target *sql.DB, p
}
}
}
if err := synchronizeSequences(ctx, transaction, plan, sequences); err != nil {
return err
}
if err := transaction.Commit(); err != nil {
return ErrSQLDataVerification
}
return nil
}
func synchronizeSequences(ctx context.Context, transaction *sql.Tx, plan Plan, sequences []sqliteSequence) error {
for _, sequence := range sequences {
if sequence.Name == "schema_migration" {
continue
}
switch plan.Target.Type {
case config.DatabaseTypeSQLite:
var current int64
err := transaction.QueryRowContext(ctx, "SELECT seq FROM sqlite_sequence WHERE name=?", sequence.Name).Scan(&current)
if errors.Is(err, sql.ErrNoRows) {
if _, err := transaction.ExecContext(ctx, "INSERT INTO sqlite_sequence(name, seq) VALUES(?, ?)", sequence.Name, sequence.Seq); err != nil {
return ErrSQLDataVerification
}
continue
}
if err != nil {
return ErrSQLDataVerification
}
if current < sequence.Seq {
if _, err := transaction.ExecContext(ctx, "UPDATE sqlite_sequence SET seq=? WHERE name=?", sequence.Seq, sequence.Name); err != nil {
return ErrSQLDataVerification
}
}
case config.DatabaseTypeMySQL:
query := "ALTER TABLE " + quoteMySQLIdentifier(sequence.Name) + " AUTO_INCREMENT = " + strconv.FormatInt(sequence.Seq+1, 10)
if _, err := transaction.ExecContext(ctx, query); err != nil {
return ErrSQLDataVerification
}
case config.DatabaseTypePostgreSQL:
query := "SELECT setval(pg_get_serial_sequence($1, 'id'), GREATEST(COALESCE(MAX(id), 0), $2), true) FROM " + quotePostgreSQLIdentifier(sequence.Name)
if _, err := transaction.ExecContext(ctx, query, sequence.Name, sequence.Seq); err != nil {
return ErrSQLDataVerification
}
}
}
return nil
}
func portableColumnList(kind config.DatabaseType, columns []string) string {
result := make([]string, len(columns))
for i, column := range columns {

View File

@ -9,6 +9,9 @@ import (
// 仅迁移 Smail 版本化业务表;不能静默忽略自定义对象。
func validatePortableObjects(ctx context.Context, database portableQuery, kind config.DatabaseType) error {
queries := []string{"SELECT COUNT(*) FROM sqlite_master WHERE type IN ('view','trigger')"}
if kind == config.DatabaseTypeSQLite {
queries = append(queries, "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND sql IS NOT NULL")
}
switch kind {
case config.DatabaseTypeMySQL:
queries = []string{

View File

@ -81,7 +81,7 @@ type Runner interface {
}
type Service struct {
requests sync.RWMutex
requests requestGate
mu sync.RWMutex
records map[string]*record
runner Runner
@ -102,7 +102,9 @@ var current struct {
}
func NewService(runner Runner) *Service {
return &Service{records: make(map[string]*record), runner: runner, now: func() time.Time { return time.Now().UTC() }}
service := &Service{records: make(map[string]*record), runner: runner, now: func() time.Time { return time.Now().UTC() }}
service.requests.initialize()
return service
}
func SetCurrent(service *Service) {
@ -223,6 +225,9 @@ func (s *Service) execute(ctx context.Context, id, token string, actor config.Au
releaseRequests := s.drainRequests(ctx)
defer releaseRequests()
if err := contextError(ctx); err != nil {
return s.fail(id, err)
}
if err := s.setStatus(id, StatusCopying); err != nil {
return s.fail(id, err)
}

View File

@ -145,7 +145,7 @@ func (e *SQLiteExecutor) Run(ctx context.Context, plan Plan, update func(Status)
}
func (e *SQLiteExecutor) resolveReference(reference string) (string, error) {
if e == nil || strings.TrimSpace(e.root) == "" || !sqliteTargetNamePattern.MatchString(reference) {
if e == nil || strings.TrimSpace(e.root) == "" || !validSQLiteReference(reference) {
return "", ErrInvalidTargetName
}
root, err := filepath.Abs(filepath.Clean(e.root))
@ -159,12 +159,62 @@ func (e *SQLiteExecutor) resolveReference(reference string) (string, error) {
}
path := filepath.Join(configDirectory, reference)
absolute, err := filepath.Abs(filepath.Clean(path))
if err != nil || filepath.Dir(absolute) != configDirectory {
if err != nil || !pathWithinDirectory(configDirectory, absolute) {
return "", ErrInvalidTargetName
}
if realConfig, configErr := filepath.EvalSymlinks(configDirectory); configErr == nil {
if realTarget, targetErr := evalWithExistingAncestor(absolute); targetErr == nil && !pathWithinDirectory(realConfig, realTarget) {
return "", ErrInvalidTargetName
}
}
return absolute, nil
}
func validSQLiteReference(value string) bool {
value = filepath.ToSlash(strings.TrimSpace(value))
if value == "" || filepath.IsAbs(value) || isWindowsAbsoluteReference(value) || strings.ContainsAny(value, `\\:`) {
return false
}
for _, part := range strings.Split(value, "/") {
if part == "" || part == "." || part == ".." || !sqliteTargetNamePattern.MatchString(part) {
return false
}
}
return true
}
func isWindowsAbsoluteReference(value string) bool {
return strings.HasPrefix(value, "//") || (len(value) >= 3 && ((value[0] >= 'A' && value[0] <= 'Z') || (value[0] >= 'a' && value[0] <= 'z')) && value[1] == ':' && value[2] == '/')
}
func pathWithinDirectory(directory, target string) bool {
relative, err := filepath.Rel(directory, target)
return err == nil && !filepath.IsAbs(relative) && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
}
func evalWithExistingAncestor(target string) (string, error) {
current := target
parts := make([]string, 0)
for {
resolved, err := filepath.EvalSymlinks(current)
if err == nil {
for index := len(parts) - 1; index >= 0; index-- {
resolved = filepath.Join(resolved, parts[index])
}
return filepath.Abs(resolved)
}
if !os.IsNotExist(err) {
return "", err
}
parent := filepath.Dir(current)
if parent == current {
return "", err
}
parts = append(parts, filepath.Base(current))
current = parent
}
}
func (e *SQLiteExecutor) temporaryPath(target string) (string, error) {
for attempt := 0; attempt < 8; attempt++ {
id, err := randomID(".migration-")