This commit is contained in:
parent
8ca7187721
commit
8bd6f1edfb
@ -581,12 +581,8 @@ func legacyDatabaseConfig(root string, vault *credential.Vault, legacy legacyMig
|
||||
func legacyDatabasePlan(legacy legacyMigrationConfig) (DatabaseConfig, string, []string) {
|
||||
switch legacy.DbType {
|
||||
case "", DBTypeSQLite:
|
||||
name := strings.TrimSpace(legacy.DbDSN)
|
||||
if name == "" {
|
||||
name = "smail.db"
|
||||
}
|
||||
name = filepath.Base(filepath.Clean(name))
|
||||
if !databaseLocalRefPattern.MatchString(name) {
|
||||
name, ok := legacySQLiteLocalRef(legacy.DbDSN)
|
||||
if !ok {
|
||||
return DatabaseConfig{}, "", []string{"dbDSN"}
|
||||
}
|
||||
return DatabaseConfig{Type: DatabaseTypeSQLite, LocalRef: name}, "", nil
|
||||
@ -607,6 +603,29 @@ func legacyDatabasePlan(legacy legacyMigrationConfig) (DatabaseConfig, string, [
|
||||
}
|
||||
}
|
||||
|
||||
func legacySQLiteLocalRef(value string) (string, bool) {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
value = "smail.db"
|
||||
}
|
||||
if filepath.IsAbs(value) || isWindowsAbsolute(value) {
|
||||
return "", false
|
||||
}
|
||||
value = filepath.ToSlash(value)
|
||||
value = strings.TrimPrefix(value, "./")
|
||||
value = strings.TrimPrefix(value, "config/")
|
||||
if value == "" || strings.HasPrefix(value, "../") || value == ".." || strings.Contains(value, "/../") || strings.Contains(value, "\\") || strings.Contains(value, ":") {
|
||||
return "", false
|
||||
}
|
||||
parts := strings.Split(value, "/")
|
||||
for _, part := range parts {
|
||||
if part == "" || part == "." || !databaseLocalRefPattern.MatchString(part) {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
return value, true
|
||||
}
|
||||
|
||||
func parseLegacyMySQLDSN(value string) (DatabaseConfig, string, bool) {
|
||||
matches := legacyMySQLDSNPattern.FindStringSubmatch(value)
|
||||
if matches == nil {
|
||||
|
||||
@ -226,6 +226,14 @@ func (s *TransactionStore) Recover() error {
|
||||
if err := validateTransactionFiles(transactionDirectory, "original", manifest.OriginalFiles); err != nil {
|
||||
return err
|
||||
}
|
||||
loader := NewConfigLoader(s.root, ConfigLoaderOptions{Validation: s.validation})
|
||||
runtime, loadErr := loader.Load()
|
||||
if loadErr == nil && runtime.Generation >= manifest.ToGeneration {
|
||||
if err := os.RemoveAll(transactionDirectory); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
switch manifest.Phase {
|
||||
case TransactionPrepared, TransactionSectionsCommitted:
|
||||
if err := restoreTransactionOriginal(directory, transactionDirectory, manifest); err != nil {
|
||||
@ -239,14 +247,6 @@ func (s *TransactionStore) Recover() error {
|
||||
return err
|
||||
}
|
||||
case TransactionRootCommitted, TransactionVerified:
|
||||
loader := NewConfigLoader(s.root, ConfigLoaderOptions{Validation: s.validation})
|
||||
runtime, loadErr := loader.Load()
|
||||
if loadErr == nil && runtime.Generation == manifest.ToGeneration {
|
||||
if err := os.RemoveAll(transactionDirectory); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := restoreTransactionOriginal(directory, transactionDirectory, manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -75,7 +75,7 @@ func validateDatabase(database DatabaseConfig) error {
|
||||
database = NormalizeDatabaseConfig(database)
|
||||
switch database.Type {
|
||||
case DatabaseTypeSQLite:
|
||||
if !databaseLocalRefPattern.MatchString(database.LocalRef) ||
|
||||
if !validSQLiteLocalRef(database.LocalRef) ||
|
||||
filepath.IsAbs(database.LocalRef) ||
|
||||
isWindowsAbsolute(database.LocalRef) ||
|
||||
strings.ContainsAny(database.LocalRef, `/\\:`) ||
|
||||
@ -99,6 +99,20 @@ func validateDatabase(database DatabaseConfig) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validSQLiteLocalRef(value string) bool {
|
||||
value = filepath.ToSlash(value)
|
||||
if value == "" || filepath.IsAbs(value) || isWindowsAbsolute(value) || strings.HasPrefix(value, "config/") || strings.ContainsAny(value, `\\:`) {
|
||||
return false
|
||||
}
|
||||
parts := strings.Split(value, "/")
|
||||
for _, part := range parts {
|
||||
if part == "" || part == "." || part == ".." || !databaseLocalRefPattern.MatchString(part) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func validateSMTP(smtp SMTPConfig) error {
|
||||
if !validLoginVerificationPolicy(smtp.LoginPolicy) {
|
||||
return ErrInvalidSMTP
|
||||
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"database/sql"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@ -50,6 +51,9 @@ type requestLease struct {
|
||||
}
|
||||
|
||||
func (lease *requestLease) release() {
|
||||
if lease == nil || lease.service == nil {
|
||||
return
|
||||
}
|
||||
if lease.exclusive {
|
||||
lease.service.requests.Unlock()
|
||||
} else {
|
||||
@ -62,11 +66,16 @@ func (lease *requestLease) release() {
|
||||
func (s *Service) drainRequests(ctx context.Context) func() {
|
||||
if lease, ok := ctx.Value(requestLeaseKey{}).(*requestLease); ok && lease.service == s {
|
||||
lease.service.requests.RUnlock()
|
||||
lease.service.requests.Lock()
|
||||
if !lease.service.requests.lockContext(ctx) {
|
||||
lease.service = nil
|
||||
return func() {}
|
||||
}
|
||||
lease.exclusive = true
|
||||
return func() {}
|
||||
}
|
||||
s.requests.Lock()
|
||||
if !s.requests.lockContext(ctx) {
|
||||
return func() {}
|
||||
}
|
||||
return s.requests.Unlock
|
||||
}
|
||||
|
||||
@ -84,6 +93,94 @@ func maintenanceRequest(r *http.Request) bool {
|
||||
path == "/api/v1/system/restart" || path == "/api/v1/system/restart/preview")
|
||||
}
|
||||
|
||||
type requestGate struct {
|
||||
mu sync.Mutex
|
||||
readers int
|
||||
writer bool
|
||||
changed chan struct{}
|
||||
}
|
||||
|
||||
func (gate *requestGate) initialize() {
|
||||
gate.mu.Lock()
|
||||
if gate.changed == nil {
|
||||
gate.changed = make(chan struct{})
|
||||
}
|
||||
gate.mu.Unlock()
|
||||
}
|
||||
|
||||
func (gate *requestGate) RLock() {
|
||||
gate.initialize()
|
||||
for {
|
||||
gate.mu.Lock()
|
||||
if !gate.writer {
|
||||
gate.readers++
|
||||
gate.mu.Unlock()
|
||||
return
|
||||
}
|
||||
changed := gate.changed
|
||||
gate.mu.Unlock()
|
||||
<-changed
|
||||
}
|
||||
}
|
||||
|
||||
func (gate *requestGate) RUnlock() {
|
||||
gate.mu.Lock()
|
||||
if gate.readers > 0 {
|
||||
gate.readers--
|
||||
}
|
||||
gate.notifyLocked()
|
||||
gate.mu.Unlock()
|
||||
}
|
||||
|
||||
func (gate *requestGate) Lock() {
|
||||
gate.initialize()
|
||||
for {
|
||||
gate.mu.Lock()
|
||||
if !gate.writer && gate.readers == 0 {
|
||||
gate.writer = true
|
||||
gate.mu.Unlock()
|
||||
return
|
||||
}
|
||||
changed := gate.changed
|
||||
gate.mu.Unlock()
|
||||
<-changed
|
||||
}
|
||||
}
|
||||
|
||||
func (gate *requestGate) lockContext(ctx context.Context) bool {
|
||||
gate.initialize()
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
for {
|
||||
gate.mu.Lock()
|
||||
if !gate.writer && gate.readers == 0 {
|
||||
gate.writer = true
|
||||
gate.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
changed := gate.changed
|
||||
gate.mu.Unlock()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-changed:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (gate *requestGate) Unlock() {
|
||||
gate.mu.Lock()
|
||||
gate.writer = false
|
||||
gate.notifyLocked()
|
||||
gate.mu.Unlock()
|
||||
}
|
||||
|
||||
func (gate *requestGate) notifyLocked() {
|
||||
close(gate.changed)
|
||||
gate.changed = make(chan struct{})
|
||||
}
|
||||
|
||||
// WhileWritable 用于会话后台清理,与 HTTP 迁移执行共享排空边界。
|
||||
func WhileWritable(action func() error) error {
|
||||
service := Current()
|
||||
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
@ -20,6 +21,34 @@ type portableQuery interface {
|
||||
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
|
||||
}
|
||||
|
||||
type sqliteSequence struct {
|
||||
Name string
|
||||
Seq int64
|
||||
}
|
||||
|
||||
func readSQLiteSequences(ctx context.Context, database portableQuery) ([]sqliteSequence, error) {
|
||||
rows, err := database.QueryContext(ctx, "SELECT name, seq FROM sqlite_sequence")
|
||||
if err != nil {
|
||||
if strings.Contains(strings.ToLower(err.Error()), "no such table") {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, ErrSQLUnsupportedObject
|
||||
}
|
||||
defer rows.Close()
|
||||
var sequences []sqliteSequence
|
||||
for rows.Next() {
|
||||
var item sqliteSequence
|
||||
if err := rows.Scan(&item.Name, &item.Seq); err != nil || item.Seq < 0 {
|
||||
return nil, ErrSQLUnsupportedObject
|
||||
}
|
||||
sequences = append(sequences, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, ErrSQLUnsupportedObject
|
||||
}
|
||||
return sequences, nil
|
||||
}
|
||||
|
||||
func portableTables(ctx context.Context, database portableQuery, kind config.DatabaseType) ([]string, error) {
|
||||
query := "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
|
||||
if kind == config.DatabaseTypeMySQL {
|
||||
@ -104,6 +133,13 @@ func copyPortableTables(ctx context.Context, source *sql.Conn, target *sql.DB, p
|
||||
return ErrSQLConnectionUnavailable
|
||||
}
|
||||
defer transaction.Rollback()
|
||||
var sequences []sqliteSequence
|
||||
if plan.Source.Type == config.DatabaseTypeSQLite {
|
||||
sequences, err = readSQLiteSequences(ctx, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, table := range tables {
|
||||
if table == "schema_migration" {
|
||||
continue
|
||||
@ -154,12 +190,53 @@ func copyPortableTables(ctx context.Context, source *sql.Conn, target *sql.DB, p
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := synchronizeSequences(ctx, transaction, plan, sequences); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := transaction.Commit(); err != nil {
|
||||
return ErrSQLDataVerification
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func synchronizeSequences(ctx context.Context, transaction *sql.Tx, plan Plan, sequences []sqliteSequence) error {
|
||||
for _, sequence := range sequences {
|
||||
if sequence.Name == "schema_migration" {
|
||||
continue
|
||||
}
|
||||
switch plan.Target.Type {
|
||||
case config.DatabaseTypeSQLite:
|
||||
var current int64
|
||||
err := transaction.QueryRowContext(ctx, "SELECT seq FROM sqlite_sequence WHERE name=?", sequence.Name).Scan(¤t)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
if _, err := transaction.ExecContext(ctx, "INSERT INTO sqlite_sequence(name, seq) VALUES(?, ?)", sequence.Name, sequence.Seq); err != nil {
|
||||
return ErrSQLDataVerification
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return ErrSQLDataVerification
|
||||
}
|
||||
if current < sequence.Seq {
|
||||
if _, err := transaction.ExecContext(ctx, "UPDATE sqlite_sequence SET seq=? WHERE name=?", sequence.Seq, sequence.Name); err != nil {
|
||||
return ErrSQLDataVerification
|
||||
}
|
||||
}
|
||||
case config.DatabaseTypeMySQL:
|
||||
query := "ALTER TABLE " + quoteMySQLIdentifier(sequence.Name) + " AUTO_INCREMENT = " + strconv.FormatInt(sequence.Seq+1, 10)
|
||||
if _, err := transaction.ExecContext(ctx, query); err != nil {
|
||||
return ErrSQLDataVerification
|
||||
}
|
||||
case config.DatabaseTypePostgreSQL:
|
||||
query := "SELECT setval(pg_get_serial_sequence($1, 'id'), GREATEST(COALESCE(MAX(id), 0), $2), true) FROM " + quotePostgreSQLIdentifier(sequence.Name)
|
||||
if _, err := transaction.ExecContext(ctx, query, sequence.Name, sequence.Seq); err != nil {
|
||||
return ErrSQLDataVerification
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func portableColumnList(kind config.DatabaseType, columns []string) string {
|
||||
result := make([]string, len(columns))
|
||||
for i, column := range columns {
|
||||
|
||||
@ -9,6 +9,9 @@ import (
|
||||
// 仅迁移 Smail 版本化业务表;不能静默忽略自定义对象。
|
||||
func validatePortableObjects(ctx context.Context, database portableQuery, kind config.DatabaseType) error {
|
||||
queries := []string{"SELECT COUNT(*) FROM sqlite_master WHERE type IN ('view','trigger')"}
|
||||
if kind == config.DatabaseTypeSQLite {
|
||||
queries = append(queries, "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND sql IS NOT NULL")
|
||||
}
|
||||
switch kind {
|
||||
case config.DatabaseTypeMySQL:
|
||||
queries = []string{
|
||||
|
||||
@ -81,7 +81,7 @@ type Runner interface {
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
requests sync.RWMutex
|
||||
requests requestGate
|
||||
mu sync.RWMutex
|
||||
records map[string]*record
|
||||
runner Runner
|
||||
@ -102,7 +102,9 @@ var current struct {
|
||||
}
|
||||
|
||||
func NewService(runner Runner) *Service {
|
||||
return &Service{records: make(map[string]*record), runner: runner, now: func() time.Time { return time.Now().UTC() }}
|
||||
service := &Service{records: make(map[string]*record), runner: runner, now: func() time.Time { return time.Now().UTC() }}
|
||||
service.requests.initialize()
|
||||
return service
|
||||
}
|
||||
|
||||
func SetCurrent(service *Service) {
|
||||
@ -223,6 +225,9 @@ func (s *Service) execute(ctx context.Context, id, token string, actor config.Au
|
||||
|
||||
releaseRequests := s.drainRequests(ctx)
|
||||
defer releaseRequests()
|
||||
if err := contextError(ctx); err != nil {
|
||||
return s.fail(id, err)
|
||||
}
|
||||
if err := s.setStatus(id, StatusCopying); err != nil {
|
||||
return s.fail(id, err)
|
||||
}
|
||||
|
||||
@ -145,7 +145,7 @@ func (e *SQLiteExecutor) Run(ctx context.Context, plan Plan, update func(Status)
|
||||
}
|
||||
|
||||
func (e *SQLiteExecutor) resolveReference(reference string) (string, error) {
|
||||
if e == nil || strings.TrimSpace(e.root) == "" || !sqliteTargetNamePattern.MatchString(reference) {
|
||||
if e == nil || strings.TrimSpace(e.root) == "" || !validSQLiteReference(reference) {
|
||||
return "", ErrInvalidTargetName
|
||||
}
|
||||
root, err := filepath.Abs(filepath.Clean(e.root))
|
||||
@ -159,12 +159,62 @@ func (e *SQLiteExecutor) resolveReference(reference string) (string, error) {
|
||||
}
|
||||
path := filepath.Join(configDirectory, reference)
|
||||
absolute, err := filepath.Abs(filepath.Clean(path))
|
||||
if err != nil || filepath.Dir(absolute) != configDirectory {
|
||||
if err != nil || !pathWithinDirectory(configDirectory, absolute) {
|
||||
return "", ErrInvalidTargetName
|
||||
}
|
||||
if realConfig, configErr := filepath.EvalSymlinks(configDirectory); configErr == nil {
|
||||
if realTarget, targetErr := evalWithExistingAncestor(absolute); targetErr == nil && !pathWithinDirectory(realConfig, realTarget) {
|
||||
return "", ErrInvalidTargetName
|
||||
}
|
||||
}
|
||||
return absolute, nil
|
||||
}
|
||||
|
||||
func validSQLiteReference(value string) bool {
|
||||
value = filepath.ToSlash(strings.TrimSpace(value))
|
||||
if value == "" || filepath.IsAbs(value) || isWindowsAbsoluteReference(value) || strings.ContainsAny(value, `\\:`) {
|
||||
return false
|
||||
}
|
||||
for _, part := range strings.Split(value, "/") {
|
||||
if part == "" || part == "." || part == ".." || !sqliteTargetNamePattern.MatchString(part) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func isWindowsAbsoluteReference(value string) bool {
|
||||
return strings.HasPrefix(value, "//") || (len(value) >= 3 && ((value[0] >= 'A' && value[0] <= 'Z') || (value[0] >= 'a' && value[0] <= 'z')) && value[1] == ':' && value[2] == '/')
|
||||
}
|
||||
|
||||
func pathWithinDirectory(directory, target string) bool {
|
||||
relative, err := filepath.Rel(directory, target)
|
||||
return err == nil && !filepath.IsAbs(relative) && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
|
||||
}
|
||||
|
||||
func evalWithExistingAncestor(target string) (string, error) {
|
||||
current := target
|
||||
parts := make([]string, 0)
|
||||
for {
|
||||
resolved, err := filepath.EvalSymlinks(current)
|
||||
if err == nil {
|
||||
for index := len(parts) - 1; index >= 0; index-- {
|
||||
resolved = filepath.Join(resolved, parts[index])
|
||||
}
|
||||
return filepath.Abs(resolved)
|
||||
}
|
||||
if !os.IsNotExist(err) {
|
||||
return "", err
|
||||
}
|
||||
parent := filepath.Dir(current)
|
||||
if parent == current {
|
||||
return "", err
|
||||
}
|
||||
parts = append(parts, filepath.Base(current))
|
||||
current = parent
|
||||
}
|
||||
}
|
||||
|
||||
func (e *SQLiteExecutor) temporaryPath(target string) (string, error) {
|
||||
for attempt := 0; attempt < 8; attempt++ {
|
||||
id, err := randomID(".migration-")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user