完善配置安全校验与数据库迁移处理
Security Checks / security (push) Waiting to run

This commit is contained in:
xuans committed 2026-08-30 17:45:52 +08:00
1 parent 3af1fc4147
commit 29cfebd620
36 files changed
+2838 -118

No files matched your search

+62 -9
View File
@@ -37,6 +37,9 @@ var (
ErrMigrationState = errors.New("database migration state is invalid")
ErrExecutorUnavailable = errors.New("database migration executor unavailable")
ErrSourceDatabase = errors.New("source database configuration is invalid")
ErrTargetExists = errors.New("database migration target already exists")
ErrSwitchUnavailable = errors.New("database migration configuration switch unavailable")
ErrUnsupportedDatabase = errors.New("database migration database type is unsupported")
)
const previewLifetime = 10 * time.Minute
@@ -54,8 +57,12 @@ type DatabaseIdentity struct {
}
type Plan struct {
Source DatabaseIdentity
Target DatabaseIdentity
Source DatabaseIdentity
Target DatabaseIdentity
sourceConfig config.DatabaseConfig
targetConfig config.DatabaseConfig
expectedGeneration uint64
actor config.AuditActor
}
type Record struct {
@@ -82,8 +89,10 @@ type Service struct {
type record struct {
Record
tokenHash [32]byte
expiresAt time.Time
tokenHash [32]byte
expiresAt time.Time
sourceConfig config.DatabaseConfig
targetConfig config.DatabaseConfig
}
var current struct {
@@ -131,7 +140,7 @@ func (s *Service) Preview(_ context.Context, source config.DatabaseConfig, curre
return Record{}, "", ErrInvalidRequest
}
now := s.currentTime()
item := &record{Record: Record{ID: id, Status: StatusPreviewed, Source: plan.Source, Target: plan.Target, CurrentGeneration: currentGeneration, CreatedAt: now, UpdatedAt: now}, tokenHash: sha256.Sum256([]byte(token)), expiresAt: now.Add(previewLifetime)}
item := &record{Record: Record{ID: id, Status: StatusPreviewed, Source: plan.Source, Target: plan.Target, CurrentGeneration: currentGeneration, CreatedAt: now, UpdatedAt: now}, tokenHash: sha256.Sum256([]byte(token)), expiresAt: now.Add(previewLifetime), sourceConfig: plan.sourceConfig, targetConfig: plan.targetConfig}
s.mu.Lock()
s.records[id] = item
s.mu.Unlock()
@@ -139,6 +148,19 @@ func (s *Service) Preview(_ context.Context, source config.DatabaseConfig, curre
}
func (s *Service) Execute(ctx context.Context, id, token string) (Record, error) {
return s.execute(ctx, id, token, config.AuditActor{})
}
// ExecuteAuthorized executes a migration while preserving the authenticated
// operator for the final configuration switch.
func (s *Service) ExecuteAuthorized(ctx context.Context, id, token string, actor config.AuditActor) (Record, error) {
if actor.UserID == "" || actor.SessionID == "" {
return Record{}, config.ErrAuthorizationRequired
}
return s.execute(ctx, id, token, actor)
}
func (s *Service) execute(ctx context.Context, id, token string, actor config.AuditActor) (Record, error) {
if s == nil || strings.TrimSpace(id) == "" || strings.TrimSpace(token) == "" {
return Record{}, ErrInvalidRequest
}
@@ -169,11 +191,11 @@ func (s *Service) Execute(ctx context.Context, id, token string) (Record, error)
item.tokenHash = [32]byte{}
item.Status = StatusCreated
item.UpdatedAt = now
plan := Plan{Source: item.Source, Target: item.Target}
plan := Plan{Source: item.Source, Target: item.Target, sourceConfig: item.sourceConfig, targetConfig: item.targetConfig, expectedGeneration: item.CurrentGeneration, actor: actor}
s.mu.Unlock()
if err := s.setStatus(id, StatusCopying); err != nil {
return Record{}, err
return s.fail(id, err)
}
if s.runner == nil {
return s.fail(id, ErrExecutorUnavailable)
@@ -182,7 +204,7 @@ func (s *Service) Execute(ctx context.Context, id, token string) (Record, error)
return s.fail(id, err)
}
if err := s.setStatus(id, StatusSucceeded); err != nil {
return Record{}, err
return s.fail(id, err)
}
return s.Get(id)
}
@@ -273,7 +295,18 @@ func buildPlan(source config.DatabaseConfig, targetName string) (Plan, error) {
if target.Type != source.Type || target.Host != source.Host || target.Port != source.Port || target.Username != source.Username {
return Plan{}, ErrImmutableSetting
}
return Plan{Source: identity, Target: target}, nil
if target.Name == source.Name && target.LocalRef == source.LocalRef {
return Plan{}, ErrInvalidTargetName
}
return Plan{Source: identity, Target: target, sourceConfig: source, targetConfig: func() config.DatabaseConfig {
candidate := source
if source.Type == config.DatabaseTypeSQLite {
candidate.LocalRef = targetName
} else {
candidate.Name = targetName
}
return candidate
}()}, nil
}
func randomID(prefix string) (string, error) {
@@ -294,6 +327,26 @@ func migrationErrorCode(err error) string {
return "EXECUTOR_UNAVAILABLE"
case errors.Is(err, ErrPreviewExpired):
return "PREVIEW_EXPIRED"
case errors.Is(err, ErrTargetExists):
return "TARGET_EXISTS"
case errors.Is(err, ErrSQLTargetExists):
return "TARGET_EXISTS"
case errors.Is(err, ErrSQLCopyUnavailable):
return "COPIER_UNAVAILABLE"
case errors.Is(err, ErrSQLConnectionUnavailable):
return "CONNECTION_UNAVAILABLE"
case errors.Is(err, ErrSQLUnsupportedObject):
return "UNSUPPORTED_OBJECT"
case errors.Is(err, ErrSQLDataVerification):
return "DATA_VERIFICATION_FAILED"
case errors.Is(err, ErrSQLSourceBusy):
return "SOURCE_BUSY"
case errors.Is(err, ErrSwitchUnavailable):
return "SWITCH_UNAVAILABLE"
case errors.Is(err, ErrSwitchOutcomeUnknown):
return "SWITCH_OUTCOME_UNKNOWN"
case errors.Is(err, ErrUnsupportedDatabase):
return "UNSUPPORTED_DATABASE"
default:
return "MIGRATION_FAILED"
}
@@ -2,12 +2,16 @@ package dbmigration
import (
"context"
"database/sql"
"errors"
"os"
"path/filepath"
"testing"
"time"
"git.pcux.cn/xuans/Smail/config"
"git.pcux.cn/xuans/Smail/utils/credential"
_ "modernc.org/sqlite"
)
type testRunner struct{}
@@ -92,6 +96,29 @@ func TestFailedExecutionKeepsSafeState(t *testing.T) {
}
}
func TestMigrationErrorCodeMapsExecutorFailures(t *testing.T) {
tests := []struct {
err error
code string
}{
{err: ErrSQLTargetExists, code: "TARGET_EXISTS"},
{err: ErrSQLCopyUnavailable, code: "COPIER_UNAVAILABLE"},
{err: ErrSQLConnectionUnavailable, code: "CONNECTION_UNAVAILABLE"},
{err: ErrSQLUnsupportedObject, code: "UNSUPPORTED_OBJECT"},
{err: ErrSQLDataVerification, code: "DATA_VERIFICATION_FAILED"},
{err: ErrSQLSourceBusy, code: "SOURCE_BUSY"},
{err: ErrSwitchUnavailable, code: "SWITCH_UNAVAILABLE"},
{err: ErrSwitchOutcomeUnknown, code: "SWITCH_OUTCOME_UNKNOWN"},
}
for _, test := range tests {
t.Run(test.code, func(t *testing.T) {
if got := migrationErrorCode(test.err); got != test.code {
t.Fatalf("migration error code = %q, want %q", got, test.code)
}
})
}
}
func TestSQLitePreviewChangesOnlyLocalReference(t *testing.T) {
service := NewService(testRunner{})
source := config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}
@@ -177,3 +204,52 @@ func TestMigrationRecordsRemainIndependentDuringConcurrentExecution(t *testing.T
t.Fatalf("migration records crossed: first=%+v second=%+v", firstState.Target, secondState.Target)
}
}
func TestExecuteAuthorizedPassesActorToSQLiteSwitch(t *testing.T) {
var received config.AuditActor
var receivedGeneration uint64
root := t.TempDir()
switchConfig := func(_ context.Context, generation uint64, _ config.DatabaseConfig, actor config.AuditActor) error {
receivedGeneration = generation
received = actor
return nil
}
executor := NewAuthorizedSQLiteExecutor(root, switchConfig)
service := NewService(executor)
actor := config.AuditActor{UserID: "admin-1", Role: "super_admin", SessionID: "session-1"}
if _, err := service.ExecuteAuthorized(context.Background(), "missing", "missing", actor); !errors.Is(err, ErrMigrationNotFound) {
t.Fatalf("missing migration error = %v, want migration not found", err)
}
if _, err := service.ExecuteAuthorized(context.Background(), "missing", "missing", config.AuditActor{UserID: "admin-1"}); !errors.Is(err, config.ErrAuthorizationRequired) {
t.Fatalf("missing session error = %v, want authorization required", err)
}
root = t.TempDir()
executor = NewAuthorizedSQLiteExecutor(root, switchConfig)
service = NewService(executor)
configDirectory := filepath.Join(root, "config")
if err := os.Mkdir(configDirectory, 0700); err != nil {
t.Fatal(err)
}
database, err := sql.Open("sqlite", filepath.Join(configDirectory, "smail.db"))
if err != nil {
t.Fatal(err)
}
if _, err := database.Exec("CREATE TABLE messages (id INTEGER PRIMARY KEY, body TEXT NOT NULL)"); err != nil {
database.Close()
t.Fatal(err)
}
if err := database.Close(); err != nil {
t.Fatal(err)
}
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "next.db")
if err != nil {
t.Fatal(err)
}
completed, err := service.ExecuteAuthorized(context.Background(), record.ID, token, actor)
if err != nil {
t.Fatal(err)
}
if completed.Status != StatusSucceeded || received != actor || receivedGeneration != record.CurrentGeneration {
t.Fatalf("authorized execution = %+v, received actor = %+v, generation = %d", completed, received, receivedGeneration)
}
}
+667
View File
@@ -0,0 +1,667 @@
package dbmigration
import (
"context"
"database/sql"
"errors"
"fmt"
"net"
"net/url"
"regexp"
"strconv"
"strings"
"sync"
"time"
"git.pcux.cn/xuans/Smail/config"
"git.pcux.cn/xuans/Smail/utils/credential"
mysql "github.com/go-sql-driver/mysql"
"github.com/lib/pq"
)
var (
ErrSQLCopyUnavailable = errors.New("structured database copier unavailable")
ErrSQLTargetExists = errors.New("structured database migration target already exists")
ErrSQLUnsupportedObject = errors.New("structured database contains unsupported object")
ErrSQLConnectionUnavailable = errors.New("structured database connection unavailable")
ErrSQLDataVerification = errors.New("structured database data verification failed")
ErrSQLSourceBusy = errors.New("structured database source has active connections")
ErrSwitchOutcomeUnknown = errors.New("database configuration switch outcome is unknown")
)
type AuthorizedDatabaseSwitch func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error
type StructuredDatabaseCopier interface {
Copy(context.Context, config.DatabaseConfig, config.DatabaseConfig) error
}
type StructuredDatabaseCleanup interface {
Cleanup(context.Context, config.DatabaseConfig, config.DatabaseConfig) error
Complete(config.DatabaseConfig, config.DatabaseConfig)
}
type DatabaseExecutor struct {
sqlite Runner
structured Runner
}
func NewDatabaseExecutor(sqlite, structured Runner) Runner {
return &DatabaseExecutor{sqlite: sqlite, structured: structured}
}
func (e *DatabaseExecutor) Run(ctx context.Context, plan Plan, update func(Status) error) error {
if e == nil {
return ErrExecutorUnavailable
}
if plan.sourceConfig.Type == config.DatabaseTypeSQLite {
if e.sqlite == nil {
return ErrExecutorUnavailable
}
return e.sqlite.Run(ctx, plan, update)
}
if e.structured == nil {
return ErrExecutorUnavailable
}
return e.structured.Run(ctx, plan, update)
}
type SQLExecutor struct {
copier StructuredDatabaseCopier
authorizedSwitch AuthorizedDatabaseSwitch
}
func NewSQLExecutor(copier StructuredDatabaseCopier, switchConfig AuthorizedDatabaseSwitch) *SQLExecutor {
return &SQLExecutor{copier: copier, authorizedSwitch: switchConfig}
}
func (e *SQLExecutor) Run(ctx context.Context, plan Plan, update func(Status) error) error {
if ctx == nil {
ctx = context.Background()
}
if e == nil || update == nil {
return ErrInvalidRequest
}
if err := contextError(ctx); err != nil {
return err
}
if plan.sourceConfig.Type != plan.targetConfig.Type ||
(plan.sourceConfig.Type != config.DatabaseTypeMySQL && plan.sourceConfig.Type != config.DatabaseTypePostgreSQL) {
return ErrUnsupportedDatabase
}
if err := config.ValidateDatabaseConfig(plan.sourceConfig); err != nil {
return ErrSourceDatabase
}
if err := config.ValidateDatabaseConfig(plan.targetConfig); err != nil {
return ErrInvalidTargetName
}
if !validStructuredDatabaseName(plan.sourceConfig.Name) ||
!validStructuredDatabaseName(plan.targetConfig.Name) ||
plan.sourceConfig.Name == plan.targetConfig.Name ||
!sameStructuredConnection(plan.sourceConfig, plan.targetConfig) {
return ErrImmutableSetting
}
if e.copier == nil {
return ErrSQLCopyUnavailable
}
if e.authorizedSwitch == nil {
return ErrSwitchUnavailable
}
if err := update(StatusVerifying); err != nil {
return err
}
if err := e.copier.Copy(ctx, plan.sourceConfig, plan.targetConfig); err != nil {
cleanupStructuredTarget(e.copier, plan.sourceConfig, plan.targetConfig)
return err
}
if err := contextError(ctx); err != nil {
cleanupStructuredTarget(e.copier, plan.sourceConfig, plan.targetConfig)
return err
}
if err := update(StatusSwitching); err != nil {
cleanupStructuredTarget(e.copier, plan.sourceConfig, plan.targetConfig)
return err
}
if e.authorizedSwitch == nil {
cleanupStructuredTarget(e.copier, plan.sourceConfig, plan.targetConfig)
return ErrSwitchUnavailable
}
if err := e.authorizedSwitch(ctx, plan.expectedGeneration, plan.targetConfig, plan.actor); err != nil {
return errors.Join(ErrSwitchOutcomeUnknown, err)
}
if cleanup, ok := e.copier.(StructuredDatabaseCleanup); ok {
cleanup.Complete(plan.sourceConfig, plan.targetConfig)
}
return nil
}
func cleanupStructuredTarget(copier StructuredDatabaseCopier, source, target config.DatabaseConfig) {
cleanup, ok := copier.(StructuredDatabaseCleanup)
if !ok {
return
}
cleanupContext, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_ = cleanup.Cleanup(cleanupContext, source, target)
}
func sameStructuredConnection(source, target config.DatabaseConfig) bool {
return source.Type == target.Type &&
source.Host == target.Host &&
source.Port == target.Port &&
source.Username == target.Username &&
source.Password == target.Password &&
source.LocalRef == target.LocalRef
}
type SQLConnectionFactory func(context.Context, config.DatabaseConfig, string) (*sql.DB, error)
type SQLCopier struct {
open SQLConnectionFactory
mu sync.Mutex
created map[string]struct{}
}
func NewSQLCopier(open SQLConnectionFactory) *SQLCopier {
return &SQLCopier{open: open, created: make(map[string]struct{})}
}
func (c *SQLCopier) Copy(ctx context.Context, source, target config.DatabaseConfig) error {
if c == nil || c.open == nil {
return ErrSQLConnectionUnavailable
}
if ctx == nil {
ctx = context.Background()
}
adminDB, err := c.open(ctx, source, "")
if err != nil {
return ErrSQLConnectionUnavailable
}
defer adminDB.Close()
switch source.Type {
case config.DatabaseTypeMySQL:
sourceDB, err := c.open(ctx, source, source.Name)
if err != nil {
return ErrSQLConnectionUnavailable
}
defer sourceDB.Close()
return c.copyMySQL(ctx, sourceDB, adminDB, source, target)
case config.DatabaseTypePostgreSQL:
return c.copyPostgreSQL(ctx, adminDB, source, target)
default:
return ErrUnsupportedDatabase
}
}
func (c *SQLCopier) copyMySQL(ctx context.Context, sourceDB, adminDB *sql.DB, source, target config.DatabaseConfig) error {
sourceName := source.Name
targetName := target.Name
if err := c.validateMySQLObjects(ctx, adminDB, sourceName); err != nil {
return err
}
var exists int
if err := adminDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = ?", targetName).Scan(&exists); err != nil {
return fmt.Errorf("check mysql migration target: %w", err)
}
if exists != 0 {
return ErrSQLTargetExists
}
if _, err := adminDB.ExecContext(ctx, "CREATE DATABASE "+quoteMySQLIdentifier(targetName)); err != nil {
if isDatabaseAlreadyExistsError(err, config.DatabaseTypeMySQL) {
return ErrSQLTargetExists
}
return fmt.Errorf("create mysql migration target: %w", err)
}
c.markCreated(source, target)
created := true
defer func() {
if created {
cleanupContext, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if _, err := adminDB.ExecContext(cleanupContext, "DROP DATABASE "+quoteMySQLIdentifier(targetName)); err == nil {
c.forgetCreated(source, target)
}
}
}()
rows, err := sourceDB.QueryContext(ctx, "SHOW FULL TABLES FROM "+quoteMySQLIdentifier(sourceName))
if err != nil {
return fmt.Errorf("list mysql tables: %w", err)
}
defer rows.Close()
tableNames := make([]string, 0)
sourceRowCounts := make(map[string]int64)
for rows.Next() {
var tableName, tableType string
if err := rows.Scan(&tableName, &tableType); err != nil {
return fmt.Errorf("read mysql table: %w", err)
}
if !strings.EqualFold(tableType, "BASE TABLE") {
return ErrSQLUnsupportedObject
}
tableNames = append(tableNames, tableName)
sourceTable := quoteMySQLIdentifier(sourceName) + "." + quoteMySQLIdentifier(tableName)
targetTable := quoteMySQLIdentifier(targetName) + "." + quoteMySQLIdentifier(tableName)
rowCount, err := countRows(ctx, sourceDB, sourceTable)
if err != nil {
return fmt.Errorf("count mysql source rows: %w", err)
}
sourceRowCounts[tableName] = rowCount
if _, err := adminDB.ExecContext(ctx, "CREATE TABLE "+targetTable+" LIKE "+sourceTable); err != nil {
return fmt.Errorf("copy mysql table structure: %w", err)
}
}
if err := rows.Err(); err != nil {
return fmt.Errorf("read mysql tables: %w", err)
}
for _, tableName := range tableNames {
sourceTable := quoteMySQLIdentifier(sourceName) + "." + quoteMySQLIdentifier(tableName)
targetTable := quoteMySQLIdentifier(targetName) + "." + quoteMySQLIdentifier(tableName)
if _, err := adminDB.ExecContext(ctx, "INSERT INTO "+targetTable+" SELECT * FROM "+sourceTable); err != nil {
return fmt.Errorf("copy mysql table data: %w", err)
}
targetRowCount, err := countRows(ctx, adminDB, targetTable)
if err != nil {
return fmt.Errorf("count mysql target rows: %w", err)
}
finalSourceRowCount, err := countRows(ctx, sourceDB, sourceTable)
if err != nil {
return fmt.Errorf("verify mysql source rows: %w", err)
}
if sourceRowCounts[tableName] != targetRowCount || sourceRowCounts[tableName] != finalSourceRowCount {
return fmt.Errorf("%w: mysql table %s row count changed during migration", ErrSQLDataVerification, tableName)
}
}
if err := c.copyMySQLForeignKeys(ctx, adminDB, sourceName, targetName, tableNames); err != nil {
return err
}
created = false
return nil
}
type mysqlForeignKey struct {
tableName string
constraintName string
columns []string
referencedSchema string
referencedTable string
referencedColumns []string
updateRule string
deleteRule string
}
func (c *SQLCopier) validateMySQLObjects(ctx context.Context, database *sql.DB, sourceName string) error {
checks := []struct {
query string
args []any
}{
{
query: "SELECT COUNT(*) FROM information_schema.triggers WHERE trigger_schema = ?",
args: []any{sourceName},
},
{
query: "SELECT COUNT(*) FROM information_schema.events WHERE event_schema = ?",
args: []any{sourceName},
},
{
query: "SELECT COUNT(*) FROM information_schema.routines WHERE routine_schema = ?",
args: []any{sourceName},
},
}
for _, check := range checks {
var count int
if err := database.QueryRowContext(ctx, check.query, check.args...).Scan(&count); err != nil {
return fmt.Errorf("%w: inspect mysql database objects", ErrSQLConnectionUnavailable)
}
if count > 0 {
return ErrSQLUnsupportedObject
}
}
return nil
}
func (c *SQLCopier) copyMySQLForeignKeys(ctx context.Context, database *sql.DB, sourceName, targetName string, tableNames []string) error {
if len(tableNames) == 0 {
return nil
}
rows, err := database.QueryContext(ctx, "SELECT kcu.TABLE_NAME, kcu.CONSTRAINT_NAME, kcu.COLUMN_NAME, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME, rc.UPDATE_RULE, rc.DELETE_RULE FROM information_schema.KEY_COLUMN_USAGE kcu JOIN information_schema.TABLE_CONSTRAINTS tc ON tc.CONSTRAINT_SCHEMA = kcu.CONSTRAINT_SCHEMA AND tc.TABLE_NAME = kcu.TABLE_NAME AND tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME JOIN information_schema.REFERENTIAL_CONSTRAINTS rc ON rc.CONSTRAINT_SCHEMA = kcu.CONSTRAINT_SCHEMA AND rc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME AND rc.TABLE_NAME = kcu.TABLE_NAME WHERE tc.CONSTRAINT_SCHEMA = ? AND tc.CONSTRAINT_TYPE = 'FOREIGN KEY' AND kcu.REFERENCED_TABLE_NAME IS NOT NULL ORDER BY kcu.TABLE_NAME, kcu.CONSTRAINT_NAME, kcu.ORDINAL_POSITION", sourceName)
if err != nil {
return fmt.Errorf("%w: inspect mysql foreign keys", ErrSQLConnectionUnavailable)
}
defer rows.Close()
foreignKeys := make([]mysqlForeignKey, 0)
indexes := make(map[string]int)
for rows.Next() {
var item mysqlForeignKey
var column, referencedColumn string
if err := rows.Scan(&item.tableName, &item.constraintName, &column, &item.referencedSchema, &item.referencedTable, &referencedColumn, &item.updateRule, &item.deleteRule); err != nil {
return fmt.Errorf("%w: read mysql foreign keys", ErrSQLConnectionUnavailable)
}
if !validMySQLReferenceRule(item.updateRule) || !validMySQLReferenceRule(item.deleteRule) {
return ErrSQLUnsupportedObject
}
key := item.tableName + "\x00" + item.constraintName
index, ok := indexes[key]
if !ok {
indexes[key] = len(foreignKeys)
foreignKeys = append(foreignKeys, item)
index = len(foreignKeys) - 1
}
foreignKeys[index].columns = append(foreignKeys[index].columns, column)
foreignKeys[index].referencedColumns = append(foreignKeys[index].referencedColumns, referencedColumn)
}
if err := rows.Err(); err != nil {
return fmt.Errorf("%w: read mysql foreign keys", ErrSQLConnectionUnavailable)
}
for _, foreignKey := range foreignKeys {
referencedSchema := foreignKey.referencedSchema
if referencedSchema == sourceName {
referencedSchema = targetName
}
columns := quoteMySQLIdentifiers(foreignKey.columns)
referencedColumns := quoteMySQLIdentifiers(foreignKey.referencedColumns)
query := "ALTER TABLE " + quoteMySQLIdentifier(targetName) + "." + quoteMySQLIdentifier(foreignKey.tableName) + " ADD CONSTRAINT " + quoteMySQLIdentifier(foreignKey.constraintName) + " FOREIGN KEY (" + strings.Join(columns, ", ") + ") REFERENCES " + quoteMySQLIdentifier(referencedSchema) + "." + quoteMySQLIdentifier(foreignKey.referencedTable) + " (" + strings.Join(referencedColumns, ", ") + ") ON UPDATE " + foreignKey.updateRule + " ON DELETE " + foreignKey.deleteRule
if _, err := database.ExecContext(ctx, query); err != nil {
return fmt.Errorf("copy mysql foreign key: %w", err)
}
}
return nil
}
func validMySQLReferenceRule(value string) bool {
switch strings.ToUpper(strings.TrimSpace(value)) {
case "CASCADE", "SET NULL", "SET DEFAULT", "RESTRICT", "NO ACTION":
return true
default:
return false
}
}
func quoteMySQLIdentifiers(values []string) []string {
quoted := make([]string, len(values))
for index, value := range values {
quoted[index] = quoteMySQLIdentifier(value)
}
return quoted
}
func (c *SQLCopier) copyPostgreSQL(ctx context.Context, adminDB *sql.DB, source, target config.DatabaseConfig) error {
sourceName := source.Name
targetName := target.Name
var exists bool
if err := adminDB.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM pg_database WHERE datname = $1)", targetName).Scan(&exists); err != nil {
return fmt.Errorf("check postgresql migration target: %w", err)
}
if exists {
return ErrSQLTargetExists
}
var activeConnections int
if err := adminDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM pg_stat_activity WHERE datname = $1 AND pid <> pg_backend_pid()", sourceName).Scan(&activeConnections); err != nil {
return fmt.Errorf("inspect postgresql source connections: %w", err)
}
if activeConnections > 0 {
return ErrSQLSourceBusy
}
if _, err := adminDB.ExecContext(ctx, "CREATE DATABASE "+quotePostgreSQLIdentifier(targetName)+" TEMPLATE "+quotePostgreSQLIdentifier(sourceName)); err != nil {
if isDatabaseAlreadyExistsError(err, config.DatabaseTypePostgreSQL) {
return ErrSQLTargetExists
}
return fmt.Errorf("copy postgresql database: %w", err)
}
c.markCreated(source, target)
created := true
defer func() {
if created {
cleanupContext, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := c.Cleanup(cleanupContext, source, target); err == nil {
created = false
}
}
}()
if err := c.verifyPostgreSQLClone(ctx, source, target); err != nil {
return err
}
created = false
return nil
}
func (c *SQLCopier) verifyPostgreSQLClone(ctx context.Context, source, target config.DatabaseConfig) error {
if c == nil || c.open == nil {
return ErrSQLConnectionUnavailable
}
sourceDB, err := c.open(ctx, source, source.Name)
if err != nil {
return ErrSQLConnectionUnavailable
}
defer sourceDB.Close()
targetDB, err := c.open(ctx, target, target.Name)
if err != nil {
return ErrSQLConnectionUnavailable
}
defer targetDB.Close()
tables, err := listPostgreSQLTables(ctx, sourceDB)
if err != nil {
return fmt.Errorf("%w: list postgresql source tables", ErrSQLConnectionUnavailable)
}
targetTables, err := listPostgreSQLTables(ctx, targetDB)
if err != nil {
return fmt.Errorf("%w: list postgresql target tables", ErrSQLConnectionUnavailable)
}
if len(tables) != len(targetTables) {
return fmt.Errorf("%w: postgresql table count changed during migration", ErrSQLDataVerification)
}
for _, table := range tables {
sourceCount, err := countRows(ctx, sourceDB, quotePostgreSQLIdentifier(table.schema)+"."+quotePostgreSQLIdentifier(table.name))
if err != nil {
return fmt.Errorf("%w: count postgresql source rows", ErrSQLConnectionUnavailable)
}
targetCount, err := countRows(ctx, targetDB, quotePostgreSQLIdentifier(table.schema)+"."+quotePostgreSQLIdentifier(table.name))
if err != nil {
return fmt.Errorf("%w: count postgresql target rows", ErrSQLConnectionUnavailable)
}
if sourceCount != targetCount {
return fmt.Errorf("%w: postgresql table %s row count changed during migration", ErrSQLDataVerification, table.name)
}
}
return nil
}
type postgreSQLTable struct {
schema string
name string
}
func listPostgreSQLTables(ctx context.Context, database *sql.DB) ([]postgreSQLTable, error) {
rows, err := database.QueryContext(ctx, "SELECT table_schema, table_name FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema NOT IN ('pg_catalog', 'information_schema') ORDER BY table_schema, table_name")
if err != nil {
return nil, err
}
defer rows.Close()
tables := make([]postgreSQLTable, 0)
for rows.Next() {
var table postgreSQLTable
if err := rows.Scan(&table.schema, &table.name); err != nil {
return nil, err
}
tables = append(tables, table)
}
if err := rows.Err(); err != nil {
return nil, err
}
return tables, nil
}
func isDatabaseAlreadyExistsError(err error, databaseType config.DatabaseType) bool {
if err == nil {
return false
}
switch databaseType {
case config.DatabaseTypeMySQL:
var mysqlErr *mysql.MySQLError
return errors.As(err, &mysqlErr) && mysqlErr.Number == 1007
case config.DatabaseTypePostgreSQL:
var postgresErr *pq.Error
return errors.As(err, &postgresErr) && string(postgresErr.Code) == "42P04"
default:
return false
}
}
func (c *SQLCopier) markCreated(source, target config.DatabaseConfig) {
if c == nil {
return
}
c.mu.Lock()
defer c.mu.Unlock()
if c.created == nil {
c.created = make(map[string]struct{})
}
c.created[databaseTargetKey(source, target)] = struct{}{}
}
func (c *SQLCopier) hasCreated(source, target config.DatabaseConfig) bool {
if c == nil {
return false
}
c.mu.Lock()
defer c.mu.Unlock()
_, ok := c.created[databaseTargetKey(source, target)]
return ok
}
func (c *SQLCopier) forgetCreated(source, target config.DatabaseConfig) {
if c == nil {
return
}
c.mu.Lock()
delete(c.created, databaseTargetKey(source, target))
c.mu.Unlock()
}
func (c *SQLCopier) Complete(source, target config.DatabaseConfig) {
c.forgetCreated(source, target)
}
func (c *SQLCopier) Cleanup(ctx context.Context, source, target config.DatabaseConfig) error {
if !c.hasCreated(source, target) {
return nil
}
if c.open == nil {
return ErrSQLConnectionUnavailable
}
adminDB, err := c.open(ctx, source, "")
if err != nil {
return ErrSQLConnectionUnavailable
}
defer adminDB.Close()
var query string
switch target.Type {
case config.DatabaseTypeMySQL:
query = "DROP DATABASE " + quoteMySQLIdentifier(target.Name)
case config.DatabaseTypePostgreSQL:
query = "DROP DATABASE " + quotePostgreSQLIdentifier(target.Name)
default:
return ErrUnsupportedDatabase
}
if _, err := adminDB.ExecContext(ctx, query); err != nil {
return fmt.Errorf("cleanup structured migration target: %w", err)
}
c.forgetCreated(source, target)
return nil
}
func countRows(ctx context.Context, database *sql.DB, table string) (int64, error) {
var count int64
if err := database.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+table).Scan(&count); err != nil {
return 0, err
}
return count, nil
}
func databaseTargetKey(source, target config.DatabaseConfig) string {
return strings.Join([]string{
string(target.Type),
target.Host,
strconv.Itoa(int(target.Port)),
target.Username,
target.Name,
source.Name,
}, "\x00")
}
func quoteMySQLIdentifier(value string) string {
return "`" + strings.ReplaceAll(value, "`", "``") + "`"
}
func quotePostgreSQLIdentifier(value string) string {
return "\"" + strings.ReplaceAll(value, "\"", "\"\"") + "\""
}
func NewRuntimeSQLConnectionFactory(root string) (SQLConnectionFactory, error) {
vault, err := credential.NewVault(root)
if err != nil {
return nil, err
}
return func(ctx context.Context, database config.DatabaseConfig, databaseName string) (*sql.DB, error) {
password, err := config.OpenStoredSecret(vault, database.Password)
if err != nil {
return nil, ErrSQLConnectionUnavailable
}
driverName, dsn, err := runtimeSQLDSN(database, password, databaseName)
if err != nil {
return nil, err
}
connection, err := sql.Open(driverName, dsn)
if err != nil {
return nil, ErrSQLConnectionUnavailable
}
if err := connection.PingContext(ctx); err != nil {
_ = connection.Close()
return nil, ErrSQLConnectionUnavailable
}
return connection, nil
}, nil
}
func runtimeSQLDSN(database config.DatabaseConfig, password, databaseName string) (string, string, error) {
if database.Host == "" || database.Port == 0 || database.Username == "" || password == "" {
return "", "", ErrSQLConnectionUnavailable
}
address := net.JoinHostPort(database.Host, strconv.Itoa(int(database.Port)))
switch database.Type {
case config.DatabaseTypeMySQL:
settings := mysql.Config{
User: database.Username,
Passwd: password,
Net: "tcp",
Addr: address,
DBName: databaseName,
ParseTime: true,
Params: map[string]string{"charset": "utf8mb4"},
}
return "mysql", settings.FormatDSN(), nil
case config.DatabaseTypePostgreSQL:
maintenanceDatabase := databaseName
if maintenanceDatabase == "" {
maintenanceDatabase = "postgres"
}
databaseURL := url.URL{
Scheme: "postgres",
User: url.UserPassword(database.Username, password),
Host: address,
Path: "/" + maintenanceDatabase,
RawQuery: "sslmode=prefer",
}
return "postgres", databaseURL.String(), nil
default:
return "", "", ErrUnsupportedDatabase
}
}
var structuredDatabaseNamePattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,62}$`)
var mysqlOptionIdentifierPattern = regexp.MustCompile(`^[A-Za-z0-9_]+$`)
func validStructuredDatabaseName(value string) bool {
return structuredDatabaseNamePattern.MatchString(strings.TrimSpace(value))
}
@@ -0,0 +1,305 @@
package dbmigration
import (
"context"
"errors"
"net/url"
"strings"
"testing"
"git.pcux.cn/xuans/Smail/config"
mysql "github.com/go-sql-driver/mysql"
"github.com/lib/pq"
)
type recordingStructuredCopier struct {
called bool
source config.DatabaseConfig
target config.DatabaseConfig
err error
}
func (c *recordingStructuredCopier) Copy(_ context.Context, source, target config.DatabaseConfig) error {
c.called = true
c.source = source
c.target = target
return c.err
}
func validStructuredPlan(t *testing.T) Plan {
t.Helper()
database := testDatabase(t)
database.Name = "smail"
target := database
target.Name = "smail_next"
return Plan{
sourceConfig: database,
targetConfig: target,
actor: config.AuditActor{UserID: "admin-1", Role: "super_admin", SessionID: "session-1"},
}
}
func TestSQLExecutorCopiesAndSwitchesWithOrderedStatuses(t *testing.T) {
copier := &recordingStructuredCopier{}
var actor config.AuditActor
var generation uint64
executor := NewSQLExecutor(copier, func(_ context.Context, receivedGeneration uint64, _ config.DatabaseConfig, received config.AuditActor) error {
generation = receivedGeneration
actor = received
return nil
})
statuses := make([]Status, 0, 2)
plan := validStructuredPlan(t)
plan.expectedGeneration = 42
if err := executor.Run(context.Background(), plan, func(status Status) error {
statuses = append(statuses, status)
return nil
}); err != nil {
t.Fatal(err)
}
if len(statuses) != 2 || statuses[0] != StatusVerifying || statuses[1] != StatusSwitching {
t.Fatalf("statuses = %v, want verifying then switching", statuses)
}
if !copier.called || copier.source.Name != "smail" || copier.target.Name != "smail_next" {
t.Fatalf("copier call = %+v", copier)
}
if actor != plan.actor {
t.Fatalf("switch actor = %+v, want %+v", actor, plan.actor)
}
if generation != plan.expectedGeneration {
t.Fatalf("switch generation = %d, want %d", generation, plan.expectedGeneration)
}
}
func TestSQLExecutorRejectsImmutablePlanBeforeCopy(t *testing.T) {
copier := &recordingStructuredCopier{}
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error { return nil })
plan := validStructuredPlan(t)
plan.targetConfig.Host = "other.example"
if err := executor.Run(context.Background(), plan, func(Status) error { return nil }); !errors.Is(err, ErrImmutableSetting) {
t.Fatalf("immutable plan error = %v, want immutable setting", err)
}
if copier.called {
t.Fatal("copier ran for immutable plan")
}
}
func TestSQLExecutorPreservesCopierError(t *testing.T) {
copyErr := errors.New("复制失败")
copier := &recordingStructuredCopier{err: copyErr}
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error { return nil })
if err := executor.Run(context.Background(), validStructuredPlan(t), func(Status) error { return nil }); !errors.Is(err, copyErr) {
t.Fatalf("copy error = %v, want copier error", err)
}
}
func TestSQLExecutorCleansStructuredTargetWhenCopyFails(t *testing.T) {
copier := &cleanupRecordingCopier{copyErr: errors.New("复制失败")}
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error {
return nil
})
if err := executor.Run(context.Background(), validStructuredPlan(t), func(Status) error { return nil }); !errors.Is(err, copier.copyErr) {
t.Fatalf("copy error = %v, want %v", err, copier.copyErr)
}
if !copier.cleanupCalled {
t.Fatal("structured target cleanup was not requested after copy failure")
}
}
func TestStructuredDatabaseNameValidation(t *testing.T) {
for _, value := range []string{"smail", "Smail_2026", "mail-prod"} {
if !validStructuredDatabaseName(value) {
t.Fatalf("valid database name rejected: %q", value)
}
}
for _, value := range []string{"", "smail;drop", "smail name", "../smail"} {
if validStructuredDatabaseName(value) {
t.Fatalf("unsafe database name accepted: %q", value)
}
}
}
func TestDatabaseExecutorRoutesByDatabaseType(t *testing.T) {
var sqliteCalled, structuredCalled bool
sqlite := runnerFunc(func(context.Context, Plan, func(Status) error) error {
sqliteCalled = true
return nil
})
structured := runnerFunc(func(context.Context, Plan, func(Status) error) error {
structuredCalled = true
return nil
})
router := NewDatabaseExecutor(sqlite, structured)
if err := router.Run(context.Background(), Plan{sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite}}, func(Status) error { return nil }); err != nil {
t.Fatal(err)
}
if err := router.Run(context.Background(), Plan{sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeMySQL}}, func(Status) error { return nil }); err != nil {
t.Fatal(err)
}
if !sqliteCalled || !structuredCalled {
t.Fatalf("router calls: sqlite=%v structured=%v", sqliteCalled, structuredCalled)
}
}
type runnerFunc func(context.Context, Plan, func(Status) error) error
func (f runnerFunc) Run(ctx context.Context, plan Plan, update func(Status) error) error {
return f(ctx, plan, update)
}
func TestSQLExecutorRejectsMissingConfigurationSwitchBeforeCopy(t *testing.T) {
copier := &recordingStructuredCopier{}
executor := NewSQLExecutor(copier, nil)
if err := executor.Run(context.Background(), validStructuredPlan(t), func(Status) error { return nil }); !errors.Is(err, ErrSwitchUnavailable) {
t.Fatalf("missing switch error = %v, want switch unavailable", err)
}
if copier.called {
t.Fatal("copier ran without a configuration switch")
}
}
type cleanupRecordingCopier struct {
cleanupCalled bool
completeCalled bool
copyErr error
}
func (c *cleanupRecordingCopier) Copy(context.Context, config.DatabaseConfig, config.DatabaseConfig) error {
return c.copyErr
}
func TestDatabaseAlreadyExistsErrorClassification(t *testing.T) {
mysqlError := &mysql.MySQLError{Number: 1007}
if !isDatabaseAlreadyExistsError(mysqlError, config.DatabaseTypeMySQL) {
t.Fatal("mysql duplicate database error was not classified")
}
postgresError := &pq.Error{Code: "42P04"}
if !isDatabaseAlreadyExistsError(postgresError, config.DatabaseTypePostgreSQL) {
t.Fatal("postgresql duplicate database error was not classified")
}
if isDatabaseAlreadyExistsError(errors.New("duplicate database"), config.DatabaseTypeMySQL) {
t.Fatal("generic mysql error was incorrectly classified")
}
}
func (c *cleanupRecordingCopier) Cleanup(context.Context, config.DatabaseConfig, config.DatabaseConfig) error {
c.cleanupCalled = true
return nil
}
func (c *cleanupRecordingCopier) Complete(config.DatabaseConfig, config.DatabaseConfig) {
c.completeCalled = true
}
func TestSQLExecutorPreservesStructuredTargetWhenConfigurationSwitchOutcomeIsUnknown(t *testing.T) {
copier := &cleanupRecordingCopier{}
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error {
return errors.New("配置切换失败")
})
if err := executor.Run(context.Background(), validStructuredPlan(t), func(Status) error { return nil }); !errors.Is(err, ErrSwitchOutcomeUnknown) {
t.Fatal("configuration switch failure was not returned")
}
if copier.cleanupCalled {
t.Fatal("structured target was cleaned up even though switch outcome was unknown")
}
if copier.completeCalled {
t.Fatal("structured target was finalized after a failed switch")
}
}
func TestSQLExecutorCleansStructuredTargetWhenStatusUpdateFails(t *testing.T) {
copier := &cleanupRecordingCopier{}
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error {
return nil
})
statusError := errors.New("状态写入失败")
if err := executor.Run(context.Background(), validStructuredPlan(t), func(status Status) error {
if status == StatusSwitching {
return statusError
}
return nil
}); !errors.Is(err, statusError) {
t.Fatalf("status update error = %v, want %v", err, statusError)
}
if !copier.cleanupCalled {
t.Fatal("structured target cleanup was not requested after status update failure")
}
}
func TestSQLExecutorCleansStructuredTargetWhenContextIsCanceledAfterCopy(t *testing.T) {
copier := &cleanupRecordingCopier{}
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error {
return nil
})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if err := executor.Run(ctx, validStructuredPlan(t), func(status Status) error {
if status == StatusVerifying {
cancel()
}
return nil
}); !errors.Is(err, context.Canceled) {
t.Fatalf("canceled execution error = %v, want context canceled", err)
}
if !copier.cleanupCalled {
t.Fatal("structured target cleanup was not requested after cancellation")
}
}
func TestRuntimeSQLDSNSupportsIPv6AndEscapesCredentials(t *testing.T) {
database := config.DatabaseConfig{
Type: config.DatabaseTypeMySQL,
Host: "2001:db8::3",
Port: 3306,
Username: "mail user",
}
driver, dsn, err := runtimeSQLDSN(database, "p@ss/word", "smail_next")
if err != nil {
t.Fatal(err)
}
if driver != "mysql" {
t.Fatalf("mysql driver = %q", driver)
}
parsedMySQL, err := mysql.ParseDSN(dsn)
if err != nil {
t.Fatal(err)
}
if parsedMySQL.Addr != "[2001:db8::3]:3306" || parsedMySQL.User != "mail user" || parsedMySQL.Passwd != "p@ss/word" || parsedMySQL.DBName != "smail_next" {
t.Fatalf("mysql dsn config = %+v", parsedMySQL)
}
database.Type = config.DatabaseTypePostgreSQL
driver, dsn, err = runtimeSQLDSN(database, "p@ss/word", "smail_next")
if err != nil {
t.Fatal(err)
}
if driver != "postgres" {
t.Fatalf("postgres driver = %q", driver)
}
parsedPostgres, err := url.Parse(dsn)
if err != nil {
t.Fatal(err)
}
password, ok := parsedPostgres.User.Password()
if !ok || parsedPostgres.Host != "[2001:db8::3]:3306" || parsedPostgres.User.Username() != "mail user" || password != "p@ss/word" || parsedPostgres.Path != "/smail_next" {
t.Fatalf("postgres dsn = %s", dsn)
}
}
func TestMySQLReferenceRulesAndIdentifiersAreRestrictedSafely(t *testing.T) {
for _, rule := range []string{"CASCADE", "SET NULL", "SET DEFAULT", "RESTRICT", "NO ACTION"} {
if !validMySQLReferenceRule(rule) {
t.Fatalf("valid mysql reference rule rejected: %q", rule)
}
}
for _, rule := range []string{"DROP DATABASE", "CASCADE; DROP TABLE users"} {
if validMySQLReferenceRule(rule) {
t.Fatalf("unsafe mysql reference rule accepted: %q", rule)
}
}
if got := quoteMySQLIdentifier("mail`prod"); got != "`mail``prod`" {
t.Fatalf("quoted mysql identifier = %q", got)
}
if got := strings.Join(quoteMySQLIdentifiers([]string{"user", "tenant"}), ", "); got != "`user`, `tenant`" {
t.Fatalf("quoted mysql identifiers = %q", got)
}
}
@@ -0,0 +1,249 @@
package dbmigration
import (
"context"
"database/sql"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"git.pcux.cn/xuans/Smail/config"
_ "modernc.org/sqlite"
)
var ErrSQLiteIntegrity = errors.New("sqlite database integrity check failed")
type SQLiteSwitch func(context.Context, config.DatabaseConfig) error
type AuthorizedSQLiteSwitch func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error
type SQLiteExecutor struct {
root string
switchConfig SQLiteSwitch
authorizedSwitch AuthorizedSQLiteSwitch
}
func NewSQLiteExecutor(root string, switchConfig SQLiteSwitch) *SQLiteExecutor {
return &SQLiteExecutor{root: root, switchConfig: switchConfig}
}
func NewAuthorizedSQLiteExecutor(root string, switchConfig AuthorizedSQLiteSwitch) *SQLiteExecutor {
return &SQLiteExecutor{root: root, authorizedSwitch: switchConfig}
}
func (e *SQLiteExecutor) Run(ctx context.Context, plan Plan, update func(Status) error) error {
if ctx == nil {
ctx = context.Background()
}
if e == nil || update == nil {
return ErrInvalidRequest
}
if err := contextError(ctx); err != nil {
return err
}
if plan.sourceConfig.Type != config.DatabaseTypeSQLite || plan.targetConfig.Type != config.DatabaseTypeSQLite {
return ErrUnsupportedDatabase
}
if err := config.ValidateDatabaseConfig(plan.sourceConfig); err != nil {
return ErrSourceDatabase
}
if err := config.ValidateDatabaseConfig(plan.targetConfig); err != nil {
return ErrInvalidTargetName
}
if plan.sourceConfig.LocalRef == plan.targetConfig.LocalRef {
return ErrInvalidTargetName
}
sourcePath, err := e.resolveReference(plan.sourceConfig.LocalRef)
if err != nil {
return err
}
targetPath, err := e.resolveReference(plan.targetConfig.LocalRef)
if err != nil {
return err
}
if sourcePath == targetPath {
return ErrInvalidTargetName
}
if err := checkRegularFile(sourcePath, false); err != nil {
return fmt.Errorf("open source database: %w", err)
}
if err := checkRegularFile(targetPath, true); err != nil {
return err
}
if err := update(StatusVerifying); err != nil {
return err
}
if err := verifySQLite(ctx, sourcePath); err != nil {
return err
}
if err := contextError(ctx); err != nil {
return err
}
temporary, err := e.temporaryPath(targetPath)
if err != nil {
return err
}
defer os.Remove(temporary)
activated := false
switched := false
switchStarted := false
defer func() {
if activated && !switched && !switchStarted {
_ = os.Remove(targetPath)
}
}()
if err := vacuumSQLite(ctx, sourcePath, temporary); err != nil {
return err
}
if err := os.Chmod(temporary, 0600); err != nil {
return fmt.Errorf("set migrated database permissions: %w", err)
}
if err := verifySQLite(ctx, temporary); err != nil {
return err
}
if err := contextError(ctx); err != nil {
return err
}
if err := update(StatusSwitching); err != nil {
return err
}
if err := os.Link(temporary, targetPath); err != nil {
if os.IsExist(err) {
return ErrTargetExists
}
return fmt.Errorf("activate migrated database file: %w", err)
}
activated = true
if err := os.Remove(temporary); err != nil {
return fmt.Errorf("finalize migrated database file: %w", err)
}
if err := verifySQLite(ctx, targetPath); err != nil {
return err
}
if e.authorizedSwitch == nil && e.switchConfig == nil {
return ErrSwitchUnavailable
}
var switchErr error
switchStarted = true
if e.authorizedSwitch != nil {
switchErr = e.authorizedSwitch(ctx, plan.expectedGeneration, plan.targetConfig, plan.actor)
} else {
switchErr = e.switchConfig(ctx, plan.targetConfig)
}
if switchErr != nil {
return errors.Join(ErrSwitchOutcomeUnknown, switchErr)
}
switched = true
return nil
}
func (e *SQLiteExecutor) resolveReference(reference string) (string, error) {
if e == nil || strings.TrimSpace(e.root) == "" || !sqliteTargetNamePattern.MatchString(reference) {
return "", ErrInvalidTargetName
}
root, err := filepath.Abs(filepath.Clean(e.root))
if err != nil {
return "", ErrInvalidTargetName
}
configDirectory := filepath.Join(root, "config")
info, err := os.Lstat(configDirectory)
if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
return "", ErrInvalidTargetName
}
path := filepath.Join(configDirectory, reference)
absolute, err := filepath.Abs(filepath.Clean(path))
if err != nil || filepath.Dir(absolute) != configDirectory {
return "", ErrInvalidTargetName
}
return absolute, nil
}
func (e *SQLiteExecutor) temporaryPath(target string) (string, error) {
for attempt := 0; attempt < 8; attempt++ {
id, err := randomID(".migration-")
if err != nil {
return "", err
}
candidate := filepath.Join(filepath.Dir(target), id+".tmp")
if _, err := os.Lstat(candidate); os.IsNotExist(err) {
return candidate, nil
} else if err != nil {
return "", err
}
}
return "", ErrTargetExists
}
func checkRegularFile(path string, target bool) error {
info, err := os.Lstat(path)
if os.IsNotExist(err) {
if target {
return nil
}
return fmt.Errorf("source database does not exist: %w", err)
}
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
if target {
return ErrTargetExists
}
return errors.New("source database is not a regular file")
}
if target {
return ErrTargetExists
}
return nil
}
func verifySQLite(ctx context.Context, path string) error {
database, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)&mode=ro")
if err != nil {
return fmt.Errorf("open sqlite database: %w", err)
}
defer database.Close()
if err := database.PingContext(ctx); err != nil {
if ctx != nil && ctx.Err() != nil {
return ctx.Err()
}
return fmt.Errorf("%w: ping sqlite database: %v", ErrSQLiteIntegrity, err)
}
var result string
if err := database.QueryRowContext(ctx, "PRAGMA integrity_check").Scan(&result); err != nil {
if ctx != nil && ctx.Err() != nil {
return ctx.Err()
}
return fmt.Errorf("%w: check sqlite database integrity: %v", ErrSQLiteIntegrity, err)
}
if !strings.EqualFold(strings.TrimSpace(result), "ok") {
return fmt.Errorf("%w: %s", ErrSQLiteIntegrity, result)
}
return nil
}
func vacuumSQLite(ctx context.Context, source, target string) error {
database, err := sql.Open("sqlite", source+"?_pragma=busy_timeout(5000)")
if err != nil {
return fmt.Errorf("open sqlite source: %w", err)
}
defer database.Close()
if err := database.PingContext(ctx); err != nil {
return fmt.Errorf("ping sqlite source: %w", err)
}
query := "VACUUM INTO '" + strings.ReplaceAll(target, "'", "''") + "'"
if _, err := database.ExecContext(ctx, query); err != nil {
return fmt.Errorf("copy sqlite database: %w", err)
}
return nil
}
func contextError(ctx context.Context) error {
if ctx == nil {
return nil
}
return ctx.Err()
}
@@ -0,0 +1,228 @@
package dbmigration
import (
"context"
"database/sql"
"encoding/json"
"errors"
"os"
"path/filepath"
"runtime"
"testing"
"git.pcux.cn/xuans/Smail/config"
_ "modernc.org/sqlite"
)
func newSQLiteMigrationFixture(t *testing.T, switchConfig SQLiteSwitch) (*Service, string) {
t.Helper()
root := t.TempDir()
configDirectory := filepath.Join(root, "config")
if err := os.Mkdir(configDirectory, 0700); err != nil {
t.Fatal(err)
}
sourcePath := filepath.Join(configDirectory, "smail.db")
database, err := sql.Open("sqlite", sourcePath)
if err != nil {
t.Fatal(err)
}
if _, err := database.Exec(`CREATE TABLE messages (id INTEGER PRIMARY KEY, body TEXT NOT NULL); INSERT INTO messages(body) VALUES ('测试邮件');`); err != nil {
database.Close()
t.Fatal(err)
}
if err := database.Close(); err != nil {
t.Fatal(err)
}
executor := NewSQLiteExecutor(root, switchConfig)
return NewService(executor), root
}
func TestSQLiteExecutorCopiesDatabaseAndSwitchesConfiguration(t *testing.T) {
var switched config.DatabaseConfig
service, root := newSQLiteMigrationFixture(t, func(_ context.Context, candidate config.DatabaseConfig) error {
switched = candidate
return nil
})
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 3, "smail-next.db")
if err != nil {
t.Fatal(err)
}
completed, err := service.Execute(context.Background(), record.ID, token)
if err != nil {
t.Fatal(err)
}
if completed.Status != StatusSucceeded {
t.Fatalf("migration status = %s, want succeeded", completed.Status)
}
if switched.LocalRef != "smail-next.db" || switched.Type != config.DatabaseTypeSQLite {
t.Fatalf("unexpected switched configuration: %+v", switched)
}
database, err := sql.Open("sqlite", filepath.Join(root, "config", "smail-next.db"))
if err != nil {
t.Fatal(err)
}
defer database.Close()
var body string
if err := database.QueryRow("SELECT body FROM messages WHERE id = 1").Scan(&body); err != nil {
t.Fatal(err)
}
if body != "测试邮件" {
t.Fatalf("copied row = %q, want copied content", body)
}
info, err := os.Stat(filepath.Join(root, "config", "smail-next.db"))
if err != nil {
t.Fatal(err)
}
if runtime.GOOS != "windows" && info.Mode().Perm() != 0600 {
t.Fatalf("migrated database permissions = %o, want 600", info.Mode().Perm())
}
}
func TestSQLiteExecutorRejectsExistingTarget(t *testing.T) {
service, root := newSQLiteMigrationFixture(t, func(context.Context, config.DatabaseConfig) error { return nil })
if err := os.WriteFile(filepath.Join(root, "config", "smail-next.db"), []byte("existing"), 0600); err != nil {
t.Fatal(err)
}
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "smail-next.db")
if err != nil {
t.Fatal(err)
}
failed, err := service.Execute(context.Background(), record.ID, token)
if !errors.Is(err, ErrTargetExists) {
t.Fatalf("execution error = %v, want target exists", err)
}
if failed.Status != StatusFailed || failed.ErrorCode != "TARGET_EXISTS" {
t.Fatalf("unexpected failed migration: %+v", failed)
}
}
func TestSQLiteExecutorRejectsCorruptSource(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "config"), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "config", "smail.db"), []byte("not sqlite"), 0600); err != nil {
t.Fatal(err)
}
executor := NewSQLiteExecutor(root, func(context.Context, config.DatabaseConfig) error { return nil })
plan := Plan{sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, targetConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "next.db"}}
if err := executor.Run(context.Background(), plan, func(Status) error { return nil }); err == nil {
t.Fatal("corrupt source unexpectedly migrated")
} else if !errors.Is(err, ErrSQLiteIntegrity) {
t.Fatalf("corrupt source error = %v, want integrity error", err)
}
}
func TestSQLiteExecutorRejectsUnsupportedDatabase(t *testing.T) {
executor := NewSQLiteExecutor(t.TempDir(), func(context.Context, config.DatabaseConfig) error { return nil })
plan := Plan{sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeMySQL}, targetConfig: config.DatabaseConfig{Type: config.DatabaseTypeMySQL}}
if err := executor.Run(context.Background(), plan, func(Status) error { return nil }); !errors.Is(err, ErrUnsupportedDatabase) {
t.Fatalf("unsupported database error = %v", err)
}
}
func TestSQLiteExecutorRequiresConfigurationSwitch(t *testing.T) {
service, root := newSQLiteMigrationFixture(t, nil)
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "next.db")
if err != nil {
t.Fatal(err)
}
failed, err := service.Execute(context.Background(), record.ID, token)
if !errors.Is(err, ErrSwitchUnavailable) {
t.Fatalf("execution error = %v, want switch unavailable", err)
}
if failed.ErrorCode != "SWITCH_UNAVAILABLE" {
t.Fatalf("error code = %q, want SWITCH_UNAVAILABLE", failed.ErrorCode)
}
if _, err := os.Stat(filepath.Join(root, "config", "next.db")); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("target database remained after switch failure, stat error = %v", err)
}
}
func TestSQLiteExecutorPreservesTargetWhenConfigurationSwitchOutcomeIsUnknown(t *testing.T) {
service, root := newSQLiteMigrationFixture(t, func(context.Context, config.DatabaseConfig) error {
return errors.New("配置切换失败")
})
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "next.db")
if err != nil {
t.Fatal(err)
}
failed, err := service.Execute(context.Background(), record.ID, token)
if err == nil || failed.Status != StatusFailed {
t.Fatalf("switch failure result = %+v, error = %v", failed, err)
}
if !errors.Is(err, ErrSwitchOutcomeUnknown) {
t.Fatalf("switch failure error = %v, want unknown switch outcome", err)
}
if _, statErr := os.Stat(filepath.Join(root, "config", "next.db")); statErr != nil {
t.Fatalf("target database was removed after unknown switch outcome: %v", statErr)
}
}
func TestSQLiteExecutorRejectsTargetSymlink(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("Windows 创建符号链接需要额外权限")
}
service, root := newSQLiteMigrationFixture(t, func(context.Context, config.DatabaseConfig) error { return nil })
target := filepath.Join(root, "config", "next.db")
if err := os.Symlink(filepath.Join(root, "config", "smail.db"), target); err != nil {
t.Fatal(err)
}
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "next.db")
if err != nil {
t.Fatal(err)
}
failed, err := service.Execute(context.Background(), record.ID, token)
if !errors.Is(err, ErrTargetExists) || failed.Status != StatusFailed {
t.Fatalf("symlink target result = %+v, error = %v", failed, err)
}
}
func TestSQLiteExecutorHonorsCancellation(t *testing.T) {
service, _ := newSQLiteMigrationFixture(t, func(context.Context, config.DatabaseConfig) error { return nil })
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "next.db")
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
failed, err := service.Execute(ctx, record.ID, token)
if !errors.Is(err, context.Canceled) {
t.Fatalf("execution error = %v, want context canceled", err)
}
if failed.Status != StatusFailed {
t.Fatalf("migration status = %s, want failed", failed.Status)
}
}
func TestMigrationPlanDoesNotSerializePrivateDatabaseConfig(t *testing.T) {
plan := Plan{Source: DatabaseIdentity{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, Target: DatabaseIdentity{Type: config.DatabaseTypeSQLite, LocalRef: "next.db"}, sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}}
data, err := json.Marshal(plan)
if err != nil {
t.Fatal(err)
}
if string(data) != `{"Source":{"type":"sqlite","localRef":"smail.db"},"Target":{"type":"sqlite","localRef":"next.db"}}` {
t.Fatalf("serialized plan = %s", data)
}
}
func TestSQLiteExecutorReportsOrderedStatuses(t *testing.T) {
service, root := newSQLiteMigrationFixture(t, func(context.Context, config.DatabaseConfig) error { return nil })
executor := service.runner.(*SQLiteExecutor)
statuses := make([]Status, 0, 2)
plan := Plan{
sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"},
targetConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "next.db"},
}
if err := executor.Run(context.Background(), plan, func(status Status) error {
statuses = append(statuses, status)
return nil
}); err != nil {
t.Fatal(err)
}
if len(statuses) != 2 || statuses[0] != StatusVerifying || statuses[1] != StatusSwitching {
t.Fatalf("statuses = %v, want verifying then switching", statuses)
}
if _, err := os.Stat(filepath.Join(root, "config", "next.db")); err != nil {
t.Fatal(err)
}
}
+32 -28
View File
@@ -9,11 +9,11 @@ import (
var ErrCleanupAlreadyStarted = errors.New("verification cleanup already started")
type CleanupRunner struct {
store *Store
once sync.Once
mu sync.Mutex
stop chan struct{}
done chan struct{}
store *Store
mu sync.Mutex
stop chan struct{}
done chan struct{}
stopping bool
}
func NewCleanupRunner(store *Store) *CleanupRunner {
@@ -25,31 +25,28 @@ func (r *CleanupRunner) Start(interval time.Duration) error {
if r == nil || r.store == nil || interval <= 0 {
return ErrInvalidChallenge
}
started := false
r.once.Do(func() {
started = true
r.mu.Lock()
r.stop = make(chan struct{})
r.done = make(chan struct{})
stop, done := r.stop, r.done
r.mu.Lock()
if r.stop != nil || r.stopping {
r.mu.Unlock()
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
defer close(done)
for {
select {
case <-ticker.C:
r.store.Cleanup()
case <-stop:
return
}
}
}()
})
if !started {
return ErrCleanupAlreadyStarted
}
r.stop = make(chan struct{})
r.done = make(chan struct{})
stop, done := r.stop, r.done
r.mu.Unlock()
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
defer close(done)
for {
select {
case <-ticker.C:
r.store.Cleanup()
case <-stop:
return
}
}
}()
return nil
}
@@ -59,10 +56,17 @@ func (r *CleanupRunner) Stop() {
}
r.mu.Lock()
stop, done := r.stop, r.done
r.stop, r.done = nil, nil
if stop == nil || r.stopping {
r.mu.Unlock()
return
}
r.stopping = true
r.mu.Unlock()
if stop != nil {
close(stop)
<-done
}
r.mu.Lock()
r.stop, r.done, r.stopping = nil, nil, false
r.mu.Unlock()
}
@@ -19,3 +19,19 @@ func TestCleanupRunnerHasSingleInstance(t *testing.T) {
t.Fatalf("second cleanup runner start error = %v", err)
}
}
func TestCleanupRunnerCanRestartAfterStop(t *testing.T) {
store, err := NewStore(4)
if err != nil {
t.Fatal(err)
}
runner := NewCleanupRunner(store)
if err := runner.Start(time.Hour); err != nil {
t.Fatal(err)
}
runner.Stop()
if err := runner.Start(time.Hour); err != nil {
t.Fatalf("cleanup runner restart error = %v", err)
}
runner.Stop()
}
+30 -22
View File
@@ -13,7 +13,8 @@ type OperationalCleanupRunner struct {
store *EventStore
rateLimits *PersistentRateLimiter
interval time.Duration
once sync.Once
running bool
stopping bool
stop chan struct{}
done chan struct{}
now func() time.Time
@@ -31,29 +32,32 @@ func (r *OperationalCleanupRunner) Start() {
if r == nil || r.store == nil {
return
}
r.once.Do(func() {
r.mu.Lock()
r.stop = make(chan struct{})
r.done = make(chan struct{})
stop, done := r.stop, r.done
interval := r.interval
r.mu.Lock()
if r.running || r.stopping {
r.mu.Unlock()
return
}
r.running = true
r.stop = make(chan struct{})
r.done = make(chan struct{})
stop, done := r.stop, r.done
interval := r.interval
r.mu.Unlock()
go func() {
defer close(done)
r.runOnce()
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
r.runOnce()
case <-stop:
return
}
go func() {
defer close(done)
r.runOnce()
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
r.runOnce()
case <-stop:
return
}
}()
})
}
}()
}
func (r *OperationalCleanupRunner) Stop() {
@@ -66,11 +70,15 @@ func (r *OperationalCleanupRunner) Stop() {
r.mu.Unlock()
return
}
r.stop, r.done = nil, nil
r.stop, r.done, r.running = nil, nil, false
r.stopping = true
r.mu.Unlock()
close(stop)
<-done
r.mu.Lock()
r.stop, r.done, r.stopping = nil, nil, false
r.mu.Unlock()
}
func (r *OperationalCleanupRunner) runOnce() {
@@ -53,6 +53,20 @@ func TestCleanupLeaseIsSingleInstanceAndHasPersistedSchedule(t *testing.T) {
}
}
func TestOperationalCleanupRunnerCanRestartAfterStop(t *testing.T) {
engine := newCleanupEngine(t)
defer engine.Close()
store, err := NewEventStore(engine)
if err != nil {
t.Fatal(err)
}
runner := NewOperationalCleanupRunner(store)
runner.Start()
runner.Stop()
runner.Start()
runner.Stop()
}
func TestCleanupFailureSchedulesSixHourRetry(t *testing.T) {
engine := newCleanupEngine(t)
defer engine.Close()