1 parent
3af1fc4147
commit
29cfebd620
36 files changed
+2838
-118
No files matched your search
@@ -37,6 +37,9 @@ var (
|
||||
ErrMigrationState = errors.New("database migration state is invalid")
|
||||
ErrExecutorUnavailable = errors.New("database migration executor unavailable")
|
||||
ErrSourceDatabase = errors.New("source database configuration is invalid")
|
||||
ErrTargetExists = errors.New("database migration target already exists")
|
||||
ErrSwitchUnavailable = errors.New("database migration configuration switch unavailable")
|
||||
ErrUnsupportedDatabase = errors.New("database migration database type is unsupported")
|
||||
)
|
||||
|
||||
const previewLifetime = 10 * time.Minute
|
||||
@@ -54,8 +57,12 @@ type DatabaseIdentity struct {
|
||||
}
|
||||
|
||||
type Plan struct {
|
||||
Source DatabaseIdentity
|
||||
Target DatabaseIdentity
|
||||
Source DatabaseIdentity
|
||||
Target DatabaseIdentity
|
||||
sourceConfig config.DatabaseConfig
|
||||
targetConfig config.DatabaseConfig
|
||||
expectedGeneration uint64
|
||||
actor config.AuditActor
|
||||
}
|
||||
|
||||
type Record struct {
|
||||
@@ -82,8 +89,10 @@ type Service struct {
|
||||
|
||||
type record struct {
|
||||
Record
|
||||
tokenHash [32]byte
|
||||
expiresAt time.Time
|
||||
tokenHash [32]byte
|
||||
expiresAt time.Time
|
||||
sourceConfig config.DatabaseConfig
|
||||
targetConfig config.DatabaseConfig
|
||||
}
|
||||
|
||||
var current struct {
|
||||
@@ -131,7 +140,7 @@ func (s *Service) Preview(_ context.Context, source config.DatabaseConfig, curre
|
||||
return Record{}, "", ErrInvalidRequest
|
||||
}
|
||||
now := s.currentTime()
|
||||
item := &record{Record: Record{ID: id, Status: StatusPreviewed, Source: plan.Source, Target: plan.Target, CurrentGeneration: currentGeneration, CreatedAt: now, UpdatedAt: now}, tokenHash: sha256.Sum256([]byte(token)), expiresAt: now.Add(previewLifetime)}
|
||||
item := &record{Record: Record{ID: id, Status: StatusPreviewed, Source: plan.Source, Target: plan.Target, CurrentGeneration: currentGeneration, CreatedAt: now, UpdatedAt: now}, tokenHash: sha256.Sum256([]byte(token)), expiresAt: now.Add(previewLifetime), sourceConfig: plan.sourceConfig, targetConfig: plan.targetConfig}
|
||||
s.mu.Lock()
|
||||
s.records[id] = item
|
||||
s.mu.Unlock()
|
||||
@@ -139,6 +148,19 @@ func (s *Service) Preview(_ context.Context, source config.DatabaseConfig, curre
|
||||
}
|
||||
|
||||
func (s *Service) Execute(ctx context.Context, id, token string) (Record, error) {
|
||||
return s.execute(ctx, id, token, config.AuditActor{})
|
||||
}
|
||||
|
||||
// ExecuteAuthorized executes a migration while preserving the authenticated
|
||||
// operator for the final configuration switch.
|
||||
func (s *Service) ExecuteAuthorized(ctx context.Context, id, token string, actor config.AuditActor) (Record, error) {
|
||||
if actor.UserID == "" || actor.SessionID == "" {
|
||||
return Record{}, config.ErrAuthorizationRequired
|
||||
}
|
||||
return s.execute(ctx, id, token, actor)
|
||||
}
|
||||
|
||||
func (s *Service) execute(ctx context.Context, id, token string, actor config.AuditActor) (Record, error) {
|
||||
if s == nil || strings.TrimSpace(id) == "" || strings.TrimSpace(token) == "" {
|
||||
return Record{}, ErrInvalidRequest
|
||||
}
|
||||
@@ -169,11 +191,11 @@ func (s *Service) Execute(ctx context.Context, id, token string) (Record, error)
|
||||
item.tokenHash = [32]byte{}
|
||||
item.Status = StatusCreated
|
||||
item.UpdatedAt = now
|
||||
plan := Plan{Source: item.Source, Target: item.Target}
|
||||
plan := Plan{Source: item.Source, Target: item.Target, sourceConfig: item.sourceConfig, targetConfig: item.targetConfig, expectedGeneration: item.CurrentGeneration, actor: actor}
|
||||
s.mu.Unlock()
|
||||
|
||||
if err := s.setStatus(id, StatusCopying); err != nil {
|
||||
return Record{}, err
|
||||
return s.fail(id, err)
|
||||
}
|
||||
if s.runner == nil {
|
||||
return s.fail(id, ErrExecutorUnavailable)
|
||||
@@ -182,7 +204,7 @@ func (s *Service) Execute(ctx context.Context, id, token string) (Record, error)
|
||||
return s.fail(id, err)
|
||||
}
|
||||
if err := s.setStatus(id, StatusSucceeded); err != nil {
|
||||
return Record{}, err
|
||||
return s.fail(id, err)
|
||||
}
|
||||
return s.Get(id)
|
||||
}
|
||||
@@ -273,7 +295,18 @@ func buildPlan(source config.DatabaseConfig, targetName string) (Plan, error) {
|
||||
if target.Type != source.Type || target.Host != source.Host || target.Port != source.Port || target.Username != source.Username {
|
||||
return Plan{}, ErrImmutableSetting
|
||||
}
|
||||
return Plan{Source: identity, Target: target}, nil
|
||||
if target.Name == source.Name && target.LocalRef == source.LocalRef {
|
||||
return Plan{}, ErrInvalidTargetName
|
||||
}
|
||||
return Plan{Source: identity, Target: target, sourceConfig: source, targetConfig: func() config.DatabaseConfig {
|
||||
candidate := source
|
||||
if source.Type == config.DatabaseTypeSQLite {
|
||||
candidate.LocalRef = targetName
|
||||
} else {
|
||||
candidate.Name = targetName
|
||||
}
|
||||
return candidate
|
||||
}()}, nil
|
||||
}
|
||||
|
||||
func randomID(prefix string) (string, error) {
|
||||
@@ -294,6 +327,26 @@ func migrationErrorCode(err error) string {
|
||||
return "EXECUTOR_UNAVAILABLE"
|
||||
case errors.Is(err, ErrPreviewExpired):
|
||||
return "PREVIEW_EXPIRED"
|
||||
case errors.Is(err, ErrTargetExists):
|
||||
return "TARGET_EXISTS"
|
||||
case errors.Is(err, ErrSQLTargetExists):
|
||||
return "TARGET_EXISTS"
|
||||
case errors.Is(err, ErrSQLCopyUnavailable):
|
||||
return "COPIER_UNAVAILABLE"
|
||||
case errors.Is(err, ErrSQLConnectionUnavailable):
|
||||
return "CONNECTION_UNAVAILABLE"
|
||||
case errors.Is(err, ErrSQLUnsupportedObject):
|
||||
return "UNSUPPORTED_OBJECT"
|
||||
case errors.Is(err, ErrSQLDataVerification):
|
||||
return "DATA_VERIFICATION_FAILED"
|
||||
case errors.Is(err, ErrSQLSourceBusy):
|
||||
return "SOURCE_BUSY"
|
||||
case errors.Is(err, ErrSwitchUnavailable):
|
||||
return "SWITCH_UNAVAILABLE"
|
||||
case errors.Is(err, ErrSwitchOutcomeUnknown):
|
||||
return "SWITCH_OUTCOME_UNKNOWN"
|
||||
case errors.Is(err, ErrUnsupportedDatabase):
|
||||
return "UNSUPPORTED_DATABASE"
|
||||
default:
|
||||
return "MIGRATION_FAILED"
|
||||
}
|
||||
|
||||
@@ -2,12 +2,16 @@ package dbmigration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/config"
|
||||
"git.pcux.cn/xuans/Smail/utils/credential"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
type testRunner struct{}
|
||||
@@ -92,6 +96,29 @@ func TestFailedExecutionKeepsSafeState(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrationErrorCodeMapsExecutorFailures(t *testing.T) {
|
||||
tests := []struct {
|
||||
err error
|
||||
code string
|
||||
}{
|
||||
{err: ErrSQLTargetExists, code: "TARGET_EXISTS"},
|
||||
{err: ErrSQLCopyUnavailable, code: "COPIER_UNAVAILABLE"},
|
||||
{err: ErrSQLConnectionUnavailable, code: "CONNECTION_UNAVAILABLE"},
|
||||
{err: ErrSQLUnsupportedObject, code: "UNSUPPORTED_OBJECT"},
|
||||
{err: ErrSQLDataVerification, code: "DATA_VERIFICATION_FAILED"},
|
||||
{err: ErrSQLSourceBusy, code: "SOURCE_BUSY"},
|
||||
{err: ErrSwitchUnavailable, code: "SWITCH_UNAVAILABLE"},
|
||||
{err: ErrSwitchOutcomeUnknown, code: "SWITCH_OUTCOME_UNKNOWN"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.code, func(t *testing.T) {
|
||||
if got := migrationErrorCode(test.err); got != test.code {
|
||||
t.Fatalf("migration error code = %q, want %q", got, test.code)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLitePreviewChangesOnlyLocalReference(t *testing.T) {
|
||||
service := NewService(testRunner{})
|
||||
source := config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}
|
||||
@@ -177,3 +204,52 @@ func TestMigrationRecordsRemainIndependentDuringConcurrentExecution(t *testing.T
|
||||
t.Fatalf("migration records crossed: first=%+v second=%+v", firstState.Target, secondState.Target)
|
||||
}
|
||||
}
|
||||
func TestExecuteAuthorizedPassesActorToSQLiteSwitch(t *testing.T) {
|
||||
var received config.AuditActor
|
||||
var receivedGeneration uint64
|
||||
root := t.TempDir()
|
||||
switchConfig := func(_ context.Context, generation uint64, _ config.DatabaseConfig, actor config.AuditActor) error {
|
||||
receivedGeneration = generation
|
||||
received = actor
|
||||
return nil
|
||||
}
|
||||
executor := NewAuthorizedSQLiteExecutor(root, switchConfig)
|
||||
service := NewService(executor)
|
||||
actor := config.AuditActor{UserID: "admin-1", Role: "super_admin", SessionID: "session-1"}
|
||||
if _, err := service.ExecuteAuthorized(context.Background(), "missing", "missing", actor); !errors.Is(err, ErrMigrationNotFound) {
|
||||
t.Fatalf("missing migration error = %v, want migration not found", err)
|
||||
}
|
||||
if _, err := service.ExecuteAuthorized(context.Background(), "missing", "missing", config.AuditActor{UserID: "admin-1"}); !errors.Is(err, config.ErrAuthorizationRequired) {
|
||||
t.Fatalf("missing session error = %v, want authorization required", err)
|
||||
}
|
||||
|
||||
root = t.TempDir()
|
||||
executor = NewAuthorizedSQLiteExecutor(root, switchConfig)
|
||||
service = NewService(executor)
|
||||
configDirectory := filepath.Join(root, "config")
|
||||
if err := os.Mkdir(configDirectory, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
database, err := sql.Open("sqlite", filepath.Join(configDirectory, "smail.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := database.Exec("CREATE TABLE messages (id INTEGER PRIMARY KEY, body TEXT NOT NULL)"); err != nil {
|
||||
database.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := database.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "next.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
completed, err := service.ExecuteAuthorized(context.Background(), record.ID, token, actor)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if completed.Status != StatusSucceeded || received != actor || receivedGeneration != record.CurrentGeneration {
|
||||
t.Fatalf("authorized execution = %+v, received actor = %+v, generation = %d", completed, received, receivedGeneration)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,667 @@
|
||||
package dbmigration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/config"
|
||||
"git.pcux.cn/xuans/Smail/utils/credential"
|
||||
mysql "github.com/go-sql-driver/mysql"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrSQLCopyUnavailable = errors.New("structured database copier unavailable")
|
||||
ErrSQLTargetExists = errors.New("structured database migration target already exists")
|
||||
ErrSQLUnsupportedObject = errors.New("structured database contains unsupported object")
|
||||
ErrSQLConnectionUnavailable = errors.New("structured database connection unavailable")
|
||||
ErrSQLDataVerification = errors.New("structured database data verification failed")
|
||||
ErrSQLSourceBusy = errors.New("structured database source has active connections")
|
||||
ErrSwitchOutcomeUnknown = errors.New("database configuration switch outcome is unknown")
|
||||
)
|
||||
|
||||
type AuthorizedDatabaseSwitch func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error
|
||||
|
||||
type StructuredDatabaseCopier interface {
|
||||
Copy(context.Context, config.DatabaseConfig, config.DatabaseConfig) error
|
||||
}
|
||||
|
||||
type StructuredDatabaseCleanup interface {
|
||||
Cleanup(context.Context, config.DatabaseConfig, config.DatabaseConfig) error
|
||||
Complete(config.DatabaseConfig, config.DatabaseConfig)
|
||||
}
|
||||
|
||||
type DatabaseExecutor struct {
|
||||
sqlite Runner
|
||||
structured Runner
|
||||
}
|
||||
|
||||
func NewDatabaseExecutor(sqlite, structured Runner) Runner {
|
||||
return &DatabaseExecutor{sqlite: sqlite, structured: structured}
|
||||
}
|
||||
|
||||
func (e *DatabaseExecutor) Run(ctx context.Context, plan Plan, update func(Status) error) error {
|
||||
if e == nil {
|
||||
return ErrExecutorUnavailable
|
||||
}
|
||||
if plan.sourceConfig.Type == config.DatabaseTypeSQLite {
|
||||
if e.sqlite == nil {
|
||||
return ErrExecutorUnavailable
|
||||
}
|
||||
return e.sqlite.Run(ctx, plan, update)
|
||||
}
|
||||
if e.structured == nil {
|
||||
return ErrExecutorUnavailable
|
||||
}
|
||||
return e.structured.Run(ctx, plan, update)
|
||||
}
|
||||
|
||||
type SQLExecutor struct {
|
||||
copier StructuredDatabaseCopier
|
||||
authorizedSwitch AuthorizedDatabaseSwitch
|
||||
}
|
||||
|
||||
func NewSQLExecutor(copier StructuredDatabaseCopier, switchConfig AuthorizedDatabaseSwitch) *SQLExecutor {
|
||||
return &SQLExecutor{copier: copier, authorizedSwitch: switchConfig}
|
||||
}
|
||||
|
||||
func (e *SQLExecutor) Run(ctx context.Context, plan Plan, update func(Status) error) error {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if e == nil || update == nil {
|
||||
return ErrInvalidRequest
|
||||
}
|
||||
if err := contextError(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if plan.sourceConfig.Type != plan.targetConfig.Type ||
|
||||
(plan.sourceConfig.Type != config.DatabaseTypeMySQL && plan.sourceConfig.Type != config.DatabaseTypePostgreSQL) {
|
||||
return ErrUnsupportedDatabase
|
||||
}
|
||||
if err := config.ValidateDatabaseConfig(plan.sourceConfig); err != nil {
|
||||
return ErrSourceDatabase
|
||||
}
|
||||
if err := config.ValidateDatabaseConfig(plan.targetConfig); err != nil {
|
||||
return ErrInvalidTargetName
|
||||
}
|
||||
if !validStructuredDatabaseName(plan.sourceConfig.Name) ||
|
||||
!validStructuredDatabaseName(plan.targetConfig.Name) ||
|
||||
plan.sourceConfig.Name == plan.targetConfig.Name ||
|
||||
!sameStructuredConnection(plan.sourceConfig, plan.targetConfig) {
|
||||
return ErrImmutableSetting
|
||||
}
|
||||
if e.copier == nil {
|
||||
return ErrSQLCopyUnavailable
|
||||
}
|
||||
if e.authorizedSwitch == nil {
|
||||
return ErrSwitchUnavailable
|
||||
}
|
||||
if err := update(StatusVerifying); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.copier.Copy(ctx, plan.sourceConfig, plan.targetConfig); err != nil {
|
||||
cleanupStructuredTarget(e.copier, plan.sourceConfig, plan.targetConfig)
|
||||
return err
|
||||
}
|
||||
if err := contextError(ctx); err != nil {
|
||||
cleanupStructuredTarget(e.copier, plan.sourceConfig, plan.targetConfig)
|
||||
return err
|
||||
}
|
||||
if err := update(StatusSwitching); err != nil {
|
||||
cleanupStructuredTarget(e.copier, plan.sourceConfig, plan.targetConfig)
|
||||
return err
|
||||
}
|
||||
if e.authorizedSwitch == nil {
|
||||
cleanupStructuredTarget(e.copier, plan.sourceConfig, plan.targetConfig)
|
||||
return ErrSwitchUnavailable
|
||||
}
|
||||
if err := e.authorizedSwitch(ctx, plan.expectedGeneration, plan.targetConfig, plan.actor); err != nil {
|
||||
return errors.Join(ErrSwitchOutcomeUnknown, err)
|
||||
}
|
||||
if cleanup, ok := e.copier.(StructuredDatabaseCleanup); ok {
|
||||
cleanup.Complete(plan.sourceConfig, plan.targetConfig)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanupStructuredTarget(copier StructuredDatabaseCopier, source, target config.DatabaseConfig) {
|
||||
cleanup, ok := copier.(StructuredDatabaseCleanup)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
cleanupContext, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
_ = cleanup.Cleanup(cleanupContext, source, target)
|
||||
}
|
||||
|
||||
func sameStructuredConnection(source, target config.DatabaseConfig) bool {
|
||||
return source.Type == target.Type &&
|
||||
source.Host == target.Host &&
|
||||
source.Port == target.Port &&
|
||||
source.Username == target.Username &&
|
||||
source.Password == target.Password &&
|
||||
source.LocalRef == target.LocalRef
|
||||
}
|
||||
|
||||
type SQLConnectionFactory func(context.Context, config.DatabaseConfig, string) (*sql.DB, error)
|
||||
|
||||
type SQLCopier struct {
|
||||
open SQLConnectionFactory
|
||||
mu sync.Mutex
|
||||
created map[string]struct{}
|
||||
}
|
||||
|
||||
func NewSQLCopier(open SQLConnectionFactory) *SQLCopier {
|
||||
return &SQLCopier{open: open, created: make(map[string]struct{})}
|
||||
}
|
||||
|
||||
func (c *SQLCopier) Copy(ctx context.Context, source, target config.DatabaseConfig) error {
|
||||
if c == nil || c.open == nil {
|
||||
return ErrSQLConnectionUnavailable
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
adminDB, err := c.open(ctx, source, "")
|
||||
if err != nil {
|
||||
return ErrSQLConnectionUnavailable
|
||||
}
|
||||
defer adminDB.Close()
|
||||
switch source.Type {
|
||||
case config.DatabaseTypeMySQL:
|
||||
sourceDB, err := c.open(ctx, source, source.Name)
|
||||
if err != nil {
|
||||
return ErrSQLConnectionUnavailable
|
||||
}
|
||||
defer sourceDB.Close()
|
||||
return c.copyMySQL(ctx, sourceDB, adminDB, source, target)
|
||||
case config.DatabaseTypePostgreSQL:
|
||||
return c.copyPostgreSQL(ctx, adminDB, source, target)
|
||||
default:
|
||||
return ErrUnsupportedDatabase
|
||||
}
|
||||
}
|
||||
|
||||
func (c *SQLCopier) copyMySQL(ctx context.Context, sourceDB, adminDB *sql.DB, source, target config.DatabaseConfig) error {
|
||||
sourceName := source.Name
|
||||
targetName := target.Name
|
||||
if err := c.validateMySQLObjects(ctx, adminDB, sourceName); err != nil {
|
||||
return err
|
||||
}
|
||||
var exists int
|
||||
if err := adminDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = ?", targetName).Scan(&exists); err != nil {
|
||||
return fmt.Errorf("check mysql migration target: %w", err)
|
||||
}
|
||||
if exists != 0 {
|
||||
return ErrSQLTargetExists
|
||||
}
|
||||
if _, err := adminDB.ExecContext(ctx, "CREATE DATABASE "+quoteMySQLIdentifier(targetName)); err != nil {
|
||||
if isDatabaseAlreadyExistsError(err, config.DatabaseTypeMySQL) {
|
||||
return ErrSQLTargetExists
|
||||
}
|
||||
return fmt.Errorf("create mysql migration target: %w", err)
|
||||
}
|
||||
c.markCreated(source, target)
|
||||
created := true
|
||||
defer func() {
|
||||
if created {
|
||||
cleanupContext, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := adminDB.ExecContext(cleanupContext, "DROP DATABASE "+quoteMySQLIdentifier(targetName)); err == nil {
|
||||
c.forgetCreated(source, target)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
rows, err := sourceDB.QueryContext(ctx, "SHOW FULL TABLES FROM "+quoteMySQLIdentifier(sourceName))
|
||||
if err != nil {
|
||||
return fmt.Errorf("list mysql tables: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
tableNames := make([]string, 0)
|
||||
sourceRowCounts := make(map[string]int64)
|
||||
for rows.Next() {
|
||||
var tableName, tableType string
|
||||
if err := rows.Scan(&tableName, &tableType); err != nil {
|
||||
return fmt.Errorf("read mysql table: %w", err)
|
||||
}
|
||||
if !strings.EqualFold(tableType, "BASE TABLE") {
|
||||
return ErrSQLUnsupportedObject
|
||||
}
|
||||
tableNames = append(tableNames, tableName)
|
||||
sourceTable := quoteMySQLIdentifier(sourceName) + "." + quoteMySQLIdentifier(tableName)
|
||||
targetTable := quoteMySQLIdentifier(targetName) + "." + quoteMySQLIdentifier(tableName)
|
||||
rowCount, err := countRows(ctx, sourceDB, sourceTable)
|
||||
if err != nil {
|
||||
return fmt.Errorf("count mysql source rows: %w", err)
|
||||
}
|
||||
sourceRowCounts[tableName] = rowCount
|
||||
if _, err := adminDB.ExecContext(ctx, "CREATE TABLE "+targetTable+" LIKE "+sourceTable); err != nil {
|
||||
return fmt.Errorf("copy mysql table structure: %w", err)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("read mysql tables: %w", err)
|
||||
}
|
||||
for _, tableName := range tableNames {
|
||||
sourceTable := quoteMySQLIdentifier(sourceName) + "." + quoteMySQLIdentifier(tableName)
|
||||
targetTable := quoteMySQLIdentifier(targetName) + "." + quoteMySQLIdentifier(tableName)
|
||||
if _, err := adminDB.ExecContext(ctx, "INSERT INTO "+targetTable+" SELECT * FROM "+sourceTable); err != nil {
|
||||
return fmt.Errorf("copy mysql table data: %w", err)
|
||||
}
|
||||
targetRowCount, err := countRows(ctx, adminDB, targetTable)
|
||||
if err != nil {
|
||||
return fmt.Errorf("count mysql target rows: %w", err)
|
||||
}
|
||||
finalSourceRowCount, err := countRows(ctx, sourceDB, sourceTable)
|
||||
if err != nil {
|
||||
return fmt.Errorf("verify mysql source rows: %w", err)
|
||||
}
|
||||
if sourceRowCounts[tableName] != targetRowCount || sourceRowCounts[tableName] != finalSourceRowCount {
|
||||
return fmt.Errorf("%w: mysql table %s row count changed during migration", ErrSQLDataVerification, tableName)
|
||||
}
|
||||
}
|
||||
if err := c.copyMySQLForeignKeys(ctx, adminDB, sourceName, targetName, tableNames); err != nil {
|
||||
return err
|
||||
}
|
||||
created = false
|
||||
return nil
|
||||
}
|
||||
|
||||
type mysqlForeignKey struct {
|
||||
tableName string
|
||||
constraintName string
|
||||
columns []string
|
||||
referencedSchema string
|
||||
referencedTable string
|
||||
referencedColumns []string
|
||||
updateRule string
|
||||
deleteRule string
|
||||
}
|
||||
|
||||
func (c *SQLCopier) validateMySQLObjects(ctx context.Context, database *sql.DB, sourceName string) error {
|
||||
checks := []struct {
|
||||
query string
|
||||
args []any
|
||||
}{
|
||||
{
|
||||
query: "SELECT COUNT(*) FROM information_schema.triggers WHERE trigger_schema = ?",
|
||||
args: []any{sourceName},
|
||||
},
|
||||
{
|
||||
query: "SELECT COUNT(*) FROM information_schema.events WHERE event_schema = ?",
|
||||
args: []any{sourceName},
|
||||
},
|
||||
{
|
||||
query: "SELECT COUNT(*) FROM information_schema.routines WHERE routine_schema = ?",
|
||||
args: []any{sourceName},
|
||||
},
|
||||
}
|
||||
for _, check := range checks {
|
||||
var count int
|
||||
if err := database.QueryRowContext(ctx, check.query, check.args...).Scan(&count); err != nil {
|
||||
return fmt.Errorf("%w: inspect mysql database objects", ErrSQLConnectionUnavailable)
|
||||
}
|
||||
if count > 0 {
|
||||
return ErrSQLUnsupportedObject
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *SQLCopier) copyMySQLForeignKeys(ctx context.Context, database *sql.DB, sourceName, targetName string, tableNames []string) error {
|
||||
if len(tableNames) == 0 {
|
||||
return nil
|
||||
}
|
||||
rows, err := database.QueryContext(ctx, "SELECT kcu.TABLE_NAME, kcu.CONSTRAINT_NAME, kcu.COLUMN_NAME, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME, rc.UPDATE_RULE, rc.DELETE_RULE FROM information_schema.KEY_COLUMN_USAGE kcu JOIN information_schema.TABLE_CONSTRAINTS tc ON tc.CONSTRAINT_SCHEMA = kcu.CONSTRAINT_SCHEMA AND tc.TABLE_NAME = kcu.TABLE_NAME AND tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME JOIN information_schema.REFERENTIAL_CONSTRAINTS rc ON rc.CONSTRAINT_SCHEMA = kcu.CONSTRAINT_SCHEMA AND rc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME AND rc.TABLE_NAME = kcu.TABLE_NAME WHERE tc.CONSTRAINT_SCHEMA = ? AND tc.CONSTRAINT_TYPE = 'FOREIGN KEY' AND kcu.REFERENCED_TABLE_NAME IS NOT NULL ORDER BY kcu.TABLE_NAME, kcu.CONSTRAINT_NAME, kcu.ORDINAL_POSITION", sourceName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: inspect mysql foreign keys", ErrSQLConnectionUnavailable)
|
||||
}
|
||||
defer rows.Close()
|
||||
foreignKeys := make([]mysqlForeignKey, 0)
|
||||
indexes := make(map[string]int)
|
||||
for rows.Next() {
|
||||
var item mysqlForeignKey
|
||||
var column, referencedColumn string
|
||||
if err := rows.Scan(&item.tableName, &item.constraintName, &column, &item.referencedSchema, &item.referencedTable, &referencedColumn, &item.updateRule, &item.deleteRule); err != nil {
|
||||
return fmt.Errorf("%w: read mysql foreign keys", ErrSQLConnectionUnavailable)
|
||||
}
|
||||
if !validMySQLReferenceRule(item.updateRule) || !validMySQLReferenceRule(item.deleteRule) {
|
||||
return ErrSQLUnsupportedObject
|
||||
}
|
||||
key := item.tableName + "\x00" + item.constraintName
|
||||
index, ok := indexes[key]
|
||||
if !ok {
|
||||
indexes[key] = len(foreignKeys)
|
||||
foreignKeys = append(foreignKeys, item)
|
||||
index = len(foreignKeys) - 1
|
||||
}
|
||||
foreignKeys[index].columns = append(foreignKeys[index].columns, column)
|
||||
foreignKeys[index].referencedColumns = append(foreignKeys[index].referencedColumns, referencedColumn)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("%w: read mysql foreign keys", ErrSQLConnectionUnavailable)
|
||||
}
|
||||
for _, foreignKey := range foreignKeys {
|
||||
referencedSchema := foreignKey.referencedSchema
|
||||
if referencedSchema == sourceName {
|
||||
referencedSchema = targetName
|
||||
}
|
||||
columns := quoteMySQLIdentifiers(foreignKey.columns)
|
||||
referencedColumns := quoteMySQLIdentifiers(foreignKey.referencedColumns)
|
||||
query := "ALTER TABLE " + quoteMySQLIdentifier(targetName) + "." + quoteMySQLIdentifier(foreignKey.tableName) + " ADD CONSTRAINT " + quoteMySQLIdentifier(foreignKey.constraintName) + " FOREIGN KEY (" + strings.Join(columns, ", ") + ") REFERENCES " + quoteMySQLIdentifier(referencedSchema) + "." + quoteMySQLIdentifier(foreignKey.referencedTable) + " (" + strings.Join(referencedColumns, ", ") + ") ON UPDATE " + foreignKey.updateRule + " ON DELETE " + foreignKey.deleteRule
|
||||
if _, err := database.ExecContext(ctx, query); err != nil {
|
||||
return fmt.Errorf("copy mysql foreign key: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validMySQLReferenceRule(value string) bool {
|
||||
switch strings.ToUpper(strings.TrimSpace(value)) {
|
||||
case "CASCADE", "SET NULL", "SET DEFAULT", "RESTRICT", "NO ACTION":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func quoteMySQLIdentifiers(values []string) []string {
|
||||
quoted := make([]string, len(values))
|
||||
for index, value := range values {
|
||||
quoted[index] = quoteMySQLIdentifier(value)
|
||||
}
|
||||
return quoted
|
||||
}
|
||||
|
||||
func (c *SQLCopier) copyPostgreSQL(ctx context.Context, adminDB *sql.DB, source, target config.DatabaseConfig) error {
|
||||
sourceName := source.Name
|
||||
targetName := target.Name
|
||||
var exists bool
|
||||
if err := adminDB.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM pg_database WHERE datname = $1)", targetName).Scan(&exists); err != nil {
|
||||
return fmt.Errorf("check postgresql migration target: %w", err)
|
||||
}
|
||||
if exists {
|
||||
return ErrSQLTargetExists
|
||||
}
|
||||
var activeConnections int
|
||||
if err := adminDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM pg_stat_activity WHERE datname = $1 AND pid <> pg_backend_pid()", sourceName).Scan(&activeConnections); err != nil {
|
||||
return fmt.Errorf("inspect postgresql source connections: %w", err)
|
||||
}
|
||||
if activeConnections > 0 {
|
||||
return ErrSQLSourceBusy
|
||||
}
|
||||
if _, err := adminDB.ExecContext(ctx, "CREATE DATABASE "+quotePostgreSQLIdentifier(targetName)+" TEMPLATE "+quotePostgreSQLIdentifier(sourceName)); err != nil {
|
||||
if isDatabaseAlreadyExistsError(err, config.DatabaseTypePostgreSQL) {
|
||||
return ErrSQLTargetExists
|
||||
}
|
||||
return fmt.Errorf("copy postgresql database: %w", err)
|
||||
}
|
||||
c.markCreated(source, target)
|
||||
created := true
|
||||
defer func() {
|
||||
if created {
|
||||
cleanupContext, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := c.Cleanup(cleanupContext, source, target); err == nil {
|
||||
created = false
|
||||
}
|
||||
}
|
||||
}()
|
||||
if err := c.verifyPostgreSQLClone(ctx, source, target); err != nil {
|
||||
return err
|
||||
}
|
||||
created = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *SQLCopier) verifyPostgreSQLClone(ctx context.Context, source, target config.DatabaseConfig) error {
|
||||
if c == nil || c.open == nil {
|
||||
return ErrSQLConnectionUnavailable
|
||||
}
|
||||
sourceDB, err := c.open(ctx, source, source.Name)
|
||||
if err != nil {
|
||||
return ErrSQLConnectionUnavailable
|
||||
}
|
||||
defer sourceDB.Close()
|
||||
targetDB, err := c.open(ctx, target, target.Name)
|
||||
if err != nil {
|
||||
return ErrSQLConnectionUnavailable
|
||||
}
|
||||
defer targetDB.Close()
|
||||
|
||||
tables, err := listPostgreSQLTables(ctx, sourceDB)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: list postgresql source tables", ErrSQLConnectionUnavailable)
|
||||
}
|
||||
targetTables, err := listPostgreSQLTables(ctx, targetDB)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: list postgresql target tables", ErrSQLConnectionUnavailable)
|
||||
}
|
||||
if len(tables) != len(targetTables) {
|
||||
return fmt.Errorf("%w: postgresql table count changed during migration", ErrSQLDataVerification)
|
||||
}
|
||||
for _, table := range tables {
|
||||
sourceCount, err := countRows(ctx, sourceDB, quotePostgreSQLIdentifier(table.schema)+"."+quotePostgreSQLIdentifier(table.name))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: count postgresql source rows", ErrSQLConnectionUnavailable)
|
||||
}
|
||||
targetCount, err := countRows(ctx, targetDB, quotePostgreSQLIdentifier(table.schema)+"."+quotePostgreSQLIdentifier(table.name))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: count postgresql target rows", ErrSQLConnectionUnavailable)
|
||||
}
|
||||
if sourceCount != targetCount {
|
||||
return fmt.Errorf("%w: postgresql table %s row count changed during migration", ErrSQLDataVerification, table.name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type postgreSQLTable struct {
|
||||
schema string
|
||||
name string
|
||||
}
|
||||
|
||||
func listPostgreSQLTables(ctx context.Context, database *sql.DB) ([]postgreSQLTable, error) {
|
||||
rows, err := database.QueryContext(ctx, "SELECT table_schema, table_name FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema NOT IN ('pg_catalog', 'information_schema') ORDER BY table_schema, table_name")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
tables := make([]postgreSQLTable, 0)
|
||||
for rows.Next() {
|
||||
var table postgreSQLTable
|
||||
if err := rows.Scan(&table.schema, &table.name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tables = append(tables, table)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return tables, nil
|
||||
}
|
||||
|
||||
func isDatabaseAlreadyExistsError(err error, databaseType config.DatabaseType) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
switch databaseType {
|
||||
case config.DatabaseTypeMySQL:
|
||||
var mysqlErr *mysql.MySQLError
|
||||
return errors.As(err, &mysqlErr) && mysqlErr.Number == 1007
|
||||
case config.DatabaseTypePostgreSQL:
|
||||
var postgresErr *pq.Error
|
||||
return errors.As(err, &postgresErr) && string(postgresErr.Code) == "42P04"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (c *SQLCopier) markCreated(source, target config.DatabaseConfig) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.created == nil {
|
||||
c.created = make(map[string]struct{})
|
||||
}
|
||||
c.created[databaseTargetKey(source, target)] = struct{}{}
|
||||
}
|
||||
|
||||
func (c *SQLCopier) hasCreated(source, target config.DatabaseConfig) bool {
|
||||
if c == nil {
|
||||
return false
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
_, ok := c.created[databaseTargetKey(source, target)]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (c *SQLCopier) forgetCreated(source, target config.DatabaseConfig) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
delete(c.created, databaseTargetKey(source, target))
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
func (c *SQLCopier) Complete(source, target config.DatabaseConfig) {
|
||||
c.forgetCreated(source, target)
|
||||
}
|
||||
|
||||
func (c *SQLCopier) Cleanup(ctx context.Context, source, target config.DatabaseConfig) error {
|
||||
if !c.hasCreated(source, target) {
|
||||
return nil
|
||||
}
|
||||
if c.open == nil {
|
||||
return ErrSQLConnectionUnavailable
|
||||
}
|
||||
adminDB, err := c.open(ctx, source, "")
|
||||
if err != nil {
|
||||
return ErrSQLConnectionUnavailable
|
||||
}
|
||||
defer adminDB.Close()
|
||||
var query string
|
||||
switch target.Type {
|
||||
case config.DatabaseTypeMySQL:
|
||||
query = "DROP DATABASE " + quoteMySQLIdentifier(target.Name)
|
||||
case config.DatabaseTypePostgreSQL:
|
||||
query = "DROP DATABASE " + quotePostgreSQLIdentifier(target.Name)
|
||||
default:
|
||||
return ErrUnsupportedDatabase
|
||||
}
|
||||
if _, err := adminDB.ExecContext(ctx, query); err != nil {
|
||||
return fmt.Errorf("cleanup structured migration target: %w", err)
|
||||
}
|
||||
c.forgetCreated(source, target)
|
||||
return nil
|
||||
}
|
||||
|
||||
func countRows(ctx context.Context, database *sql.DB, table string) (int64, error) {
|
||||
var count int64
|
||||
if err := database.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+table).Scan(&count); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func databaseTargetKey(source, target config.DatabaseConfig) string {
|
||||
return strings.Join([]string{
|
||||
string(target.Type),
|
||||
target.Host,
|
||||
strconv.Itoa(int(target.Port)),
|
||||
target.Username,
|
||||
target.Name,
|
||||
source.Name,
|
||||
}, "\x00")
|
||||
}
|
||||
|
||||
func quoteMySQLIdentifier(value string) string {
|
||||
return "`" + strings.ReplaceAll(value, "`", "``") + "`"
|
||||
}
|
||||
|
||||
func quotePostgreSQLIdentifier(value string) string {
|
||||
return "\"" + strings.ReplaceAll(value, "\"", "\"\"") + "\""
|
||||
}
|
||||
|
||||
func NewRuntimeSQLConnectionFactory(root string) (SQLConnectionFactory, error) {
|
||||
vault, err := credential.NewVault(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return func(ctx context.Context, database config.DatabaseConfig, databaseName string) (*sql.DB, error) {
|
||||
password, err := config.OpenStoredSecret(vault, database.Password)
|
||||
if err != nil {
|
||||
return nil, ErrSQLConnectionUnavailable
|
||||
}
|
||||
driverName, dsn, err := runtimeSQLDSN(database, password, databaseName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
connection, err := sql.Open(driverName, dsn)
|
||||
if err != nil {
|
||||
return nil, ErrSQLConnectionUnavailable
|
||||
}
|
||||
if err := connection.PingContext(ctx); err != nil {
|
||||
_ = connection.Close()
|
||||
return nil, ErrSQLConnectionUnavailable
|
||||
}
|
||||
return connection, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
func runtimeSQLDSN(database config.DatabaseConfig, password, databaseName string) (string, string, error) {
|
||||
if database.Host == "" || database.Port == 0 || database.Username == "" || password == "" {
|
||||
return "", "", ErrSQLConnectionUnavailable
|
||||
}
|
||||
address := net.JoinHostPort(database.Host, strconv.Itoa(int(database.Port)))
|
||||
switch database.Type {
|
||||
case config.DatabaseTypeMySQL:
|
||||
settings := mysql.Config{
|
||||
User: database.Username,
|
||||
Passwd: password,
|
||||
Net: "tcp",
|
||||
Addr: address,
|
||||
DBName: databaseName,
|
||||
ParseTime: true,
|
||||
Params: map[string]string{"charset": "utf8mb4"},
|
||||
}
|
||||
return "mysql", settings.FormatDSN(), nil
|
||||
case config.DatabaseTypePostgreSQL:
|
||||
maintenanceDatabase := databaseName
|
||||
if maintenanceDatabase == "" {
|
||||
maintenanceDatabase = "postgres"
|
||||
}
|
||||
databaseURL := url.URL{
|
||||
Scheme: "postgres",
|
||||
User: url.UserPassword(database.Username, password),
|
||||
Host: address,
|
||||
Path: "/" + maintenanceDatabase,
|
||||
RawQuery: "sslmode=prefer",
|
||||
}
|
||||
return "postgres", databaseURL.String(), nil
|
||||
default:
|
||||
return "", "", ErrUnsupportedDatabase
|
||||
}
|
||||
}
|
||||
|
||||
var structuredDatabaseNamePattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,62}$`)
|
||||
var mysqlOptionIdentifierPattern = regexp.MustCompile(`^[A-Za-z0-9_]+$`)
|
||||
|
||||
func validStructuredDatabaseName(value string) bool {
|
||||
return structuredDatabaseNamePattern.MatchString(strings.TrimSpace(value))
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
package dbmigration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/config"
|
||||
mysql "github.com/go-sql-driver/mysql"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
type recordingStructuredCopier struct {
|
||||
called bool
|
||||
source config.DatabaseConfig
|
||||
target config.DatabaseConfig
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *recordingStructuredCopier) Copy(_ context.Context, source, target config.DatabaseConfig) error {
|
||||
c.called = true
|
||||
c.source = source
|
||||
c.target = target
|
||||
return c.err
|
||||
}
|
||||
|
||||
func validStructuredPlan(t *testing.T) Plan {
|
||||
t.Helper()
|
||||
database := testDatabase(t)
|
||||
database.Name = "smail"
|
||||
target := database
|
||||
target.Name = "smail_next"
|
||||
return Plan{
|
||||
sourceConfig: database,
|
||||
targetConfig: target,
|
||||
actor: config.AuditActor{UserID: "admin-1", Role: "super_admin", SessionID: "session-1"},
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLExecutorCopiesAndSwitchesWithOrderedStatuses(t *testing.T) {
|
||||
copier := &recordingStructuredCopier{}
|
||||
var actor config.AuditActor
|
||||
var generation uint64
|
||||
executor := NewSQLExecutor(copier, func(_ context.Context, receivedGeneration uint64, _ config.DatabaseConfig, received config.AuditActor) error {
|
||||
generation = receivedGeneration
|
||||
actor = received
|
||||
return nil
|
||||
})
|
||||
statuses := make([]Status, 0, 2)
|
||||
plan := validStructuredPlan(t)
|
||||
plan.expectedGeneration = 42
|
||||
if err := executor.Run(context.Background(), plan, func(status Status) error {
|
||||
statuses = append(statuses, status)
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(statuses) != 2 || statuses[0] != StatusVerifying || statuses[1] != StatusSwitching {
|
||||
t.Fatalf("statuses = %v, want verifying then switching", statuses)
|
||||
}
|
||||
if !copier.called || copier.source.Name != "smail" || copier.target.Name != "smail_next" {
|
||||
t.Fatalf("copier call = %+v", copier)
|
||||
}
|
||||
if actor != plan.actor {
|
||||
t.Fatalf("switch actor = %+v, want %+v", actor, plan.actor)
|
||||
}
|
||||
if generation != plan.expectedGeneration {
|
||||
t.Fatalf("switch generation = %d, want %d", generation, plan.expectedGeneration)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLExecutorRejectsImmutablePlanBeforeCopy(t *testing.T) {
|
||||
copier := &recordingStructuredCopier{}
|
||||
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error { return nil })
|
||||
plan := validStructuredPlan(t)
|
||||
plan.targetConfig.Host = "other.example"
|
||||
if err := executor.Run(context.Background(), plan, func(Status) error { return nil }); !errors.Is(err, ErrImmutableSetting) {
|
||||
t.Fatalf("immutable plan error = %v, want immutable setting", err)
|
||||
}
|
||||
if copier.called {
|
||||
t.Fatal("copier ran for immutable plan")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLExecutorPreservesCopierError(t *testing.T) {
|
||||
copyErr := errors.New("复制失败")
|
||||
copier := &recordingStructuredCopier{err: copyErr}
|
||||
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error { return nil })
|
||||
if err := executor.Run(context.Background(), validStructuredPlan(t), func(Status) error { return nil }); !errors.Is(err, copyErr) {
|
||||
t.Fatalf("copy error = %v, want copier error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLExecutorCleansStructuredTargetWhenCopyFails(t *testing.T) {
|
||||
copier := &cleanupRecordingCopier{copyErr: errors.New("复制失败")}
|
||||
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error {
|
||||
return nil
|
||||
})
|
||||
if err := executor.Run(context.Background(), validStructuredPlan(t), func(Status) error { return nil }); !errors.Is(err, copier.copyErr) {
|
||||
t.Fatalf("copy error = %v, want %v", err, copier.copyErr)
|
||||
}
|
||||
if !copier.cleanupCalled {
|
||||
t.Fatal("structured target cleanup was not requested after copy failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStructuredDatabaseNameValidation(t *testing.T) {
|
||||
for _, value := range []string{"smail", "Smail_2026", "mail-prod"} {
|
||||
if !validStructuredDatabaseName(value) {
|
||||
t.Fatalf("valid database name rejected: %q", value)
|
||||
}
|
||||
}
|
||||
for _, value := range []string{"", "smail;drop", "smail name", "../smail"} {
|
||||
if validStructuredDatabaseName(value) {
|
||||
t.Fatalf("unsafe database name accepted: %q", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatabaseExecutorRoutesByDatabaseType(t *testing.T) {
|
||||
var sqliteCalled, structuredCalled bool
|
||||
sqlite := runnerFunc(func(context.Context, Plan, func(Status) error) error {
|
||||
sqliteCalled = true
|
||||
return nil
|
||||
})
|
||||
structured := runnerFunc(func(context.Context, Plan, func(Status) error) error {
|
||||
structuredCalled = true
|
||||
return nil
|
||||
})
|
||||
router := NewDatabaseExecutor(sqlite, structured)
|
||||
if err := router.Run(context.Background(), Plan{sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite}}, func(Status) error { return nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := router.Run(context.Background(), Plan{sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeMySQL}}, func(Status) error { return nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !sqliteCalled || !structuredCalled {
|
||||
t.Fatalf("router calls: sqlite=%v structured=%v", sqliteCalled, structuredCalled)
|
||||
}
|
||||
}
|
||||
|
||||
type runnerFunc func(context.Context, Plan, func(Status) error) error
|
||||
|
||||
func (f runnerFunc) Run(ctx context.Context, plan Plan, update func(Status) error) error {
|
||||
return f(ctx, plan, update)
|
||||
}
|
||||
func TestSQLExecutorRejectsMissingConfigurationSwitchBeforeCopy(t *testing.T) {
|
||||
copier := &recordingStructuredCopier{}
|
||||
executor := NewSQLExecutor(copier, nil)
|
||||
if err := executor.Run(context.Background(), validStructuredPlan(t), func(Status) error { return nil }); !errors.Is(err, ErrSwitchUnavailable) {
|
||||
t.Fatalf("missing switch error = %v, want switch unavailable", err)
|
||||
}
|
||||
if copier.called {
|
||||
t.Fatal("copier ran without a configuration switch")
|
||||
}
|
||||
}
|
||||
|
||||
type cleanupRecordingCopier struct {
|
||||
cleanupCalled bool
|
||||
completeCalled bool
|
||||
copyErr error
|
||||
}
|
||||
|
||||
func (c *cleanupRecordingCopier) Copy(context.Context, config.DatabaseConfig, config.DatabaseConfig) error {
|
||||
return c.copyErr
|
||||
}
|
||||
|
||||
func TestDatabaseAlreadyExistsErrorClassification(t *testing.T) {
|
||||
mysqlError := &mysql.MySQLError{Number: 1007}
|
||||
if !isDatabaseAlreadyExistsError(mysqlError, config.DatabaseTypeMySQL) {
|
||||
t.Fatal("mysql duplicate database error was not classified")
|
||||
}
|
||||
postgresError := &pq.Error{Code: "42P04"}
|
||||
if !isDatabaseAlreadyExistsError(postgresError, config.DatabaseTypePostgreSQL) {
|
||||
t.Fatal("postgresql duplicate database error was not classified")
|
||||
}
|
||||
if isDatabaseAlreadyExistsError(errors.New("duplicate database"), config.DatabaseTypeMySQL) {
|
||||
t.Fatal("generic mysql error was incorrectly classified")
|
||||
}
|
||||
}
|
||||
|
||||
func (c *cleanupRecordingCopier) Cleanup(context.Context, config.DatabaseConfig, config.DatabaseConfig) error {
|
||||
c.cleanupCalled = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *cleanupRecordingCopier) Complete(config.DatabaseConfig, config.DatabaseConfig) {
|
||||
c.completeCalled = true
|
||||
}
|
||||
|
||||
func TestSQLExecutorPreservesStructuredTargetWhenConfigurationSwitchOutcomeIsUnknown(t *testing.T) {
|
||||
copier := &cleanupRecordingCopier{}
|
||||
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error {
|
||||
return errors.New("配置切换失败")
|
||||
})
|
||||
if err := executor.Run(context.Background(), validStructuredPlan(t), func(Status) error { return nil }); !errors.Is(err, ErrSwitchOutcomeUnknown) {
|
||||
t.Fatal("configuration switch failure was not returned")
|
||||
}
|
||||
if copier.cleanupCalled {
|
||||
t.Fatal("structured target was cleaned up even though switch outcome was unknown")
|
||||
}
|
||||
if copier.completeCalled {
|
||||
t.Fatal("structured target was finalized after a failed switch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLExecutorCleansStructuredTargetWhenStatusUpdateFails(t *testing.T) {
|
||||
copier := &cleanupRecordingCopier{}
|
||||
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error {
|
||||
return nil
|
||||
})
|
||||
statusError := errors.New("状态写入失败")
|
||||
if err := executor.Run(context.Background(), validStructuredPlan(t), func(status Status) error {
|
||||
if status == StatusSwitching {
|
||||
return statusError
|
||||
}
|
||||
return nil
|
||||
}); !errors.Is(err, statusError) {
|
||||
t.Fatalf("status update error = %v, want %v", err, statusError)
|
||||
}
|
||||
if !copier.cleanupCalled {
|
||||
t.Fatal("structured target cleanup was not requested after status update failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLExecutorCleansStructuredTargetWhenContextIsCanceledAfterCopy(t *testing.T) {
|
||||
copier := &cleanupRecordingCopier{}
|
||||
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error {
|
||||
return nil
|
||||
})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
if err := executor.Run(ctx, validStructuredPlan(t), func(status Status) error {
|
||||
if status == StatusVerifying {
|
||||
cancel()
|
||||
}
|
||||
return nil
|
||||
}); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("canceled execution error = %v, want context canceled", err)
|
||||
}
|
||||
if !copier.cleanupCalled {
|
||||
t.Fatal("structured target cleanup was not requested after cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeSQLDSNSupportsIPv6AndEscapesCredentials(t *testing.T) {
|
||||
database := config.DatabaseConfig{
|
||||
Type: config.DatabaseTypeMySQL,
|
||||
Host: "2001:db8::3",
|
||||
Port: 3306,
|
||||
Username: "mail user",
|
||||
}
|
||||
driver, dsn, err := runtimeSQLDSN(database, "p@ss/word", "smail_next")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if driver != "mysql" {
|
||||
t.Fatalf("mysql driver = %q", driver)
|
||||
}
|
||||
parsedMySQL, err := mysql.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if parsedMySQL.Addr != "[2001:db8::3]:3306" || parsedMySQL.User != "mail user" || parsedMySQL.Passwd != "p@ss/word" || parsedMySQL.DBName != "smail_next" {
|
||||
t.Fatalf("mysql dsn config = %+v", parsedMySQL)
|
||||
}
|
||||
|
||||
database.Type = config.DatabaseTypePostgreSQL
|
||||
driver, dsn, err = runtimeSQLDSN(database, "p@ss/word", "smail_next")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if driver != "postgres" {
|
||||
t.Fatalf("postgres driver = %q", driver)
|
||||
}
|
||||
parsedPostgres, err := url.Parse(dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
password, ok := parsedPostgres.User.Password()
|
||||
if !ok || parsedPostgres.Host != "[2001:db8::3]:3306" || parsedPostgres.User.Username() != "mail user" || password != "p@ss/word" || parsedPostgres.Path != "/smail_next" {
|
||||
t.Fatalf("postgres dsn = %s", dsn)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLReferenceRulesAndIdentifiersAreRestrictedSafely(t *testing.T) {
|
||||
for _, rule := range []string{"CASCADE", "SET NULL", "SET DEFAULT", "RESTRICT", "NO ACTION"} {
|
||||
if !validMySQLReferenceRule(rule) {
|
||||
t.Fatalf("valid mysql reference rule rejected: %q", rule)
|
||||
}
|
||||
}
|
||||
for _, rule := range []string{"DROP DATABASE", "CASCADE; DROP TABLE users"} {
|
||||
if validMySQLReferenceRule(rule) {
|
||||
t.Fatalf("unsafe mysql reference rule accepted: %q", rule)
|
||||
}
|
||||
}
|
||||
if got := quoteMySQLIdentifier("mail`prod"); got != "`mail``prod`" {
|
||||
t.Fatalf("quoted mysql identifier = %q", got)
|
||||
}
|
||||
if got := strings.Join(quoteMySQLIdentifiers([]string{"user", "tenant"}), ", "); got != "`user`, `tenant`" {
|
||||
t.Fatalf("quoted mysql identifiers = %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
package dbmigration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/config"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
var ErrSQLiteIntegrity = errors.New("sqlite database integrity check failed")
|
||||
|
||||
type SQLiteSwitch func(context.Context, config.DatabaseConfig) error
|
||||
type AuthorizedSQLiteSwitch func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error
|
||||
|
||||
type SQLiteExecutor struct {
|
||||
root string
|
||||
switchConfig SQLiteSwitch
|
||||
authorizedSwitch AuthorizedSQLiteSwitch
|
||||
}
|
||||
|
||||
func NewSQLiteExecutor(root string, switchConfig SQLiteSwitch) *SQLiteExecutor {
|
||||
return &SQLiteExecutor{root: root, switchConfig: switchConfig}
|
||||
}
|
||||
|
||||
func NewAuthorizedSQLiteExecutor(root string, switchConfig AuthorizedSQLiteSwitch) *SQLiteExecutor {
|
||||
return &SQLiteExecutor{root: root, authorizedSwitch: switchConfig}
|
||||
}
|
||||
|
||||
func (e *SQLiteExecutor) Run(ctx context.Context, plan Plan, update func(Status) error) error {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if e == nil || update == nil {
|
||||
return ErrInvalidRequest
|
||||
}
|
||||
if err := contextError(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if plan.sourceConfig.Type != config.DatabaseTypeSQLite || plan.targetConfig.Type != config.DatabaseTypeSQLite {
|
||||
return ErrUnsupportedDatabase
|
||||
}
|
||||
if err := config.ValidateDatabaseConfig(plan.sourceConfig); err != nil {
|
||||
return ErrSourceDatabase
|
||||
}
|
||||
if err := config.ValidateDatabaseConfig(plan.targetConfig); err != nil {
|
||||
return ErrInvalidTargetName
|
||||
}
|
||||
if plan.sourceConfig.LocalRef == plan.targetConfig.LocalRef {
|
||||
return ErrInvalidTargetName
|
||||
}
|
||||
|
||||
sourcePath, err := e.resolveReference(plan.sourceConfig.LocalRef)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
targetPath, err := e.resolveReference(plan.targetConfig.LocalRef)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if sourcePath == targetPath {
|
||||
return ErrInvalidTargetName
|
||||
}
|
||||
if err := checkRegularFile(sourcePath, false); err != nil {
|
||||
return fmt.Errorf("open source database: %w", err)
|
||||
}
|
||||
if err := checkRegularFile(targetPath, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := update(StatusVerifying); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verifySQLite(ctx, sourcePath); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := contextError(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
temporary, err := e.temporaryPath(targetPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(temporary)
|
||||
activated := false
|
||||
switched := false
|
||||
switchStarted := false
|
||||
defer func() {
|
||||
if activated && !switched && !switchStarted {
|
||||
_ = os.Remove(targetPath)
|
||||
}
|
||||
}()
|
||||
if err := vacuumSQLite(ctx, sourcePath, temporary); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Chmod(temporary, 0600); err != nil {
|
||||
return fmt.Errorf("set migrated database permissions: %w", err)
|
||||
}
|
||||
if err := verifySQLite(ctx, temporary); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := contextError(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := update(StatusSwitching); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Link(temporary, targetPath); err != nil {
|
||||
if os.IsExist(err) {
|
||||
return ErrTargetExists
|
||||
}
|
||||
return fmt.Errorf("activate migrated database file: %w", err)
|
||||
}
|
||||
activated = true
|
||||
if err := os.Remove(temporary); err != nil {
|
||||
return fmt.Errorf("finalize migrated database file: %w", err)
|
||||
}
|
||||
if err := verifySQLite(ctx, targetPath); err != nil {
|
||||
return err
|
||||
}
|
||||
if e.authorizedSwitch == nil && e.switchConfig == nil {
|
||||
return ErrSwitchUnavailable
|
||||
}
|
||||
var switchErr error
|
||||
switchStarted = true
|
||||
if e.authorizedSwitch != nil {
|
||||
switchErr = e.authorizedSwitch(ctx, plan.expectedGeneration, plan.targetConfig, plan.actor)
|
||||
} else {
|
||||
switchErr = e.switchConfig(ctx, plan.targetConfig)
|
||||
}
|
||||
if switchErr != nil {
|
||||
return errors.Join(ErrSwitchOutcomeUnknown, switchErr)
|
||||
}
|
||||
switched = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *SQLiteExecutor) resolveReference(reference string) (string, error) {
|
||||
if e == nil || strings.TrimSpace(e.root) == "" || !sqliteTargetNamePattern.MatchString(reference) {
|
||||
return "", ErrInvalidTargetName
|
||||
}
|
||||
root, err := filepath.Abs(filepath.Clean(e.root))
|
||||
if err != nil {
|
||||
return "", ErrInvalidTargetName
|
||||
}
|
||||
configDirectory := filepath.Join(root, "config")
|
||||
info, err := os.Lstat(configDirectory)
|
||||
if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||
return "", ErrInvalidTargetName
|
||||
}
|
||||
path := filepath.Join(configDirectory, reference)
|
||||
absolute, err := filepath.Abs(filepath.Clean(path))
|
||||
if err != nil || filepath.Dir(absolute) != configDirectory {
|
||||
return "", ErrInvalidTargetName
|
||||
}
|
||||
return absolute, nil
|
||||
}
|
||||
|
||||
func (e *SQLiteExecutor) temporaryPath(target string) (string, error) {
|
||||
for attempt := 0; attempt < 8; attempt++ {
|
||||
id, err := randomID(".migration-")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
candidate := filepath.Join(filepath.Dir(target), id+".tmp")
|
||||
if _, err := os.Lstat(candidate); os.IsNotExist(err) {
|
||||
return candidate, nil
|
||||
} else if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return "", ErrTargetExists
|
||||
}
|
||||
|
||||
func checkRegularFile(path string, target bool) error {
|
||||
info, err := os.Lstat(path)
|
||||
if os.IsNotExist(err) {
|
||||
if target {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("source database does not exist: %w", err)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
if target {
|
||||
return ErrTargetExists
|
||||
}
|
||||
return errors.New("source database is not a regular file")
|
||||
}
|
||||
if target {
|
||||
return ErrTargetExists
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifySQLite(ctx context.Context, path string) error {
|
||||
database, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)&mode=ro")
|
||||
if err != nil {
|
||||
return fmt.Errorf("open sqlite database: %w", err)
|
||||
}
|
||||
defer database.Close()
|
||||
if err := database.PingContext(ctx); err != nil {
|
||||
if ctx != nil && ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
return fmt.Errorf("%w: ping sqlite database: %v", ErrSQLiteIntegrity, err)
|
||||
}
|
||||
var result string
|
||||
if err := database.QueryRowContext(ctx, "PRAGMA integrity_check").Scan(&result); err != nil {
|
||||
if ctx != nil && ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
return fmt.Errorf("%w: check sqlite database integrity: %v", ErrSQLiteIntegrity, err)
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(result), "ok") {
|
||||
return fmt.Errorf("%w: %s", ErrSQLiteIntegrity, result)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func vacuumSQLite(ctx context.Context, source, target string) error {
|
||||
database, err := sql.Open("sqlite", source+"?_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
return fmt.Errorf("open sqlite source: %w", err)
|
||||
}
|
||||
defer database.Close()
|
||||
if err := database.PingContext(ctx); err != nil {
|
||||
return fmt.Errorf("ping sqlite source: %w", err)
|
||||
}
|
||||
query := "VACUUM INTO '" + strings.ReplaceAll(target, "'", "''") + "'"
|
||||
if _, err := database.ExecContext(ctx, query); err != nil {
|
||||
return fmt.Errorf("copy sqlite database: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func contextError(ctx context.Context) error {
|
||||
if ctx == nil {
|
||||
return nil
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package dbmigration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/config"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func newSQLiteMigrationFixture(t *testing.T, switchConfig SQLiteSwitch) (*Service, string) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
configDirectory := filepath.Join(root, "config")
|
||||
if err := os.Mkdir(configDirectory, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourcePath := filepath.Join(configDirectory, "smail.db")
|
||||
database, err := sql.Open("sqlite", sourcePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := database.Exec(`CREATE TABLE messages (id INTEGER PRIMARY KEY, body TEXT NOT NULL); INSERT INTO messages(body) VALUES ('测试邮件');`); err != nil {
|
||||
database.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := database.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
executor := NewSQLiteExecutor(root, switchConfig)
|
||||
return NewService(executor), root
|
||||
}
|
||||
|
||||
func TestSQLiteExecutorCopiesDatabaseAndSwitchesConfiguration(t *testing.T) {
|
||||
var switched config.DatabaseConfig
|
||||
service, root := newSQLiteMigrationFixture(t, func(_ context.Context, candidate config.DatabaseConfig) error {
|
||||
switched = candidate
|
||||
return nil
|
||||
})
|
||||
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 3, "smail-next.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
completed, err := service.Execute(context.Background(), record.ID, token)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if completed.Status != StatusSucceeded {
|
||||
t.Fatalf("migration status = %s, want succeeded", completed.Status)
|
||||
}
|
||||
if switched.LocalRef != "smail-next.db" || switched.Type != config.DatabaseTypeSQLite {
|
||||
t.Fatalf("unexpected switched configuration: %+v", switched)
|
||||
}
|
||||
database, err := sql.Open("sqlite", filepath.Join(root, "config", "smail-next.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
var body string
|
||||
if err := database.QueryRow("SELECT body FROM messages WHERE id = 1").Scan(&body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if body != "测试邮件" {
|
||||
t.Fatalf("copied row = %q, want copied content", body)
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(root, "config", "smail-next.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if runtime.GOOS != "windows" && info.Mode().Perm() != 0600 {
|
||||
t.Fatalf("migrated database permissions = %o, want 600", info.Mode().Perm())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteExecutorRejectsExistingTarget(t *testing.T) {
|
||||
service, root := newSQLiteMigrationFixture(t, func(context.Context, config.DatabaseConfig) error { return nil })
|
||||
if err := os.WriteFile(filepath.Join(root, "config", "smail-next.db"), []byte("existing"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "smail-next.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed, err := service.Execute(context.Background(), record.ID, token)
|
||||
if !errors.Is(err, ErrTargetExists) {
|
||||
t.Fatalf("execution error = %v, want target exists", err)
|
||||
}
|
||||
if failed.Status != StatusFailed || failed.ErrorCode != "TARGET_EXISTS" {
|
||||
t.Fatalf("unexpected failed migration: %+v", failed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteExecutorRejectsCorruptSource(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "config"), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "config", "smail.db"), []byte("not sqlite"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
executor := NewSQLiteExecutor(root, func(context.Context, config.DatabaseConfig) error { return nil })
|
||||
plan := Plan{sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, targetConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "next.db"}}
|
||||
if err := executor.Run(context.Background(), plan, func(Status) error { return nil }); err == nil {
|
||||
t.Fatal("corrupt source unexpectedly migrated")
|
||||
} else if !errors.Is(err, ErrSQLiteIntegrity) {
|
||||
t.Fatalf("corrupt source error = %v, want integrity error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteExecutorRejectsUnsupportedDatabase(t *testing.T) {
|
||||
executor := NewSQLiteExecutor(t.TempDir(), func(context.Context, config.DatabaseConfig) error { return nil })
|
||||
plan := Plan{sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeMySQL}, targetConfig: config.DatabaseConfig{Type: config.DatabaseTypeMySQL}}
|
||||
if err := executor.Run(context.Background(), plan, func(Status) error { return nil }); !errors.Is(err, ErrUnsupportedDatabase) {
|
||||
t.Fatalf("unsupported database error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteExecutorRequiresConfigurationSwitch(t *testing.T) {
|
||||
service, root := newSQLiteMigrationFixture(t, nil)
|
||||
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "next.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed, err := service.Execute(context.Background(), record.ID, token)
|
||||
if !errors.Is(err, ErrSwitchUnavailable) {
|
||||
t.Fatalf("execution error = %v, want switch unavailable", err)
|
||||
}
|
||||
if failed.ErrorCode != "SWITCH_UNAVAILABLE" {
|
||||
t.Fatalf("error code = %q, want SWITCH_UNAVAILABLE", failed.ErrorCode)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "config", "next.db")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("target database remained after switch failure, stat error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteExecutorPreservesTargetWhenConfigurationSwitchOutcomeIsUnknown(t *testing.T) {
|
||||
service, root := newSQLiteMigrationFixture(t, func(context.Context, config.DatabaseConfig) error {
|
||||
return errors.New("配置切换失败")
|
||||
})
|
||||
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "next.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed, err := service.Execute(context.Background(), record.ID, token)
|
||||
if err == nil || failed.Status != StatusFailed {
|
||||
t.Fatalf("switch failure result = %+v, error = %v", failed, err)
|
||||
}
|
||||
if !errors.Is(err, ErrSwitchOutcomeUnknown) {
|
||||
t.Fatalf("switch failure error = %v, want unknown switch outcome", err)
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(root, "config", "next.db")); statErr != nil {
|
||||
t.Fatalf("target database was removed after unknown switch outcome: %v", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteExecutorRejectsTargetSymlink(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Windows 创建符号链接需要额外权限")
|
||||
}
|
||||
service, root := newSQLiteMigrationFixture(t, func(context.Context, config.DatabaseConfig) error { return nil })
|
||||
target := filepath.Join(root, "config", "next.db")
|
||||
if err := os.Symlink(filepath.Join(root, "config", "smail.db"), target); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "next.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed, err := service.Execute(context.Background(), record.ID, token)
|
||||
if !errors.Is(err, ErrTargetExists) || failed.Status != StatusFailed {
|
||||
t.Fatalf("symlink target result = %+v, error = %v", failed, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteExecutorHonorsCancellation(t *testing.T) {
|
||||
service, _ := newSQLiteMigrationFixture(t, func(context.Context, config.DatabaseConfig) error { return nil })
|
||||
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "next.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
failed, err := service.Execute(ctx, record.ID, token)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("execution error = %v, want context canceled", err)
|
||||
}
|
||||
if failed.Status != StatusFailed {
|
||||
t.Fatalf("migration status = %s, want failed", failed.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrationPlanDoesNotSerializePrivateDatabaseConfig(t *testing.T) {
|
||||
plan := Plan{Source: DatabaseIdentity{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, Target: DatabaseIdentity{Type: config.DatabaseTypeSQLite, LocalRef: "next.db"}, sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}}
|
||||
data, err := json.Marshal(plan)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != `{"Source":{"type":"sqlite","localRef":"smail.db"},"Target":{"type":"sqlite","localRef":"next.db"}}` {
|
||||
t.Fatalf("serialized plan = %s", data)
|
||||
}
|
||||
}
|
||||
func TestSQLiteExecutorReportsOrderedStatuses(t *testing.T) {
|
||||
service, root := newSQLiteMigrationFixture(t, func(context.Context, config.DatabaseConfig) error { return nil })
|
||||
executor := service.runner.(*SQLiteExecutor)
|
||||
statuses := make([]Status, 0, 2)
|
||||
plan := Plan{
|
||||
sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"},
|
||||
targetConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "next.db"},
|
||||
}
|
||||
if err := executor.Run(context.Background(), plan, func(status Status) error {
|
||||
statuses = append(statuses, status)
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(statuses) != 2 || statuses[0] != StatusVerifying || statuses[1] != StatusSwitching {
|
||||
t.Fatalf("statuses = %v, want verifying then switching", statuses)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "config", "next.db")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -9,11 +9,11 @@ import (
|
||||
var ErrCleanupAlreadyStarted = errors.New("verification cleanup already started")
|
||||
|
||||
type CleanupRunner struct {
|
||||
store *Store
|
||||
once sync.Once
|
||||
mu sync.Mutex
|
||||
stop chan struct{}
|
||||
done chan struct{}
|
||||
store *Store
|
||||
mu sync.Mutex
|
||||
stop chan struct{}
|
||||
done chan struct{}
|
||||
stopping bool
|
||||
}
|
||||
|
||||
func NewCleanupRunner(store *Store) *CleanupRunner {
|
||||
@@ -25,31 +25,28 @@ func (r *CleanupRunner) Start(interval time.Duration) error {
|
||||
if r == nil || r.store == nil || interval <= 0 {
|
||||
return ErrInvalidChallenge
|
||||
}
|
||||
started := false
|
||||
r.once.Do(func() {
|
||||
started = true
|
||||
r.mu.Lock()
|
||||
r.stop = make(chan struct{})
|
||||
r.done = make(chan struct{})
|
||||
stop, done := r.stop, r.done
|
||||
r.mu.Lock()
|
||||
if r.stop != nil || r.stopping {
|
||||
r.mu.Unlock()
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
defer close(done)
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
r.store.Cleanup()
|
||||
case <-stop:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
if !started {
|
||||
return ErrCleanupAlreadyStarted
|
||||
}
|
||||
r.stop = make(chan struct{})
|
||||
r.done = make(chan struct{})
|
||||
stop, done := r.stop, r.done
|
||||
r.mu.Unlock()
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
defer close(done)
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
r.store.Cleanup()
|
||||
case <-stop:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -59,10 +56,17 @@ func (r *CleanupRunner) Stop() {
|
||||
}
|
||||
r.mu.Lock()
|
||||
stop, done := r.stop, r.done
|
||||
r.stop, r.done = nil, nil
|
||||
if stop == nil || r.stopping {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
r.stopping = true
|
||||
r.mu.Unlock()
|
||||
if stop != nil {
|
||||
close(stop)
|
||||
<-done
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.stop, r.done, r.stopping = nil, nil, false
|
||||
r.mu.Unlock()
|
||||
}
|
||||
@@ -19,3 +19,19 @@ func TestCleanupRunnerHasSingleInstance(t *testing.T) {
|
||||
t.Fatalf("second cleanup runner start error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupRunnerCanRestartAfterStop(t *testing.T) {
|
||||
store, err := NewStore(4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner := NewCleanupRunner(store)
|
||||
if err := runner.Start(time.Hour); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner.Stop()
|
||||
if err := runner.Start(time.Hour); err != nil {
|
||||
t.Fatalf("cleanup runner restart error = %v", err)
|
||||
}
|
||||
runner.Stop()
|
||||
}
|
||||
@@ -13,7 +13,8 @@ type OperationalCleanupRunner struct {
|
||||
store *EventStore
|
||||
rateLimits *PersistentRateLimiter
|
||||
interval time.Duration
|
||||
once sync.Once
|
||||
running bool
|
||||
stopping bool
|
||||
stop chan struct{}
|
||||
done chan struct{}
|
||||
now func() time.Time
|
||||
@@ -31,29 +32,32 @@ func (r *OperationalCleanupRunner) Start() {
|
||||
if r == nil || r.store == nil {
|
||||
return
|
||||
}
|
||||
r.once.Do(func() {
|
||||
r.mu.Lock()
|
||||
r.stop = make(chan struct{})
|
||||
r.done = make(chan struct{})
|
||||
stop, done := r.stop, r.done
|
||||
interval := r.interval
|
||||
r.mu.Lock()
|
||||
if r.running || r.stopping {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
r.running = true
|
||||
r.stop = make(chan struct{})
|
||||
r.done = make(chan struct{})
|
||||
stop, done := r.stop, r.done
|
||||
interval := r.interval
|
||||
r.mu.Unlock()
|
||||
|
||||
go func() {
|
||||
defer close(done)
|
||||
r.runOnce()
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
r.runOnce()
|
||||
case <-stop:
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer close(done)
|
||||
r.runOnce()
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
r.runOnce()
|
||||
case <-stop:
|
||||
return
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (r *OperationalCleanupRunner) Stop() {
|
||||
@@ -66,11 +70,15 @@ func (r *OperationalCleanupRunner) Stop() {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
r.stop, r.done = nil, nil
|
||||
r.stop, r.done, r.running = nil, nil, false
|
||||
r.stopping = true
|
||||
r.mu.Unlock()
|
||||
|
||||
close(stop)
|
||||
<-done
|
||||
r.mu.Lock()
|
||||
r.stop, r.done, r.stopping = nil, nil, false
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
func (r *OperationalCleanupRunner) runOnce() {
|
||||
|
||||
@@ -53,6 +53,20 @@ func TestCleanupLeaseIsSingleInstanceAndHasPersistedSchedule(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOperationalCleanupRunnerCanRestartAfterStop(t *testing.T) {
|
||||
engine := newCleanupEngine(t)
|
||||
defer engine.Close()
|
||||
store, err := NewEventStore(engine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner := NewOperationalCleanupRunner(store)
|
||||
runner.Start()
|
||||
runner.Stop()
|
||||
runner.Start()
|
||||
runner.Stop()
|
||||
}
|
||||
|
||||
func TestCleanupFailureSchedulesSixHourRetry(t *testing.T) {
|
||||
engine := newCleanupEngine(t)
|
||||
defer engine.Close()
|
||||
|
||||
Reference in new issue
Block a user