完善配置安全校验与数据库迁移处理
Security Checks / security (push) Waiting to run

This commit is contained in:
xuans committed 2026-08-30 17:45:52 +08:00
1 parent 3af1fc4147
commit 29cfebd620
36 files changed
+2838 -118

No files matched your search

+2
View File
@@ -35,6 +35,8 @@ type Config struct {
SpamFilterLevel int `json:"spamFilterLevel"` //垃圾邮件过滤级别,0不过滤、1 spf dkim 校验均失败时过滤,2 spf校验不通过时过滤 3,dkim 校验不过的时候过滤
HttpPort int `json:"httpPort"` //http服务端口设置,默认80
HttpsPort int `json:"httpsPort"` //https服务端口,默认443
TLSMode string `json:"tlsMode,omitempty"`
TrustedProxyCIDRs []string `json:"trustedProxyCidrs,omitempty"`
WeChatPushAppId string `json:"weChatPushAppId"`
WeChatPushSecret string `json:"weChatPushSecret"`
WeChatPushTemplateId string `json:"weChatPushTemplateId"`
+112
View File
@@ -0,0 +1,112 @@
package config
import (
"context"
"errors"
"time"
)
var ErrDatabaseMigrationTarget = errors.New("database migration target is invalid")
// SwitchDatabase commits only the database name or SQLite local reference.
// It deliberately does not open a connection or restart the service; the
// caller must complete the physical migration before invoking this method.
func (m *ConfigManager) SwitchDatabase(ctx context.Context, expectedGeneration uint64, target DatabaseConfig, actor AuditActor) error {
if err := contextError(ctx); err != nil {
return err
}
if m == nil || expectedGeneration == 0 {
return ErrManagerStorage
}
if actor.UserID == "" || actor.SessionID == "" {
return ErrAuthorizationRequired
}
if err := m.authorize(ctx, actor, ConfigOperationMigration, SectionRoot); err != nil {
return err
}
m.writeMu.Lock()
defer m.writeMu.Unlock()
m.mu.RLock()
current := cloneRuntimeConfig(m.current)
pending := m.pending != nil
m.mu.RUnlock()
if pending || expectedGeneration != current.Generation {
return ErrGenerationConflict
}
target = NormalizeDatabaseConfig(target)
if err := ValidateDatabaseConfig(target); err != nil {
return ErrDatabaseMigrationTarget
}
if !sameDatabaseConnection(current.Root.Database, target) {
return ErrDatabaseMigrationTarget
}
if target.Type == DatabaseTypeSQLite {
if target.LocalRef == current.Root.Database.LocalRef {
return ErrDatabaseMigrationTarget
}
} else if target.Name == current.Root.Database.Name {
return ErrDatabaseMigrationTarget
}
candidate := cloneRuntimeConfig(current)
candidate.Root.Database = target
setCandidateGeneration(&candidate, current.Generation+1)
if err := ValidateRuntimeConfig(candidate, m.options); err != nil {
return err
}
if err := ValidateRuntimeSecrets(m.vault, candidate); err != nil {
return err
}
if _, _, err := m.transactions.CreateBackupIfDue(current, time.Now().UTC()); err != nil {
return err
}
if err := m.transactions.Commit(current, candidate); err != nil {
return err
}
if err := m.snapshots.Publish(candidate); err != nil {
return err
}
now := time.Now().UTC()
m.mu.Lock()
m.current = cloneRuntimeConfig(candidate)
m.pending = nil
m.status = ConfigStatus{
SetupState: candidate.SetupState,
ActiveGeneration: candidate.Generation,
LastValidAt: &now,
PendingRestart: []ServiceName{ServiceDatabase},
}
m.mu.Unlock()
changedField := "root.database.name"
if target.Type == DatabaseTypeSQLite {
changedField = "root.database.localRef"
}
return m.writeAudit(ctx, ConfigAuditEvent{
Operation: ConfigOperationMigration,
Section: SectionRoot,
ActorID: actor.UserID,
ActorRole: actor.Role,
FromGeneration: current.Generation,
ToGeneration: candidate.Generation,
TargetGeneration: candidate.Generation,
ChangedFields: []string{changedField},
Result: "success",
CreatedAt: now,
})
}
func sameDatabaseConnection(current, target DatabaseConfig) bool {
if current.Type != target.Type ||
current.Host != target.Host ||
current.Port != target.Port ||
current.Username != target.Username ||
current.Password != target.Password {
return false
}
if current.Type == DatabaseTypeSQLite {
return true
}
return current.LocalRef == target.LocalRef
}
+71
View File
@@ -0,0 +1,71 @@
package config
import (
"context"
"errors"
"testing"
)
func TestConfigManagerSwitchDatabaseCommitsSQLiteReference(t *testing.T) {
root := t.TempDir()
writeRuntimeFixture(t, root, validRuntimeConfig())
manager, err := NewConfigManager(root, ConfigLoaderOptions{Validation: RuntimeValidationOptions{HasSuperAdmin: true, Production: true}})
if err != nil {
t.Fatal(err)
}
target := manager.Snapshot().Root.Database
target.LocalRef = "smail-next.db"
if err := manager.SwitchDatabase(context.Background(), 1, target, authorizedConfigActor()); err != nil {
t.Fatal(err)
}
if got := manager.Snapshot(); got.Generation != 2 || got.Root.Database.LocalRef != "smail-next.db" {
t.Fatalf("switched snapshot = %+v", got)
}
status := manager.Status()
if len(status.PendingRestart) != 1 || status.PendingRestart[0] != ServiceDatabase {
t.Fatalf("pending restart services = %+v", status.PendingRestart)
}
audit := manager.security.Audit.(*FileAuditWriter)
events, err := audit.Query(context.Background(), ConfigAuditQuery{Operation: ConfigOperationMigration, Limit: 10})
if err != nil {
t.Fatal(err)
}
if len(events) != 1 || events[0].ChangedFields[0] != "root.database.localRef" {
t.Fatalf("migration audit events = %+v", events)
}
}
func TestConfigManagerSwitchDatabaseRejectsImmutableChangesAndStaleGeneration(t *testing.T) {
root := t.TempDir()
writeRuntimeFixture(t, root, validRuntimeConfig())
manager, err := NewConfigManager(root, ConfigLoaderOptions{Validation: RuntimeValidationOptions{HasSuperAdmin: true, Production: true}})
if err != nil {
t.Fatal(err)
}
target := manager.Snapshot().Root.Database
target.LocalRef = "next.db"
target.Host = "127.0.0.1"
if err := manager.SwitchDatabase(context.Background(), 1, target, authorizedConfigActor()); !errors.Is(err, ErrDatabaseMigrationTarget) {
t.Fatalf("immutable target error = %v, want invalid migration target", err)
}
target = manager.Snapshot().Root.Database
target.LocalRef = "next.db"
if err := manager.SwitchDatabase(context.Background(), 2, target, authorizedConfigActor()); !errors.Is(err, ErrGenerationConflict) {
t.Fatalf("stale generation error = %v, want generation conflict", err)
}
}
func TestConfigManagerSwitchDatabaseRequiresAuthorizedActor(t *testing.T) {
root := t.TempDir()
writeRuntimeFixture(t, root, validRuntimeConfig())
manager, err := NewConfigManager(root, ConfigLoaderOptions{Validation: RuntimeValidationOptions{HasSuperAdmin: true, Production: true}})
if err != nil {
t.Fatal(err)
}
target := manager.Snapshot().Root.Database
target.LocalRef = "next.db"
if err := manager.SwitchDatabase(context.Background(), 1, target, AuditActor{}); !errors.Is(err, ErrAuthorizationRequired) {
t.Fatalf("missing actor error = %v, want authorization required", err)
}
}
+1
View File
@@ -119,6 +119,7 @@ func cloneRootConfig(source RootConfig) RootConfig {
clone.Security.WebAuthnRPOrigins = append([]string(nil), source.Security.WebAuthnRPOrigins...)
clone.Domains.Accepted = append([]string(nil), source.Domains.Accepted...)
clone.Domains.AccessHosts = append([]string(nil), source.Domains.AccessHosts...)
clone.HTTP.TrustedProxyCIDRs = append([]string(nil), source.HTTP.TrustedProxyCIDRs...)
if source.Sections != nil {
clone.Sections = make(map[Section]SectionRef, len(source.Sections))
for section, reference := range source.Sections {
+15 -2
View File
@@ -59,6 +59,8 @@ type legacyMigrationConfig struct {
SpamFilterLevel int `json:"spamFilterLevel"`
HttpPort int `json:"httpPort"`
HttpsPort int `json:"httpsPort"`
TLSMode string `json:"tlsMode"`
TrustedProxyCIDRs []string `json:"trustedProxyCidrs"`
WeChatPushAppID string `json:"weChatPushAppId"`
WeChatPushSecret string `json:"weChatPushSecret"`
WeChatPushTemplateID string `json:"weChatPushTemplateId"`
@@ -86,7 +88,7 @@ type legacyMigrationConfig struct {
var legacyMigrationKnownFields = map[string]struct{}{
"logLevel": {}, "domain": {}, "domains": {}, "webDomain": {}, "accessHosts": {},
"dkimPrivateKeyPath": {}, "sslType": {}, "SSLPrivateKeyPath": {}, "SSLPublicKeyPath": {},
"dbDSN": {}, "dbType": {}, "httpsEnabled": {}, "spamFilterLevel": {}, "httpPort": {}, "httpsPort": {},
"dbDSN": {}, "dbType": {}, "httpsEnabled": {}, "spamFilterLevel": {}, "httpPort": {}, "httpsPort": {}, "tlsMode": {}, "trustedProxyCidrs": {},
"weChatPushAppId": {}, "weChatPushSecret": {}, "weChatPushTemplateId": {}, "weChatPushUserId": {},
"tgBotToken": {}, "tgChatId": {}, "isInit": {}, "setupState": {}, "webPushUrl": {}, "webPushToken": {},
"smtpHost": {}, "smtpPort": {}, "smtpUsername": {}, "smtpPassword": {}, "smtpSecurity": {}, "smtpFrom": {},
@@ -454,6 +456,11 @@ func legacyRuntimeConfig(root string, legacy legacyMigrationConfig) (RuntimeConf
}
httpsEnabled := legacy.HttpsEnabled != 2
httpsPort := legacy.HttpsPort
tlsMode := strings.ToLower(strings.TrimSpace(legacy.TLSMode))
if tlsMode == HTTPSTLSModeProxy {
httpsEnabled = false
httpsPort = 0
}
if httpsEnabled && httpsPort == 0 {
httpsPort = 443
}
@@ -485,7 +492,13 @@ func legacyRuntimeConfig(root string, legacy legacyMigrationConfig) (RuntimeConf
Web: legacy.WebDomain,
AccessHosts: append([]string(nil), legacy.AccessHosts...),
},
HTTP: HTTPConfig{Port: legacy.HttpPort, HTTPSPort: httpsPort, HTTPSEnabled: httpsEnabled},
HTTP: HTTPConfig{
Port: legacy.HttpPort,
HTTPSPort: httpsPort,
HTTPSEnabled: httpsEnabled,
TLSMode: tlsMode,
TrustedProxyCIDRs: append([]string(nil), legacy.TrustedProxyCIDRs...),
},
Security: SecurityConfig{PasswordVersion: "argon2id", EncryptionKeyRef: "credential.key"},
UpdatedAt: time.Now().UTC(),
}
+50 -1
View File
@@ -1,6 +1,9 @@
package config
import "sync/atomic"
import (
"strings"
"sync/atomic"
)
// publishedConfig 是旧配置模型的只读兼容快照。
// 新业务应通过 CurrentConfig 读取,禁止在请求处理中直接持有并修改 Instance。
@@ -43,6 +46,45 @@ func CurrentConfig() *Config {
return cloneConfig(Instance)
}
// CurrentHTTPConfig 返回当前进程实际使用的 HTTP 配置。
// 分区配置存在时优先读取分区配置,旧版配置仅作为兼容回退。
func CurrentHTTPConfig() HTTPConfig {
if manager := CurrentManager(); manager != nil {
runtime := manager.Snapshot()
if runtime.Generation != 0 {
return cloneHTTPConfig(runtime.Root.HTTP)
}
}
cfg := CurrentConfig()
if cfg == nil {
return HTTPConfig{}
}
mode := strings.ToLower(strings.TrimSpace(cfg.TLSMode))
return HTTPConfig{
Port: cfg.HttpPort,
HTTPSPort: cfg.HttpsPort,
HTTPSEnabled: cfg.HttpsEnabled != 2,
TLSMode: mode,
TrustedProxyCIDRs: append([]string(nil), cfg.TrustedProxyCIDRs...),
}
}
// CurrentWebDomain 返回当前进程实际使用的 Web 域名。
// 分区配置存在时优先读取分区配置,旧版配置仅作为兼容回退。
func CurrentWebDomain() string {
if manager := CurrentManager(); manager != nil {
runtime := manager.Snapshot()
if runtime.Generation != 0 {
return runtime.Root.Domains.Web
}
}
cfg := CurrentConfig()
if cfg == nil {
return ""
}
return cfg.WebDomain
}
// SetRuntimeSetupPort stores the temporary setup listener port outside the persisted configuration.
func SetRuntimeSetupPort(port int) {
setupPort.Store(int64(port))
@@ -59,6 +101,7 @@ func cloneConfig(source *Config) *Config {
clone := *source
clone.Domains = append([]string(nil), source.Domains...)
clone.AccessHosts = append([]string(nil), source.AccessHosts...)
clone.TrustedProxyCIDRs = append([]string(nil), source.TrustedProxyCIDRs...)
clone.PluginAllowList = append([]string(nil), source.PluginAllowList...)
if source.Tables != nil {
clone.Tables = make(map[string]string, len(source.Tables))
@@ -74,3 +117,9 @@ func cloneConfig(source *Config) *Config {
}
return &clone
}
func cloneHTTPConfig(source HTTPConfig) HTTPConfig {
clone := source
clone.TrustedProxyCIDRs = append([]string(nil), source.TrustedProxyCIDRs...)
return clone
}
+1 -1
View File
@@ -72,7 +72,7 @@ func ValidateRuntimeConfig(runtime RuntimeConfig, options RuntimeValidationOptio
return ErrInvalidRuntimeConfig
}
}
if options.Production && runtime.SetupState == SetupStateDegraded && !runtime.Root.HTTP.HTTPSEnabled {
if options.Production && runtime.SetupState == SetupStateDegraded && !runtime.Root.HTTP.HTTPSEnabled && runtime.Root.HTTP.TLSMode != HTTPSTLSModeProxy {
return ErrInvalidRuntimeConfig
}
return nil
+32
View File
@@ -118,3 +118,35 @@ func TestProductionDegradedRuntimeRequiresHTTPS(t *testing.T) {
t.Fatal(err)
}
}
func TestValidateHTTPProxyMode(t *testing.T) {
runtime := validRuntimeConfig()
runtime.Root.HTTP = HTTPConfig{
Port: 15184,
TLSMode: HTTPSTLSModeProxy,
TrustedProxyCIDRs: []string{"127.0.0.1/32", "::1/128"},
}
if err := ValidateRuntimeConfig(runtime, RuntimeValidationOptions{HasSuperAdmin: true, Production: true}); err != nil {
t.Fatalf("proxy mode was rejected: %v", err)
}
invalid := runtime
invalid.Root.HTTP.HTTPSPort = 443
if err := ValidateRuntimeConfig(invalid, RuntimeValidationOptions{HasSuperAdmin: true, Production: true}); !errors.Is(err, ErrInvalidHTTP) {
t.Fatalf("proxy mode with HTTPS port returned %v, want %v", err, ErrInvalidHTTP)
}
}
func TestValidateHTTPRejectsInvalidProxySettings(t *testing.T) {
cases := []HTTPConfig{
{Port: 15184, TLSMode: "unsupported"},
{Port: 15184, TLSMode: HTTPSTLSModeProxy, HTTPSEnabled: true},
{Port: 15184, TLSMode: HTTPSTLSModeProxy, TrustedProxyCIDRs: []string{"0.0.0.0"}},
{Port: 15184, TLSMode: HTTPSTLSModeProxy, TrustedProxyCIDRs: []string{" 127.0.0.1/32"}},
}
for index, candidate := range cases {
if err := validateHTTP(candidate); !errors.Is(err, ErrInvalidHTTP) {
t.Fatalf("case %d returned %v, want %v", index, err, ErrInvalidHTTP)
}
}
}
+10 -3
View File
@@ -41,11 +41,18 @@ type DomainConfig struct {
}
type HTTPConfig struct {
Port int `json:"port,omitempty"`
HTTPSPort int `json:"httpsPort,omitempty"`
HTTPSEnabled bool `json:"httpsEnabled"`
Port int `json:"port,omitempty"`
HTTPSPort int `json:"httpsPort,omitempty"`
HTTPSEnabled bool `json:"httpsEnabled"`
TLSMode string `json:"tlsMode,omitempty"`
TrustedProxyCIDRs []string `json:"trustedProxyCidrs,omitempty"`
}
const (
HTTPSTLSModeDirect = "direct"
HTTPSTLSModeProxy = "proxy"
)
type SMTPSectionConfig struct {
Meta SectionMeta `json:"meta"`
Relay SMTPConfig `json:"relay"`
+2
View File
@@ -144,6 +144,8 @@ func DefaultConfigAuthorization(_ context.Context, actor AuditActor, operation C
capability := access.CapabilityConfigWriteGlobal
if operation == ConfigOperationRollback {
capability = access.CapabilityConfigRollback
} else if operation == ConfigOperationMigration {
capability = access.CapabilityDatabaseMigrate
}
request := access.Request{
ActorRole: access.Role(actor.Role),
+15 -1
View File
@@ -1,4 +1,4 @@
package config
package config
import (
"errors"
@@ -199,6 +199,20 @@ func validateHTTP(http HTTPConfig) error {
if http.HTTPSPort != 0 && !validPort(http.HTTPSPort) {
return ErrInvalidHTTP
}
if http.TLSMode != "" && http.TLSMode != HTTPSTLSModeDirect && http.TLSMode != HTTPSTLSModeProxy {
return ErrInvalidHTTP
}
if http.TLSMode == HTTPSTLSModeProxy && (http.HTTPSEnabled || http.HTTPSPort != 0) {
return ErrInvalidHTTP
}
for _, cidr := range http.TrustedProxyCIDRs {
if strings.TrimSpace(cidr) != cidr {
return ErrInvalidHTTP
}
if _, _, err := net.ParseCIDR(cidr); err != nil {
return ErrInvalidHTTP
}
}
if http.HTTPSEnabled && http.HTTPSPort == 0 {
return ErrInvalidHTTP
}
+41 -1
View File
@@ -93,6 +93,46 @@ func configErrorDetails(err error) (string, int, string, map[string]string) {
code = "DATABASE_MIGRATION_UNAVAILABLE"
status = http.StatusServiceUnavailable
message = "数据库迁移执行器暂未配置,请先完成服务部署"
case errors.Is(err, dbmigration.ErrSwitchUnavailable):
code = "DATABASE_MIGRATION_SWITCH_UNAVAILABLE"
status = http.StatusServiceUnavailable
message = "数据库迁移已完成校验,但配置切换通道暂不可用,当前配置未切换"
case errors.Is(err, dbmigration.ErrSwitchOutcomeUnknown):
code = "SWITCH_OUTCOME_UNKNOWN"
status = http.StatusServiceUnavailable
message = "数据库迁移配置切换结果暂不确定,请先确认当前配置和目标数据库状态"
case errors.Is(err, dbmigration.ErrTargetExists):
code = "DATABASE_MIGRATION_TARGET_EXISTS"
status = http.StatusConflict
message = "目标数据库已存在,请更换数据库名称"
case errors.Is(err, dbmigration.ErrSQLTargetExists):
code = "DATABASE_MIGRATION_TARGET_EXISTS"
status = http.StatusConflict
message = "目标数据库已存在,请更换数据库名称"
case errors.Is(err, dbmigration.ErrSQLCopyUnavailable):
code = "DATABASE_MIGRATION_COPIER_UNAVAILABLE"
status = http.StatusServiceUnavailable
message = "数据库迁移复制器暂不可用,请先完成服务部署"
case errors.Is(err, dbmigration.ErrSQLConnectionUnavailable):
code = "DATABASE_MIGRATION_CONNECTION_UNAVAILABLE"
status = http.StatusServiceUnavailable
message = "数据库连接暂不可用,请检查数据库配置和服务状态"
case errors.Is(err, dbmigration.ErrSQLUnsupportedObject):
code = "DATABASE_MIGRATION_UNSUPPORTED_OBJECT"
status = http.StatusUnprocessableEntity
message = "源数据库包含暂不支持迁移的对象,请先处理后重试"
case errors.Is(err, dbmigration.ErrSQLDataVerification):
code = "SQL_DATA_VERIFICATION_FAILED"
status = http.StatusConflict
message = "数据库迁移数据校验未通过,源数据库可能在迁移期间发生变化"
case errors.Is(err, dbmigration.ErrSQLSourceBusy):
code = "SQL_SOURCE_BUSY"
status = http.StatusConflict
message = "源数据库存在活动连接,请停止相关连接后重试迁移"
case errors.Is(err, dbmigration.ErrUnsupportedDatabase):
code = "DATABASE_MIGRATION_UNSUPPORTED"
status = http.StatusUnprocessableEntity
message = "当前数据库类型暂不支持在线迁移"
case errors.Is(err, dbmigration.ErrMigrationState):
code = "DATABASE_MIGRATION_STATE_INVALID"
status = http.StatusConflict
@@ -101,7 +141,7 @@ func configErrorDetails(err error) (string, int, string, map[string]string) {
code = "DATABASE_MIGRATION_PREVIEW_INVALID"
status = http.StatusForbidden
message = "数据库迁移预览票据无效"
case errors.Is(err, dbmigration.ErrInvalidRequest), errors.Is(err, dbmigration.ErrInvalidTargetName), errors.Is(err, dbmigration.ErrImmutableSetting), errors.Is(err, dbmigration.ErrSourceDatabase):
case errors.Is(err, config.ErrDatabaseMigrationTarget), errors.Is(err, dbmigration.ErrInvalidRequest), errors.Is(err, dbmigration.ErrInvalidTargetName), errors.Is(err, dbmigration.ErrImmutableSetting), errors.Is(err, dbmigration.ErrSourceDatabase):
code = "DATABASE_MIGRATION_VALIDATION_FAILED"
status = http.StatusUnprocessableEntity
message = "数据库迁移参数校验未通过"
+77
View File
@@ -10,6 +10,7 @@ import (
"git.pcux.cn/xuans/Smail/dto/response"
"git.pcux.cn/xuans/Smail/services/access"
"git.pcux.cn/xuans/Smail/services/dbmigration"
)
func TestWriteConfigErrorKeepsStableAuthorizationCode(t *testing.T) {
@@ -90,3 +91,79 @@ func TestWriteConfigErrorUsesStableCodeAndHTTPStatus(t *testing.T) {
t.Fatalf("configuration error message is not a Chinese public message: %q", payload.ErrorMsg)
}
}
func TestConfigErrorDetailsMapsDatabaseMigrationFailures(t *testing.T) {
tests := []struct {
name string
err error
code string
status int
message string
}{
{
name: "invalid target",
err: config.ErrDatabaseMigrationTarget,
code: "DATABASE_MIGRATION_VALIDATION_FAILED",
status: 422,
message: "数据库迁移参数校验未通过",
},
{
name: "structured target exists",
err: dbmigration.ErrSQLTargetExists,
code: "DATABASE_MIGRATION_TARGET_EXISTS",
status: 409,
message: "目标数据库已存在,请更换数据库名称",
},
{
name: "copier unavailable",
err: dbmigration.ErrSQLCopyUnavailable,
code: "DATABASE_MIGRATION_COPIER_UNAVAILABLE",
status: 503,
message: "数据库迁移复制器暂不可用,请先完成服务部署",
},
{
name: "connection unavailable",
err: dbmigration.ErrSQLConnectionUnavailable,
code: "DATABASE_MIGRATION_CONNECTION_UNAVAILABLE",
status: 503,
message: "数据库连接暂不可用,请检查数据库配置和服务状态",
},
{
name: "unsupported object",
err: dbmigration.ErrSQLUnsupportedObject,
code: "DATABASE_MIGRATION_UNSUPPORTED_OBJECT",
status: 422,
message: "源数据库包含暂不支持迁移的对象,请先处理后重试",
},
{
name: "data verification failed",
err: dbmigration.ErrSQLDataVerification,
code: "SQL_DATA_VERIFICATION_FAILED",
status: 409,
message: "数据库迁移数据校验未通过,源数据库可能在迁移期间发生变化",
},
{
name: "source busy",
err: dbmigration.ErrSQLSourceBusy,
code: "SQL_SOURCE_BUSY",
status: 409,
message: "源数据库存在活动连接,请停止相关连接后重试迁移",
},
{
name: "switch outcome unknown",
err: dbmigration.ErrSwitchOutcomeUnknown,
code: "SWITCH_OUTCOME_UNKNOWN",
status: 503,
message: "数据库迁移配置切换结果暂不确定,请先确认当前配置和目标数据库状态",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
code, status, message, fields := configErrorDetails(test.err)
if code != test.code || status != test.status || message != test.message || fields != nil {
t.Fatalf("config error details = (%q, %d, %q, %#v), want (%q, %d, %q, nil)", code, status, message, fields, test.code, test.status, test.message)
}
})
}
}
+1 -1
View File
@@ -69,7 +69,7 @@ func ExecuteDatabaseMigration(ctx *appcontext.Context, w http.ResponseWriter, re
writeConfigError(w, dbmigration.ErrInvalidRequest)
return
}
record, err := dbmigration.Current().Execute(req.Context(), input.ID, input.Token)
record, err := dbmigration.Current().ExecuteAuthorized(req.Context(), input.ID, input.Token, configActor(ctx, req))
if err != nil {
writeConfigError(w, err)
return
+32 -21
View File
@@ -246,31 +246,23 @@ func allowRouteMethod(w http.ResponseWriter, r *http.Request, allowed ...string)
}
func HttpStart() error {
cfg := config.CurrentConfig()
if cfg == nil {
if config.CurrentConfig() == nil && config.CurrentManager() == nil {
return fmt.Errorf("runtime configuration is unavailable")
}
mux := http.NewServeMux()
cfg := config.CurrentHTTPConfig()
httpPort := 80
if cfg.HttpPort > 0 {
httpPort = cfg.HttpPort
if cfg.Port > 0 {
httpPort = cfg.Port
}
var handler http.Handler = mux
if config.DevelopmentMode && cfg.HttpsEnabled == 2 {
if err := router(mux); err != nil {
return err
}
handler = session.Instance.LoadAndSave(mux)
} else {
mux.HandleFunc("/.well-known/", controllers.AcmeChallenge)
mux.HandleFunc("/api/ping", controllers.Ping)
mux.HandleFunc("/", redirectToHTTPS)
handler, err := newHTTPHandler(cfg)
if err != nil {
return err
}
httpServer = &http.Server{
Addr: fmt.Sprintf(":%d", httpPort),
Handler: securityHeaders(handler),
Handler: handler,
ReadTimeout: time.Second * 90,
ReadHeaderTimeout: time.Second * 10,
WriteTimeout: time.Second * 90,
@@ -284,14 +276,32 @@ func HttpStart() error {
return nil
}
func newHTTPHandler(cfg config.HTTPConfig) (http.Handler, error) {
mux := http.NewServeMux()
var handler http.Handler = mux
if cfg.TLSMode == config.HTTPSTLSModeProxy || (config.DevelopmentMode && !cfg.HTTPSEnabled) {
if err := router(mux); err != nil {
return nil, err
}
if session.Instance == nil {
return nil, fmt.Errorf("session manager is unavailable")
}
handler = session.Instance.LoadAndSave(mux)
} else {
mux.HandleFunc("/.well-known/", controllers.AcmeChallenge)
mux.HandleFunc("/api/ping", controllers.Ping)
mux.HandleFunc("/", redirectToHTTPS)
}
return securityHeaders(handler), nil
}
func redirectToHTTPS(w http.ResponseWriter, r *http.Request) {
cfg := config.CurrentConfig()
if cfg == nil {
if config.CurrentConfig() == nil && config.CurrentManager() == nil {
w.Header().Set("Connection", "close")
http.Error(w, "HTTPS is required", http.StatusUpgradeRequired)
return
}
host := strings.TrimSpace(cfg.WebDomain)
host := strings.TrimSpace(config.CurrentWebDomain())
if host == "" {
w.Header().Set("Connection", "close")
http.Error(w, "HTTPS is required", http.StatusUpgradeRequired)
@@ -308,8 +318,9 @@ func redirectToHTTPS(w http.ResponseWriter, r *http.Request) {
if strings.Contains(host, ":") {
host = "[" + host + "]"
}
if cfg.HttpsPort > 0 && cfg.HttpsPort != 443 {
host = fmt.Sprintf("%s:%d", host, cfg.HttpsPort)
httpsPort := config.CurrentHTTPConfig().HTTPSPort
if httpsPort > 0 && httpsPort != 443 {
host = fmt.Sprintf("%s:%d", host, httpsPort)
}
target := "https://" + host + r.URL.EscapedPath()
if r.URL.RawQuery != "" {
+65 -12
View File
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"mime"
"net"
"net/http"
"net/url"
"strconv"
@@ -39,10 +40,15 @@ func (w *nullWrite) Write(p []byte) (int, error) {
// HttpsStart 启动仅承载业务的 HTTPS 服务。
func HttpsStart() error {
cfg := config.CurrentConfig()
if cfg == nil {
legacyCfg := config.CurrentConfig()
if legacyCfg == nil && config.CurrentManager() == nil {
return fmt.Errorf("runtime configuration is unavailable")
}
httpCfg := config.CurrentHTTPConfig()
if httpCfg.TLSMode == config.HTTPSTLSModeProxy {
log.Info("HTTPS 使用反向代理终止模式,跳过 Smail 内部 HTTPS 监听")
return nil
}
mux := http.NewServeMux()
if err := router(mux); err != nil {
return err
@@ -50,14 +56,17 @@ func HttpsStart() error {
nullLog := olog.New(&nullWrite{}, "", olog.Ldate)
httpsPort := 443
if cfg.HttpsPort > 0 {
httpsPort = cfg.HttpsPort
if httpCfg.HTTPSPort > 0 {
httpsPort = httpCfg.HTTPSPort
}
if config.DevelopmentMode && cfg.HttpsEnabled == 2 {
if config.DevelopmentMode && legacyCfg != nil && legacyCfg.HttpsEnabled == 2 {
return nil
}
certificatePath, privateKeyPath, err := config.ResolveRuntimeSSLPaths(cfg)
if legacyCfg == nil {
return fmt.Errorf("legacy SSL configuration is unavailable")
}
certificatePath, privateKeyPath, err := config.ResolveRuntimeSSLPaths(legacyCfg)
if err != nil {
return fmt.Errorf("resolve HTTPS certificate: %w", err)
}
@@ -281,10 +290,9 @@ func sameOriginRequest(r *http.Request) bool {
return false
}
u, err := url.Parse(origin)
expectedScheme := "http"
expectedScheme := requestScheme(r)
defaultPort := "80"
if r.TLS != nil {
expectedScheme = "https"
if expectedScheme == "https" {
defaultPort = "443"
}
if err != nil || u.User != nil || u.Scheme != expectedScheme || u.Host == "" {
@@ -301,14 +309,59 @@ func sameOriginRequest(r *http.Request) bool {
if config.DevelopmentMode {
return true
}
cfg := config.CurrentConfig()
if cfg == nil {
if config.CurrentConfig() == nil && config.CurrentManager() == nil {
return false
}
configuredHost, _, ok := normalizeAuthority(strings.TrimSpace(cfg.WebDomain), defaultPort)
configuredHost, _, ok := normalizeAuthority(strings.TrimSpace(config.CurrentWebDomain()), defaultPort)
return ok && requestHost == configuredHost
}
func requestScheme(r *http.Request) string {
if r != nil && r.TLS != nil {
return "https"
}
if r == nil || config.CurrentHTTPConfig().TLSMode != config.HTTPSTLSModeProxy || !isTrustedProxyRequest(r) {
return "http"
}
forwarded := strings.TrimSpace(r.Header.Get("X-Forwarded-Proto"))
if strings.Contains(forwarded, ",") {
return "http"
}
switch strings.ToLower(forwarded) {
case "https":
return "https"
case "http", "":
return "http"
default:
return "http"
}
}
func isTrustedProxyRequest(r *http.Request) bool {
if r == nil {
return false
}
remoteHost := strings.TrimSpace(r.RemoteAddr)
if host, _, err := net.SplitHostPort(remoteHost); err == nil {
remoteHost = host
}
remoteIP := net.ParseIP(strings.Trim(remoteHost, "[]"))
if remoteIP == nil {
return false
}
cfg := config.CurrentHTTPConfig()
if len(cfg.TrustedProxyCIDRs) == 0 {
return remoteIP.IsLoopback()
}
for _, cidr := range cfg.TrustedProxyCIDRs {
_, network, err := net.ParseCIDR(strings.TrimSpace(cidr))
if err == nil && network.Contains(remoteIP) {
return true
}
}
return false
}
func normalizeAuthority(value string, defaultPort string) (string, string, bool) {
parsed, err := url.Parse("//" + strings.TrimSpace(value))
if err != nil || parsed.User != nil || parsed.Hostname() == "" || parsed.Path != "" || parsed.RawQuery != "" || parsed.Fragment != "" {
@@ -179,6 +179,103 @@ func TestCSRFAllowsConfiguredHostOnNonStandardPort(t *testing.T) {
}
}
func TestSameOriginUsesTrustedForwardedProto(t *testing.T) {
oldConfig := config.Instance
oldManager := config.CurrentManager()
config.SetRuntimeManager(nil)
config.Instance = &config.Config{
WebDomain: "xuans.com",
TLSMode: config.HTTPSTLSModeProxy,
TrustedProxyCIDRs: []string{"127.0.0.1/32"},
}
t.Cleanup(func() {
config.Instance = oldConfig
config.SetRuntimeManager(oldManager)
})
request := httptest.NewRequest(http.MethodPost, "http://xuans.com/api/settings", nil)
request.Host = "xuans.com"
request.RemoteAddr = "127.0.0.1:15184"
request.Header.Set("X-Forwarded-Proto", "https")
request.Header.Set("Origin", "https://xuans.com")
if requestScheme(request) != "https" || !sameOriginRequest(request) {
t.Fatal("trusted HTTPS proxy request was rejected")
}
}
func TestSameOriginDoesNotTrustForwardedProtoFromUntrustedClient(t *testing.T) {
oldConfig := config.Instance
oldManager := config.CurrentManager()
config.SetRuntimeManager(nil)
config.Instance = &config.Config{
WebDomain: "xuans.com",
TLSMode: config.HTTPSTLSModeProxy,
TrustedProxyCIDRs: []string{"127.0.0.1/32"},
}
t.Cleanup(func() {
config.Instance = oldConfig
config.SetRuntimeManager(oldManager)
})
request := httptest.NewRequest(http.MethodPost, "http://xuans.com/api/settings", nil)
request.Host = "xuans.com"
request.RemoteAddr = "192.0.2.10:15184"
request.Header.Set("X-Forwarded-Proto", "https")
request.Header.Set("Origin", "https://xuans.com")
if requestScheme(request) != "http" || sameOriginRequest(request) {
t.Fatal("untrusted client controlled the forwarded protocol")
}
}
func TestSameOriginRejectsAmbiguousForwardedProto(t *testing.T) {
oldConfig := config.Instance
oldManager := config.CurrentManager()
config.SetRuntimeManager(nil)
config.Instance = &config.Config{
WebDomain: "xuans.com",
TLSMode: config.HTTPSTLSModeProxy,
TrustedProxyCIDRs: []string{"127.0.0.1/32"},
}
t.Cleanup(func() {
config.Instance = oldConfig
config.SetRuntimeManager(oldManager)
})
request := httptest.NewRequest(http.MethodPost, "http://xuans.com/api/settings", nil)
request.Host = "xuans.com"
request.RemoteAddr = "127.0.0.1:15184"
request.Header.Set("X-Forwarded-Proto", "https, http")
request.Header.Set("Origin", "https://xuans.com")
if requestScheme(request) != "http" || sameOriginRequest(request) {
t.Fatal("ambiguous forwarded protocol was trusted")
}
}
func TestHttpsStartSkipsListenerInProxyMode(t *testing.T) {
oldConfig := config.Instance
oldManager := config.CurrentManager()
config.SetRuntimeManager(nil)
config.Instance = &config.Config{
WebDomain: "xuans.com",
TLSMode: config.HTTPSTLSModeProxy,
HttpsEnabled: 1,
HttpsPort: 443,
}
httpsServer = nil
t.Cleanup(func() {
config.Instance = oldConfig
config.SetRuntimeManager(oldManager)
httpsServer = nil
})
if err := HttpsStart(); err != nil {
t.Fatalf("proxy mode started HTTPS listener: %v", err)
}
if httpsServer != nil {
t.Fatal("proxy mode created an internal HTTPS server")
}
}
func TestSecurityHeadersRespectPluginFrameBoundary(t *testing.T) {
oldConfig := config.Instance
defer func() { config.Instance = oldConfig }()
@@ -6,6 +6,8 @@ import (
"testing"
"git.pcux.cn/xuans/Smail/config"
"git.pcux.cn/xuans/Smail/session"
"github.com/alexedwards/scs/v2"
)
func TestRedirectToHTTPSPreservesPathQueryAndPort(t *testing.T) {
@@ -25,6 +27,51 @@ func TestRedirectToHTTPSPreservesPathQueryAndPort(t *testing.T) {
}
}
func TestProxyHTTPHandlerServesBusinessRoutesWithoutRedirect(t *testing.T) {
oldConfig := config.Instance
oldManager := config.CurrentManager()
oldSession := session.Instance
config.SetRuntimeManager(nil)
config.Instance = &config.Config{WebDomain: "xuans.com", TLSMode: config.HTTPSTLSModeProxy}
session.Instance = scs.New()
t.Cleanup(func() {
config.Instance = oldConfig
config.SetRuntimeManager(oldManager)
session.Instance = oldSession
})
handler, err := newHTTPHandler(config.CurrentHTTPConfig())
if err != nil {
t.Fatal(err)
}
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "http://xuans.com/", nil))
if recorder.Code == http.StatusPermanentRedirect || recorder.Code == http.StatusTemporaryRedirect {
t.Fatalf("proxy HTTP handler redirected the request: %d", recorder.Code)
}
}
func TestDirectHTTPHandlerKeepsHTTPSRedirect(t *testing.T) {
oldConfig := config.Instance
oldManager := config.CurrentManager()
config.SetRuntimeManager(nil)
config.Instance = &config.Config{WebDomain: "xuans.com", HttpsEnabled: 1, HttpsPort: 443}
t.Cleanup(func() {
config.Instance = oldConfig
config.SetRuntimeManager(oldManager)
})
handler, err := newHTTPHandler(config.CurrentHTTPConfig())
if err != nil {
t.Fatal(err)
}
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "http://xuans.com/inbox", nil))
if recorder.Code != http.StatusPermanentRedirect || recorder.Header().Get("Location") != "https://xuans.com/inbox" {
t.Fatalf("unexpected direct HTTP redirect: status=%d location=%q", recorder.Code, recorder.Header().Get("Location"))
}
}
func TestRedirectToHTTPSAllowsConfiguredIPHost(t *testing.T) {
old := config.Instance
config.Instance = &config.Config{WebDomain: "192.0.2.10", HttpsPort: 443}
+4 -6
View File
@@ -111,12 +111,10 @@ func (s *setupSecurity) bootstrap(next http.Handler) http.Handler {
http.Error(w, "forbidden", http.StatusForbidden)
return
}
if setupRemoteIP(r).IsLoopback() {
http.SetCookie(w, &http.Cookie{
Name: setupCookieName, Value: s.token, Path: "/api/",
HttpOnly: true, Secure: false, SameSite: http.SameSiteStrictMode,
})
}
http.SetCookie(w, &http.Cookie{
Name: setupCookieName, Value: s.token, Path: "/api/",
HttpOnly: true, Secure: r.TLS != nil, SameSite: http.SameSiteStrictMode,
})
next.ServeHTTP(w, r)
})
}
@@ -227,3 +227,41 @@ func TestSetupAPIRoutesBecomeInvalidAfterSetupCompletes(t *testing.T) {
}
}
}
func TestSetupSecurityBootstrapCookieForAllowedRemote(t *testing.T) {
const token = "0123456789abcdefghijklmnop"
oldToken := os.Getenv("SMAIL_SETUP_TOKEN")
oldCIDRs := os.Getenv("SMAIL_SETUP_ALLOWED_CIDRS")
oldInit := config.IsInit
if err := os.Setenv("SMAIL_SETUP_TOKEN", token); err != nil {
t.Fatal(err)
}
if err := os.Setenv("SMAIL_SETUP_ALLOWED_CIDRS", "192.168.2.0/24"); err != nil {
t.Fatal(err)
}
config.IsInit = false
t.Cleanup(func() {
config.IsInit = oldInit
_ = os.Setenv("SMAIL_SETUP_TOKEN", oldToken)
_ = os.Setenv("SMAIL_SETUP_ALLOWED_CIDRS", oldCIDRs)
})
security, err := newSetupSecurity()
if err != nil {
t.Fatal(err)
}
handler := security.bootstrap(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
request := httptest.NewRequest(http.MethodGet, "http://192.168.2.3/", nil)
request.RemoteAddr = "192.168.2.2:12345"
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusNoContent {
t.Fatalf("remote bootstrap returned %d", recorder.Code)
}
cookies := recorder.Result().Cookies()
if len(cookies) != 1 || cookies[0].Name != setupCookieName || cookies[0].Value != token || !cookies[0].HttpOnly || cookies[0].Secure || cookies[0].SameSite != http.SameSiteStrictMode || cookies[0].Path != "/api/" {
t.Fatalf("unsafe remote setup cookie: %+v", cookies)
}
}
+22
View File
@@ -20,6 +20,7 @@ import (
"git.pcux.cn/xuans/Smail/listen/smtp_server"
"git.pcux.cn/xuans/Smail/services/adminreset"
"git.pcux.cn/xuans/Smail/services/credentials"
"git.pcux.cn/xuans/Smail/services/dbmigration"
"git.pcux.cn/xuans/Smail/services/externalemail"
pluginservice "git.pcux.cn/xuans/Smail/services/plugin"
"git.pcux.cn/xuans/Smail/services/recovery"
@@ -49,6 +50,7 @@ func startConfigManager() error {
if _, err := os.Stat(rootConfig); err != nil {
if os.IsNotExist(err) {
config.SetRuntimeManager(nil)
dbmigration.SetCurrent(nil)
return nil
}
return err
@@ -66,6 +68,7 @@ func startConfigManager() error {
if !partitionFound {
// 未发现分区配置时继续使用旧版配置读取路径。
config.SetRuntimeManager(nil)
dbmigration.SetCurrent(nil)
return nil
}
manager, err := config.NewConfigManager(root, config.ConfigLoaderOptions{Validation: config.RuntimeValidationOptions{
@@ -76,11 +79,30 @@ func startConfigManager() error {
return fmt.Errorf("load partitioned configuration: %w", err)
}
config.SetRuntimeManager(manager)
authorizedSwitch := func(ctx context.Context, expectedGeneration uint64, target config.DatabaseConfig, actor config.AuditActor) error {
return manager.SwitchDatabase(ctx, expectedGeneration, target, actor)
}
sqliteExecutor := dbmigration.NewAuthorizedSQLiteExecutor(config.ROOT_PATH, authorizedSwitch)
var structuredExecutor dbmigration.Runner
if manager.Snapshot().Root.Database.Type != config.DatabaseTypeSQLite {
factory, factoryErr := dbmigration.NewRuntimeSQLConnectionFactory(config.ROOT_PATH)
if factoryErr != nil {
return fmt.Errorf("initialize structured database migration: %w", factoryErr)
}
structuredExecutor = dbmigration.NewSQLExecutor(dbmigration.NewSQLCopier(factory), authorizedSwitch)
}
dbmigration.SetCurrent(
dbmigration.NewService(
dbmigration.NewDatabaseExecutor(sqliteExecutor, structuredExecutor),
),
)
return nil
}
func stopConfigManager() {
config.SetRuntimeManager(nil)
dbmigration.SetCurrent(nil)
}
func startPluginConfigMigration() error {
hooks.SetPluginConfigService(nil)
+62 -9
View File
@@ -37,6 +37,9 @@ var (
ErrMigrationState = errors.New("database migration state is invalid")
ErrExecutorUnavailable = errors.New("database migration executor unavailable")
ErrSourceDatabase = errors.New("source database configuration is invalid")
ErrTargetExists = errors.New("database migration target already exists")
ErrSwitchUnavailable = errors.New("database migration configuration switch unavailable")
ErrUnsupportedDatabase = errors.New("database migration database type is unsupported")
)
const previewLifetime = 10 * time.Minute
@@ -54,8 +57,12 @@ type DatabaseIdentity struct {
}
type Plan struct {
Source DatabaseIdentity
Target DatabaseIdentity
Source DatabaseIdentity
Target DatabaseIdentity
sourceConfig config.DatabaseConfig
targetConfig config.DatabaseConfig
expectedGeneration uint64
actor config.AuditActor
}
type Record struct {
@@ -82,8 +89,10 @@ type Service struct {
type record struct {
Record
tokenHash [32]byte
expiresAt time.Time
tokenHash [32]byte
expiresAt time.Time
sourceConfig config.DatabaseConfig
targetConfig config.DatabaseConfig
}
var current struct {
@@ -131,7 +140,7 @@ func (s *Service) Preview(_ context.Context, source config.DatabaseConfig, curre
return Record{}, "", ErrInvalidRequest
}
now := s.currentTime()
item := &record{Record: Record{ID: id, Status: StatusPreviewed, Source: plan.Source, Target: plan.Target, CurrentGeneration: currentGeneration, CreatedAt: now, UpdatedAt: now}, tokenHash: sha256.Sum256([]byte(token)), expiresAt: now.Add(previewLifetime)}
item := &record{Record: Record{ID: id, Status: StatusPreviewed, Source: plan.Source, Target: plan.Target, CurrentGeneration: currentGeneration, CreatedAt: now, UpdatedAt: now}, tokenHash: sha256.Sum256([]byte(token)), expiresAt: now.Add(previewLifetime), sourceConfig: plan.sourceConfig, targetConfig: plan.targetConfig}
s.mu.Lock()
s.records[id] = item
s.mu.Unlock()
@@ -139,6 +148,19 @@ func (s *Service) Preview(_ context.Context, source config.DatabaseConfig, curre
}
func (s *Service) Execute(ctx context.Context, id, token string) (Record, error) {
return s.execute(ctx, id, token, config.AuditActor{})
}
// ExecuteAuthorized executes a migration while preserving the authenticated
// operator for the final configuration switch.
func (s *Service) ExecuteAuthorized(ctx context.Context, id, token string, actor config.AuditActor) (Record, error) {
if actor.UserID == "" || actor.SessionID == "" {
return Record{}, config.ErrAuthorizationRequired
}
return s.execute(ctx, id, token, actor)
}
func (s *Service) execute(ctx context.Context, id, token string, actor config.AuditActor) (Record, error) {
if s == nil || strings.TrimSpace(id) == "" || strings.TrimSpace(token) == "" {
return Record{}, ErrInvalidRequest
}
@@ -169,11 +191,11 @@ func (s *Service) Execute(ctx context.Context, id, token string) (Record, error)
item.tokenHash = [32]byte{}
item.Status = StatusCreated
item.UpdatedAt = now
plan := Plan{Source: item.Source, Target: item.Target}
plan := Plan{Source: item.Source, Target: item.Target, sourceConfig: item.sourceConfig, targetConfig: item.targetConfig, expectedGeneration: item.CurrentGeneration, actor: actor}
s.mu.Unlock()
if err := s.setStatus(id, StatusCopying); err != nil {
return Record{}, err
return s.fail(id, err)
}
if s.runner == nil {
return s.fail(id, ErrExecutorUnavailable)
@@ -182,7 +204,7 @@ func (s *Service) Execute(ctx context.Context, id, token string) (Record, error)
return s.fail(id, err)
}
if err := s.setStatus(id, StatusSucceeded); err != nil {
return Record{}, err
return s.fail(id, err)
}
return s.Get(id)
}
@@ -273,7 +295,18 @@ func buildPlan(source config.DatabaseConfig, targetName string) (Plan, error) {
if target.Type != source.Type || target.Host != source.Host || target.Port != source.Port || target.Username != source.Username {
return Plan{}, ErrImmutableSetting
}
return Plan{Source: identity, Target: target}, nil
if target.Name == source.Name && target.LocalRef == source.LocalRef {
return Plan{}, ErrInvalidTargetName
}
return Plan{Source: identity, Target: target, sourceConfig: source, targetConfig: func() config.DatabaseConfig {
candidate := source
if source.Type == config.DatabaseTypeSQLite {
candidate.LocalRef = targetName
} else {
candidate.Name = targetName
}
return candidate
}()}, nil
}
func randomID(prefix string) (string, error) {
@@ -294,6 +327,26 @@ func migrationErrorCode(err error) string {
return "EXECUTOR_UNAVAILABLE"
case errors.Is(err, ErrPreviewExpired):
return "PREVIEW_EXPIRED"
case errors.Is(err, ErrTargetExists):
return "TARGET_EXISTS"
case errors.Is(err, ErrSQLTargetExists):
return "TARGET_EXISTS"
case errors.Is(err, ErrSQLCopyUnavailable):
return "COPIER_UNAVAILABLE"
case errors.Is(err, ErrSQLConnectionUnavailable):
return "CONNECTION_UNAVAILABLE"
case errors.Is(err, ErrSQLUnsupportedObject):
return "UNSUPPORTED_OBJECT"
case errors.Is(err, ErrSQLDataVerification):
return "DATA_VERIFICATION_FAILED"
case errors.Is(err, ErrSQLSourceBusy):
return "SOURCE_BUSY"
case errors.Is(err, ErrSwitchUnavailable):
return "SWITCH_UNAVAILABLE"
case errors.Is(err, ErrSwitchOutcomeUnknown):
return "SWITCH_OUTCOME_UNKNOWN"
case errors.Is(err, ErrUnsupportedDatabase):
return "UNSUPPORTED_DATABASE"
default:
return "MIGRATION_FAILED"
}
@@ -2,12 +2,16 @@ package dbmigration
import (
"context"
"database/sql"
"errors"
"os"
"path/filepath"
"testing"
"time"
"git.pcux.cn/xuans/Smail/config"
"git.pcux.cn/xuans/Smail/utils/credential"
_ "modernc.org/sqlite"
)
type testRunner struct{}
@@ -92,6 +96,29 @@ func TestFailedExecutionKeepsSafeState(t *testing.T) {
}
}
func TestMigrationErrorCodeMapsExecutorFailures(t *testing.T) {
tests := []struct {
err error
code string
}{
{err: ErrSQLTargetExists, code: "TARGET_EXISTS"},
{err: ErrSQLCopyUnavailable, code: "COPIER_UNAVAILABLE"},
{err: ErrSQLConnectionUnavailable, code: "CONNECTION_UNAVAILABLE"},
{err: ErrSQLUnsupportedObject, code: "UNSUPPORTED_OBJECT"},
{err: ErrSQLDataVerification, code: "DATA_VERIFICATION_FAILED"},
{err: ErrSQLSourceBusy, code: "SOURCE_BUSY"},
{err: ErrSwitchUnavailable, code: "SWITCH_UNAVAILABLE"},
{err: ErrSwitchOutcomeUnknown, code: "SWITCH_OUTCOME_UNKNOWN"},
}
for _, test := range tests {
t.Run(test.code, func(t *testing.T) {
if got := migrationErrorCode(test.err); got != test.code {
t.Fatalf("migration error code = %q, want %q", got, test.code)
}
})
}
}
func TestSQLitePreviewChangesOnlyLocalReference(t *testing.T) {
service := NewService(testRunner{})
source := config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}
@@ -177,3 +204,52 @@ func TestMigrationRecordsRemainIndependentDuringConcurrentExecution(t *testing.T
t.Fatalf("migration records crossed: first=%+v second=%+v", firstState.Target, secondState.Target)
}
}
func TestExecuteAuthorizedPassesActorToSQLiteSwitch(t *testing.T) {
var received config.AuditActor
var receivedGeneration uint64
root := t.TempDir()
switchConfig := func(_ context.Context, generation uint64, _ config.DatabaseConfig, actor config.AuditActor) error {
receivedGeneration = generation
received = actor
return nil
}
executor := NewAuthorizedSQLiteExecutor(root, switchConfig)
service := NewService(executor)
actor := config.AuditActor{UserID: "admin-1", Role: "super_admin", SessionID: "session-1"}
if _, err := service.ExecuteAuthorized(context.Background(), "missing", "missing", actor); !errors.Is(err, ErrMigrationNotFound) {
t.Fatalf("missing migration error = %v, want migration not found", err)
}
if _, err := service.ExecuteAuthorized(context.Background(), "missing", "missing", config.AuditActor{UserID: "admin-1"}); !errors.Is(err, config.ErrAuthorizationRequired) {
t.Fatalf("missing session error = %v, want authorization required", err)
}
root = t.TempDir()
executor = NewAuthorizedSQLiteExecutor(root, switchConfig)
service = NewService(executor)
configDirectory := filepath.Join(root, "config")
if err := os.Mkdir(configDirectory, 0700); err != nil {
t.Fatal(err)
}
database, err := sql.Open("sqlite", filepath.Join(configDirectory, "smail.db"))
if err != nil {
t.Fatal(err)
}
if _, err := database.Exec("CREATE TABLE messages (id INTEGER PRIMARY KEY, body TEXT NOT NULL)"); err != nil {
database.Close()
t.Fatal(err)
}
if err := database.Close(); err != nil {
t.Fatal(err)
}
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "next.db")
if err != nil {
t.Fatal(err)
}
completed, err := service.ExecuteAuthorized(context.Background(), record.ID, token, actor)
if err != nil {
t.Fatal(err)
}
if completed.Status != StatusSucceeded || received != actor || receivedGeneration != record.CurrentGeneration {
t.Fatalf("authorized execution = %+v, received actor = %+v, generation = %d", completed, received, receivedGeneration)
}
}
+667
View File
@@ -0,0 +1,667 @@
package dbmigration
import (
"context"
"database/sql"
"errors"
"fmt"
"net"
"net/url"
"regexp"
"strconv"
"strings"
"sync"
"time"
"git.pcux.cn/xuans/Smail/config"
"git.pcux.cn/xuans/Smail/utils/credential"
mysql "github.com/go-sql-driver/mysql"
"github.com/lib/pq"
)
var (
ErrSQLCopyUnavailable = errors.New("structured database copier unavailable")
ErrSQLTargetExists = errors.New("structured database migration target already exists")
ErrSQLUnsupportedObject = errors.New("structured database contains unsupported object")
ErrSQLConnectionUnavailable = errors.New("structured database connection unavailable")
ErrSQLDataVerification = errors.New("structured database data verification failed")
ErrSQLSourceBusy = errors.New("structured database source has active connections")
ErrSwitchOutcomeUnknown = errors.New("database configuration switch outcome is unknown")
)
type AuthorizedDatabaseSwitch func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error
type StructuredDatabaseCopier interface {
Copy(context.Context, config.DatabaseConfig, config.DatabaseConfig) error
}
type StructuredDatabaseCleanup interface {
Cleanup(context.Context, config.DatabaseConfig, config.DatabaseConfig) error
Complete(config.DatabaseConfig, config.DatabaseConfig)
}
type DatabaseExecutor struct {
sqlite Runner
structured Runner
}
func NewDatabaseExecutor(sqlite, structured Runner) Runner {
return &DatabaseExecutor{sqlite: sqlite, structured: structured}
}
func (e *DatabaseExecutor) Run(ctx context.Context, plan Plan, update func(Status) error) error {
if e == nil {
return ErrExecutorUnavailable
}
if plan.sourceConfig.Type == config.DatabaseTypeSQLite {
if e.sqlite == nil {
return ErrExecutorUnavailable
}
return e.sqlite.Run(ctx, plan, update)
}
if e.structured == nil {
return ErrExecutorUnavailable
}
return e.structured.Run(ctx, plan, update)
}
type SQLExecutor struct {
copier StructuredDatabaseCopier
authorizedSwitch AuthorizedDatabaseSwitch
}
func NewSQLExecutor(copier StructuredDatabaseCopier, switchConfig AuthorizedDatabaseSwitch) *SQLExecutor {
return &SQLExecutor{copier: copier, authorizedSwitch: switchConfig}
}
func (e *SQLExecutor) Run(ctx context.Context, plan Plan, update func(Status) error) error {
if ctx == nil {
ctx = context.Background()
}
if e == nil || update == nil {
return ErrInvalidRequest
}
if err := contextError(ctx); err != nil {
return err
}
if plan.sourceConfig.Type != plan.targetConfig.Type ||
(plan.sourceConfig.Type != config.DatabaseTypeMySQL && plan.sourceConfig.Type != config.DatabaseTypePostgreSQL) {
return ErrUnsupportedDatabase
}
if err := config.ValidateDatabaseConfig(plan.sourceConfig); err != nil {
return ErrSourceDatabase
}
if err := config.ValidateDatabaseConfig(plan.targetConfig); err != nil {
return ErrInvalidTargetName
}
if !validStructuredDatabaseName(plan.sourceConfig.Name) ||
!validStructuredDatabaseName(plan.targetConfig.Name) ||
plan.sourceConfig.Name == plan.targetConfig.Name ||
!sameStructuredConnection(plan.sourceConfig, plan.targetConfig) {
return ErrImmutableSetting
}
if e.copier == nil {
return ErrSQLCopyUnavailable
}
if e.authorizedSwitch == nil {
return ErrSwitchUnavailable
}
if err := update(StatusVerifying); err != nil {
return err
}
if err := e.copier.Copy(ctx, plan.sourceConfig, plan.targetConfig); err != nil {
cleanupStructuredTarget(e.copier, plan.sourceConfig, plan.targetConfig)
return err
}
if err := contextError(ctx); err != nil {
cleanupStructuredTarget(e.copier, plan.sourceConfig, plan.targetConfig)
return err
}
if err := update(StatusSwitching); err != nil {
cleanupStructuredTarget(e.copier, plan.sourceConfig, plan.targetConfig)
return err
}
if e.authorizedSwitch == nil {
cleanupStructuredTarget(e.copier, plan.sourceConfig, plan.targetConfig)
return ErrSwitchUnavailable
}
if err := e.authorizedSwitch(ctx, plan.expectedGeneration, plan.targetConfig, plan.actor); err != nil {
return errors.Join(ErrSwitchOutcomeUnknown, err)
}
if cleanup, ok := e.copier.(StructuredDatabaseCleanup); ok {
cleanup.Complete(plan.sourceConfig, plan.targetConfig)
}
return nil
}
func cleanupStructuredTarget(copier StructuredDatabaseCopier, source, target config.DatabaseConfig) {
cleanup, ok := copier.(StructuredDatabaseCleanup)
if !ok {
return
}
cleanupContext, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_ = cleanup.Cleanup(cleanupContext, source, target)
}
func sameStructuredConnection(source, target config.DatabaseConfig) bool {
return source.Type == target.Type &&
source.Host == target.Host &&
source.Port == target.Port &&
source.Username == target.Username &&
source.Password == target.Password &&
source.LocalRef == target.LocalRef
}
type SQLConnectionFactory func(context.Context, config.DatabaseConfig, string) (*sql.DB, error)
type SQLCopier struct {
open SQLConnectionFactory
mu sync.Mutex
created map[string]struct{}
}
func NewSQLCopier(open SQLConnectionFactory) *SQLCopier {
return &SQLCopier{open: open, created: make(map[string]struct{})}
}
func (c *SQLCopier) Copy(ctx context.Context, source, target config.DatabaseConfig) error {
if c == nil || c.open == nil {
return ErrSQLConnectionUnavailable
}
if ctx == nil {
ctx = context.Background()
}
adminDB, err := c.open(ctx, source, "")
if err != nil {
return ErrSQLConnectionUnavailable
}
defer adminDB.Close()
switch source.Type {
case config.DatabaseTypeMySQL:
sourceDB, err := c.open(ctx, source, source.Name)
if err != nil {
return ErrSQLConnectionUnavailable
}
defer sourceDB.Close()
return c.copyMySQL(ctx, sourceDB, adminDB, source, target)
case config.DatabaseTypePostgreSQL:
return c.copyPostgreSQL(ctx, adminDB, source, target)
default:
return ErrUnsupportedDatabase
}
}
func (c *SQLCopier) copyMySQL(ctx context.Context, sourceDB, adminDB *sql.DB, source, target config.DatabaseConfig) error {
sourceName := source.Name
targetName := target.Name
if err := c.validateMySQLObjects(ctx, adminDB, sourceName); err != nil {
return err
}
var exists int
if err := adminDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = ?", targetName).Scan(&exists); err != nil {
return fmt.Errorf("check mysql migration target: %w", err)
}
if exists != 0 {
return ErrSQLTargetExists
}
if _, err := adminDB.ExecContext(ctx, "CREATE DATABASE "+quoteMySQLIdentifier(targetName)); err != nil {
if isDatabaseAlreadyExistsError(err, config.DatabaseTypeMySQL) {
return ErrSQLTargetExists
}
return fmt.Errorf("create mysql migration target: %w", err)
}
c.markCreated(source, target)
created := true
defer func() {
if created {
cleanupContext, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if _, err := adminDB.ExecContext(cleanupContext, "DROP DATABASE "+quoteMySQLIdentifier(targetName)); err == nil {
c.forgetCreated(source, target)
}
}
}()
rows, err := sourceDB.QueryContext(ctx, "SHOW FULL TABLES FROM "+quoteMySQLIdentifier(sourceName))
if err != nil {
return fmt.Errorf("list mysql tables: %w", err)
}
defer rows.Close()
tableNames := make([]string, 0)
sourceRowCounts := make(map[string]int64)
for rows.Next() {
var tableName, tableType string
if err := rows.Scan(&tableName, &tableType); err != nil {
return fmt.Errorf("read mysql table: %w", err)
}
if !strings.EqualFold(tableType, "BASE TABLE") {
return ErrSQLUnsupportedObject
}
tableNames = append(tableNames, tableName)
sourceTable := quoteMySQLIdentifier(sourceName) + "." + quoteMySQLIdentifier(tableName)
targetTable := quoteMySQLIdentifier(targetName) + "." + quoteMySQLIdentifier(tableName)
rowCount, err := countRows(ctx, sourceDB, sourceTable)
if err != nil {
return fmt.Errorf("count mysql source rows: %w", err)
}
sourceRowCounts[tableName] = rowCount
if _, err := adminDB.ExecContext(ctx, "CREATE TABLE "+targetTable+" LIKE "+sourceTable); err != nil {
return fmt.Errorf("copy mysql table structure: %w", err)
}
}
if err := rows.Err(); err != nil {
return fmt.Errorf("read mysql tables: %w", err)
}
for _, tableName := range tableNames {
sourceTable := quoteMySQLIdentifier(sourceName) + "." + quoteMySQLIdentifier(tableName)
targetTable := quoteMySQLIdentifier(targetName) + "." + quoteMySQLIdentifier(tableName)
if _, err := adminDB.ExecContext(ctx, "INSERT INTO "+targetTable+" SELECT * FROM "+sourceTable); err != nil {
return fmt.Errorf("copy mysql table data: %w", err)
}
targetRowCount, err := countRows(ctx, adminDB, targetTable)
if err != nil {
return fmt.Errorf("count mysql target rows: %w", err)
}
finalSourceRowCount, err := countRows(ctx, sourceDB, sourceTable)
if err != nil {
return fmt.Errorf("verify mysql source rows: %w", err)
}
if sourceRowCounts[tableName] != targetRowCount || sourceRowCounts[tableName] != finalSourceRowCount {
return fmt.Errorf("%w: mysql table %s row count changed during migration", ErrSQLDataVerification, tableName)
}
}
if err := c.copyMySQLForeignKeys(ctx, adminDB, sourceName, targetName, tableNames); err != nil {
return err
}
created = false
return nil
}
type mysqlForeignKey struct {
tableName string
constraintName string
columns []string
referencedSchema string
referencedTable string
referencedColumns []string
updateRule string
deleteRule string
}
func (c *SQLCopier) validateMySQLObjects(ctx context.Context, database *sql.DB, sourceName string) error {
checks := []struct {
query string
args []any
}{
{
query: "SELECT COUNT(*) FROM information_schema.triggers WHERE trigger_schema = ?",
args: []any{sourceName},
},
{
query: "SELECT COUNT(*) FROM information_schema.events WHERE event_schema = ?",
args: []any{sourceName},
},
{
query: "SELECT COUNT(*) FROM information_schema.routines WHERE routine_schema = ?",
args: []any{sourceName},
},
}
for _, check := range checks {
var count int
if err := database.QueryRowContext(ctx, check.query, check.args...).Scan(&count); err != nil {
return fmt.Errorf("%w: inspect mysql database objects", ErrSQLConnectionUnavailable)
}
if count > 0 {
return ErrSQLUnsupportedObject
}
}
return nil
}
func (c *SQLCopier) copyMySQLForeignKeys(ctx context.Context, database *sql.DB, sourceName, targetName string, tableNames []string) error {
if len(tableNames) == 0 {
return nil
}
rows, err := database.QueryContext(ctx, "SELECT kcu.TABLE_NAME, kcu.CONSTRAINT_NAME, kcu.COLUMN_NAME, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME, rc.UPDATE_RULE, rc.DELETE_RULE FROM information_schema.KEY_COLUMN_USAGE kcu JOIN information_schema.TABLE_CONSTRAINTS tc ON tc.CONSTRAINT_SCHEMA = kcu.CONSTRAINT_SCHEMA AND tc.TABLE_NAME = kcu.TABLE_NAME AND tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME JOIN information_schema.REFERENTIAL_CONSTRAINTS rc ON rc.CONSTRAINT_SCHEMA = kcu.CONSTRAINT_SCHEMA AND rc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME AND rc.TABLE_NAME = kcu.TABLE_NAME WHERE tc.CONSTRAINT_SCHEMA = ? AND tc.CONSTRAINT_TYPE = 'FOREIGN KEY' AND kcu.REFERENCED_TABLE_NAME IS NOT NULL ORDER BY kcu.TABLE_NAME, kcu.CONSTRAINT_NAME, kcu.ORDINAL_POSITION", sourceName)
if err != nil {
return fmt.Errorf("%w: inspect mysql foreign keys", ErrSQLConnectionUnavailable)
}
defer rows.Close()
foreignKeys := make([]mysqlForeignKey, 0)
indexes := make(map[string]int)
for rows.Next() {
var item mysqlForeignKey
var column, referencedColumn string
if err := rows.Scan(&item.tableName, &item.constraintName, &column, &item.referencedSchema, &item.referencedTable, &referencedColumn, &item.updateRule, &item.deleteRule); err != nil {
return fmt.Errorf("%w: read mysql foreign keys", ErrSQLConnectionUnavailable)
}
if !validMySQLReferenceRule(item.updateRule) || !validMySQLReferenceRule(item.deleteRule) {
return ErrSQLUnsupportedObject
}
key := item.tableName + "\x00" + item.constraintName
index, ok := indexes[key]
if !ok {
indexes[key] = len(foreignKeys)
foreignKeys = append(foreignKeys, item)
index = len(foreignKeys) - 1
}
foreignKeys[index].columns = append(foreignKeys[index].columns, column)
foreignKeys[index].referencedColumns = append(foreignKeys[index].referencedColumns, referencedColumn)
}
if err := rows.Err(); err != nil {
return fmt.Errorf("%w: read mysql foreign keys", ErrSQLConnectionUnavailable)
}
for _, foreignKey := range foreignKeys {
referencedSchema := foreignKey.referencedSchema
if referencedSchema == sourceName {
referencedSchema = targetName
}
columns := quoteMySQLIdentifiers(foreignKey.columns)
referencedColumns := quoteMySQLIdentifiers(foreignKey.referencedColumns)
query := "ALTER TABLE " + quoteMySQLIdentifier(targetName) + "." + quoteMySQLIdentifier(foreignKey.tableName) + " ADD CONSTRAINT " + quoteMySQLIdentifier(foreignKey.constraintName) + " FOREIGN KEY (" + strings.Join(columns, ", ") + ") REFERENCES " + quoteMySQLIdentifier(referencedSchema) + "." + quoteMySQLIdentifier(foreignKey.referencedTable) + " (" + strings.Join(referencedColumns, ", ") + ") ON UPDATE " + foreignKey.updateRule + " ON DELETE " + foreignKey.deleteRule
if _, err := database.ExecContext(ctx, query); err != nil {
return fmt.Errorf("copy mysql foreign key: %w", err)
}
}
return nil
}
func validMySQLReferenceRule(value string) bool {
switch strings.ToUpper(strings.TrimSpace(value)) {
case "CASCADE", "SET NULL", "SET DEFAULT", "RESTRICT", "NO ACTION":
return true
default:
return false
}
}
func quoteMySQLIdentifiers(values []string) []string {
quoted := make([]string, len(values))
for index, value := range values {
quoted[index] = quoteMySQLIdentifier(value)
}
return quoted
}
func (c *SQLCopier) copyPostgreSQL(ctx context.Context, adminDB *sql.DB, source, target config.DatabaseConfig) error {
sourceName := source.Name
targetName := target.Name
var exists bool
if err := adminDB.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM pg_database WHERE datname = $1)", targetName).Scan(&exists); err != nil {
return fmt.Errorf("check postgresql migration target: %w", err)
}
if exists {
return ErrSQLTargetExists
}
var activeConnections int
if err := adminDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM pg_stat_activity WHERE datname = $1 AND pid <> pg_backend_pid()", sourceName).Scan(&activeConnections); err != nil {
return fmt.Errorf("inspect postgresql source connections: %w", err)
}
if activeConnections > 0 {
return ErrSQLSourceBusy
}
if _, err := adminDB.ExecContext(ctx, "CREATE DATABASE "+quotePostgreSQLIdentifier(targetName)+" TEMPLATE "+quotePostgreSQLIdentifier(sourceName)); err != nil {
if isDatabaseAlreadyExistsError(err, config.DatabaseTypePostgreSQL) {
return ErrSQLTargetExists
}
return fmt.Errorf("copy postgresql database: %w", err)
}
c.markCreated(source, target)
created := true
defer func() {
if created {
cleanupContext, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := c.Cleanup(cleanupContext, source, target); err == nil {
created = false
}
}
}()
if err := c.verifyPostgreSQLClone(ctx, source, target); err != nil {
return err
}
created = false
return nil
}
func (c *SQLCopier) verifyPostgreSQLClone(ctx context.Context, source, target config.DatabaseConfig) error {
if c == nil || c.open == nil {
return ErrSQLConnectionUnavailable
}
sourceDB, err := c.open(ctx, source, source.Name)
if err != nil {
return ErrSQLConnectionUnavailable
}
defer sourceDB.Close()
targetDB, err := c.open(ctx, target, target.Name)
if err != nil {
return ErrSQLConnectionUnavailable
}
defer targetDB.Close()
tables, err := listPostgreSQLTables(ctx, sourceDB)
if err != nil {
return fmt.Errorf("%w: list postgresql source tables", ErrSQLConnectionUnavailable)
}
targetTables, err := listPostgreSQLTables(ctx, targetDB)
if err != nil {
return fmt.Errorf("%w: list postgresql target tables", ErrSQLConnectionUnavailable)
}
if len(tables) != len(targetTables) {
return fmt.Errorf("%w: postgresql table count changed during migration", ErrSQLDataVerification)
}
for _, table := range tables {
sourceCount, err := countRows(ctx, sourceDB, quotePostgreSQLIdentifier(table.schema)+"."+quotePostgreSQLIdentifier(table.name))
if err != nil {
return fmt.Errorf("%w: count postgresql source rows", ErrSQLConnectionUnavailable)
}
targetCount, err := countRows(ctx, targetDB, quotePostgreSQLIdentifier(table.schema)+"."+quotePostgreSQLIdentifier(table.name))
if err != nil {
return fmt.Errorf("%w: count postgresql target rows", ErrSQLConnectionUnavailable)
}
if sourceCount != targetCount {
return fmt.Errorf("%w: postgresql table %s row count changed during migration", ErrSQLDataVerification, table.name)
}
}
return nil
}
type postgreSQLTable struct {
schema string
name string
}
func listPostgreSQLTables(ctx context.Context, database *sql.DB) ([]postgreSQLTable, error) {
rows, err := database.QueryContext(ctx, "SELECT table_schema, table_name FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema NOT IN ('pg_catalog', 'information_schema') ORDER BY table_schema, table_name")
if err != nil {
return nil, err
}
defer rows.Close()
tables := make([]postgreSQLTable, 0)
for rows.Next() {
var table postgreSQLTable
if err := rows.Scan(&table.schema, &table.name); err != nil {
return nil, err
}
tables = append(tables, table)
}
if err := rows.Err(); err != nil {
return nil, err
}
return tables, nil
}
func isDatabaseAlreadyExistsError(err error, databaseType config.DatabaseType) bool {
if err == nil {
return false
}
switch databaseType {
case config.DatabaseTypeMySQL:
var mysqlErr *mysql.MySQLError
return errors.As(err, &mysqlErr) && mysqlErr.Number == 1007
case config.DatabaseTypePostgreSQL:
var postgresErr *pq.Error
return errors.As(err, &postgresErr) && string(postgresErr.Code) == "42P04"
default:
return false
}
}
func (c *SQLCopier) markCreated(source, target config.DatabaseConfig) {
if c == nil {
return
}
c.mu.Lock()
defer c.mu.Unlock()
if c.created == nil {
c.created = make(map[string]struct{})
}
c.created[databaseTargetKey(source, target)] = struct{}{}
}
func (c *SQLCopier) hasCreated(source, target config.DatabaseConfig) bool {
if c == nil {
return false
}
c.mu.Lock()
defer c.mu.Unlock()
_, ok := c.created[databaseTargetKey(source, target)]
return ok
}
func (c *SQLCopier) forgetCreated(source, target config.DatabaseConfig) {
if c == nil {
return
}
c.mu.Lock()
delete(c.created, databaseTargetKey(source, target))
c.mu.Unlock()
}
func (c *SQLCopier) Complete(source, target config.DatabaseConfig) {
c.forgetCreated(source, target)
}
func (c *SQLCopier) Cleanup(ctx context.Context, source, target config.DatabaseConfig) error {
if !c.hasCreated(source, target) {
return nil
}
if c.open == nil {
return ErrSQLConnectionUnavailable
}
adminDB, err := c.open(ctx, source, "")
if err != nil {
return ErrSQLConnectionUnavailable
}
defer adminDB.Close()
var query string
switch target.Type {
case config.DatabaseTypeMySQL:
query = "DROP DATABASE " + quoteMySQLIdentifier(target.Name)
case config.DatabaseTypePostgreSQL:
query = "DROP DATABASE " + quotePostgreSQLIdentifier(target.Name)
default:
return ErrUnsupportedDatabase
}
if _, err := adminDB.ExecContext(ctx, query); err != nil {
return fmt.Errorf("cleanup structured migration target: %w", err)
}
c.forgetCreated(source, target)
return nil
}
func countRows(ctx context.Context, database *sql.DB, table string) (int64, error) {
var count int64
if err := database.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+table).Scan(&count); err != nil {
return 0, err
}
return count, nil
}
func databaseTargetKey(source, target config.DatabaseConfig) string {
return strings.Join([]string{
string(target.Type),
target.Host,
strconv.Itoa(int(target.Port)),
target.Username,
target.Name,
source.Name,
}, "\x00")
}
func quoteMySQLIdentifier(value string) string {
return "`" + strings.ReplaceAll(value, "`", "``") + "`"
}
func quotePostgreSQLIdentifier(value string) string {
return "\"" + strings.ReplaceAll(value, "\"", "\"\"") + "\""
}
func NewRuntimeSQLConnectionFactory(root string) (SQLConnectionFactory, error) {
vault, err := credential.NewVault(root)
if err != nil {
return nil, err
}
return func(ctx context.Context, database config.DatabaseConfig, databaseName string) (*sql.DB, error) {
password, err := config.OpenStoredSecret(vault, database.Password)
if err != nil {
return nil, ErrSQLConnectionUnavailable
}
driverName, dsn, err := runtimeSQLDSN(database, password, databaseName)
if err != nil {
return nil, err
}
connection, err := sql.Open(driverName, dsn)
if err != nil {
return nil, ErrSQLConnectionUnavailable
}
if err := connection.PingContext(ctx); err != nil {
_ = connection.Close()
return nil, ErrSQLConnectionUnavailable
}
return connection, nil
}, nil
}
func runtimeSQLDSN(database config.DatabaseConfig, password, databaseName string) (string, string, error) {
if database.Host == "" || database.Port == 0 || database.Username == "" || password == "" {
return "", "", ErrSQLConnectionUnavailable
}
address := net.JoinHostPort(database.Host, strconv.Itoa(int(database.Port)))
switch database.Type {
case config.DatabaseTypeMySQL:
settings := mysql.Config{
User: database.Username,
Passwd: password,
Net: "tcp",
Addr: address,
DBName: databaseName,
ParseTime: true,
Params: map[string]string{"charset": "utf8mb4"},
}
return "mysql", settings.FormatDSN(), nil
case config.DatabaseTypePostgreSQL:
maintenanceDatabase := databaseName
if maintenanceDatabase == "" {
maintenanceDatabase = "postgres"
}
databaseURL := url.URL{
Scheme: "postgres",
User: url.UserPassword(database.Username, password),
Host: address,
Path: "/" + maintenanceDatabase,
RawQuery: "sslmode=prefer",
}
return "postgres", databaseURL.String(), nil
default:
return "", "", ErrUnsupportedDatabase
}
}
var structuredDatabaseNamePattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,62}$`)
var mysqlOptionIdentifierPattern = regexp.MustCompile(`^[A-Za-z0-9_]+$`)
func validStructuredDatabaseName(value string) bool {
return structuredDatabaseNamePattern.MatchString(strings.TrimSpace(value))
}
@@ -0,0 +1,305 @@
package dbmigration
import (
"context"
"errors"
"net/url"
"strings"
"testing"
"git.pcux.cn/xuans/Smail/config"
mysql "github.com/go-sql-driver/mysql"
"github.com/lib/pq"
)
type recordingStructuredCopier struct {
called bool
source config.DatabaseConfig
target config.DatabaseConfig
err error
}
func (c *recordingStructuredCopier) Copy(_ context.Context, source, target config.DatabaseConfig) error {
c.called = true
c.source = source
c.target = target
return c.err
}
func validStructuredPlan(t *testing.T) Plan {
t.Helper()
database := testDatabase(t)
database.Name = "smail"
target := database
target.Name = "smail_next"
return Plan{
sourceConfig: database,
targetConfig: target,
actor: config.AuditActor{UserID: "admin-1", Role: "super_admin", SessionID: "session-1"},
}
}
func TestSQLExecutorCopiesAndSwitchesWithOrderedStatuses(t *testing.T) {
copier := &recordingStructuredCopier{}
var actor config.AuditActor
var generation uint64
executor := NewSQLExecutor(copier, func(_ context.Context, receivedGeneration uint64, _ config.DatabaseConfig, received config.AuditActor) error {
generation = receivedGeneration
actor = received
return nil
})
statuses := make([]Status, 0, 2)
plan := validStructuredPlan(t)
plan.expectedGeneration = 42
if err := executor.Run(context.Background(), plan, func(status Status) error {
statuses = append(statuses, status)
return nil
}); err != nil {
t.Fatal(err)
}
if len(statuses) != 2 || statuses[0] != StatusVerifying || statuses[1] != StatusSwitching {
t.Fatalf("statuses = %v, want verifying then switching", statuses)
}
if !copier.called || copier.source.Name != "smail" || copier.target.Name != "smail_next" {
t.Fatalf("copier call = %+v", copier)
}
if actor != plan.actor {
t.Fatalf("switch actor = %+v, want %+v", actor, plan.actor)
}
if generation != plan.expectedGeneration {
t.Fatalf("switch generation = %d, want %d", generation, plan.expectedGeneration)
}
}
func TestSQLExecutorRejectsImmutablePlanBeforeCopy(t *testing.T) {
copier := &recordingStructuredCopier{}
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error { return nil })
plan := validStructuredPlan(t)
plan.targetConfig.Host = "other.example"
if err := executor.Run(context.Background(), plan, func(Status) error { return nil }); !errors.Is(err, ErrImmutableSetting) {
t.Fatalf("immutable plan error = %v, want immutable setting", err)
}
if copier.called {
t.Fatal("copier ran for immutable plan")
}
}
func TestSQLExecutorPreservesCopierError(t *testing.T) {
copyErr := errors.New("复制失败")
copier := &recordingStructuredCopier{err: copyErr}
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error { return nil })
if err := executor.Run(context.Background(), validStructuredPlan(t), func(Status) error { return nil }); !errors.Is(err, copyErr) {
t.Fatalf("copy error = %v, want copier error", err)
}
}
func TestSQLExecutorCleansStructuredTargetWhenCopyFails(t *testing.T) {
copier := &cleanupRecordingCopier{copyErr: errors.New("复制失败")}
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error {
return nil
})
if err := executor.Run(context.Background(), validStructuredPlan(t), func(Status) error { return nil }); !errors.Is(err, copier.copyErr) {
t.Fatalf("copy error = %v, want %v", err, copier.copyErr)
}
if !copier.cleanupCalled {
t.Fatal("structured target cleanup was not requested after copy failure")
}
}
func TestStructuredDatabaseNameValidation(t *testing.T) {
for _, value := range []string{"smail", "Smail_2026", "mail-prod"} {
if !validStructuredDatabaseName(value) {
t.Fatalf("valid database name rejected: %q", value)
}
}
for _, value := range []string{"", "smail;drop", "smail name", "../smail"} {
if validStructuredDatabaseName(value) {
t.Fatalf("unsafe database name accepted: %q", value)
}
}
}
func TestDatabaseExecutorRoutesByDatabaseType(t *testing.T) {
var sqliteCalled, structuredCalled bool
sqlite := runnerFunc(func(context.Context, Plan, func(Status) error) error {
sqliteCalled = true
return nil
})
structured := runnerFunc(func(context.Context, Plan, func(Status) error) error {
structuredCalled = true
return nil
})
router := NewDatabaseExecutor(sqlite, structured)
if err := router.Run(context.Background(), Plan{sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite}}, func(Status) error { return nil }); err != nil {
t.Fatal(err)
}
if err := router.Run(context.Background(), Plan{sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeMySQL}}, func(Status) error { return nil }); err != nil {
t.Fatal(err)
}
if !sqliteCalled || !structuredCalled {
t.Fatalf("router calls: sqlite=%v structured=%v", sqliteCalled, structuredCalled)
}
}
type runnerFunc func(context.Context, Plan, func(Status) error) error
func (f runnerFunc) Run(ctx context.Context, plan Plan, update func(Status) error) error {
return f(ctx, plan, update)
}
func TestSQLExecutorRejectsMissingConfigurationSwitchBeforeCopy(t *testing.T) {
copier := &recordingStructuredCopier{}
executor := NewSQLExecutor(copier, nil)
if err := executor.Run(context.Background(), validStructuredPlan(t), func(Status) error { return nil }); !errors.Is(err, ErrSwitchUnavailable) {
t.Fatalf("missing switch error = %v, want switch unavailable", err)
}
if copier.called {
t.Fatal("copier ran without a configuration switch")
}
}
type cleanupRecordingCopier struct {
cleanupCalled bool
completeCalled bool
copyErr error
}
func (c *cleanupRecordingCopier) Copy(context.Context, config.DatabaseConfig, config.DatabaseConfig) error {
return c.copyErr
}
func TestDatabaseAlreadyExistsErrorClassification(t *testing.T) {
mysqlError := &mysql.MySQLError{Number: 1007}
if !isDatabaseAlreadyExistsError(mysqlError, config.DatabaseTypeMySQL) {
t.Fatal("mysql duplicate database error was not classified")
}
postgresError := &pq.Error{Code: "42P04"}
if !isDatabaseAlreadyExistsError(postgresError, config.DatabaseTypePostgreSQL) {
t.Fatal("postgresql duplicate database error was not classified")
}
if isDatabaseAlreadyExistsError(errors.New("duplicate database"), config.DatabaseTypeMySQL) {
t.Fatal("generic mysql error was incorrectly classified")
}
}
func (c *cleanupRecordingCopier) Cleanup(context.Context, config.DatabaseConfig, config.DatabaseConfig) error {
c.cleanupCalled = true
return nil
}
func (c *cleanupRecordingCopier) Complete(config.DatabaseConfig, config.DatabaseConfig) {
c.completeCalled = true
}
func TestSQLExecutorPreservesStructuredTargetWhenConfigurationSwitchOutcomeIsUnknown(t *testing.T) {
copier := &cleanupRecordingCopier{}
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error {
return errors.New("配置切换失败")
})
if err := executor.Run(context.Background(), validStructuredPlan(t), func(Status) error { return nil }); !errors.Is(err, ErrSwitchOutcomeUnknown) {
t.Fatal("configuration switch failure was not returned")
}
if copier.cleanupCalled {
t.Fatal("structured target was cleaned up even though switch outcome was unknown")
}
if copier.completeCalled {
t.Fatal("structured target was finalized after a failed switch")
}
}
func TestSQLExecutorCleansStructuredTargetWhenStatusUpdateFails(t *testing.T) {
copier := &cleanupRecordingCopier{}
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error {
return nil
})
statusError := errors.New("状态写入失败")
if err := executor.Run(context.Background(), validStructuredPlan(t), func(status Status) error {
if status == StatusSwitching {
return statusError
}
return nil
}); !errors.Is(err, statusError) {
t.Fatalf("status update error = %v, want %v", err, statusError)
}
if !copier.cleanupCalled {
t.Fatal("structured target cleanup was not requested after status update failure")
}
}
func TestSQLExecutorCleansStructuredTargetWhenContextIsCanceledAfterCopy(t *testing.T) {
copier := &cleanupRecordingCopier{}
executor := NewSQLExecutor(copier, func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error {
return nil
})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if err := executor.Run(ctx, validStructuredPlan(t), func(status Status) error {
if status == StatusVerifying {
cancel()
}
return nil
}); !errors.Is(err, context.Canceled) {
t.Fatalf("canceled execution error = %v, want context canceled", err)
}
if !copier.cleanupCalled {
t.Fatal("structured target cleanup was not requested after cancellation")
}
}
func TestRuntimeSQLDSNSupportsIPv6AndEscapesCredentials(t *testing.T) {
database := config.DatabaseConfig{
Type: config.DatabaseTypeMySQL,
Host: "2001:db8::3",
Port: 3306,
Username: "mail user",
}
driver, dsn, err := runtimeSQLDSN(database, "p@ss/word", "smail_next")
if err != nil {
t.Fatal(err)
}
if driver != "mysql" {
t.Fatalf("mysql driver = %q", driver)
}
parsedMySQL, err := mysql.ParseDSN(dsn)
if err != nil {
t.Fatal(err)
}
if parsedMySQL.Addr != "[2001:db8::3]:3306" || parsedMySQL.User != "mail user" || parsedMySQL.Passwd != "p@ss/word" || parsedMySQL.DBName != "smail_next" {
t.Fatalf("mysql dsn config = %+v", parsedMySQL)
}
database.Type = config.DatabaseTypePostgreSQL
driver, dsn, err = runtimeSQLDSN(database, "p@ss/word", "smail_next")
if err != nil {
t.Fatal(err)
}
if driver != "postgres" {
t.Fatalf("postgres driver = %q", driver)
}
parsedPostgres, err := url.Parse(dsn)
if err != nil {
t.Fatal(err)
}
password, ok := parsedPostgres.User.Password()
if !ok || parsedPostgres.Host != "[2001:db8::3]:3306" || parsedPostgres.User.Username() != "mail user" || password != "p@ss/word" || parsedPostgres.Path != "/smail_next" {
t.Fatalf("postgres dsn = %s", dsn)
}
}
func TestMySQLReferenceRulesAndIdentifiersAreRestrictedSafely(t *testing.T) {
for _, rule := range []string{"CASCADE", "SET NULL", "SET DEFAULT", "RESTRICT", "NO ACTION"} {
if !validMySQLReferenceRule(rule) {
t.Fatalf("valid mysql reference rule rejected: %q", rule)
}
}
for _, rule := range []string{"DROP DATABASE", "CASCADE; DROP TABLE users"} {
if validMySQLReferenceRule(rule) {
t.Fatalf("unsafe mysql reference rule accepted: %q", rule)
}
}
if got := quoteMySQLIdentifier("mail`prod"); got != "`mail``prod`" {
t.Fatalf("quoted mysql identifier = %q", got)
}
if got := strings.Join(quoteMySQLIdentifiers([]string{"user", "tenant"}), ", "); got != "`user`, `tenant`" {
t.Fatalf("quoted mysql identifiers = %q", got)
}
}
@@ -0,0 +1,249 @@
package dbmigration
import (
"context"
"database/sql"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"git.pcux.cn/xuans/Smail/config"
_ "modernc.org/sqlite"
)
var ErrSQLiteIntegrity = errors.New("sqlite database integrity check failed")
type SQLiteSwitch func(context.Context, config.DatabaseConfig) error
type AuthorizedSQLiteSwitch func(context.Context, uint64, config.DatabaseConfig, config.AuditActor) error
type SQLiteExecutor struct {
root string
switchConfig SQLiteSwitch
authorizedSwitch AuthorizedSQLiteSwitch
}
func NewSQLiteExecutor(root string, switchConfig SQLiteSwitch) *SQLiteExecutor {
return &SQLiteExecutor{root: root, switchConfig: switchConfig}
}
func NewAuthorizedSQLiteExecutor(root string, switchConfig AuthorizedSQLiteSwitch) *SQLiteExecutor {
return &SQLiteExecutor{root: root, authorizedSwitch: switchConfig}
}
func (e *SQLiteExecutor) Run(ctx context.Context, plan Plan, update func(Status) error) error {
if ctx == nil {
ctx = context.Background()
}
if e == nil || update == nil {
return ErrInvalidRequest
}
if err := contextError(ctx); err != nil {
return err
}
if plan.sourceConfig.Type != config.DatabaseTypeSQLite || plan.targetConfig.Type != config.DatabaseTypeSQLite {
return ErrUnsupportedDatabase
}
if err := config.ValidateDatabaseConfig(plan.sourceConfig); err != nil {
return ErrSourceDatabase
}
if err := config.ValidateDatabaseConfig(plan.targetConfig); err != nil {
return ErrInvalidTargetName
}
if plan.sourceConfig.LocalRef == plan.targetConfig.LocalRef {
return ErrInvalidTargetName
}
sourcePath, err := e.resolveReference(plan.sourceConfig.LocalRef)
if err != nil {
return err
}
targetPath, err := e.resolveReference(plan.targetConfig.LocalRef)
if err != nil {
return err
}
if sourcePath == targetPath {
return ErrInvalidTargetName
}
if err := checkRegularFile(sourcePath, false); err != nil {
return fmt.Errorf("open source database: %w", err)
}
if err := checkRegularFile(targetPath, true); err != nil {
return err
}
if err := update(StatusVerifying); err != nil {
return err
}
if err := verifySQLite(ctx, sourcePath); err != nil {
return err
}
if err := contextError(ctx); err != nil {
return err
}
temporary, err := e.temporaryPath(targetPath)
if err != nil {
return err
}
defer os.Remove(temporary)
activated := false
switched := false
switchStarted := false
defer func() {
if activated && !switched && !switchStarted {
_ = os.Remove(targetPath)
}
}()
if err := vacuumSQLite(ctx, sourcePath, temporary); err != nil {
return err
}
if err := os.Chmod(temporary, 0600); err != nil {
return fmt.Errorf("set migrated database permissions: %w", err)
}
if err := verifySQLite(ctx, temporary); err != nil {
return err
}
if err := contextError(ctx); err != nil {
return err
}
if err := update(StatusSwitching); err != nil {
return err
}
if err := os.Link(temporary, targetPath); err != nil {
if os.IsExist(err) {
return ErrTargetExists
}
return fmt.Errorf("activate migrated database file: %w", err)
}
activated = true
if err := os.Remove(temporary); err != nil {
return fmt.Errorf("finalize migrated database file: %w", err)
}
if err := verifySQLite(ctx, targetPath); err != nil {
return err
}
if e.authorizedSwitch == nil && e.switchConfig == nil {
return ErrSwitchUnavailable
}
var switchErr error
switchStarted = true
if e.authorizedSwitch != nil {
switchErr = e.authorizedSwitch(ctx, plan.expectedGeneration, plan.targetConfig, plan.actor)
} else {
switchErr = e.switchConfig(ctx, plan.targetConfig)
}
if switchErr != nil {
return errors.Join(ErrSwitchOutcomeUnknown, switchErr)
}
switched = true
return nil
}
func (e *SQLiteExecutor) resolveReference(reference string) (string, error) {
if e == nil || strings.TrimSpace(e.root) == "" || !sqliteTargetNamePattern.MatchString(reference) {
return "", ErrInvalidTargetName
}
root, err := filepath.Abs(filepath.Clean(e.root))
if err != nil {
return "", ErrInvalidTargetName
}
configDirectory := filepath.Join(root, "config")
info, err := os.Lstat(configDirectory)
if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
return "", ErrInvalidTargetName
}
path := filepath.Join(configDirectory, reference)
absolute, err := filepath.Abs(filepath.Clean(path))
if err != nil || filepath.Dir(absolute) != configDirectory {
return "", ErrInvalidTargetName
}
return absolute, nil
}
func (e *SQLiteExecutor) temporaryPath(target string) (string, error) {
for attempt := 0; attempt < 8; attempt++ {
id, err := randomID(".migration-")
if err != nil {
return "", err
}
candidate := filepath.Join(filepath.Dir(target), id+".tmp")
if _, err := os.Lstat(candidate); os.IsNotExist(err) {
return candidate, nil
} else if err != nil {
return "", err
}
}
return "", ErrTargetExists
}
func checkRegularFile(path string, target bool) error {
info, err := os.Lstat(path)
if os.IsNotExist(err) {
if target {
return nil
}
return fmt.Errorf("source database does not exist: %w", err)
}
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
if target {
return ErrTargetExists
}
return errors.New("source database is not a regular file")
}
if target {
return ErrTargetExists
}
return nil
}
func verifySQLite(ctx context.Context, path string) error {
database, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)&mode=ro")
if err != nil {
return fmt.Errorf("open sqlite database: %w", err)
}
defer database.Close()
if err := database.PingContext(ctx); err != nil {
if ctx != nil && ctx.Err() != nil {
return ctx.Err()
}
return fmt.Errorf("%w: ping sqlite database: %v", ErrSQLiteIntegrity, err)
}
var result string
if err := database.QueryRowContext(ctx, "PRAGMA integrity_check").Scan(&result); err != nil {
if ctx != nil && ctx.Err() != nil {
return ctx.Err()
}
return fmt.Errorf("%w: check sqlite database integrity: %v", ErrSQLiteIntegrity, err)
}
if !strings.EqualFold(strings.TrimSpace(result), "ok") {
return fmt.Errorf("%w: %s", ErrSQLiteIntegrity, result)
}
return nil
}
func vacuumSQLite(ctx context.Context, source, target string) error {
database, err := sql.Open("sqlite", source+"?_pragma=busy_timeout(5000)")
if err != nil {
return fmt.Errorf("open sqlite source: %w", err)
}
defer database.Close()
if err := database.PingContext(ctx); err != nil {
return fmt.Errorf("ping sqlite source: %w", err)
}
query := "VACUUM INTO '" + strings.ReplaceAll(target, "'", "''") + "'"
if _, err := database.ExecContext(ctx, query); err != nil {
return fmt.Errorf("copy sqlite database: %w", err)
}
return nil
}
func contextError(ctx context.Context) error {
if ctx == nil {
return nil
}
return ctx.Err()
}
@@ -0,0 +1,228 @@
package dbmigration
import (
"context"
"database/sql"
"encoding/json"
"errors"
"os"
"path/filepath"
"runtime"
"testing"
"git.pcux.cn/xuans/Smail/config"
_ "modernc.org/sqlite"
)
func newSQLiteMigrationFixture(t *testing.T, switchConfig SQLiteSwitch) (*Service, string) {
t.Helper()
root := t.TempDir()
configDirectory := filepath.Join(root, "config")
if err := os.Mkdir(configDirectory, 0700); err != nil {
t.Fatal(err)
}
sourcePath := filepath.Join(configDirectory, "smail.db")
database, err := sql.Open("sqlite", sourcePath)
if err != nil {
t.Fatal(err)
}
if _, err := database.Exec(`CREATE TABLE messages (id INTEGER PRIMARY KEY, body TEXT NOT NULL); INSERT INTO messages(body) VALUES ('测试邮件');`); err != nil {
database.Close()
t.Fatal(err)
}
if err := database.Close(); err != nil {
t.Fatal(err)
}
executor := NewSQLiteExecutor(root, switchConfig)
return NewService(executor), root
}
func TestSQLiteExecutorCopiesDatabaseAndSwitchesConfiguration(t *testing.T) {
var switched config.DatabaseConfig
service, root := newSQLiteMigrationFixture(t, func(_ context.Context, candidate config.DatabaseConfig) error {
switched = candidate
return nil
})
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 3, "smail-next.db")
if err != nil {
t.Fatal(err)
}
completed, err := service.Execute(context.Background(), record.ID, token)
if err != nil {
t.Fatal(err)
}
if completed.Status != StatusSucceeded {
t.Fatalf("migration status = %s, want succeeded", completed.Status)
}
if switched.LocalRef != "smail-next.db" || switched.Type != config.DatabaseTypeSQLite {
t.Fatalf("unexpected switched configuration: %+v", switched)
}
database, err := sql.Open("sqlite", filepath.Join(root, "config", "smail-next.db"))
if err != nil {
t.Fatal(err)
}
defer database.Close()
var body string
if err := database.QueryRow("SELECT body FROM messages WHERE id = 1").Scan(&body); err != nil {
t.Fatal(err)
}
if body != "测试邮件" {
t.Fatalf("copied row = %q, want copied content", body)
}
info, err := os.Stat(filepath.Join(root, "config", "smail-next.db"))
if err != nil {
t.Fatal(err)
}
if runtime.GOOS != "windows" && info.Mode().Perm() != 0600 {
t.Fatalf("migrated database permissions = %o, want 600", info.Mode().Perm())
}
}
func TestSQLiteExecutorRejectsExistingTarget(t *testing.T) {
service, root := newSQLiteMigrationFixture(t, func(context.Context, config.DatabaseConfig) error { return nil })
if err := os.WriteFile(filepath.Join(root, "config", "smail-next.db"), []byte("existing"), 0600); err != nil {
t.Fatal(err)
}
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "smail-next.db")
if err != nil {
t.Fatal(err)
}
failed, err := service.Execute(context.Background(), record.ID, token)
if !errors.Is(err, ErrTargetExists) {
t.Fatalf("execution error = %v, want target exists", err)
}
if failed.Status != StatusFailed || failed.ErrorCode != "TARGET_EXISTS" {
t.Fatalf("unexpected failed migration: %+v", failed)
}
}
func TestSQLiteExecutorRejectsCorruptSource(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "config"), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "config", "smail.db"), []byte("not sqlite"), 0600); err != nil {
t.Fatal(err)
}
executor := NewSQLiteExecutor(root, func(context.Context, config.DatabaseConfig) error { return nil })
plan := Plan{sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, targetConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "next.db"}}
if err := executor.Run(context.Background(), plan, func(Status) error { return nil }); err == nil {
t.Fatal("corrupt source unexpectedly migrated")
} else if !errors.Is(err, ErrSQLiteIntegrity) {
t.Fatalf("corrupt source error = %v, want integrity error", err)
}
}
func TestSQLiteExecutorRejectsUnsupportedDatabase(t *testing.T) {
executor := NewSQLiteExecutor(t.TempDir(), func(context.Context, config.DatabaseConfig) error { return nil })
plan := Plan{sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeMySQL}, targetConfig: config.DatabaseConfig{Type: config.DatabaseTypeMySQL}}
if err := executor.Run(context.Background(), plan, func(Status) error { return nil }); !errors.Is(err, ErrUnsupportedDatabase) {
t.Fatalf("unsupported database error = %v", err)
}
}
func TestSQLiteExecutorRequiresConfigurationSwitch(t *testing.T) {
service, root := newSQLiteMigrationFixture(t, nil)
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "next.db")
if err != nil {
t.Fatal(err)
}
failed, err := service.Execute(context.Background(), record.ID, token)
if !errors.Is(err, ErrSwitchUnavailable) {
t.Fatalf("execution error = %v, want switch unavailable", err)
}
if failed.ErrorCode != "SWITCH_UNAVAILABLE" {
t.Fatalf("error code = %q, want SWITCH_UNAVAILABLE", failed.ErrorCode)
}
if _, err := os.Stat(filepath.Join(root, "config", "next.db")); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("target database remained after switch failure, stat error = %v", err)
}
}
func TestSQLiteExecutorPreservesTargetWhenConfigurationSwitchOutcomeIsUnknown(t *testing.T) {
service, root := newSQLiteMigrationFixture(t, func(context.Context, config.DatabaseConfig) error {
return errors.New("配置切换失败")
})
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "next.db")
if err != nil {
t.Fatal(err)
}
failed, err := service.Execute(context.Background(), record.ID, token)
if err == nil || failed.Status != StatusFailed {
t.Fatalf("switch failure result = %+v, error = %v", failed, err)
}
if !errors.Is(err, ErrSwitchOutcomeUnknown) {
t.Fatalf("switch failure error = %v, want unknown switch outcome", err)
}
if _, statErr := os.Stat(filepath.Join(root, "config", "next.db")); statErr != nil {
t.Fatalf("target database was removed after unknown switch outcome: %v", statErr)
}
}
func TestSQLiteExecutorRejectsTargetSymlink(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("Windows 创建符号链接需要额外权限")
}
service, root := newSQLiteMigrationFixture(t, func(context.Context, config.DatabaseConfig) error { return nil })
target := filepath.Join(root, "config", "next.db")
if err := os.Symlink(filepath.Join(root, "config", "smail.db"), target); err != nil {
t.Fatal(err)
}
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "next.db")
if err != nil {
t.Fatal(err)
}
failed, err := service.Execute(context.Background(), record.ID, token)
if !errors.Is(err, ErrTargetExists) || failed.Status != StatusFailed {
t.Fatalf("symlink target result = %+v, error = %v", failed, err)
}
}
func TestSQLiteExecutorHonorsCancellation(t *testing.T) {
service, _ := newSQLiteMigrationFixture(t, func(context.Context, config.DatabaseConfig) error { return nil })
record, token, err := service.Preview(context.Background(), config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, 1, "next.db")
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
failed, err := service.Execute(ctx, record.ID, token)
if !errors.Is(err, context.Canceled) {
t.Fatalf("execution error = %v, want context canceled", err)
}
if failed.Status != StatusFailed {
t.Fatalf("migration status = %s, want failed", failed.Status)
}
}
func TestMigrationPlanDoesNotSerializePrivateDatabaseConfig(t *testing.T) {
plan := Plan{Source: DatabaseIdentity{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}, Target: DatabaseIdentity{Type: config.DatabaseTypeSQLite, LocalRef: "next.db"}, sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"}}
data, err := json.Marshal(plan)
if err != nil {
t.Fatal(err)
}
if string(data) != `{"Source":{"type":"sqlite","localRef":"smail.db"},"Target":{"type":"sqlite","localRef":"next.db"}}` {
t.Fatalf("serialized plan = %s", data)
}
}
func TestSQLiteExecutorReportsOrderedStatuses(t *testing.T) {
service, root := newSQLiteMigrationFixture(t, func(context.Context, config.DatabaseConfig) error { return nil })
executor := service.runner.(*SQLiteExecutor)
statuses := make([]Status, 0, 2)
plan := Plan{
sourceConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "smail.db"},
targetConfig: config.DatabaseConfig{Type: config.DatabaseTypeSQLite, LocalRef: "next.db"},
}
if err := executor.Run(context.Background(), plan, func(status Status) error {
statuses = append(statuses, status)
return nil
}); err != nil {
t.Fatal(err)
}
if len(statuses) != 2 || statuses[0] != StatusVerifying || statuses[1] != StatusSwitching {
t.Fatalf("statuses = %v, want verifying then switching", statuses)
}
if _, err := os.Stat(filepath.Join(root, "config", "next.db")); err != nil {
t.Fatal(err)
}
}
+32 -28
View File
@@ -9,11 +9,11 @@ import (
var ErrCleanupAlreadyStarted = errors.New("verification cleanup already started")
type CleanupRunner struct {
store *Store
once sync.Once
mu sync.Mutex
stop chan struct{}
done chan struct{}
store *Store
mu sync.Mutex
stop chan struct{}
done chan struct{}
stopping bool
}
func NewCleanupRunner(store *Store) *CleanupRunner {
@@ -25,31 +25,28 @@ func (r *CleanupRunner) Start(interval time.Duration) error {
if r == nil || r.store == nil || interval <= 0 {
return ErrInvalidChallenge
}
started := false
r.once.Do(func() {
started = true
r.mu.Lock()
r.stop = make(chan struct{})
r.done = make(chan struct{})
stop, done := r.stop, r.done
r.mu.Lock()
if r.stop != nil || r.stopping {
r.mu.Unlock()
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
defer close(done)
for {
select {
case <-ticker.C:
r.store.Cleanup()
case <-stop:
return
}
}
}()
})
if !started {
return ErrCleanupAlreadyStarted
}
r.stop = make(chan struct{})
r.done = make(chan struct{})
stop, done := r.stop, r.done
r.mu.Unlock()
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
defer close(done)
for {
select {
case <-ticker.C:
r.store.Cleanup()
case <-stop:
return
}
}
}()
return nil
}
@@ -59,10 +56,17 @@ func (r *CleanupRunner) Stop() {
}
r.mu.Lock()
stop, done := r.stop, r.done
r.stop, r.done = nil, nil
if stop == nil || r.stopping {
r.mu.Unlock()
return
}
r.stopping = true
r.mu.Unlock()
if stop != nil {
close(stop)
<-done
}
r.mu.Lock()
r.stop, r.done, r.stopping = nil, nil, false
r.mu.Unlock()
}
@@ -19,3 +19,19 @@ func TestCleanupRunnerHasSingleInstance(t *testing.T) {
t.Fatalf("second cleanup runner start error = %v", err)
}
}
func TestCleanupRunnerCanRestartAfterStop(t *testing.T) {
store, err := NewStore(4)
if err != nil {
t.Fatal(err)
}
runner := NewCleanupRunner(store)
if err := runner.Start(time.Hour); err != nil {
t.Fatal(err)
}
runner.Stop()
if err := runner.Start(time.Hour); err != nil {
t.Fatalf("cleanup runner restart error = %v", err)
}
runner.Stop()
}
+30 -22
View File
@@ -13,7 +13,8 @@ type OperationalCleanupRunner struct {
store *EventStore
rateLimits *PersistentRateLimiter
interval time.Duration
once sync.Once
running bool
stopping bool
stop chan struct{}
done chan struct{}
now func() time.Time
@@ -31,29 +32,32 @@ func (r *OperationalCleanupRunner) Start() {
if r == nil || r.store == nil {
return
}
r.once.Do(func() {
r.mu.Lock()
r.stop = make(chan struct{})
r.done = make(chan struct{})
stop, done := r.stop, r.done
interval := r.interval
r.mu.Lock()
if r.running || r.stopping {
r.mu.Unlock()
return
}
r.running = true
r.stop = make(chan struct{})
r.done = make(chan struct{})
stop, done := r.stop, r.done
interval := r.interval
r.mu.Unlock()
go func() {
defer close(done)
r.runOnce()
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
r.runOnce()
case <-stop:
return
}
go func() {
defer close(done)
r.runOnce()
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
r.runOnce()
case <-stop:
return
}
}()
})
}
}()
}
func (r *OperationalCleanupRunner) Stop() {
@@ -66,11 +70,15 @@ func (r *OperationalCleanupRunner) Stop() {
r.mu.Unlock()
return
}
r.stop, r.done = nil, nil
r.stop, r.done, r.running = nil, nil, false
r.stopping = true
r.mu.Unlock()
close(stop)
<-done
r.mu.Lock()
r.stop, r.done, r.stopping = nil, nil, false
r.mu.Unlock()
}
func (r *OperationalCleanupRunner) runOnce() {
@@ -53,6 +53,20 @@ func TestCleanupLeaseIsSingleInstanceAndHasPersistedSchedule(t *testing.T) {
}
}
func TestOperationalCleanupRunnerCanRestartAfterStop(t *testing.T) {
engine := newCleanupEngine(t)
defer engine.Close()
store, err := NewEventStore(engine)
if err != nil {
t.Fatal(err)
}
runner := NewOperationalCleanupRunner(store)
runner.Start()
runner.Stop()
runner.Start()
runner.Stop()
}
func TestCleanupFailureSchedulesSixHourRetry(t *testing.T) {
engine := newCleanupEngine(t)
defer engine.Close()