整理前后端与测试构建目录并完善配置和安全引导
Security Checks / security (push) Waiting to run

统一 frontend、backend、test、test-log 和 out 目录,更新构建、部署及测试入口;纳入配置管理兼容与首次安全引导修复;排除本机配置、密钥和测试产物。
This commit is contained in:
xuans committed 2026-09-05 19:19:52 +08:00
1 parent 28cd9d3406
commit f60d3b0880
603 files changed
+2351 -660

No files matched your search

+215
View File
@@ -0,0 +1,215 @@
package db
import (
stdcontext "context"
"fmt"
"os"
"strings"
"time"
"git.pcux.cn/xuans/Smail/config"
"git.pcux.cn/xuans/Smail/db/migration"
"git.pcux.cn/xuans/Smail/models"
"git.pcux.cn/xuans/Smail/services/access"
requestcontext "git.pcux.cn/xuans/Smail/utils/context"
"git.pcux.cn/xuans/Smail/utils/errors"
_ "github.com/go-sql-driver/mysql"
_ "github.com/lib/pq"
log "github.com/sirupsen/logrus"
"xorm.io/xorm"
)
var Instance *xorm.Engine
func Init(version string) error {
cfg := config.CurrentConfig()
if cfg == nil {
return errors.New("runtime configuration is unavailable")
}
dsn := cfg.DbDSN
var err error
if cfg.DbType == config.DBTypeSQLite {
dsn, err = sqliteDSNWithReliabilityOptions(dsn)
if err != nil {
return errors.Wrap(err)
}
}
switch cfg.DbType {
case "mysql":
Instance, err = xorm.NewEngine("mysql", dsn)
case "sqlite":
Instance, err = xorm.NewEngine("sqlite", dsn)
case "postgres":
Instance, err = xorm.NewEngine("postgres", dsn)
default:
return errors.New("Database Type Error!")
}
if err != nil {
log.Errorf("DB init Error! %s", err.Error())
return errors.Wrap(err)
}
if cfg.DbType == config.DBTypeSQLite {
Instance.SetMaxOpenConns(1)
Instance.SetMaxIdleConns(1)
} else {
Instance.SetMaxOpenConns(100)
Instance.SetMaxIdleConns(10)
}
Instance.SetConnMaxLifetime(30 * time.Minute)
Instance.ShowSQL(cfg.LogLevel == "debug")
dialect, err := migration.ParseDialect(cfg.DbType)
if err != nil {
return errors.Wrap(err)
}
if legacySyncEnabled(version) {
// 先完成兼容期表结构引导,再由有版本迁移记录的迁移器执行数据迁移。
// 新版业务迁移不得在此处继续增加无版本的结构变更。
if err := syncTables(); err != nil {
return errors.Wrap(err)
}
}
report, err := (migration.Runner{
DB: Instance.DB().DB,
Dialect: dialect,
Migrations: migration.DefaultMigrations(),
}).Run(stdcontext.Background())
if err != nil {
return errors.Wrap(err)
}
log.Infof("\u5f53\u524d\u6570\u636e\u5e93\u7ed3\u6784\u7248\u672c: %d", report.CurrentVersion)
// 更新历史数据
if err := fixHistoryData(); err != nil {
return errors.Wrap(err)
}
// 在数据库中记录程序版本
var v models.Version
_, err = Instance.Get(&v)
if err != nil {
return errors.Wrap(err)
}
if version != "" && v.Info != version && version != "test" {
v.Info = version
if v.Id == 0 {
if _, err := Instance.Insert(&v); err != nil {
return errors.Wrap(err)
}
} else {
if _, err := Instance.ID(v.Id).Cols("info").Update(&v); err != nil {
return errors.Wrap(err)
}
}
}
//if config.Instance.LogLevel == "debug" {
// Instance.ShowSQL(true)
//}
return nil
}
func WithContext(ctx *requestcontext.Context, sql string) string {
if ctx != nil {
logId := ctx.GetValue(requestcontext.LogID)
return fmt.Sprintf("/* %s */ %s", logId, sql)
}
return sql
}
func legacySyncEnabled(version string) bool {
value := strings.TrimSpace(os.Getenv("SMAIL_ENABLE_LEGACY_SYNC2"))
if value != "1" && !strings.EqualFold(value, "true") {
return false
}
if version != "test" && !config.DevelopmentMode {
log.Warn("\u751f\u4ea7\u73af\u5883\u5df2\u62d2\u7edd\u542f\u7528\u65e7 Sync2 \u517c\u5bb9\u6a21\u5f0f")
return false
}
return true
}
func syncTables() error {
tables := []any{
&models.User{},
&models.Email{},
&models.Group{},
&models.Rule{},
&models.Sessions{},
&models.UserEmail{},
&models.ExternalEmail{},
&models.VerificationEvent{},
&models.SecurityQuestion{},
&models.RecoveryCode{},
&models.WebAuthnCredential{},
&models.ProtocolCredential{},
&models.Version{},
}
for _, table := range tables {
if err := Instance.Sync2(table); err != nil {
return fmt.Errorf("sync database table %T: %w", table, err)
}
}
return nil
}
func fixHistoryData() error {
var ueNum int
_, err := Instance.Table(&models.UserEmail{}).Select("count(1)").Get(&ueNum)
if err != nil {
return errors.Wrap(err)
}
if ueNum > 0 {
return nil
}
// 只有一个管理员用户
var user []models.User
err = Instance.Table(&models.User{}).OrderBy("id asc").Find(&user)
if err != nil {
return errors.Wrap(err)
}
// 只有一个账号,且不是管理员账号,将账号提权为管理员
if len(user) == 1 && user[0].IsAdmin == 0 {
u := user[0]
u.ApplyRole(access.RoleSuperAdmin)
_, err = Instance.Update(&u)
if err != nil {
return errors.Wrap(err)
}
}
if len(user) != 1 {
return nil
}
// 以前有邮件
var emails []*models.Email
err = Instance.Table(&models.Email{}).Select("id,status").OrderBy("id asc").Find(&emails)
if err != nil {
return errors.Wrap(err)
}
if len(emails) == 0 {
return nil
}
log.Infof("Sync History Data!Please Wait!")
// 把以前的邮件,全部分到管理员账号下面去
for _, email := range emails {
ue := models.UserEmail{
UserID: user[0].ID,
EmailID: email.Id,
Status: email.Status,
}
_, err = Instance.Insert(&ue)
if err != nil {
return errors.Wrap(err)
}
}
log.Infof("Sync History Data Finished. Num: %d", len(emails))
return nil
}
+69
View File
@@ -0,0 +1,69 @@
package db
import (
"context"
"path/filepath"
"testing"
"git.pcux.cn/xuans/Smail/config"
"git.pcux.cn/xuans/Smail/db/migration"
)
func TestInitAppliesSchemaMigration(t *testing.T) {
databasePath := filepath.Join(t.TempDir(), "init.db")
previousConfig := config.Instance
previousInstance := Instance
t.Cleanup(func() {
if Instance != nil && Instance != previousInstance {
_ = Instance.Close()
}
Instance = previousInstance
config.Instance = previousConfig
})
config.Instance = &config.Config{DbType: config.DBTypeSQLite, DbDSN: databasePath}
if err := Init("test"); err != nil {
t.Fatal(err)
}
runner := migration.Runner{DB: Instance.DB().DB, Dialect: migration.DialectSQLite, Migrations: migration.DefaultMigrations()}
wantVersion := int64(len(migration.DefaultMigrations()))
version, err := runner.CurrentVersion(context.Background())
if err != nil {
t.Fatal(err)
}
if version != wantVersion {
t.Fatalf("schema version = %d, want %d", version, wantVersion)
}
var count int
if _, err = Instance.SQL("SELECT count(*) FROM schema_migration").Get(&count); err != nil {
t.Fatal(err)
}
if count != int(wantVersion) {
t.Fatalf("schema migration rows = %d, want %d", count, wantVersion)
}
}
func TestLegacySyncRequiresExplicitDevelopmentOrTestMode(t *testing.T) {
previousDevelopment := config.DevelopmentMode
t.Cleanup(func() { config.DevelopmentMode = previousDevelopment })
t.Setenv("SMAIL_ENABLE_LEGACY_SYNC2", "")
config.DevelopmentMode = true
if legacySyncEnabled("test") {
t.Fatal("legacy Sync2 was enabled without the compatibility flag")
}
t.Setenv("SMAIL_ENABLE_LEGACY_SYNC2", "1")
config.DevelopmentMode = false
if legacySyncEnabled("release") {
t.Fatal("legacy Sync2 was enabled in production")
}
if !legacySyncEnabled("test") {
t.Fatal("legacy Sync2 was not enabled for an explicit test run")
}
config.DevelopmentMode = true
if !legacySyncEnabled("release") {
t.Fatal("legacy Sync2 was not enabled in development mode")
}
}
+45
View File
@@ -0,0 +1,45 @@
package migration
import "fmt"
type Dialect string
const (
DialectMySQL Dialect = "mysql"
DialectSQLite Dialect = "sqlite"
DialectPostgres Dialect = "postgres"
)
func ParseDialect(value string) (Dialect, error) {
dialect := Dialect(value)
switch dialect {
case DialectMySQL, DialectSQLite, DialectPostgres:
return dialect, nil
default:
return "", fmt.Errorf("unsupported migration dialect %q", value)
}
}
func schemaTableDDL(dialect Dialect) (string, error) {
switch dialect {
case DialectMySQL:
return "CREATE TABLE IF NOT EXISTS schema_migration (version BIGINT NOT NULL PRIMARY KEY, name VARCHAR(255) NOT NULL, checksum CHAR(64) NOT NULL, applied_at TIMESTAMP(6) NOT NULL, execution_ms BIGINT NOT NULL)", nil
case DialectSQLite:
return "CREATE TABLE IF NOT EXISTS schema_migration (version INTEGER NOT NULL PRIMARY KEY, name TEXT NOT NULL, checksum TEXT NOT NULL, applied_at TIMESTAMP NOT NULL, execution_ms INTEGER NOT NULL)", nil
case DialectPostgres:
return "CREATE TABLE IF NOT EXISTS schema_migration (version BIGINT PRIMARY KEY, name VARCHAR(255) NOT NULL, checksum CHAR(64) NOT NULL, applied_at TIMESTAMPTZ NOT NULL, execution_ms BIGINT NOT NULL)", nil
default:
return "", fmt.Errorf("unsupported migration dialect %q", dialect)
}
}
func insertSQL(dialect Dialect) (string, error) {
switch dialect {
case DialectMySQL, DialectSQLite:
return "INSERT INTO schema_migration(version, name, checksum, applied_at, execution_ms) VALUES (?, ?, ?, ?, ?)", nil
case DialectPostgres:
return "INSERT INTO schema_migration(version, name, checksum, applied_at, execution_ms) VALUES ($1, $2, $3, $4, $5)", nil
default:
return "", fmt.Errorf("unsupported migration dialect %q", dialect)
}
}
+32
View File
@@ -0,0 +1,32 @@
package migration
import "testing"
func TestDialectSQL(t *testing.T) {
for _, dialect := range []Dialect{DialectMySQL, DialectSQLite, DialectPostgres} {
if _, err := schemaTableDDL(dialect); err != nil {
t.Fatalf("DDL for %s: %v", dialect, err)
}
if _, err := insertSQL(dialect); err != nil {
t.Fatalf("insert SQL for %s: %v", dialect, err)
}
if parsed, err := ParseDialect(string(dialect)); err != nil || parsed != dialect {
t.Fatalf("ParseDialect(%s) = %s, %v", dialect, parsed, err)
}
}
if _, err := ParseDialect("oracle"); err == nil {
t.Fatal("unsupported dialect accepted")
}
}
func TestMigrationChecksumIsStableAndContentBound(t *testing.T) {
base := Define(1, "create", "CREATE TABLE sample(id INTEGER)")
copyValue := Define(1, "create", "CREATE TABLE sample(id INTEGER)")
changed := Define(1, "create", "CREATE TABLE sample(id BIGINT)")
if base.Checksum() != copyValue.Checksum() {
t.Fatal("equivalent migrations have different checksums")
}
if base.Checksum() == changed.Checksum() {
t.Fatal("changed migration retained checksum")
}
}
+19
View File
@@ -0,0 +1,19 @@
package migration
import (
"context"
"fmt"
)
func applyIMAPDeletedFlagSchema(ctx context.Context, exec executor, dialect Dialect) error {
definitions := map[Dialect]string{
DialectSQLite: `INTEGER NOT NULL DEFAULT 0`,
DialectMySQL: `TINYINT NOT NULL DEFAULT 0`,
DialectPostgres: `SMALLINT NOT NULL DEFAULT 0`,
}
definition, ok := definitions[dialect]
if !ok {
return fmt.Errorf("unsupported migration dialect %q", dialect)
}
return ensureColumn(ctx, exec, dialect, "user_email", "is_deleted", definition)
}
+59
View File
@@ -0,0 +1,59 @@
package migration
import (
"context"
"fmt"
"strings"
)
func ensureIndex(ctx context.Context, exec executor, dialect Dialect, table, index string, unique bool, columns []string) error {
exists, err := indexExists(ctx, exec, dialect, table, index)
if err != nil {
return fmt.Errorf("check index %s: %w", index, err)
}
if exists {
return nil
}
quotedTable, quotedIndex, err := quotedIdentifiers(dialect, table, index)
if err != nil {
return err
}
quotedColumns := make([]string, 0, len(columns))
for _, column := range columns {
_, quotedColumn, err := quotedIdentifiers(dialect, table, column)
if err != nil {
return err
}
quotedColumns = append(quotedColumns, quotedColumn)
}
indexType := "INDEX"
if unique {
indexType = "UNIQUE INDEX"
}
statement := fmt.Sprintf("CREATE %s %s ON %s (%s)", indexType, quotedIndex, quotedTable, strings.Join(quotedColumns, ", "))
if _, err := exec.ExecContext(ctx, statement); err != nil {
return fmt.Errorf("create index %s: %w", index, err)
}
return nil
}
func indexExists(ctx context.Context, exec executor, dialect Dialect, table, index string) (bool, error) {
var count int
switch dialect {
case DialectSQLite:
if err := exec.QueryRowContext(ctx, `SELECT COUNT(*) FROM pragma_index_list(?) WHERE name = ?`, table, index).Scan(&count); err != nil {
return false, err
}
case DialectMySQL:
if err := exec.QueryRowContext(ctx, "SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema = DATABASE() AND table_name = ? AND index_name = ?", table, index).Scan(&count); err != nil {
return false, err
}
case DialectPostgres:
if err := exec.QueryRowContext(ctx, "SELECT COUNT(*) FROM pg_indexes WHERE schemaname = current_schema() AND tablename = $1 AND indexname = $2", table, index).Scan(&count); err != nil {
return false, err
}
default:
return false, fmt.Errorf("unsupported index dialect %q", dialect)
}
return count > 0, nil
}
+265
View File
@@ -0,0 +1,265 @@
package migration
import (
"context"
"database/sql"
"fmt"
"net/url"
"os"
"strings"
"sync"
"testing"
"time"
mysqlDriver "github.com/go-sql-driver/mysql"
_ "github.com/lib/pq"
)
const integrationTimeout = 45 * time.Second
type integrationDatabaseFactory func(*testing.T) (*sql.DB, func())
func TestMySQLIntegration(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("SMAIL_TEST_MYSQL_DSN"))
if dsn == "" {
t.Skip("SMAIL_TEST_MYSQL_DSN is not set")
}
runIntegrationSuite(t, DialectMySQL, mysqlIntegrationDatabase(dsn))
}
func TestPostgresIntegration(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("SMAIL_TEST_POSTGRES_DSN"))
if dsn == "" {
t.Skip("SMAIL_TEST_POSTGRES_DSN is not set")
}
runIntegrationSuite(t, DialectPostgres, postgresIntegrationDatabase(dsn))
}
func runIntegrationSuite(t *testing.T, dialect Dialect, factory integrationDatabaseFactory) {
t.Helper()
t.Run("first_and_repeated_run", func(t *testing.T) {
db, cleanup := factory(t)
defer cleanup()
runner := Runner{DB: db, Dialect: dialect, Migrations: DefaultMigrations(), LockWait: 10 * time.Second}
wantVersion := int64(len(DefaultMigrations()))
first := runIntegrationMigrations(t, runner)
if first.CurrentVersion != wantVersion || len(first.Applied) != int(wantVersion) {
t.Fatalf("first report = %+v", first)
}
second := runIntegrationMigrations(t, runner)
if second.CurrentVersion != wantVersion || len(second.Applied) != 0 {
t.Fatalf("second report = %+v", second)
}
})
t.Run("concurrent_run", func(t *testing.T) {
db, cleanup := factory(t)
defer cleanup()
db.SetMaxOpenConns(4)
runner := Runner{DB: db, Dialect: dialect, Migrations: DefaultMigrations(), LockWait: 15 * time.Second}
ctx, cancel := context.WithTimeout(context.Background(), integrationTimeout)
defer cancel()
var wg sync.WaitGroup
reports := make(chan Report, 2)
errs := make(chan error, 2)
for i := 0; i < 2; i++ {
wg.Add(1)
go func() {
defer wg.Done()
report, err := runner.Run(ctx)
if err != nil {
errs <- err
return
}
reports <- report
}()
}
wg.Wait()
close(reports)
close(errs)
for err := range errs {
t.Fatal(err)
}
applied := 0
for report := range reports {
applied += len(report.Applied)
}
wantVersion := len(DefaultMigrations())
if applied != wantVersion {
t.Fatalf("applied migration count = %d, want %d", applied, wantVersion)
}
assertIntegrationCount(t, db, "schema_migration", wantVersion)
})
t.Run("failed_migration_recovery", func(t *testing.T) {
db, cleanup := factory(t)
defer cleanup()
if _, err := db.Exec("CREATE TABLE recovery_probe (id BIGINT PRIMARY KEY, value VARCHAR(64) NOT NULL)"); err != nil {
t.Fatal(err)
}
broken := Runner{
DB: db,
Dialect: dialect,
LockWait: 10 * time.Second,
Migrations: []Migration{
Define(1, "recoverable", "INSERT INTO recovery_probe(id, value) VALUES (1, 'pending')", "INSERT INTO missing_recovery_probe(id) VALUES (1)"),
},
}
ctx, cancel := context.WithTimeout(context.Background(), integrationTimeout)
_, err := broken.Run(ctx)
cancel()
if err == nil {
t.Fatal("expected migration failure")
}
assertIntegrationCount(t, db, "recovery_probe", 0)
assertIntegrationCount(t, db, "schema_migration", 0)
fixed := Runner{
DB: db,
Dialect: dialect,
LockWait: 10 * time.Second,
Migrations: []Migration{Define(1, "recoverable", "INSERT INTO recovery_probe(id, value) VALUES (1, 'recovered')")},
}
report := runIntegrationMigrations(t, fixed)
if report.CurrentVersion != 1 || len(report.Applied) != 1 {
t.Fatalf("recovery report = %+v", report)
}
assertIntegrationCount(t, db, "recovery_probe", 1)
assertIntegrationCount(t, db, "schema_migration", 1)
})
}
func runIntegrationMigrations(t *testing.T, runner Runner) Report {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), integrationTimeout)
defer cancel()
report, err := runner.Run(ctx)
if err != nil {
t.Fatal(err)
}
return report
}
func assertIntegrationCount(t *testing.T, db *sql.DB, table string, want int) {
t.Helper()
var count int
if err := db.QueryRow("SELECT COUNT(*) FROM " + table).Scan(&count); err != nil {
t.Fatal(err)
}
if count != want {
t.Fatalf("%s row count = %d, want %d", table, count, want)
}
}
func mysqlIntegrationDatabase(dsn string) integrationDatabaseFactory {
return func(t *testing.T) (*sql.DB, func()) {
t.Helper()
config, err := mysqlDriver.ParseDSN(dsn)
if err != nil {
t.Fatal(err)
}
admin, err := sql.Open("mysql", config.FormatDSN())
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), integrationTimeout)
defer cancel()
if err = admin.PingContext(ctx); err != nil {
_ = admin.Close()
t.Fatal(err)
}
name := integrationIdentifier("smail_migration_test")
if _, err = admin.ExecContext(ctx, "CREATE DATABASE `"+name+"` CHARACTER SET utf8mb4"); err != nil {
_ = admin.Close()
t.Fatal(err)
}
testConfig := *config
testConfig.DBName = name
testConfig.ParseTime = true
db, err := sql.Open("mysql", testConfig.FormatDSN())
if err != nil {
_, _ = admin.Exec("DROP DATABASE IF EXISTS `" + name + "`")
_ = admin.Close()
t.Fatal(err)
}
if err = db.PingContext(ctx); err != nil {
_ = db.Close()
_, _ = admin.Exec("DROP DATABASE IF EXISTS `" + name + "`")
_ = admin.Close()
t.Fatal(err)
}
cleanup := func() {
_ = db.Close()
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), integrationTimeout)
defer cleanupCancel()
_, _ = admin.ExecContext(cleanupCtx, "DROP DATABASE IF EXISTS `"+name+"`")
_ = admin.Close()
}
return db, cleanup
}
}
func postgresIntegrationDatabase(dsn string) integrationDatabaseFactory {
return func(t *testing.T) (*sql.DB, func()) {
t.Helper()
admin, err := sql.Open("postgres", dsn)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), integrationTimeout)
defer cancel()
if err = admin.PingContext(ctx); err != nil {
_ = admin.Close()
t.Fatal(err)
}
name := integrationIdentifier("smail_migration_test")
if _, err = admin.ExecContext(ctx, "CREATE SCHEMA \""+name+"\""); err != nil {
_ = admin.Close()
t.Fatal(err)
}
testDSN, err := postgresDSNWithSchema(dsn, name)
if err != nil {
_, _ = admin.Exec("DROP SCHEMA IF EXISTS \"" + name + "\" CASCADE")
_ = admin.Close()
t.Fatal(err)
}
db, err := sql.Open("postgres", testDSN)
if err != nil {
_, _ = admin.Exec("DROP SCHEMA IF EXISTS \"" + name + "\" CASCADE")
_ = admin.Close()
t.Fatal(err)
}
if err = db.PingContext(ctx); err != nil {
_ = db.Close()
_, _ = admin.Exec("DROP SCHEMA IF EXISTS \"" + name + "\" CASCADE")
_ = admin.Close()
t.Fatal(err)
}
cleanup := func() {
_ = db.Close()
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), integrationTimeout)
defer cleanupCancel()
_, _ = admin.ExecContext(cleanupCtx, "DROP SCHEMA IF EXISTS \""+name+"\" CASCADE")
_ = admin.Close()
}
return db, cleanup
}
}
func postgresDSNWithSchema(dsn, schema string) (string, error) {
if strings.HasPrefix(dsn, "postgres://") || strings.HasPrefix(dsn, "postgresql://") {
parsed, err := url.Parse(dsn)
if err != nil {
return "", err
}
query := parsed.Query()
query.Set("search_path", schema)
parsed.RawQuery = query.Encode()
return parsed.String(), nil
}
return strings.TrimSpace(dsn) + " search_path=" + schema, nil
}
func integrationIdentifier(prefix string) string {
return fmt.Sprintf("%s_%d", prefix, time.Now().UnixNano())
}
+161
View File
@@ -0,0 +1,161 @@
package migration
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strings"
)
type Migration struct {
Version int64
Name string
Statements []string
DialectStatements map[Dialect][]string
ChecksumSeed string
Apply func(context.Context, executor, Dialect) error
CompatibleChecksums []string
}
func Define(version int64, name string, statements ...string) Migration {
copied := append([]string(nil), statements...)
return Migration{Version: version, Name: strings.TrimSpace(name), Statements: copied}
}
func DefineForDialects(version int64, name string, statements map[Dialect][]string) Migration {
copied := make(map[Dialect][]string, len(statements))
for dialect, items := range statements {
copied[dialect] = append([]string(nil), items...)
}
return Migration{Version: version, Name: strings.TrimSpace(name), DialectStatements: copied}
}
func DefineCustom(version int64, name string, checksumSeed string, apply func(context.Context, executor, Dialect) error) Migration {
return Migration{
Version: version,
Name: strings.TrimSpace(name),
ChecksumSeed: strings.TrimSpace(checksumSeed),
Apply: apply,
}
}
func (m Migration) StatementsFor(dialect Dialect) []string {
if len(m.DialectStatements) > 0 {
return append([]string(nil), m.DialectStatements[dialect]...)
}
return append([]string(nil), m.Statements...)
}
func WithCompatibleChecksums(migration Migration, checksums ...string) Migration {
migration.CompatibleChecksums = make([]string, len(checksums))
for index, checksum := range checksums {
migration.CompatibleChecksums[index] = strings.TrimSpace(checksum)
}
return migration
}
func (m Migration) AcceptsChecksum(checksum string) bool {
if checksum == m.Checksum() {
return true
}
for _, compatible := range m.CompatibleChecksums {
if checksum == compatible {
return true
}
}
return false
}
func (m Migration) Checksum() string {
hash := sha256.New()
_, _ = fmt.Fprintf(hash, "%d\x00%s\x00", m.Version, m.Name)
for _, statement := range m.Statements {
_, _ = hash.Write([]byte(strings.TrimSpace(statement)))
_, _ = hash.Write([]byte{0})
}
if m.ChecksumSeed != "" {
_, _ = hash.Write([]byte(m.ChecksumSeed))
_, _ = hash.Write([]byte{0})
}
keys := make([]string, 0, len(m.DialectStatements))
for dialect := range m.DialectStatements {
keys = append(keys, string(dialect))
}
sort.Strings(keys)
for _, key := range keys {
_, _ = hash.Write([]byte(key))
_, _ = hash.Write([]byte{0})
for _, statement := range m.DialectStatements[Dialect(key)] {
_, _ = hash.Write([]byte(strings.TrimSpace(statement)))
_, _ = hash.Write([]byte{0})
}
}
return hex.EncodeToString(hash.Sum(nil))
}
func normalize(migrations []Migration) ([]Migration, error) {
result := append([]Migration(nil), migrations...)
sort.Slice(result, func(i, j int) bool { return result[i].Version < result[j].Version })
for i, item := range result {
if i == 0 && item.Version != 1 {
return nil, fmt.Errorf("migration sequence must start at version 1")
}
if i > 0 && item.Version != result[i-1].Version+1 {
return nil, fmt.Errorf("migration version %d is not contiguous", item.Version)
}
if item.Version <= 0 || item.Name == "" || !hasStatements(item) {
return nil, fmt.Errorf("invalid migration definition at index %d", i)
}
knownChecksums := map[string]struct{}{item.Checksum(): {}}
for _, checksum := range item.CompatibleChecksums {
checksum = strings.TrimSpace(checksum)
decoded, decodeErr := hex.DecodeString(checksum)
if decodeErr != nil || len(decoded) != sha256.Size {
return nil, fmt.Errorf("migration %d contains an invalid compatible checksum", item.Version)
}
if _, exists := knownChecksums[checksum]; exists {
return nil, fmt.Errorf("migration %d contains a duplicate compatible checksum", item.Version)
}
knownChecksums[checksum] = struct{}{}
}
for _, statement := range item.Statements {
if strings.TrimSpace(statement) == "" {
return nil, fmt.Errorf("migration %d contains an empty statement", item.Version)
}
}
for dialect, statements := range item.DialectStatements {
if _, err := ParseDialect(string(dialect)); err != nil {
return nil, err
}
if len(statements) == 0 {
return nil, fmt.Errorf("migration %d contains no statements for %s", item.Version, dialect)
}
for _, statement := range statements {
if strings.TrimSpace(statement) == "" {
return nil, fmt.Errorf("migration %d contains an empty statement", item.Version)
}
}
}
if i > 0 && result[i-1].Version == item.Version {
return nil, fmt.Errorf("duplicate migration version %d", item.Version)
}
}
return result, nil
}
func hasStatements(m Migration) bool {
if m.Apply != nil && m.ChecksumSeed != "" {
return true
}
if len(m.Statements) > 0 {
return true
}
for _, statements := range m.DialectStatements {
if len(statements) > 0 {
return true
}
}
return false
}
@@ -0,0 +1,19 @@
package migration
func protocolCredentialStatements() map[Dialect][]string {
return map[Dialect][]string{
DialectSQLite: {
`CREATE TABLE IF NOT EXISTS "protocol_credential" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "user_id" INTEGER NOT NULL, "protocol" TEXT NOT NULL, "enabled" INTEGER NOT NULL DEFAULT 0, "code_hash" TEXT NOT NULL DEFAULT '', "code_version" INTEGER NOT NULL DEFAULT 1, "created_at" DATETIME NOT NULL, "updated_at" DATETIME NOT NULL, "last_used_at" DATETIME NULL, "revoked_at" DATETIME NULL, UNIQUE("user_id", "protocol"))`,
`CREATE INDEX IF NOT EXISTS "idx_protocol_credential_user_id" ON "protocol_credential"("user_id")`,
`CREATE INDEX IF NOT EXISTS "idx_protocol_credential_protocol_enabled" ON "protocol_credential"("protocol", "enabled", "revoked_at")`,
},
DialectMySQL: {
"CREATE TABLE IF NOT EXISTS `protocol_credential` (`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, `user_id` BIGINT NOT NULL, `protocol` VARCHAR(16) NOT NULL, `enabled` TINYINT(1) NOT NULL DEFAULT 0, `code_hash` VARCHAR(255) NOT NULL DEFAULT '', `code_version` INT NOT NULL DEFAULT 1, `created_at` DATETIME(6) NOT NULL, `updated_at` DATETIME(6) NOT NULL, `last_used_at` DATETIME(6) NULL, `revoked_at` DATETIME(6) NULL, UNIQUE KEY `uq_protocol_credential_user_protocol` (`user_id`, `protocol`), KEY `idx_protocol_credential_user_id` (`user_id`), KEY `idx_protocol_credential_protocol_enabled` (`protocol`, `enabled`, `revoked_at`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
},
DialectPostgres: {
`CREATE TABLE IF NOT EXISTS "protocol_credential" ("id" BIGSERIAL PRIMARY KEY, "user_id" BIGINT NOT NULL, "protocol" VARCHAR(16) NOT NULL, "enabled" BOOLEAN NOT NULL DEFAULT FALSE, "code_hash" VARCHAR(255) NOT NULL DEFAULT '', "code_version" INTEGER NOT NULL DEFAULT 1, "created_at" TIMESTAMPTZ NOT NULL, "updated_at" TIMESTAMPTZ NOT NULL, "last_used_at" TIMESTAMPTZ NULL, "revoked_at" TIMESTAMPTZ NULL, UNIQUE("user_id", "protocol"))`,
`CREATE INDEX IF NOT EXISTS "idx_protocol_credential_user_id" ON "protocol_credential"("user_id")`,
`CREATE INDEX IF NOT EXISTS "idx_protocol_credential_protocol_enabled" ON "protocol_credential"("protocol", "enabled", "revoked_at")`,
},
}
}
+24
View File
@@ -0,0 +1,24 @@
package migration
func DefaultMigrations() []Migration {
return []Migration{
WithCompatibleChecksums(
DefineForDialects(1, "create_core_schema", coreSchemaStatements()),
"57b378b273f243bae44e92cad161f297248e5226e96fbd7905815aa8604146bc",
),
DefineCustom(2, "expand_user_security", "expand_user_security_v1", applyUserSecuritySchema),
DefineForDialects(3, "create_security_email_tables", securityEmailStatements()),
DefineForDialects(4, "create_account_recovery_tables", credentialStatements()),
DefineForDialects(5, "create_verification_operations_tables", verificationOperationsStatements()),
DefineCustom(6, "expand_verification_retention", "expand_verification_retention_v1", applyVerificationRetentionSchema),
DefineForDialects(7, "create_webauthn_credentials", webAuthnCredentialStatements()),
DefineCustom(8, "mark_webauthn_discoverability", "mark_webauthn_discoverability_v1", applyWebAuthnDiscoverabilitySchema),
DefineForDialects(9, "create_totp_authenticator", totpAuthenticatorStatements()),
DefineCustom(10, "expand_user_lifecycle_security", "expand_user_lifecycle_security_v1", applyUserLifecycleSchema),
DefineForDialects(11, "create_plugin_configuration_tables", pluginSettingsStatements()),
DefineCustom(12, "expand_user_email_login_addresses", "expand_user_email_login_addresses_v1", applyUserEmailAddressSchema),
DefineForDialects(13, "create_protocol_credentials", protocolCredentialStatements()),
DefineCustom(14, "expand_imap_deleted_flag", "expand_imap_deleted_flag_v1", applyIMAPDeletedFlagSchema),
DefineCustom(15, "expand_security_setup", "expand_security_setup_v1", applySecuritySetupSchema),
}
}
+279
View File
@@ -0,0 +1,279 @@
package migration
import (
"context"
"database/sql"
"strings"
"testing"
"time"
)
func TestDefaultMigrationsCreateCompleteSQLiteSchema(t *testing.T) {
db := openSQLite(t)
runner := Runner{DB: db, Dialect: DialectSQLite, Migrations: DefaultMigrations()}
report, err := runner.Run(context.Background())
if err != nil {
t.Fatal(err)
}
wantVersion := int64(len(DefaultMigrations()))
if report.CurrentVersion != wantVersion || len(report.Applied) != int(wantVersion) {
t.Fatalf("report = %+v", report)
}
for _, table := range []string{"user", "email", "group", "rule", "sessions", "user_email", "version", "external_email", "verification_event", "security_question", "recovery_code", "verification_rate_limit", "verification_cleanup_state", "webauthn_credential", "totp_authenticator", "plugin_secret", "plugin_user_setting", "schema_migration"} {
if !sqliteTableExists(t, db, table) {
t.Fatalf("table %s was not created", table)
}
}
for _, column := range []string{"role", "session_version", "security_version", "must_change_password", "password_changed_at", "security_setup_acknowledged"} {
exists, err := columnExists(context.Background(), db, DialectSQLite, "user", column)
if err != nil {
t.Fatal(err)
}
if !exists {
t.Fatalf("column user.%s was not created", column)
}
}
exists, err := columnExists(context.Background(), db, DialectSQLite, "webauthn_credential", "discoverable")
if err != nil {
t.Fatal(err)
}
if !exists {
t.Fatal("column webauthn_credential.discoverable was not created")
}
for _, column := range []string{"address", "address_canonical", "is_primary", "verified_at", "created_at", "updated_at", "is_deleted"} {
exists, err := columnExists(context.Background(), db, DialectSQLite, "user_email", column)
if err != nil {
t.Fatal(err)
}
if !exists {
t.Fatalf("column user_email.%s was not created", column)
}
}
for _, index := range []string{"uq_user_email_address_canonical", "idx_user_email_user_primary"} {
if !sqliteIndexExists(t, db, index) {
t.Fatalf("index %s was not created", index)
}
}
}
func sqliteIndexExists(t *testing.T, db *sql.DB, index string) bool {
t.Helper()
var count int
if err := db.QueryRow("SELECT count(*) FROM sqlite_master WHERE type='index' AND name=?", index).Scan(&count); err != nil {
t.Fatal(err)
}
return count == 1
}
func TestDefaultMigrationsUpgradeLegacyUserSchema(t *testing.T) {
db := openSQLite(t)
if _, err := db.Exec(`CREATE TABLE "user" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "account" TEXT NOT NULL UNIQUE, "name" TEXT NOT NULL, "password" CHAR(32) NOT NULL, "disabled" INTEGER NOT NULL DEFAULT 0, "is_admin" INTEGER NOT NULL DEFAULT 0)`); err != nil {
t.Fatal(err)
}
if _, err := db.Exec(`INSERT INTO "user"("account", "name", "password", "is_admin") VALUES ('root', 'root', 'legacy', 1)`); err != nil {
t.Fatal(err)
}
runner := Runner{DB: db, Dialect: DialectSQLite, Migrations: DefaultMigrations()}
if _, err := runner.Run(context.Background()); err != nil {
t.Fatal(err)
}
var role string
var sessionVersion int
var securityVersion int
var mustChangePassword int
var passwordChangedAt any
if err := db.QueryRow(`SELECT "role", "session_version", "security_version", "must_change_password", "password_changed_at" FROM "user" WHERE "account" = 'root'`).Scan(&role, &sessionVersion, &securityVersion, &mustChangePassword, &passwordChangedAt); err != nil {
t.Fatal(err)
}
if role != "super_admin" || sessionVersion != 1 || securityVersion != 1 || mustChangePassword != 0 || passwordChangedAt != nil {
t.Fatalf("role = %q, session version = %d, security version = %d, must change password = %d, password changed at = %v", role, sessionVersion, securityVersion, mustChangePassword, passwordChangedAt)
}
}
func TestUserEmailLoginAddressMigrationUpgradesLegacyTableAndIsIdempotent(t *testing.T) {
db := openSQLite(t)
if _, err := db.Exec(`CREATE TABLE "user_email" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "user_id" INTEGER NOT NULL, "email_id" INTEGER NOT NULL, "is_read" INTEGER NOT NULL DEFAULT 0, "group_id" INTEGER NOT NULL DEFAULT 0, "status" INTEGER NOT NULL DEFAULT 0, "create" DATETIME NULL)`); err != nil {
t.Fatal(err)
}
runner := Runner{DB: db, Dialect: DialectSQLite, Migrations: DefaultMigrations()}
if _, err := runner.Run(context.Background()); err != nil {
t.Fatal(err)
}
for _, column := range []string{"address", "address_canonical", "is_primary", "verified_at", "created_at", "updated_at", "is_deleted"} {
exists, err := columnExists(context.Background(), db, DialectSQLite, "user_email", column)
if err != nil {
t.Fatal(err)
}
if !exists {
t.Fatalf("legacy table is missing migrated column %s", column)
}
}
for _, index := range []string{"uq_user_email_address_canonical", "idx_user_email_user_primary"} {
if !sqliteIndexExists(t, db, index) {
t.Fatalf("legacy table is missing migrated index %s", index)
}
}
report, err := runner.Run(context.Background())
if err != nil {
t.Fatal(err)
}
if len(report.Applied) != 0 || report.CurrentVersion != int64(len(DefaultMigrations())) {
t.Fatalf("repeated migration report = %+v", report)
}
}
func TestUserEmailLoginAddressMigrationRejectsDuplicateCanonicalAddresses(t *testing.T) {
db := openSQLite(t)
if _, err := db.Exec(`CREATE TABLE "user_email" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "user_id" INTEGER NOT NULL, "email_id" INTEGER NOT NULL, "address" TEXT NULL, "address_canonical" TEXT NULL, "is_primary" INTEGER NOT NULL DEFAULT 0, "verified_at" DATETIME NULL, "created_at" DATETIME NULL, "updated_at" DATETIME NULL, "is_read" INTEGER NOT NULL DEFAULT 0, "group_id" INTEGER NOT NULL DEFAULT 0, "status" INTEGER NOT NULL DEFAULT 0, "create" DATETIME NULL)`); err != nil {
t.Fatal(err)
}
if _, err := db.Exec(`INSERT INTO "user_email"("user_id", "email_id", "address", "address_canonical") VALUES (1, 1, 'one@example.com', 'duplicate@example.com'), (2, 2, 'two@example.com', 'duplicate@example.com')`); err != nil {
t.Fatal(err)
}
err := applyUserEmailAddressSchema(context.Background(), db, DialectSQLite)
if err == nil || !strings.Contains(err.Error(), "conflict count: 1") {
t.Fatalf("duplicate canonical address error = %v", err)
}
if sqliteIndexExists(t, db, "uq_user_email_address_canonical") {
t.Fatal("unique index was created despite duplicate canonical addresses")
}
}
func TestIMAPDeletedFlagMigrationRejectsUnexpectedDialect(t *testing.T) {
err := applyIMAPDeletedFlagSchema(context.Background(), nil, Dialect("unsupported"))
if err == nil || !strings.Contains(err.Error(), "unsupported") {
t.Fatalf("unexpected dialect error = %v", err)
}
}
func TestUserEmailLoginAddressMigrationRejectsUnexpectedDialect(t *testing.T) {
err := applyUserEmailAddressSchema(context.Background(), nil, Dialect("unsupported"))
if err == nil || !strings.Contains(err.Error(), "unsupported") {
t.Fatalf("unexpected dialect error = %v", err)
}
}
func TestSecurityEmailMySQLSchemaDoesNotCreateDuplicateIndex(t *testing.T) {
statements := securityEmailStatements()[DialectMySQL]
for _, statement := range statements {
if strings.HasPrefix(strings.ToUpper(strings.TrimSpace(statement)), "CREATE INDEX IDX_EXTERNAL_EMAIL_USER_ID") {
t.Fatal("external email index is created twice")
}
}
}
func TestCustomMigrationChecksumIncludesSeed(t *testing.T) {
first := DefineCustom(1, "custom", "seed-a", func(context.Context, executor, Dialect) error { return nil })
second := DefineCustom(1, "custom", "seed-b", func(context.Context, executor, Dialect) error { return nil })
if first.Checksum() == second.Checksum() {
t.Fatal("custom migration checksum ignored its seed")
}
}
func TestDefaultMigrationsAreContiguousAndDialectComplete(t *testing.T) {
migrations := DefaultMigrations()
if len(migrations) == 0 {
t.Fatal("default migrations are empty")
}
for index, item := range migrations {
wantVersion := int64(index + 1)
if item.Version != wantVersion {
t.Fatalf("migration %d has version %d, want %d", index, item.Version, wantVersion)
}
if item.Name == "" {
t.Fatalf("migration %d has an empty name", item.Version)
}
for _, dialect := range []Dialect{DialectSQLite, DialectMySQL, DialectPostgres} {
if item.Apply == nil && len(item.StatementsFor(dialect)) == 0 {
t.Fatalf("migration %d (%s) has no statements for %s", item.Version, item.Name, dialect)
}
}
}
if _, err := normalize(migrations); err != nil {
t.Fatalf("default migrations failed normalization: %v", err)
}
}
func TestDefaultMigrationChecksumsAreImmutable(t *testing.T) {
expected := map[int64]string{
1: "bb9bbf1ef17f1430900b8c5bed3539d01f72d49bdfedc2528dd3cf089b2a00d8",
2: "b91524dc66ba86962372368ed7254e0c33eda6f76d487d6d3d291574cbf41345",
3: "675251af1f5cd5d03f645c4b4ac15db2b488f8a3c9a972e836a136450da67c52",
4: "30d34ff976ada3fec17173556f1b3a2798e03db6bce9fc93df6bd370f9044e7c",
5: "678da8a46273d17e5f85a12c6c3ca8d254af25ccd3c48245b11d8f80e1dba2c3",
6: "95b0ca429b6d237b1c905e8b5756363a242ead7fc0cf53783199d3e9ac317dc8",
7: "ae1e63c75bf6bde508d8711689db2d5a36f6619885fb3b0a28c1f26b9710143d",
8: "6c0697b86c9a57e5dc5b1a38bf18dc1893d850a62229c95134d931b6f4a9d59a",
9: "b600b7bd4d7f0aa72ca12f261d2019bc419500b6fde099bba7c3ccf0448f5860",
10: "701167503d518151ee73ff2ba32f561194538cc46f630551b50b36a483c77c12",
11: "b7203bbb965b5e5760704ff495c58a8938e97c52ac068e2b0e7c522a2874b3ed",
12: "b746b89a15262ecedacf8725148d77be22caa57e4dfb60a209854ecdec98382b",
13: "c51604f27347d0a1b93acad1e33af53463fceb7b6573960f881f95f29c5628ed",
14: "531398c2b8fc83a65753fae08666946bb253ac3da6596829e71cc8ccc4ddf62f",
15: "59f0ff9acfed3619c17f078d21a1362ddd5f1feacb9a2d77626f69040d25de65",
}
migrations := DefaultMigrations()
if len(migrations) != len(expected) {
t.Fatalf("migration count = %d, want %d", len(migrations), len(expected))
}
if got := migrations[0].CompatibleChecksums; len(got) != 1 || got[0] != "57b378b273f243bae44e92cad161f297248e5226e96fbd7905815aa8604146bc" {
t.Fatalf("migration 1 compatible checksums = %v", got)
}
for _, item := range migrations {
want, ok := expected[item.Version]
if !ok {
t.Fatalf("migration version %d is not registered", item.Version)
}
if got := item.Checksum(); got != want {
t.Fatalf("migration %d (%s) checksum = %s, want %s; create a new migration instead of changing an applied migration", item.Version, item.Name, got, want)
}
}
}
func TestDefaultMigrationsResumePublishedVersionOne(t *testing.T) {
migrations := DefaultMigrations()
published := migrations[0]
checksums := append([]string{published.Checksum()}, published.CompatibleChecksums...)
for _, checksum := range checksums {
checksum := checksum
t.Run(checksum[:8], func(t *testing.T) {
database := openSQLite(t)
for _, statement := range published.StatementsFor(DialectSQLite) {
if _, err := database.Exec(statement); err != nil {
t.Fatal(err)
}
}
ddl, err := schemaTableDDL(DialectSQLite)
if err != nil {
t.Fatal(err)
}
if _, err = database.Exec(ddl); err != nil {
t.Fatal(err)
}
if _, err = database.Exec(
`INSERT INTO schema_migration(version, name, checksum, applied_at, execution_ms) VALUES(?, ?, ?, ?, ?)`,
published.Version,
published.Name,
checksum,
time.Now().UTC(),
0,
); err != nil {
t.Fatal(err)
}
report, err := (Runner{DB: database, Dialect: DialectSQLite, Migrations: migrations}).Run(context.Background())
if err != nil {
t.Fatal(err)
}
if report.CurrentVersion != int64(len(migrations)) || len(report.Applied) != len(migrations)-1 {
t.Fatalf("report = %+v", report)
}
})
}
}
+334
View File
@@ -0,0 +1,334 @@
package migration
import (
"context"
"database/sql"
"errors"
"fmt"
"math"
"sync"
"time"
)
var (
ErrChecksumMismatch = errors.New("migration checksum mismatch")
ErrUnknownAppliedMigration = errors.New("unknown applied migration")
ErrLockUnavailable = errors.New("migration lock unavailable")
sqliteProcessLock sync.Mutex
)
const migrationLockID int64 = 0x534d41494c
type Applied struct {
Version int64
Name string
Checksum string
AppliedAt time.Time
ExecutionMS int64
}
type Report struct {
CurrentVersion int64
Applied []Applied
}
type Runner struct {
DB *sql.DB
Dialect Dialect
Migrations []Migration
LockWait time.Duration
Now func() time.Time
}
type executor interface {
ExecContext(context.Context, string, ...any) (sql.Result, error)
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
QueryRowContext(context.Context, string, ...any) *sql.Row
}
func (r Runner) Run(ctx context.Context) (Report, error) {
if r.DB == nil {
return Report{}, errors.New("migration database is nil")
}
if _, err := ParseDialect(string(r.Dialect)); err != nil {
return Report{}, err
}
migrations, err := normalize(r.Migrations)
if err != nil {
return Report{}, err
}
if err = validateDialectMigrations(migrations, r.Dialect); err != nil {
return Report{}, err
}
conn, err := r.DB.Conn(ctx)
if err != nil {
return Report{}, fmt.Errorf("open migration connection: %w", err)
}
defer conn.Close()
if r.Dialect == DialectSQLite {
return r.runSQLite(ctx, conn, migrations)
}
if err = r.acquireAdvisoryLock(ctx, conn); err != nil {
return Report{}, err
}
defer r.releaseAdvisoryLock(conn)
return r.runTransactional(ctx, conn, migrations)
}
func validateDialectMigrations(migrations []Migration, dialect Dialect) error {
for _, item := range migrations {
if item.Apply == nil && len(item.StatementsFor(dialect)) == 0 {
return fmt.Errorf("migration %d has no statements for %s", item.Version, dialect)
}
}
return nil
}
func (r Runner) CurrentVersion(ctx context.Context) (int64, error) {
if r.DB == nil {
return 0, errors.New("migration database is nil")
}
ddl, err := schemaTableDDL(r.Dialect)
if err != nil {
return 0, err
}
if _, err = r.DB.ExecContext(ctx, ddl); err != nil {
return 0, fmt.Errorf("create migration table: %w", err)
}
var version sql.NullInt64
if err = r.DB.QueryRowContext(ctx, "SELECT MAX(version) FROM schema_migration").Scan(&version); err != nil {
return 0, fmt.Errorf("query migration version: %w", err)
}
if !version.Valid {
return 0, nil
}
return version.Int64, nil
}
func (r Runner) runSQLite(ctx context.Context, conn *sql.Conn, migrations []Migration) (report Report, resultErr error) {
sqliteProcessLock.Lock()
defer sqliteProcessLock.Unlock()
if _, err := conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil {
return Report{}, fmt.Errorf("acquire SQLite migration lock: %w", err)
}
defer func() {
if resultErr != nil {
_, _ = conn.ExecContext(context.Background(), "ROLLBACK")
}
}()
report, resultErr = r.applyWithExecutor(ctx, conn, migrations)
if resultErr != nil {
return Report{}, resultErr
}
if _, resultErr = conn.ExecContext(ctx, "COMMIT"); resultErr != nil {
return Report{}, fmt.Errorf("commit SQLite migrations: %w", resultErr)
}
return report, nil
}
func (r Runner) runTransactional(ctx context.Context, conn *sql.Conn, migrations []Migration) (Report, error) {
ddl, err := schemaTableDDL(r.Dialect)
if err != nil {
return Report{}, err
}
if _, err = conn.ExecContext(ctx, ddl); err != nil {
return Report{}, fmt.Errorf("create migration table: %w", err)
}
applied, err := loadApplied(ctx, conn)
if err != nil {
return Report{}, err
}
if err = verifyApplied(applied, migrations); err != nil {
return Report{}, err
}
report := Report{CurrentVersion: highestVersion(applied)}
for _, item := range migrations {
if _, ok := applied[item.Version]; ok {
continue
}
tx, beginErr := conn.BeginTx(ctx, nil)
if beginErr != nil {
return Report{}, fmt.Errorf("begin migration %d: %w", item.Version, beginErr)
}
entry, applyErr := r.applyOne(ctx, tx, item)
if applyErr != nil {
_ = tx.Rollback()
return Report{}, applyErr
}
if commitErr := tx.Commit(); commitErr != nil {
return Report{}, fmt.Errorf("commit migration %d: %w", item.Version, commitErr)
}
report.Applied = append(report.Applied, entry)
report.CurrentVersion = item.Version
}
return report, nil
}
func (r Runner) applyWithExecutor(ctx context.Context, exec executor, migrations []Migration) (Report, error) {
ddl, err := schemaTableDDL(r.Dialect)
if err != nil {
return Report{}, err
}
if _, err = exec.ExecContext(ctx, ddl); err != nil {
return Report{}, fmt.Errorf("create migration table: %w", err)
}
applied, err := loadApplied(ctx, exec)
if err != nil {
return Report{}, err
}
if err = verifyApplied(applied, migrations); err != nil {
return Report{}, err
}
report := Report{CurrentVersion: highestVersion(applied)}
for _, item := range migrations {
if _, ok := applied[item.Version]; ok {
continue
}
entry, applyErr := r.applyOne(ctx, exec, item)
if applyErr != nil {
return Report{}, applyErr
}
report.Applied = append(report.Applied, entry)
report.CurrentVersion = item.Version
}
return report, nil
}
func (r Runner) applyOne(ctx context.Context, exec executor, item Migration) (Applied, error) {
started := time.Now()
if r.Now != nil {
started = r.Now()
}
if item.Apply != nil {
if err := item.Apply(ctx, exec, r.Dialect); err != nil {
return Applied{}, fmt.Errorf("execute migration %d (%s): %w", item.Version, item.Name, err)
}
} else {
for _, statement := range item.StatementsFor(r.Dialect) {
if _, err := exec.ExecContext(ctx, statement); err != nil {
return Applied{}, fmt.Errorf("execute migration %d (%s): %w", item.Version, item.Name, err)
}
}
}
finished := time.Now()
if r.Now != nil {
finished = r.Now()
}
executionMS := int64(math.Max(0, float64(finished.Sub(started).Milliseconds())))
entry := Applied{Version: item.Version, Name: item.Name, Checksum: item.Checksum(), AppliedAt: finished.UTC(), ExecutionMS: executionMS}
query, err := insertSQL(r.Dialect)
if err != nil {
return Applied{}, err
}
if _, err = exec.ExecContext(ctx, query, entry.Version, entry.Name, entry.Checksum, entry.AppliedAt, entry.ExecutionMS); err != nil {
return Applied{}, fmt.Errorf("record migration %d: %w", item.Version, err)
}
return entry, nil
}
func loadApplied(ctx context.Context, exec executor) (map[int64]Applied, error) {
rows, err := exec.QueryContext(ctx, "SELECT version, name, checksum, applied_at, execution_ms FROM schema_migration ORDER BY version")
if err != nil {
return nil, fmt.Errorf("query applied migrations: %w", err)
}
defer rows.Close()
result := map[int64]Applied{}
for rows.Next() {
var item Applied
if err = rows.Scan(&item.Version, &item.Name, &item.Checksum, &item.AppliedAt, &item.ExecutionMS); err != nil {
return nil, fmt.Errorf("scan applied migration: %w", err)
}
result[item.Version] = item
}
if err = rows.Err(); err != nil {
return nil, fmt.Errorf("iterate applied migrations: %w", err)
}
return result, nil
}
func verifyApplied(applied map[int64]Applied, migrations []Migration) error {
known := make(map[int64]Migration, len(migrations))
for _, item := range migrations {
known[item.Version] = item
}
for version, stored := range applied {
item, ok := known[version]
if !ok {
return fmt.Errorf("%w: version %d", ErrUnknownAppliedMigration, version)
}
if stored.Name != item.Name || !item.AcceptsChecksum(stored.Checksum) {
return fmt.Errorf("%w: version %d", ErrChecksumMismatch, version)
}
}
return nil
}
func highestVersion(applied map[int64]Applied) int64 {
var result int64
for version := range applied {
if version > result {
result = version
}
}
return result
}
func (r Runner) acquireAdvisoryLock(ctx context.Context, conn *sql.Conn) error {
wait := r.LockWait
if wait <= 0 {
wait = 30 * time.Second
}
switch r.Dialect {
case DialectMySQL:
var acquired sql.NullInt64
if err := conn.QueryRowContext(ctx, "SELECT GET_LOCK(?, ?)", "smail_schema_migration", int(math.Ceil(wait.Seconds()))).Scan(&acquired); err != nil {
return fmt.Errorf("acquire MySQL migration lock: %w", err)
}
if !acquired.Valid || acquired.Int64 != 1 {
return ErrLockUnavailable
}
case DialectPostgres:
deadline := time.Now().Add(wait)
for {
var acquired bool
if err := conn.QueryRowContext(ctx, "SELECT pg_try_advisory_lock($1)", migrationLockID).Scan(&acquired); err != nil {
return fmt.Errorf("acquire PostgreSQL migration lock: %w", err)
}
if acquired {
return nil
}
remaining := time.Until(deadline)
if remaining <= 0 {
return ErrLockUnavailable
}
delay := 100 * time.Millisecond
if remaining < delay {
delay = remaining
}
timer := time.NewTimer(delay)
select {
case <-ctx.Done():
if !timer.Stop() {
<-timer.C
}
return ctx.Err()
case <-timer.C:
}
}
default:
return fmt.Errorf("unsupported advisory lock dialect %q", r.Dialect)
}
return nil
}
func (r Runner) releaseAdvisoryLock(conn *sql.Conn) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
switch r.Dialect {
case DialectMySQL:
_, _ = conn.ExecContext(ctx, "SELECT RELEASE_LOCK(?)", "smail_schema_migration")
case DialectPostgres:
_, _ = conn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", migrationLockID)
}
}
+245
View File
@@ -0,0 +1,245 @@
package migration
import (
"context"
"database/sql"
"errors"
"path/filepath"
"sync"
"testing"
_ "modernc.org/sqlite"
)
func TestSQLiteFirstAndRepeatedRun(t *testing.T) {
db := openSQLite(t)
runner := Runner{DB: db, Dialect: DialectSQLite, Migrations: testMigrations()}
first, err := runner.Run(context.Background())
if err != nil {
t.Fatal(err)
}
if first.CurrentVersion != 2 || len(first.Applied) != 2 {
t.Fatalf("first report = %+v", first)
}
second, err := runner.Run(context.Background())
if err != nil {
t.Fatal(err)
}
if second.CurrentVersion != 2 || len(second.Applied) != 0 {
t.Fatalf("second report = %+v", second)
}
version, err := runner.CurrentVersion(context.Background())
if err != nil {
t.Fatal(err)
}
if version != 2 {
t.Fatalf("version = %d, want 2", version)
}
assertRowCount(t, db, "sample", 1)
}
func TestSQLiteChecksumConflict(t *testing.T) {
db := openSQLite(t)
runner := Runner{DB: db, Dialect: DialectSQLite, Migrations: testMigrations()}
if _, err := runner.Run(context.Background()); err != nil {
t.Fatal(err)
}
changed := testMigrations()
changed[0].Statements[0] = "CREATE TABLE sample (id INTEGER PRIMARY KEY, changed TEXT)"
_, err := (Runner{DB: db, Dialect: DialectSQLite, Migrations: changed}).Run(context.Background())
if !errors.Is(err, ErrChecksumMismatch) {
t.Fatalf("error = %v, want checksum mismatch", err)
}
}
func TestSQLiteCompatibleChecksumAccepted(t *testing.T) {
db := openSQLite(t)
migrations := testMigrations()
if _, err := (Runner{DB: db, Dialect: DialectSQLite, Migrations: migrations}).Run(context.Background()); err != nil {
t.Fatal(err)
}
const compatibleChecksum = "1111111111111111111111111111111111111111111111111111111111111111"
if _, err := db.Exec("UPDATE schema_migration SET checksum = ? WHERE version = 1", compatibleChecksum); err != nil {
t.Fatal(err)
}
migrations[0] = WithCompatibleChecksums(migrations[0], compatibleChecksum)
report, err := (Runner{DB: db, Dialect: DialectSQLite, Migrations: migrations}).Run(context.Background())
if err != nil {
t.Fatal(err)
}
if report.CurrentVersion != 2 || len(report.Applied) != 0 {
t.Fatalf("report = %+v", report)
}
}
func TestSQLiteCompatibleChecksumStillRequiresMatchingName(t *testing.T) {
db := openSQLite(t)
migrations := testMigrations()
if _, err := (Runner{DB: db, Dialect: DialectSQLite, Migrations: migrations}).Run(context.Background()); err != nil {
t.Fatal(err)
}
const compatibleChecksum = "2222222222222222222222222222222222222222222222222222222222222222"
if _, err := db.Exec("UPDATE schema_migration SET name = 'changed', checksum = ? WHERE version = 1", compatibleChecksum); err != nil {
t.Fatal(err)
}
migrations[0] = WithCompatibleChecksums(migrations[0], compatibleChecksum)
_, err := (Runner{DB: db, Dialect: DialectSQLite, Migrations: migrations}).Run(context.Background())
if !errors.Is(err, ErrChecksumMismatch) {
t.Fatalf("error = %v, want checksum mismatch", err)
}
}
func TestSQLiteFailedMigrationRollsBack(t *testing.T) {
db := openSQLite(t)
runner := Runner{
DB: db,
Dialect: DialectSQLite,
Migrations: []Migration{
Define(1, "broken", "CREATE TABLE should_rollback (id INTEGER PRIMARY KEY)", "INVALID SQL"),
},
}
if _, err := runner.Run(context.Background()); err == nil {
t.Fatal("expected migration failure")
}
if sqliteTableExists(t, db, "should_rollback") {
t.Fatal("failed migration left a partial table")
}
if sqliteTableExists(t, db, "schema_migration") {
assertRowCount(t, db, "schema_migration", 0)
}
}
func TestSQLiteConcurrentRunHasSingleExecutor(t *testing.T) {
db := openSQLite(t)
db.SetMaxOpenConns(4)
runner := Runner{DB: db, Dialect: DialectSQLite, Migrations: testMigrations()}
var wg sync.WaitGroup
reports := make(chan Report, 2)
errs := make(chan error, 2)
for i := 0; i < 2; i++ {
wg.Add(1)
go func() {
defer wg.Done()
report, err := runner.Run(context.Background())
if err != nil {
errs <- err
return
}
reports <- report
}()
}
wg.Wait()
close(reports)
close(errs)
for err := range errs {
t.Fatal(err)
}
appliedCount := 0
for report := range reports {
appliedCount += len(report.Applied)
}
if appliedCount != 2 {
t.Fatalf("applied migration count = %d, want 2", appliedCount)
}
assertRowCount(t, db, "schema_migration", 2)
assertRowCount(t, db, "sample", 1)
}
func TestUnknownAppliedMigrationIsRejected(t *testing.T) {
db := openSQLite(t)
runner := Runner{DB: db, Dialect: DialectSQLite, Migrations: testMigrations()}
if _, err := runner.Run(context.Background()); err != nil {
t.Fatal(err)
}
if _, err := db.Exec("INSERT INTO schema_migration(version, name, checksum, applied_at, execution_ms) VALUES (99, 'future', 'x', CURRENT_TIMESTAMP, 0)"); err != nil {
t.Fatal(err)
}
_, err := runner.Run(context.Background())
if !errors.Is(err, ErrUnknownAppliedMigration) {
t.Fatalf("error = %v, want unknown migration", err)
}
}
func TestNormalizeRejectsInvalidDefinitions(t *testing.T) {
tests := [][]Migration{
{Define(0, "invalid", "SELECT 1")},
{Define(1, "", "SELECT 1")},
{Define(1, "empty")},
{Define(1, "one", "SELECT 1"), Define(1, "duplicate", "SELECT 1")},
{Define(2, "missing_first", "SELECT 1")},
{Define(1, "one", "SELECT 1"), Define(3, "gap", "SELECT 1")},
}
for i, migrations := range tests {
if _, err := normalize(migrations); err == nil {
t.Fatalf("case %d accepted", i)
}
}
}
func TestNormalizeRejectsInvalidCompatibleChecksums(t *testing.T) {
base := Define(1, "create_sample", "CREATE TABLE sample (id INTEGER PRIMARY KEY)")
migrations := [][]Migration{
{WithCompatibleChecksums(base, "invalid")},
{WithCompatibleChecksums(base, base.Checksum())},
}
for index, items := range migrations {
if _, err := normalize(items); err == nil {
t.Fatalf("case %d accepted", index)
}
}
}
func testMigrations() []Migration {
return []Migration{
Define(1, "create_sample", "CREATE TABLE sample (id INTEGER PRIMARY KEY, value TEXT NOT NULL)"),
Define(2, "seed_sample", "INSERT INTO sample(id, value) VALUES (1, 'ok')"),
}
}
func openSQLite(t *testing.T) *sql.DB {
t.Helper()
dsn := filepath.Join(t.TempDir(), "migration.db") + "?_pragma=busy_timeout(1000)"
db, err := sql.Open("sqlite", dsn)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
return db
}
func assertRowCount(t *testing.T, db *sql.DB, table string, want int) {
t.Helper()
var count int
if err := db.QueryRow("SELECT count(*) FROM " + table).Scan(&count); err != nil {
t.Fatal(err)
}
if count != want {
t.Fatalf("%s row count = %d, want %d", table, count, want)
}
}
func sqliteTableExists(t *testing.T, db *sql.DB, table string) bool {
t.Helper()
var count int
if err := db.QueryRow("SELECT count(*) FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&count); err != nil {
t.Fatal(err)
}
return count == 1
}
func TestRunnerRejectsMissingDialectStatements(t *testing.T) {
db := openSQLite(t)
runner := Runner{
DB: db,
Dialect: DialectSQLite,
Migrations: []Migration{
DefineForDialects(1, "mysql_only", map[Dialect][]string{DialectMySQL: {"SELECT 1"}}),
},
}
if _, err := runner.Run(context.Background()); err == nil {
t.Fatal("migration without SQLite statements was accepted")
}
}
+285
View File
@@ -0,0 +1,285 @@
package migration
import (
"context"
"fmt"
"strings"
)
func coreSchemaStatements() map[Dialect][]string {
return map[Dialect][]string{
DialectSQLite: {
`CREATE TABLE IF NOT EXISTS "user" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "account" TEXT NOT NULL UNIQUE, "name" TEXT NOT NULL, "password" TEXT NOT NULL, "disabled" INTEGER NOT NULL DEFAULT 0, "role" TEXT NOT NULL DEFAULT 'user', "is_admin" INTEGER NOT NULL DEFAULT 0, "session_version" INTEGER NOT NULL DEFAULT 1)`,
`CREATE TABLE IF NOT EXISTS "email" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "type" INTEGER NOT NULL DEFAULT 0, "subject" TEXT NOT NULL DEFAULT '', "reply_to" TEXT NULL, "from_name" TEXT NOT NULL DEFAULT '', "from_address" TEXT NOT NULL DEFAULT '', "to" TEXT NULL, "bcc" TEXT NULL, "cc" TEXT NULL, "text" TEXT NULL, "html" TEXT NULL, "sender" TEXT NULL, "attachments" TEXT NULL, "spf_check" INTEGER NOT NULL DEFAULT 0, "dkim_check" INTEGER NOT NULL DEFAULT 0, "status" INTEGER NOT NULL DEFAULT 0, "cron_send_time" DATETIME NULL, "update_time" DATETIME NULL, "send_user_id" INTEGER NOT NULL DEFAULT 0, "size" INTEGER NOT NULL DEFAULT 1000, "error" TEXT NULL, "send_date" DATETIME NULL, "create_time" DATETIME NULL, "msg_id" TEXT NOT NULL DEFAULT '')`,
`CREATE TABLE IF NOT EXISTS "group" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL DEFAULT '', "parent_id" INTEGER NOT NULL DEFAULT 0, "user_id" INTEGER NOT NULL DEFAULT 0, "full_path" TEXT NULL)`,
`CREATE TABLE IF NOT EXISTS "rule" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "user_id" INTEGER NOT NULL DEFAULT 0, "name" TEXT NOT NULL DEFAULT '', "value" TEXT NULL, "action" INTEGER NOT NULL DEFAULT 0, "params" TEXT NOT NULL DEFAULT '', "sort" INTEGER NOT NULL DEFAULT 0)`,
`CREATE TABLE IF NOT EXISTS "sessions" ("token" TEXT PRIMARY KEY, "data" BLOB NOT NULL, "expiry" REAL NOT NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_sessions_expiry" ON "sessions"("expiry")`,
`CREATE TABLE IF NOT EXISTS "user_email" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "user_id" INTEGER NOT NULL, "email_id" INTEGER NOT NULL, "is_read" INTEGER NOT NULL DEFAULT 0, "group_id" INTEGER NOT NULL DEFAULT 0, "status" INTEGER NOT NULL DEFAULT 0, "create" DATETIME NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_eid" ON "user_email"("user_id", "email_id")`,
`CREATE INDEX IF NOT EXISTS "idx_create_time" ON "user_email"("create")`,
`CREATE TABLE IF NOT EXISTS "version" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "info" TEXT NOT NULL)`,
},
DialectMySQL: {
"CREATE TABLE IF NOT EXISTS `user` (`id` INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, `account` VARCHAR(20) NOT NULL, `name` VARCHAR(10) NOT NULL, `password` VARCHAR(255) NOT NULL, `disabled` INT UNSIGNED NOT NULL DEFAULT 0, `role` VARCHAR(20) NOT NULL DEFAULT 'user', `is_admin` INT UNSIGNED NOT NULL DEFAULT 0, `session_version` INT UNSIGNED NOT NULL DEFAULT 1, UNIQUE KEY `uq_user_account` (`account`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
"CREATE TABLE IF NOT EXISTS `email` (`id` INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, `type` TINYINT NOT NULL DEFAULT 0, `subject` VARCHAR(1000) NOT NULL DEFAULT '', `reply_to` TEXT NULL, `from_name` VARCHAR(50) NOT NULL DEFAULT '', `from_address` VARCHAR(100) NOT NULL DEFAULT '', `to` TEXT NULL, `bcc` TEXT NULL, `cc` TEXT NULL, `text` TEXT NULL, `html` MEDIUMTEXT NULL, `sender` TEXT NULL, `attachments` LONGTEXT NULL, `spf_check` TINYINT NOT NULL DEFAULT 0, `dkim_check` TINYINT NOT NULL DEFAULT 0, `status` TINYINT NOT NULL DEFAULT 0, `cron_send_time` DATETIME(6) NULL, `update_time` DATETIME(6) NULL, `send_user_id` INT UNSIGNED NOT NULL DEFAULT 0, `size` INT UNSIGNED NOT NULL DEFAULT 1000, `error` TEXT NULL, `send_date` DATETIME(6) NULL, `create_time` DATETIME(6) NULL, `msg_id` VARCHAR(255) NOT NULL DEFAULT '') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
"CREATE TABLE IF NOT EXISTS `group` (`id` INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, `name` VARCHAR(10) NOT NULL DEFAULT '', `parent_id` INT UNSIGNED NOT NULL DEFAULT 0, `user_id` INT UNSIGNED NOT NULL DEFAULT 0, `full_path` VARCHAR(600) NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
"CREATE TABLE IF NOT EXISTS `rule` (`id` INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, `user_id` INT NOT NULL DEFAULT 0, `name` VARCHAR(255) NOT NULL DEFAULT '', `value` TEXT NULL, `action` INT NOT NULL DEFAULT 0, `params` VARCHAR(255) NOT NULL DEFAULT '', `sort` INT NOT NULL DEFAULT 0) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
"CREATE TABLE IF NOT EXISTS `sessions` (`token` CHAR(43) NOT NULL PRIMARY KEY, `data` BLOB NOT NULL, `expiry` TIMESTAMP(6) NOT NULL, KEY `idx_sessions_expiry` (`expiry`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
"CREATE TABLE IF NOT EXISTS `user_email` (`id` INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, `user_id` INT NOT NULL, `email_id` INT NOT NULL, `is_read` TINYINT NOT NULL DEFAULT 0, `group_id` INT NOT NULL DEFAULT 0, `status` TINYINT NOT NULL DEFAULT 0, `create` DATETIME(6) NULL, KEY `idx_eid` (`user_id`, `email_id`), KEY `idx_create_time` (`create`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
"CREATE TABLE IF NOT EXISTS `version` (`id` INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, `info` VARCHAR(255) NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
},
DialectPostgres: {
`CREATE TABLE IF NOT EXISTS "user" ("id" BIGSERIAL PRIMARY KEY, "account" VARCHAR(20) NOT NULL UNIQUE, "name" VARCHAR(10) NOT NULL, "password" VARCHAR(255) NOT NULL, "disabled" INTEGER NOT NULL DEFAULT 0, "role" VARCHAR(20) NOT NULL DEFAULT 'user', "is_admin" INTEGER NOT NULL DEFAULT 0, "session_version" INTEGER NOT NULL DEFAULT 1)`,
`CREATE TABLE IF NOT EXISTS "email" ("id" BIGSERIAL PRIMARY KEY, "type" SMALLINT NOT NULL DEFAULT 0, "subject" VARCHAR(1000) NOT NULL DEFAULT '', "reply_to" TEXT NULL, "from_name" VARCHAR(50) NOT NULL DEFAULT '', "from_address" VARCHAR(100) NOT NULL DEFAULT '', "to" TEXT NULL, "bcc" TEXT NULL, "cc" TEXT NULL, "text" TEXT NULL, "html" TEXT NULL, "sender" TEXT NULL, "attachments" TEXT NULL, "spf_check" SMALLINT NOT NULL DEFAULT 0, "dkim_check" SMALLINT NOT NULL DEFAULT 0, "status" SMALLINT NOT NULL DEFAULT 0, "cron_send_time" TIMESTAMPTZ NULL, "update_time" TIMESTAMPTZ NULL, "send_user_id" BIGINT NOT NULL DEFAULT 0, "size" BIGINT NOT NULL DEFAULT 1000, "error" TEXT NULL, "send_date" TIMESTAMPTZ NULL, "create_time" TIMESTAMPTZ NULL, "msg_id" VARCHAR(255) NOT NULL DEFAULT '')`,
`CREATE TABLE IF NOT EXISTS "group" ("id" BIGSERIAL PRIMARY KEY, "name" VARCHAR(10) NOT NULL DEFAULT '', "parent_id" BIGINT NOT NULL DEFAULT 0, "user_id" BIGINT NOT NULL DEFAULT 0, "full_path" VARCHAR(600) NULL)`,
`CREATE TABLE IF NOT EXISTS "rule" ("id" BIGSERIAL PRIMARY KEY, "user_id" BIGINT NOT NULL DEFAULT 0, "name" VARCHAR(255) NOT NULL DEFAULT '', "value" TEXT NULL, "action" INTEGER NOT NULL DEFAULT 0, "params" VARCHAR(255) NOT NULL DEFAULT '', "sort" INTEGER NOT NULL DEFAULT 0)`,
`CREATE TABLE IF NOT EXISTS "sessions" ("token" VARCHAR(43) PRIMARY KEY, "data" BYTEA NOT NULL, "expiry" TIMESTAMPTZ NOT NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_sessions_expiry" ON "sessions"("expiry")`,
`CREATE TABLE IF NOT EXISTS "user_email" ("id" BIGSERIAL PRIMARY KEY, "user_id" BIGINT NOT NULL, "email_id" BIGINT NOT NULL, "is_read" SMALLINT NOT NULL DEFAULT 0, "group_id" BIGINT NOT NULL DEFAULT 0, "status" SMALLINT NOT NULL DEFAULT 0, "create" TIMESTAMPTZ NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_eid" ON "user_email"("user_id", "email_id")`,
`CREATE INDEX IF NOT EXISTS "idx_create_time" ON "user_email"("create")`,
`CREATE TABLE IF NOT EXISTS "version" ("id" BIGSERIAL PRIMARY KEY, "info" VARCHAR(255) NOT NULL)`,
},
}
}
func applyUserLifecycleSchema(ctx context.Context, exec executor, dialect Dialect) error {
columns := []struct {
name string
definition map[Dialect]string
}{
{name: "security_version", definition: map[Dialect]string{
DialectSQLite: `INTEGER NOT NULL DEFAULT 1`,
DialectMySQL: `INT UNSIGNED NOT NULL DEFAULT 1`,
DialectPostgres: `INTEGER NOT NULL DEFAULT 1`,
}},
{name: "must_change_password", definition: map[Dialect]string{
DialectSQLite: `INTEGER NOT NULL DEFAULT 0`,
DialectMySQL: `INT UNSIGNED NOT NULL DEFAULT 0`,
DialectPostgres: `INTEGER NOT NULL DEFAULT 0`,
}},
{name: "password_changed_at", definition: map[Dialect]string{
DialectSQLite: `DATETIME NULL`,
DialectMySQL: `DATETIME(6) NULL`,
DialectPostgres: `TIMESTAMP NULL`,
}},
}
for _, column := range columns {
if err := ensureColumn(ctx, exec, dialect, "user", column.name, column.definition[dialect]); err != nil {
return err
}
}
return nil
}
func verificationOperationsStatements() map[Dialect][]string {
return map[Dialect][]string{
DialectSQLite: {
`CREATE TABLE IF NOT EXISTS "verification_rate_limit" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "dimension_key" TEXT NOT NULL UNIQUE, "dimension_type" TEXT NOT NULL, "hour_started_at" DATETIME NULL, "hour_count" INTEGER NOT NULL DEFAULT 0, "day_started_at" DATETIME NULL, "day_count" INTEGER NOT NULL DEFAULT 0, "last_issued_at" DATETIME NULL, "updated_at" DATETIME NOT NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_verification_rate_limit_updated_at" ON "verification_rate_limit"("updated_at")`,
`CREATE TABLE IF NOT EXISTS "verification_cleanup_state" ("state_key" TEXT PRIMARY KEY, "next_run_at" DATETIME NOT NULL, "lease_until" DATETIME NOT NULL, "lease_token" TEXT NOT NULL, "failure_count" INTEGER NOT NULL DEFAULT 0, "updated_at" DATETIME NOT NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_verification_cleanup_state_next_run_at" ON "verification_cleanup_state"("next_run_at")`,
},
DialectMySQL: {
"CREATE TABLE IF NOT EXISTS `verification_rate_limit` (`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, `dimension_key` CHAR(64) NOT NULL, `dimension_type` VARCHAR(32) NOT NULL, `hour_started_at` DATETIME(6) NULL, `hour_count` INT NOT NULL DEFAULT 0, `day_started_at` DATETIME(6) NULL, `day_count` INT NOT NULL DEFAULT 0, `last_issued_at` DATETIME(6) NULL, `updated_at` DATETIME(6) NOT NULL, UNIQUE KEY `uq_verification_rate_limit_dimension_key` (`dimension_key`), KEY `idx_verification_rate_limit_updated_at` (`updated_at`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
"CREATE TABLE IF NOT EXISTS `verification_cleanup_state` (`state_key` VARCHAR(64) NOT NULL PRIMARY KEY, `next_run_at` DATETIME(6) NOT NULL, `lease_until` DATETIME(6) NOT NULL, `lease_token` CHAR(64) NOT NULL, `failure_count` INT NOT NULL DEFAULT 0, `updated_at` DATETIME(6) NOT NULL, KEY `idx_verification_cleanup_state_next_run_at` (`next_run_at`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
},
DialectPostgres: {
`CREATE TABLE IF NOT EXISTS "verification_rate_limit" ("id" BIGSERIAL PRIMARY KEY, "dimension_key" CHAR(64) NOT NULL UNIQUE, "dimension_type" VARCHAR(32) NOT NULL, "hour_started_at" TIMESTAMPTZ NULL, "hour_count" INTEGER NOT NULL DEFAULT 0, "day_started_at" TIMESTAMPTZ NULL, "day_count" INTEGER NOT NULL DEFAULT 0, "last_issued_at" TIMESTAMPTZ NULL, "updated_at" TIMESTAMPTZ NOT NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_verification_rate_limit_updated_at" ON "verification_rate_limit"("updated_at")`,
`CREATE TABLE IF NOT EXISTS "verification_cleanup_state" ("state_key" VARCHAR(64) PRIMARY KEY, "next_run_at" TIMESTAMPTZ NOT NULL, "lease_until" TIMESTAMPTZ NOT NULL, "lease_token" CHAR(64) NOT NULL, "failure_count" INTEGER NOT NULL DEFAULT 0, "updated_at" TIMESTAMPTZ NOT NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_verification_cleanup_state_next_run_at" ON "verification_cleanup_state"("next_run_at")`,
},
}
}
func securityEmailStatements() map[Dialect][]string {
return map[Dialect][]string{
DialectSQLite: {
`CREATE TABLE IF NOT EXISTS "external_email" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "user_id" INTEGER NOT NULL, "email_ciphertext" TEXT NOT NULL, "email_fingerprint" TEXT NOT NULL UNIQUE, "email_type" TEXT NOT NULL, "status" TEXT NOT NULL, "verified_at" DATETIME NULL, "credential_version" INTEGER NOT NULL DEFAULT 1, "created_at" DATETIME NOT NULL, "updated_at" DATETIME NOT NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_external_email_user_id" ON "external_email"("user_id")`,
`CREATE TABLE IF NOT EXISTS "verification_event" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "user_id" INTEGER NOT NULL DEFAULT 0, "purpose" TEXT NOT NULL, "action" TEXT NOT NULL, "outcome" TEXT NOT NULL, "account_fingerprint" TEXT NULL, "email_fingerprint" TEXT NULL, "ip_fingerprint" TEXT NULL, "created_at" DATETIME NOT NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_verification_event_created_at" ON "verification_event"("created_at")`,
`CREATE INDEX IF NOT EXISTS "idx_verification_event_user_id" ON "verification_event"("user_id")`,
},
DialectMySQL: {
"CREATE TABLE IF NOT EXISTS `external_email` (`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, `user_id` BIGINT NOT NULL, `email_ciphertext` VARCHAR(1024) NOT NULL, `email_fingerprint` VARCHAR(80) NOT NULL, `email_type` VARCHAR(16) NOT NULL, `status` VARCHAR(16) NOT NULL, `verified_at` DATETIME(6) NULL, `credential_version` INT NOT NULL DEFAULT 1, `created_at` DATETIME(6) NOT NULL, `updated_at` DATETIME(6) NOT NULL, UNIQUE KEY `uq_external_email_fingerprint` (`email_fingerprint`), KEY `idx_external_email_user_id` (`user_id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
"CREATE TABLE IF NOT EXISTS `verification_event` (`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, `user_id` BIGINT NOT NULL DEFAULT 0, `purpose` VARCHAR(32) NOT NULL, `action` VARCHAR(32) NOT NULL, `outcome` VARCHAR(16) NOT NULL, `account_fingerprint` VARCHAR(80) NULL, `email_fingerprint` VARCHAR(80) NULL, `ip_fingerprint` VARCHAR(80) NULL, `created_at` DATETIME(6) NOT NULL, KEY `idx_verification_event_created_at` (`created_at`), KEY `idx_verification_event_user_id` (`user_id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
},
DialectPostgres: {
`CREATE TABLE IF NOT EXISTS "external_email" ("id" BIGSERIAL PRIMARY KEY, "user_id" BIGINT NOT NULL, "email_ciphertext" VARCHAR(1024) NOT NULL, "email_fingerprint" VARCHAR(80) NOT NULL UNIQUE, "email_type" VARCHAR(16) NOT NULL, "status" VARCHAR(16) NOT NULL, "verified_at" TIMESTAMPTZ NULL, "credential_version" INTEGER NOT NULL DEFAULT 1, "created_at" TIMESTAMPTZ NOT NULL, "updated_at" TIMESTAMPTZ NOT NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_external_email_user_id" ON "external_email"("user_id")`,
`CREATE TABLE IF NOT EXISTS "verification_event" ("id" BIGSERIAL PRIMARY KEY, "user_id" BIGINT NOT NULL DEFAULT 0, "purpose" VARCHAR(32) NOT NULL, "action" VARCHAR(32) NOT NULL, "outcome" VARCHAR(16) NOT NULL, "account_fingerprint" VARCHAR(80) NULL, "email_fingerprint" VARCHAR(80) NULL, "ip_fingerprint" VARCHAR(80) NULL, "created_at" TIMESTAMPTZ NOT NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_verification_event_created_at" ON "verification_event"("created_at")`,
`CREATE INDEX IF NOT EXISTS "idx_verification_event_user_id" ON "verification_event"("user_id")`,
},
}
}
func applyUserSecuritySchema(ctx context.Context, exec executor, dialect Dialect) error {
columns := []struct {
name string
definition map[Dialect]string
}{
{name: "role", definition: map[Dialect]string{
DialectSQLite: `TEXT NOT NULL DEFAULT 'user'`,
DialectMySQL: `VARCHAR(20) NOT NULL DEFAULT 'user'`,
DialectPostgres: `VARCHAR(20) NOT NULL DEFAULT 'user'`,
}},
{name: "session_version", definition: map[Dialect]string{
DialectSQLite: `INTEGER NOT NULL DEFAULT 1`,
DialectMySQL: `INT UNSIGNED NOT NULL DEFAULT 1`,
DialectPostgres: `INTEGER NOT NULL DEFAULT 1`,
}},
}
for _, column := range columns {
if err := ensureColumn(ctx, exec, dialect, "user", column.name, column.definition[dialect]); err != nil {
return err
}
}
switch dialect {
case DialectMySQL:
if _, err := exec.ExecContext(ctx, "ALTER TABLE `user` MODIFY COLUMN `password` VARCHAR(255) NOT NULL"); err != nil {
return fmt.Errorf("expand user password column: %w", err)
}
case DialectPostgres:
if _, err := exec.ExecContext(ctx, `ALTER TABLE "user" ALTER COLUMN "password" TYPE VARCHAR(255)`); err != nil {
return fmt.Errorf("expand user password column: %w", err)
}
}
query := `UPDATE "user" SET "role" = CASE WHEN "is_admin" = 1 THEN 'super_admin' ELSE 'user' END WHERE "role" IS NULL OR "role" = '' OR ("is_admin" = 1 AND "role" = 'user')`
if dialect == DialectMySQL {
query = "UPDATE `user` SET `role` = CASE WHEN `is_admin` = 1 THEN 'super_admin' ELSE 'user' END WHERE `role` IS NULL OR `role` = '' OR (`is_admin` = 1 AND `role` = 'user')"
}
if _, err := exec.ExecContext(ctx, query); err != nil {
return fmt.Errorf("backfill user roles: %w", err)
}
return nil
}
func ensureColumn(ctx context.Context, exec executor, dialect Dialect, table string, column string, definition string) error {
exists, err := columnExists(ctx, exec, dialect, table, column)
if err != nil {
return err
}
if exists {
return nil
}
quotedTable, quotedColumn, err := quotedIdentifiers(dialect, table, column)
if err != nil {
return err
}
if _, err = exec.ExecContext(ctx, fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", quotedTable, quotedColumn, definition)); err != nil {
return fmt.Errorf("add column %s.%s: %w", table, column, err)
}
return nil
}
func columnExists(ctx context.Context, exec executor, dialect Dialect, table string, column string) (bool, error) {
switch dialect {
case DialectSQLite:
rows, err := exec.QueryContext(ctx, fmt.Sprintf(`PRAGMA table_info("%s")`, table))
if err != nil {
return false, fmt.Errorf("query SQLite columns: %w", err)
}
defer rows.Close()
for rows.Next() {
var cid int
var name, dataType string
var notNull, primaryKey int
var defaultValue any
if err = rows.Scan(&cid, &name, &dataType, &notNull, &defaultValue, &primaryKey); err != nil {
return false, fmt.Errorf("scan SQLite column: %w", err)
}
if strings.EqualFold(name, column) {
return true, nil
}
}
return false, rows.Err()
case DialectMySQL:
var count int
err := exec.QueryRowContext(ctx, "SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = ? AND column_name = ?", table, column).Scan(&count)
return count > 0, err
case DialectPostgres:
var count int
err := exec.QueryRowContext(ctx, "SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = $1 AND column_name = $2", table, column).Scan(&count)
return count > 0, err
default:
return false, fmt.Errorf("unsupported migration dialect %q", dialect)
}
}
func quotedIdentifiers(dialect Dialect, table string, column string) (string, string, error) {
if strings.ContainsAny(table+column, "`\".';\\/ ") {
return "", "", fmt.Errorf("invalid migration identifier")
}
switch dialect {
case DialectMySQL:
return "`" + table + "`", "`" + column + "`", nil
case DialectSQLite, DialectPostgres:
return `"` + table + `"`, `"` + column + `"`, nil
default:
return "", "", fmt.Errorf("unsupported migration dialect %q", dialect)
}
}
func credentialStatements() map[Dialect][]string {
return map[Dialect][]string{
DialectSQLite: {
`CREATE TABLE IF NOT EXISTS "security_question" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "user_id" INTEGER NOT NULL UNIQUE, "question" TEXT NOT NULL, "answer_hash" TEXT NOT NULL, "version" INTEGER NOT NULL DEFAULT 1, "created_at" DATETIME NOT NULL, "updated_at" DATETIME NOT NULL)`,
`CREATE TABLE IF NOT EXISTS "recovery_code" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "user_id" INTEGER NOT NULL, "code_hash" TEXT NOT NULL, "code_version" INTEGER NOT NULL DEFAULT 1, "used_at" DATETIME NULL, "created_at" DATETIME NOT NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_recovery_code_user_id" ON "recovery_code"("user_id")`,
`CREATE INDEX IF NOT EXISTS "idx_recovery_code_used_at" ON "recovery_code"("used_at")`,
},
DialectMySQL: {
"CREATE TABLE IF NOT EXISTS `security_question` (`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, `user_id` BIGINT NOT NULL, `question` VARCHAR(255) NOT NULL, `answer_hash` VARCHAR(255) NOT NULL, `version` INT NOT NULL DEFAULT 1, `created_at` DATETIME(6) NOT NULL, `updated_at` DATETIME(6) NOT NULL, UNIQUE KEY `uq_security_question_user_id` (`user_id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
"CREATE TABLE IF NOT EXISTS `recovery_code` (`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, `user_id` BIGINT NOT NULL, `code_hash` VARCHAR(255) NOT NULL, `code_version` INT NOT NULL DEFAULT 1, `used_at` DATETIME(6) NULL, `created_at` DATETIME(6) NOT NULL, KEY `idx_recovery_code_user_id` (`user_id`), KEY `idx_recovery_code_used_at` (`used_at`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
},
DialectPostgres: {
`CREATE TABLE IF NOT EXISTS "security_question" ("id" BIGSERIAL PRIMARY KEY, "user_id" BIGINT NOT NULL UNIQUE, "question" VARCHAR(255) NOT NULL, "answer_hash" VARCHAR(255) NOT NULL, "version" INTEGER NOT NULL DEFAULT 1, "created_at" TIMESTAMPTZ NOT NULL, "updated_at" TIMESTAMPTZ NOT NULL)`,
`CREATE TABLE IF NOT EXISTS "recovery_code" ("id" BIGSERIAL PRIMARY KEY, "user_id" BIGINT NOT NULL, "code_hash" VARCHAR(255) NOT NULL, "code_version" INTEGER NOT NULL DEFAULT 1, "used_at" TIMESTAMPTZ NULL, "created_at" TIMESTAMPTZ NOT NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_recovery_code_user_id" ON "recovery_code"("user_id")`,
`CREATE INDEX IF NOT EXISTS "idx_recovery_code_used_at" ON "recovery_code"("used_at")`,
},
}
}
func webAuthnCredentialStatements() map[Dialect][]string {
return map[Dialect][]string{
DialectSQLite: {
`CREATE TABLE IF NOT EXISTS "webauthn_credential" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "user_id" INTEGER NOT NULL, "credential_id" TEXT NOT NULL UNIQUE, "credential_data" TEXT NOT NULL, "display_name" TEXT NOT NULL, "created_at" DATETIME NOT NULL, "last_used_at" DATETIME NULL, "revoked_at" DATETIME NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_webauthn_credential_user_id" ON "webauthn_credential"("user_id")`,
`CREATE INDEX IF NOT EXISTS "idx_webauthn_credential_revoked_at" ON "webauthn_credential"("revoked_at")`,
},
DialectMySQL: {
"CREATE TABLE IF NOT EXISTS `webauthn_credential` (`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, `user_id` BIGINT NOT NULL, `credential_id` VARCHAR(512) NOT NULL UNIQUE, `credential_data` TEXT NOT NULL, `display_name` VARCHAR(128) NOT NULL, `created_at` DATETIME(6) NOT NULL, `last_used_at` DATETIME(6) NULL, `revoked_at` DATETIME(6) NULL, KEY `idx_webauthn_credential_user_id` (`user_id`), KEY `idx_webauthn_credential_revoked_at` (`revoked_at`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
},
DialectPostgres: {
`CREATE TABLE IF NOT EXISTS "webauthn_credential" ("id" BIGSERIAL PRIMARY KEY, "user_id" BIGINT NOT NULL, "credential_id" VARCHAR(512) NOT NULL UNIQUE, "credential_data" TEXT NOT NULL, "display_name" VARCHAR(128) NOT NULL, "created_at" TIMESTAMPTZ NOT NULL, "last_used_at" TIMESTAMPTZ NULL, "revoked_at" TIMESTAMPTZ NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_webauthn_credential_user_id" ON "webauthn_credential"("user_id")`,
`CREATE INDEX IF NOT EXISTS "idx_webauthn_credential_revoked_at" ON "webauthn_credential"("revoked_at")`,
},
}
}
func pluginSettingsStatements() map[Dialect][]string {
return map[Dialect][]string{
DialectSQLite: {
`CREATE TABLE IF NOT EXISTS "plugin_secret" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "plugin_id" TEXT NOT NULL, "field_name" TEXT NOT NULL, "ciphertext" TEXT NOT NULL, "key_version" INTEGER NOT NULL, "record_version" INTEGER NOT NULL DEFAULT 1, "created_at" DATETIME NOT NULL, "updated_at" DATETIME NOT NULL, UNIQUE("plugin_id", "field_name"))`,
`CREATE TABLE IF NOT EXISTS "plugin_user_setting" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "plugin_id" TEXT NOT NULL, "user_id" INTEGER NOT NULL, "schema_version" INTEGER NOT NULL, "settings_ciphertext" TEXT NOT NULL, "settings_hash" TEXT NOT NULL, "enabled" INTEGER NOT NULL DEFAULT 1, "version" INTEGER NOT NULL DEFAULT 1, "created_at" DATETIME NOT NULL, "updated_at" DATETIME NOT NULL, UNIQUE("plugin_id", "user_id"))`,
`CREATE INDEX IF NOT EXISTS "idx_plugin_user_setting_user_id" ON "plugin_user_setting"("user_id")`,
},
DialectMySQL: {
"CREATE TABLE IF NOT EXISTS `plugin_secret` (`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, `plugin_id` VARCHAR(64) NOT NULL, `field_name` VARCHAR(64) NOT NULL, `ciphertext` TEXT NOT NULL, `key_version` INT UNSIGNED NOT NULL, `record_version` BIGINT UNSIGNED NOT NULL DEFAULT 1, `created_at` DATETIME(6) NOT NULL, `updated_at` DATETIME(6) NOT NULL, UNIQUE KEY `uq_plugin_secret_field` (`plugin_id`, `field_name`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
"CREATE TABLE IF NOT EXISTS `plugin_user_setting` (`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, `plugin_id` VARCHAR(64) NOT NULL, `user_id` BIGINT NOT NULL, `schema_version` INT UNSIGNED NOT NULL, `settings_ciphertext` TEXT NOT NULL, `settings_hash` VARCHAR(128) NOT NULL, `enabled` TINYINT NOT NULL DEFAULT 1, `version` BIGINT UNSIGNED NOT NULL DEFAULT 1, `created_at` DATETIME(6) NOT NULL, `updated_at` DATETIME(6) NOT NULL, UNIQUE KEY `uq_plugin_user_setting_owner` (`plugin_id`, `user_id`), KEY `idx_plugin_user_setting_user_id` (`user_id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
},
DialectPostgres: {
`CREATE TABLE IF NOT EXISTS "plugin_secret" ("id" BIGSERIAL PRIMARY KEY, "plugin_id" VARCHAR(64) NOT NULL, "field_name" VARCHAR(64) NOT NULL, "ciphertext" TEXT NOT NULL, "key_version" INTEGER NOT NULL, "record_version" BIGINT NOT NULL DEFAULT 1, "created_at" TIMESTAMPTZ NOT NULL, "updated_at" TIMESTAMPTZ NOT NULL, UNIQUE("plugin_id", "field_name"))`,
`CREATE TABLE IF NOT EXISTS "plugin_user_setting" ("id" BIGSERIAL PRIMARY KEY, "plugin_id" VARCHAR(64) NOT NULL, "user_id" BIGINT NOT NULL, "schema_version" INTEGER NOT NULL, "settings_ciphertext" TEXT NOT NULL, "settings_hash" VARCHAR(128) NOT NULL, "enabled" BOOLEAN NOT NULL DEFAULT TRUE, "version" BIGINT NOT NULL DEFAULT 1, "created_at" TIMESTAMPTZ NOT NULL, "updated_at" TIMESTAMPTZ NOT NULL, UNIQUE("plugin_id", "user_id"))`,
`CREATE INDEX IF NOT EXISTS "idx_plugin_user_setting_user_id" ON "plugin_user_setting"("user_id")`,
},
}
}
+19
View File
@@ -0,0 +1,19 @@
package migration
import (
"context"
"fmt"
)
func applySecuritySetupSchema(ctx context.Context, exec executor, dialect Dialect) error {
definitions := map[Dialect]string{
DialectSQLite: "INTEGER NOT NULL DEFAULT 0",
DialectMySQL: "TINYINT NOT NULL DEFAULT 0",
DialectPostgres: "SMALLINT NOT NULL DEFAULT 0",
}
definition, ok := definitions[dialect]
if !ok {
return fmt.Errorf("unsupported migration dialect %q", dialect)
}
return ensureColumn(ctx, exec, dialect, "user", "security_setup_acknowledged", definition)
}
@@ -0,0 +1,40 @@
package migration
import (
"context"
"testing"
)
func TestSecuritySetupMigrationPreservesAcknowledgement(t *testing.T) {
db := openSQLite(t)
runner := Runner{DB: db, Dialect: DialectSQLite, Migrations: DefaultMigrations()[:14]}
if _, err := runner.Run(context.Background()); err != nil {
t.Fatal(err)
}
if _, err := db.Exec(`INSERT INTO "user" (account, name, password) VALUES ('root', 'Root', 'unchanged')`); err != nil {
t.Fatal(err)
}
runner.Migrations = DefaultMigrations()
if _, err := runner.Run(context.Background()); err != nil {
t.Fatal(err)
}
var acknowledged int
if err := db.QueryRow(`SELECT security_setup_acknowledged FROM "user" WHERE account='root'`).Scan(&acknowledged); err != nil {
t.Fatal(err)
}
if acknowledged != 0 {
t.Fatal("existing account must receive the new guide once")
}
if _, err := db.Exec(`UPDATE "user" SET security_setup_acknowledged=1 WHERE account='root'`); err != nil {
t.Fatal(err)
}
if err := applySecuritySetupSchema(context.Background(), db, DialectSQLite); err != nil {
t.Fatal(err)
}
if err := db.QueryRow(`SELECT security_setup_acknowledged FROM "user" WHERE account='root'`).Scan(&acknowledged); err != nil {
t.Fatal(err)
}
if acknowledged != 1 {
t.Fatal("repeated migration reset acknowledgement")
}
}
+17
View File
@@ -0,0 +1,17 @@
package migration
func totpAuthenticatorStatements() map[Dialect][]string {
return map[Dialect][]string{
DialectSQLite: {
`CREATE TABLE IF NOT EXISTS "totp_authenticator" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "user_id" INTEGER NOT NULL UNIQUE, "secret_ciphertext" TEXT NOT NULL, "credential_version" INTEGER NOT NULL DEFAULT 1, "enabled" INTEGER NOT NULL DEFAULT 1, "confirmed_at" DATETIME NULL, "last_used_step" INTEGER NOT NULL DEFAULT 0, "created_at" DATETIME NOT NULL, "updated_at" DATETIME NOT NULL, "revoked_at" DATETIME NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_totp_authenticator_revoked_at" ON "totp_authenticator"("revoked_at")`,
},
DialectMySQL: {
"CREATE TABLE IF NOT EXISTS `totp_authenticator` (`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, `user_id` BIGINT NOT NULL, `secret_ciphertext` TEXT NOT NULL, `credential_version` INT NOT NULL DEFAULT 1, `enabled` TINYINT(1) NOT NULL DEFAULT 1, `confirmed_at` DATETIME(6) NULL, `last_used_step` BIGINT NOT NULL DEFAULT 0, `created_at` DATETIME(6) NOT NULL, `updated_at` DATETIME(6) NOT NULL, `revoked_at` DATETIME(6) NULL, UNIQUE KEY `uq_totp_authenticator_user_id` (`user_id`), KEY `idx_totp_authenticator_revoked_at` (`revoked_at`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4",
},
DialectPostgres: {
`CREATE TABLE IF NOT EXISTS "totp_authenticator" ("id" BIGSERIAL PRIMARY KEY, "user_id" BIGINT NOT NULL UNIQUE, "secret_ciphertext" TEXT NOT NULL, "credential_version" INTEGER NOT NULL DEFAULT 1, "enabled" BOOLEAN NOT NULL DEFAULT TRUE, "confirmed_at" TIMESTAMPTZ NULL, "last_used_step" BIGINT NOT NULL DEFAULT 0, "created_at" TIMESTAMPTZ NOT NULL, "updated_at" TIMESTAMPTZ NOT NULL, "revoked_at" TIMESTAMPTZ NULL)`,
`CREATE INDEX IF NOT EXISTS "idx_totp_authenticator_revoked_at" ON "totp_authenticator"("revoked_at")`,
},
}
}
+69
View File
@@ -0,0 +1,69 @@
package migration
import (
"context"
"fmt"
)
func applyUserEmailAddressSchema(ctx context.Context, exec executor, dialect Dialect) error {
columns := []struct {
name string
definition map[Dialect]string
}{
{name: "address", definition: map[Dialect]string{
DialectSQLite: `TEXT NULL`,
DialectMySQL: `VARCHAR(254) NULL`,
DialectPostgres: `VARCHAR(254) NULL`,
}},
{name: "address_canonical", definition: map[Dialect]string{
DialectSQLite: `TEXT NULL`,
DialectMySQL: `VARCHAR(254) NULL`,
DialectPostgres: `VARCHAR(254) NULL`,
}},
{name: "is_primary", definition: map[Dialect]string{
DialectSQLite: `INTEGER NOT NULL DEFAULT 0`,
DialectMySQL: `TINYINT NOT NULL DEFAULT 0`,
DialectPostgres: `SMALLINT NOT NULL DEFAULT 0`,
}},
{name: "verified_at", definition: map[Dialect]string{
DialectSQLite: `DATETIME NULL`,
DialectMySQL: `DATETIME(6) NULL`,
DialectPostgres: `TIMESTAMPTZ NULL`,
}},
{name: "created_at", definition: map[Dialect]string{
DialectSQLite: `DATETIME NULL`,
DialectMySQL: `DATETIME(6) NULL`,
DialectPostgres: `TIMESTAMPTZ NULL`,
}},
{name: "updated_at", definition: map[Dialect]string{
DialectSQLite: `DATETIME NULL`,
DialectMySQL: `DATETIME(6) NULL`,
DialectPostgres: `TIMESTAMPTZ NULL`,
}},
}
for _, column := range columns {
if err := ensureColumn(ctx, exec, dialect, "user_email", column.name, column.definition[dialect]); err != nil {
return err
}
}
var conflicts int
conflictQuery := `SELECT COUNT(*) FROM (SELECT "address_canonical" FROM "user_email" WHERE "address_canonical" IS NOT NULL AND TRIM("address_canonical") <> '' GROUP BY "address_canonical" HAVING COUNT(*) > 1) AS conflicts`
if dialect == DialectMySQL {
conflictQuery = "SELECT COUNT(*) FROM (SELECT `address_canonical` FROM `user_email` WHERE `address_canonical` IS NOT NULL AND TRIM(`address_canonical`) <> '' GROUP BY `address_canonical` HAVING COUNT(*) > 1) AS conflicts"
}
if dialect == DialectPostgres {
conflictQuery = `SELECT COUNT(*) FROM (SELECT "address_canonical" FROM "user_email" WHERE "address_canonical" IS NOT NULL AND BTRIM("address_canonical") <> '' GROUP BY "address_canonical" HAVING COUNT(*) > 1) AS conflicts`
}
if err := exec.QueryRowContext(ctx, conflictQuery).Scan(&conflicts); err != nil {
return fmt.Errorf("check user email address conflicts: %w", err)
}
if conflicts > 0 {
return fmt.Errorf("user email address conflict count: %d", conflicts)
}
if err := ensureIndex(ctx, exec, dialect, "user_email", "uq_user_email_address_canonical", true, []string{"address_canonical"}); err != nil {
return err
}
return ensureIndex(ctx, exec, dialect, "user_email", "idx_user_email_user_primary", false, []string{"user_id", "is_primary"})
}
@@ -0,0 +1,82 @@
package migration
import (
"context"
"fmt"
)
func applyVerificationRetentionSchema(ctx context.Context, exec executor, dialect Dialect) error {
columns := []struct {
table string
name string
definition map[Dialect]string
}{
{table: "verification_rate_limit", name: "expires_at", definition: map[Dialect]string{
DialectSQLite: `DATETIME NOT NULL DEFAULT '1970-01-01 00:00:00'`,
DialectMySQL: `DATETIME(6) NOT NULL DEFAULT '1970-01-01 00:00:00'`,
DialectPostgres: `TIMESTAMPTZ NOT NULL DEFAULT '1970-01-01 00:00:00+00'`,
}},
{table: "verification_event", name: "expires_at", definition: map[Dialect]string{
DialectSQLite: `DATETIME NOT NULL DEFAULT '1970-01-01 00:00:00'`,
DialectMySQL: `DATETIME(6) NOT NULL DEFAULT '1970-01-01 00:00:00'`,
DialectPostgres: `TIMESTAMPTZ NOT NULL DEFAULT '1970-01-01 00:00:00+00'`,
}},
{table: "verification_event", name: "retention_class", definition: map[Dialect]string{
DialectSQLite: `TEXT NOT NULL DEFAULT 'detail'`,
DialectMySQL: `VARCHAR(16) NOT NULL DEFAULT 'detail'`,
DialectPostgres: `VARCHAR(16) NOT NULL DEFAULT 'detail'`,
}},
{table: "verification_event", name: "investigation_hold", definition: map[Dialect]string{
DialectSQLite: `INTEGER NOT NULL DEFAULT 0`,
DialectMySQL: `TINYINT NOT NULL DEFAULT 0`,
DialectPostgres: `BOOLEAN NOT NULL DEFAULT FALSE`,
}},
}
for _, column := range columns {
if err := ensureColumn(ctx, exec, dialect, column.table, column.name, column.definition[dialect]); err != nil {
return err
}
}
updates := map[Dialect][]string{
DialectSQLite: {
`UPDATE verification_rate_limit SET expires_at = datetime(updated_at, '+30 days') WHERE expires_at = '1970-01-01 00:00:00'`,
`UPDATE verification_event SET expires_at = datetime(created_at, '+30 days') WHERE expires_at = '1970-01-01 00:00:00'`,
},
DialectMySQL: {
`UPDATE verification_rate_limit SET expires_at = DATE_ADD(updated_at, INTERVAL 30 DAY) WHERE expires_at = '1970-01-01 00:00:00'`,
`UPDATE verification_event SET expires_at = DATE_ADD(created_at, INTERVAL 30 DAY) WHERE expires_at = '1970-01-01 00:00:00'`,
},
DialectPostgres: {
`UPDATE verification_rate_limit SET expires_at = updated_at + INTERVAL '30 days' WHERE expires_at = '1970-01-01 00:00:00+00'`,
`UPDATE verification_event SET expires_at = created_at + INTERVAL '30 days' WHERE expires_at = '1970-01-01 00:00:00+00'`,
},
}
for _, statement := range updates[dialect] {
if _, err := exec.ExecContext(ctx, statement); err != nil {
return fmt.Errorf("backfill verification retention: %w", err)
}
}
return createVerificationRetentionIndexes(ctx, exec, dialect)
}
func createVerificationRetentionIndexes(ctx context.Context, exec executor, dialect Dialect) error {
statement := `CREATE INDEX IF NOT EXISTS "idx_verification_event_expires_at" ON "verification_event"("expires_at")`
if dialect == DialectMySQL {
var count int
if err := exec.QueryRowContext(ctx, "SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema = DATABASE() AND table_name = ? AND index_name = ?", "verification_event", "idx_verification_event_expires_at").Scan(&count); err != nil {
return fmt.Errorf("check verification retention index: %w", err)
}
if count == 0 {
if _, err := exec.ExecContext(ctx, "CREATE INDEX `idx_verification_event_expires_at` ON `verification_event` (`expires_at`)"); err != nil {
return fmt.Errorf("create verification retention index: %w", err)
}
}
return nil
}
if _, err := exec.ExecContext(ctx, statement); err != nil {
return fmt.Errorf("create verification retention index: %w", err)
}
return nil
}
@@ -0,0 +1,19 @@
package migration
import (
"context"
"fmt"
)
func applyWebAuthnDiscoverabilitySchema(ctx context.Context, exec executor, dialect Dialect) error {
definitions := map[Dialect]string{
DialectSQLite: `INTEGER NOT NULL DEFAULT 0`,
DialectMySQL: `TINYINT(1) NOT NULL DEFAULT 0`,
DialectPostgres: `BOOLEAN NOT NULL DEFAULT FALSE`,
}
definition, ok := definitions[dialect]
if !ok {
return fmt.Errorf("unsupported migration dialect %q", dialect)
}
return ensureColumn(ctx, exec, dialect, "webauthn_credential", "discoverable", definition)
}
+174
View File
@@ -0,0 +1,174 @@
package db
import (
stderrors "errors"
"fmt"
"net/url"
"strconv"
"strings"
"time"
"unicode"
log "github.com/sirupsen/logrus"
"modernc.org/sqlite"
"xorm.io/xorm"
)
const (
sqliteBusyTimeoutMilliseconds = 1000
transactionAttempts = 3
transactionInitialBackoff = 50 * time.Millisecond
)
// Transaction runs fn atomically. SQLite BUSY and LOCKED failures retry the
// entire transaction, so fn must contain only work that is safe to repeat.
func Transaction(engine *xorm.Engine, fn func(*xorm.Session) error) error {
return transactionWithRetry(engine, transactionAttempts, transactionInitialBackoff, fn)
}
func transactionWithRetry(engine *xorm.Engine, attempts int, initialBackoff time.Duration, fn func(*xorm.Session) error) error {
if attempts < 1 {
attempts = 1
}
var err error
for attempt := 1; attempt <= attempts; attempt++ {
var retrySafe bool
err, retrySafe = runTransactionAttempt(engine, fn)
if err == nil {
return nil
}
if !retrySafe || engine.DriverName() != "sqlite" || !IsSQLiteBusyOrLocked(err) || attempt == attempts {
return err
}
time.Sleep(initialBackoff << (attempt - 1))
}
return err
}
func runTransaction(engine *xorm.Engine, fn func(*xorm.Session) error) error {
err, _ := runTransactionAttempt(engine, fn)
return err
}
func runTransactionAttempt(engine *xorm.Engine, fn func(*xorm.Session) error) (error, bool) {
session := engine.NewSession()
defer session.Close()
if err := session.Begin(); err != nil {
return fmt.Errorf("begin transaction: %w", err), true
}
if err := fn(session); err != nil {
if rollbackErr := session.Rollback(); rollbackErr != nil {
return stderrors.Join(err, fmt.Errorf("rollback transaction: %w", rollbackErr)), false
}
return err, true
}
if err := session.Commit(); err != nil {
// modernc.org/sqlite v1.46.1 and newer roll back internally when COMMIT
// fails, leaving the connection clean and the transaction safe to retry.
return fmt.Errorf("commit transaction: %w", err), true
}
return nil, false
}
// IsSQLiteBusyOrLocked classifies SQLite errors by result code, including
// extended BUSY and LOCKED codes. Error message text is intentionally ignored.
func IsSQLiteBusyOrLocked(err error) bool {
var sqliteErr *sqlite.Error
if !stderrors.As(err, &sqliteErr) {
return false
}
primaryCode := sqliteErr.Code() & 0xff
return primaryCode == 5 || primaryCode == 6
}
func sqliteDSNWithReliabilityOptions(dsn string) (string, error) {
dsnPrefix, rawQuery, hasQuery := strings.Cut(dsn, "?")
query, err := url.ParseQuery(rawQuery)
if err != nil {
return "", fmt.Errorf("parse SQLite DSN: %w", err)
}
pragmas := query["_pragma"]
timeoutIndex := -1
for index, pragma := range pragmas {
if sqlitePragmaName(pragma) == "busy_timeout" {
if timeoutIndex >= 0 {
return "", fmt.Errorf("parse SQLite DSN: multiple busy_timeout pragmas are ambiguous")
}
timeoutIndex = index
}
}
if timeoutIndex >= 0 {
requested, err := sqliteBusyTimeoutValue(pragmas[timeoutIndex])
if err != nil {
return "", fmt.Errorf("parse SQLite busy_timeout pragma %q: %w", pragmas[timeoutIndex], err)
}
if requested <= sqliteBusyTimeoutMilliseconds {
return dsn, nil
}
log.Warnf(
"SQLite busy_timeout requested %dms exceeds the %dms maximum; using %dms",
requested,
sqliteBusyTimeoutMilliseconds,
sqliteBusyTimeoutMilliseconds,
)
pragmas[timeoutIndex] = fmt.Sprintf("busy_timeout(%d)", sqliteBusyTimeoutMilliseconds)
query["_pragma"] = pragmas
return dsnPrefix + "?" + query.Encode(), nil
}
separator := "?"
if hasQuery {
separator = "&"
if strings.HasSuffix(dsn, "?") || strings.HasSuffix(dsn, "&") {
separator = ""
}
}
option := url.Values{"_pragma": {fmt.Sprintf("busy_timeout(%d)", sqliteBusyTimeoutMilliseconds)}}.Encode()
return dsn + separator + option, nil
}
func sqliteBusyTimeoutValue(pragma string) (int64, error) {
const name = "busy_timeout"
normalized := strings.TrimSpace(pragma)
if len(normalized) < len(name) || !strings.EqualFold(normalized[:len(name)], name) {
return 0, fmt.Errorf("not a busy_timeout pragma")
}
remainder := strings.TrimSpace(normalized[len(name):])
var valueText string
switch {
case strings.HasPrefix(remainder, "("):
if !strings.HasSuffix(remainder, ")") {
return 0, fmt.Errorf("expected busy_timeout(<milliseconds>)")
}
valueText = strings.TrimSpace(remainder[1 : len(remainder)-1])
case strings.HasPrefix(remainder, "="):
valueText = strings.TrimSpace(remainder[1:])
default:
return 0, fmt.Errorf("expected busy_timeout(<milliseconds>) or busy_timeout = <milliseconds>")
}
if valueText == "" {
return 0, fmt.Errorf("timeout value is empty")
}
timeout, err := strconv.ParseInt(valueText, 10, 64)
if err != nil {
return 0, fmt.Errorf("invalid timeout value %q: %w", valueText, err)
}
return timeout, nil
}
func sqlitePragmaName(pragma string) string {
normalized := strings.ToLower(strings.TrimSpace(pragma))
if end := strings.IndexFunc(normalized, func(character rune) bool {
return character == '(' || character == '=' || unicode.IsSpace(character)
}); end >= 0 {
normalized = normalized[:end]
}
return normalized
}
+407
View File
@@ -0,0 +1,407 @@
package db
import (
"bytes"
"context"
"database/sql"
"errors"
"fmt"
"net/url"
"path/filepath"
"strings"
"testing"
"time"
log "github.com/sirupsen/logrus"
"modernc.org/sqlite"
"xorm.io/xorm"
)
func TestSQLiteDSNOptions(t *testing.T) {
tests := []struct {
name string
dsn string
wantDSN string
wantBusyTimeout string
wantJournalMode string
wantTxLock string
}{
{
name: "plain filename gets default timeout",
dsn: "/tmp/smail.db",
wantBusyTimeout: "busy_timeout(1000)",
},
{
name: "timeout at cap is preserved byte for byte",
dsn: "file:smail.db?_txlock=deferred&_pragma=BUSY_TIMEOUT%20%3D%201000&mode=memory",
wantDSN: "file:smail.db?_txlock=deferred&_pragma=BUSY_TIMEOUT%20%3D%201000&mode=memory",
wantBusyTimeout: "BUSY_TIMEOUT = 1000",
wantTxLock: "deferred",
},
{
name: "explicit timeout is capped and other options are preserved",
dsn: "file:smail.db?mode=memory&_pragma=journal_mode(WAL)&_pragma=busy_timeout%20%3D%20600000&_txlock=deferred",
wantBusyTimeout: "busy_timeout(1000)",
wantJournalMode: "journal_mode(WAL)",
wantTxLock: "deferred",
},
{
name: "negative timeout is preserved byte for byte",
dsn: "file:smail.db?cache=shared&_pragma=%20busy_timeout%20%28%20-1%20%29%20",
wantDSN: "file:smail.db?cache=shared&_pragma=%20busy_timeout%20%28%20-1%20%29%20",
wantBusyTimeout: " busy_timeout ( -1 ) ",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := sqliteDSNWithReliabilityOptions(tt.dsn)
if err != nil {
t.Fatalf("sqliteDSNWithReliabilityOptions() error = %v", err)
}
if tt.wantDSN != "" && got != tt.wantDSN {
t.Errorf("sqliteDSNWithReliabilityOptions() = %q, want byte-for-byte %q", got, tt.wantDSN)
}
queryText := ""
if _, after, ok := strings.Cut(got, "?"); ok {
queryText = after
}
query, err := url.ParseQuery(queryText)
if err != nil {
t.Fatalf("ParseQuery(%q): %v", queryText, err)
}
if got := query.Get("_txlock"); got != tt.wantTxLock {
t.Errorf("_txlock = %q, want %q", got, tt.wantTxLock)
}
if !containsString(query["_pragma"], tt.wantBusyTimeout) {
t.Errorf("_pragma = %q, want %q", query["_pragma"], tt.wantBusyTimeout)
}
if got := countPragma(query["_pragma"], "busy_timeout"); got != 1 {
t.Errorf("busy_timeout pragma count = %d, want 1: %q", got, query["_pragma"])
}
if tt.wantJournalMode != "" && !containsString(query["_pragma"], tt.wantJournalMode) {
t.Errorf("_pragma = %q, want preserved %q", query["_pragma"], tt.wantJournalMode)
}
})
}
}
func TestSQLiteDSNRejectsInvalidBusyTimeout(t *testing.T) {
tests := []struct {
name string
dsn string
}{
{
name: "duplicate timeout pragmas",
dsn: "file:smail.db?_pragma=busy_timeout(10)&_pragma=BUSY_TIMEOUT%20%3D%2020",
},
{
name: "missing timeout value",
dsn: "file:smail.db?_pragma=busy_timeout",
},
{
name: "nonnumeric timeout",
dsn: "file:smail.db?_pragma=busy_timeout(forever)",
},
{
name: "timeout with trailing junk",
dsn: "file:smail.db?_pragma=busy_timeout(10)%20milliseconds",
},
{
name: "overflowing timeout",
dsn: "file:smail.db?_pragma=busy_timeout(9223372036854775808)",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if _, err := sqliteDSNWithReliabilityOptions(tt.dsn); err == nil {
t.Fatalf("sqliteDSNWithReliabilityOptions(%q) error = nil, want error", tt.dsn)
}
})
}
}
func TestSQLiteDSNWarnsWhenBusyTimeoutIsCapped(t *testing.T) {
logger := log.StandardLogger()
originalOutput := logger.Out
originalLevel := logger.GetLevel()
var output bytes.Buffer
logger.SetOutput(&output)
logger.SetLevel(log.WarnLevel)
t.Cleanup(func() {
logger.SetOutput(originalOutput)
logger.SetLevel(originalLevel)
})
if _, err := sqliteDSNWithReliabilityOptions("file:smail.db?_pragma=busy_timeout(600000)"); err != nil {
t.Fatalf("sqliteDSNWithReliabilityOptions() error = %v", err)
}
for _, want := range []string{"600000ms", "1000ms"} {
if !strings.Contains(output.String(), want) {
t.Errorf("warning = %q, want it to contain %q", output.String(), want)
}
}
}
func TestSQLiteBusyTimeoutAppliesToEveryConnection(t *testing.T) {
dsn, err := sqliteDSNWithReliabilityOptions(filepath.Join(t.TempDir(), "connections.db"))
if err != nil {
t.Fatal(err)
}
database, err := sql.Open("sqlite", dsn)
if err != nil {
t.Fatal(err)
}
defer database.Close()
database.SetMaxOpenConns(2)
first, err := database.Conn(context.Background())
if err != nil {
t.Fatal(err)
}
defer first.Close()
second, err := database.Conn(context.Background())
if err != nil {
t.Fatal(err)
}
defer second.Close()
for index, connection := range []*sql.Conn{first, second} {
var timeout int
if err = connection.QueryRowContext(context.Background(), "PRAGMA busy_timeout").Scan(&timeout); err != nil {
t.Fatalf("connection %d: %v", index+1, err)
}
if timeout != sqliteBusyTimeoutMilliseconds {
t.Errorf("connection %d busy_timeout = %d, want %d", index+1, timeout, sqliteBusyTimeoutMilliseconds)
}
}
}
func TestIsSQLiteBusyOrLockedUsesDriverCode(t *testing.T) {
engine, err := xorm.NewEngine("sqlite", filepath.Join(t.TempDir(), "busy.db")+"?_pragma=busy_timeout(1)")
if err != nil {
t.Fatal(err)
}
defer engine.Close()
if _, err = engine.Exec("CREATE TABLE audit (id INTEGER PRIMARY KEY, value TEXT)"); err != nil {
t.Fatal(err)
}
busyErr := sqliteBusyError(t, engine.DataSourceName())
var sqliteErr *sqlite.Error
if !errors.As(busyErr, &sqliteErr) {
t.Fatalf("error type = %T, want *sqlite.Error", busyErr)
}
if !IsSQLiteBusyOrLocked(fmt.Errorf("wrapped: %w", busyErr)) {
t.Fatalf("IsSQLiteBusyOrLocked(%v) = false", busyErr)
}
if IsSQLiteBusyOrLocked(errors.New("database is locked")) {
t.Fatal("plain error text must not be classified as SQLITE_BUSY")
}
}
func TestTransactionRetriesWholeSQLiteTransaction(t *testing.T) {
engine, err := xorm.NewEngine("sqlite", filepath.Join(t.TempDir(), "retry.db")+"?_pragma=busy_timeout(10)")
if err != nil {
t.Fatal(err)
}
defer engine.Close()
engine.SetMaxOpenConns(1)
if _, err = engine.Exec("CREATE TABLE audit (id INTEGER PRIMARY KEY, value TEXT)"); err != nil {
t.Fatal(err)
}
busyErr := sqliteBusyError(t, engine.DataSourceName())
attempts := 0
err = transactionWithRetry(engine, 3, time.Millisecond, func(session *xorm.Session) error {
attempts++
if attempts == 1 {
return busyErr
}
_, insertErr := session.Exec("INSERT INTO audit(value) VALUES ('saved')")
return insertErr
})
if err != nil {
t.Fatalf("transactionWithRetry() error = %v", err)
}
if attempts != 2 {
t.Fatalf("attempts = %d, want 2", attempts)
}
assertAuditRowCount(t, engine, 1)
}
func TestTransactionRetriesSQLiteBusyCommit(t *testing.T) {
engine, blocker, readTx := newCommitBusyTest(t, 10)
attempts := 0
err := transactionWithRetry(engine, 2, 5*time.Millisecond, func(session *xorm.Session) error {
attempts++
if attempts == 2 {
if rollbackErr := readTx.Rollback(); rollbackErr != nil {
return fmt.Errorf("release read transaction: %w", rollbackErr)
}
}
_, insertErr := session.Exec("INSERT INTO audit(value) VALUES ('saved')")
return insertErr
})
_ = blocker.Close()
if err != nil {
t.Fatalf("transactionWithRetry() error = %v", err)
}
if attempts != 2 {
t.Fatalf("attempts = %d, want 2", attempts)
}
assertAuditRowCount(t, engine, 2)
}
func TestPermanentSQLiteBusyCommitLeavesConnectionUsable(t *testing.T) {
engine, blocker, readTx := newCommitBusyTest(t, 5)
attempts := 0
err := transactionWithRetry(engine, 2, time.Millisecond, func(session *xorm.Session) error {
attempts++
_, insertErr := session.Exec("INSERT INTO audit(value) VALUES ('rolled back')")
return insertErr
})
if err == nil || !IsSQLiteBusyOrLocked(err) {
t.Fatalf("transactionWithRetry() error = %v, want SQLITE_BUSY", err)
}
if attempts != 2 {
t.Fatalf("attempts = %d, want 2", attempts)
}
if err = readTx.Rollback(); err != nil {
t.Fatal(err)
}
_ = blocker.Close()
err = runTransaction(engine, func(session *xorm.Session) error {
_, insertErr := session.Exec("INSERT INTO audit(value) VALUES ('after busy')")
return insertErr
})
if err != nil {
t.Fatalf("transaction after BUSY = %v", err)
}
assertAuditRowCount(t, engine, 2)
}
func TestTransactionDoesNotClassifyBusyText(t *testing.T) {
engine, err := xorm.NewEngine("sqlite", filepath.Join(t.TempDir(), "text.db"))
if err != nil {
t.Fatal(err)
}
defer engine.Close()
attempts := 0
wantErr := errors.New("database is locked")
err = transactionWithRetry(engine, 3, time.Millisecond, func(*xorm.Session) error {
attempts++
return wantErr
})
if !errors.Is(err, wantErr) {
t.Fatalf("transactionWithRetry() error = %v, want %v", err, wantErr)
}
if attempts != 1 {
t.Fatalf("attempts = %d, want 1", attempts)
}
}
func newCommitBusyTest(t *testing.T, timeoutMilliseconds int) (*xorm.Engine, *sql.DB, *sql.Tx) {
t.Helper()
path := filepath.Join(t.TempDir(), "commit-busy.db")
dsn := fmt.Sprintf("%s?_pragma=busy_timeout(%d)", path, timeoutMilliseconds)
engine, err := xorm.NewEngine("sqlite", dsn)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = engine.Close() })
engine.SetMaxOpenConns(1)
engine.SetMaxIdleConns(1)
if _, err = engine.Exec("CREATE TABLE audit (id INTEGER PRIMARY KEY, value TEXT)"); err != nil {
t.Fatal(err)
}
if _, err = engine.Exec("INSERT INTO audit(value) VALUES ('seed')"); err != nil {
t.Fatal(err)
}
blocker, err := sql.Open("sqlite", dsn)
if err != nil {
t.Fatal(err)
}
readTx, err := blocker.Begin()
if err != nil {
_ = blocker.Close()
t.Fatal(err)
}
var count int
if err = readTx.QueryRow("SELECT count(*) FROM audit").Scan(&count); err != nil {
_ = readTx.Rollback()
_ = blocker.Close()
t.Fatal(err)
}
t.Cleanup(func() {
_ = readTx.Rollback()
_ = blocker.Close()
})
return engine, blocker, readTx
}
func sqliteBusyError(t *testing.T, dsn string) error {
t.Helper()
first, err := xorm.NewEngine("sqlite", dsn)
if err != nil {
t.Fatal(err)
}
defer first.Close()
second, err := xorm.NewEngine("sqlite", dsn)
if err != nil {
t.Fatal(err)
}
defer second.Close()
tx := first.NewSession()
defer tx.Close()
if err = tx.Begin(); err != nil {
t.Fatal(err)
}
if _, err = tx.Exec("INSERT INTO audit(value) VALUES ('rolled back')"); err != nil {
t.Fatal(err)
}
_, busyErr := second.Exec("INSERT INTO audit(value) VALUES ('blocked')")
_ = tx.Rollback()
if busyErr == nil {
t.Fatal("expected SQLITE_BUSY")
}
return busyErr
}
func assertAuditRowCount(t *testing.T, engine *xorm.Engine, want int) {
t.Helper()
var count int
if _, err := engine.SQL("SELECT count(*) FROM audit").Get(&count); err != nil {
t.Fatal(err)
}
if count != want {
t.Fatalf("audit rows = %d, want %d", count, want)
}
}
func containsString(values []string, want string) bool {
for _, value := range values {
if value == want {
return true
}
}
return false
}
func countPragma(values []string, name string) int {
count := 0
for _, value := range values {
if sqlitePragmaName(value) == name {
count++
}
}
return count
}