diff --git a/README.md b/README.md index d3b2563..2bb98d5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Smail -当前版本的运行配置仅保存在根目录 `config` 的五个 JSON 文件中，数据库、密钥、证书和配置历史保存在根目录 `data`。升级已有部署前请先阅读 [运行目录迁移说明](docs/运行目录迁移说明.md)，停服并完整备份；不能只复制配置文件，也不能沿用下文历史容器示例中仅挂载 `config` 的方式。 +当前版本每套部署只使用一个数据库，SQLite、MySQL、PostgreSQL 三选一，配置历史与业务数据共用所选数据库，不再新建独立的 SQLite 状态库。生效配置保存在根目录 `config` 的五个 JSON 文件中；新部署的密钥和证书保存在 `ssl`，SQLite 文件及配置恢复事务日志保存在 `data`。请阅读 [单数据库部署说明](docs/单数据库部署说明.md) 和 [运行目录迁移说明](docs/运行目录迁移说明.md)，升级前停服并完整备份；不能只复制配置文件，也不能沿用下文历史容器示例中仅挂载 `config` 的方式。 > 一台服务器、一个域名、一行代码、一分钟时间，你就能够搭建出一个自己的域名邮箱。 diff --git a/backend/config/database_migration.go b/backend/config/database_migration.go index 24aba77..77d786d 100644 --- a/backend/config/database_migration.go +++ b/backend/config/database_migration.go @@ -69,7 +69,9 @@ func (m *ConfigManager) SwitchDatabase(ctx context.Context, expectedGeneration u if err := contextError(ctx); err != nil { return err } - if _, _, err := m.transactions.CreateBackupIfDue(current, time.Now().UTC()); err != nil { + backupStore := *m.transactions + backupStore.database = &target + if _, _, err := backupStore.CreateBackupIfDue(current, time.Now().UTC()); err != nil { return err } if err := contextError(ctx); err != nil { diff --git a/backend/config/deployment_state.go b/backend/config/deployment_state.go index 2ec81e7..cf8f93d 100644 --- a/backend/config/deployment_state.go +++ b/backend/config/deployment_state.go @@ -9,7 +9,7 @@ import ( "git.pcux.cn/xuans/Smail/utils/runtimepath" ) -// InitializeDeployment 为全新部署创建状态库，并自动归档已采用 ssl 布局的原有业务密钥。 +// InitializeDeployment 为全新部署准备目录和密钥；选定数据库前不创建数据库文件。 // 更旧的 config/data 布局仍需完整迁移，不能只搬密钥而切断证书或数据库引用。 func InitializeDeployment(root string) error { if runtimepath.DeploymentLayout(root) { @@ -22,13 +22,6 @@ func InitializeDeployment(root string) error { return err } } - db, err := NewTransactionStore(root, RuntimeValidationOptions{}).openState() - if err != nil { - return err - } - if err := db.Close(); err != nil { - return err - } return initializeDeploymentKeys(root) } diff --git a/backend/config/partitioned_compatibility.go b/backend/config/partitioned_compatibility.go index bd2a411..c8f62d6 100644 --- a/backend/config/partitioned_compatibility.go +++ b/backend/config/partitioned_compatibility.go @@ -98,8 +98,12 @@ func runtimeCompatibility(runtime RuntimeConfig) (*Config, error) { } func compatibilityDatabase(database DatabaseConfig, vault *credential.Vault) (string, string, error) { + return compatibilityDatabaseAt(ROOT_PATH, database, vault) +} + +func compatibilityDatabaseAt(root string, database DatabaseConfig, vault *credential.Vault) (string, string, error) { if database.Type == DatabaseTypeSQLite { - path := filepath.Join(ROOT_PATH, "data", database.LocalRef) + path := filepath.Join(root, "data", database.LocalRef) return DBTypeSQLite, path, nil } password, err := OpenStoredSecret(vault, database.Password) diff --git a/backend/config/ssl_retention.go b/backend/config/ssl_retention.go index 86830ef..2901c8c 100644 --- a/backend/config/ssl_retention.go +++ b/backend/config/ssl_retention.go @@ -121,13 +121,18 @@ func pruneManagedSSLLocked(root string, protected []SSLConfig) error { } } // 恢复事务可能仍需要旧材料；恢复未完成时不能清理任何版本。 - if runtimepath.DeploymentLayout(root) { - records, err := NewTransactionStore(root, RuntimeValidationOptions{HasSuperAdmin: true}).stateRecords("transaction") + if entries, err := os.ReadDir(filepath.Join(root, "data", "transaction")); err != nil && !os.IsNotExist(err) || len(entries) > 0 { + return ErrConfigRecoveryRequired + } + if _, err := os.Lstat(filepath.Join(root, "data", "smail-state.db")); err == nil { + store := NewTransactionStore(root, RuntimeValidationOptions{HasSuperAdmin: true}) + store.legacyState = true + records, err := store.stateRecords("transaction") if err != nil || len(records) != 0 { return errors.Join(ErrConfigRecoveryRequired, err) } - } else if entries, err := os.ReadDir(filepath.Join(root, "data", "transaction")); err != nil && !os.IsNotExist(err) || len(entries) > 0 { - return ErrConfigRecoveryRequired + } else if !os.IsNotExist(err) { + return err } keep := make(map[string]bool) for _, ssl := range protected { diff --git a/backend/config/state_database.go b/backend/config/state_database.go new file mode 100644 index 0000000..2a734c7 --- /dev/null +++ b/backend/config/state_database.go @@ -0,0 +1,182 @@ +package config + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "git.pcux.cn/xuans/Smail/internal/configstate" + "git.pcux.cn/xuans/Smail/utils/credential" + "git.pcux.cn/xuans/Smail/utils/runtimepath" + _ "github.com/lib/pq" + _ "modernc.org/sqlite" +) + +var ErrStateDatabaseUnavailable = errors.New("配置历史数据库不可用，请检查所选数据库连接；不会回退到独立 SQLite 数据库") + +type stateDatabase struct { + *sql.DB + driver string + ctx context.Context + cancel context.CancelFunc +} + +func (d *stateDatabase) Close() error { + d.cancel() + return d.DB.Close() +} + +func (d *stateDatabase) Begin() (*sql.Tx, error) { + return d.DB.BeginTx(d.ctx, nil) +} + +func (d *stateDatabase) bind(query string) string { + if d.driver != "postgres" { + return query + } + var result strings.Builder + index := 0 + for _, value := range query { + if value == '?' { + index++ + fmt.Fprintf(&result, "$%d", index) + } else { + result.WriteRune(value) + } + } + return result.String() +} + +func (d *stateDatabase) Exec(query string, args ...any) (sql.Result, error) { + ctx, cancel := context.WithTimeout(d.ctx, 10*time.Second) + defer cancel() + return d.DB.ExecContext(ctx, d.bind(query), args...) +} + +func (d *stateDatabase) Query(query string, args ...any) (*sql.Rows, error) { + return d.DB.QueryContext(d.ctx, d.bind(query), args...) +} + +func (d *stateDatabase) QueryRow(query string, args ...any) *sql.Row { + return d.DB.QueryRowContext(d.ctx, d.bind(query), args...) +} + +func (d *stateDatabase) upsert() string { + query := "INSERT INTO config_records(kind,id,payload,digest) VALUES(?,?,?,?) " + if d.driver == "mysql" { + return query + "ON DUPLICATE KEY UPDATE payload=VALUES(payload),digest=VALUES(digest)" + } + return query + "ON CONFLICT(kind,id) DO UPDATE SET payload=excluded.payload,digest=excluded.digest" +} + +func (s *TransactionStore) stateDatabaseConfig() (DatabaseConfig, error) { + if s.database != nil { + return *s.database, ValidateDatabaseConfig(*s.database) + } + directory, err := secureManagerConfigDirectory(s.root) + if err != nil { + return DatabaseConfig{}, err + } + data, err := readManagerConfigFile(directory, "config.json") + if err != nil { + return DatabaseConfig{}, err + } + var root RootConfig + if err := ParseStrict(data, &root); err != nil { + return DatabaseConfig{}, err + } + database := NormalizeDatabaseConfig(root.Database) + return database, ValidateDatabaseConfig(database) +} + +func (s *TransactionStore) openState() (*stateDatabase, error) { + var driver, dsn string + if s.legacyState { + driver = "sqlite" + dsn = filepath.Join(s.root, "data", "smail-state.db") + } else { + database, err := s.stateDatabaseConfig() + if err != nil { + return nil, err + } + vault, err := credential.NewVault(s.root) + if err != nil { + return nil, err + } + driver, dsn, err = compatibilityDatabaseAt(s.root, database, vault) + if err != nil { + return nil, err + } + } + filename := "" + if driver == "sqlite" { + if _, err := runtimepath.Directory(s.root); err != nil { + return nil, err + } + filename = dsn + relative, err := filepath.Rel(filepath.Join(s.root, "data"), filepath.Dir(filename)) + if err != nil || relative != "." && !filepath.IsLocal(relative) { + return nil, ErrManagerStorage + } + parent := filepath.Join(s.root, "data") + if relative != "." { + for _, part := range strings.Split(filepath.ToSlash(relative), "/") { + parent, err = runtimepath.SecureDirectory(parent, part) + if err != nil { + return nil, err + } + } + } + info, statErr := os.Lstat(filename) + if statErr == nil && !info.Mode().IsRegular() { + return nil, ErrManagerStorage + } + if statErr != nil && (!os.IsNotExist(statErr) || s.legacyState) { + return nil, statErr + } + path := filepath.ToSlash(filename) + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + uri := url.URL{Scheme: "file", Path: path} + query := url.Values{"_pragma": {"busy_timeout(5000)", "synchronous(FULL)"}} + if s.legacyState { + query.Set("mode", "rw") + } + uri.RawQuery = query.Encode() + dsn = uri.String() + } + connection, err := sql.Open(driver, dsn) + if err != nil { + return nil, ErrStateDatabaseUnavailable + } + operationContext, operationCancel := context.WithTimeout(context.Background(), 30*time.Second) + database := &stateDatabase{DB: connection, driver: driver, ctx: operationContext, cancel: operationCancel} + database.SetMaxOpenConns(1) + database.SetMaxIdleConns(1) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := database.PingContext(ctx); err != nil { + database.Close() + return nil, ErrStateDatabaseUnavailable + } + if !s.legacyState { + if _, err := database.Exec(configstate.Schema(driver)); err != nil { + database.Close() + return nil, ErrStateDatabaseUnavailable + } + } + if filename != "" { + if err := os.Chmod(filename, 0600); err != nil { + database.Close() + return nil, err + } + } + return database, nil +} diff --git a/backend/config/state_database_test.go b/backend/config/state_database_test.go new file mode 100644 index 0000000..dc53406 --- /dev/null +++ b/backend/config/state_database_test.go @@ -0,0 +1,271 @@ +package config + +import ( + "context" + "database/sql" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "git.pcux.cn/xuans/Smail/db/migration" + "git.pcux.cn/xuans/Smail/internal/configstate" + "git.pcux.cn/xuans/Smail/utils/credential" +) + +func newTestStateVault(root, password string) (StoredSecret, error) { + vault, err := credential.NewVault(root) + if err != nil { + return StoredSecret{}, err + } + ciphertext, err := vault.Encrypt(password) + return StoredSecret{Ciphertext: ciphertext, Version: vault.Version(), UpdatedAt: time.Now().UTC()}, err +} + +func TestSelectedSQLiteContainsConfigurationHistory(t *testing.T) { + root := t.TempDir() + if err := InitializeDeployment(root); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(filepath.Join(root, "data")) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.Name() != ".config-state.lock" { + t.Fatalf("选库前产生额外文件：%s", entry.Name()) + } + } + current := validRuntimeConfig() + current.Root.Database.LocalRef = "selected.db" + writeRuntimeFixture(t, root, current) + store := NewTransactionStore(root, RuntimeValidationOptions{HasSuperAdmin: true}) + if _, err := store.CreateBackup(current); err != nil { + t.Fatal(err) + } + database, err := store.openState() + if err != nil { + t.Fatal(err) + } + var count int + err = database.QueryRow("SELECT COUNT(*) FROM config_records WHERE kind='backup'").Scan(&count) + database.Close() + if err != nil || count != 1 { + t.Fatalf("所选数据库没有配置历史：count=%d, err=%v", count, err) + } + entries, err = os.ReadDir(filepath.Join(root, "data")) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.Name() != ".config-state.lock" && entry.Name() != "selected.db" { + t.Fatalf("出现额外数据库或文件：%s", entry.Name()) + } + } +} + +func TestDatabaseSwitchBacksUpToTargetWhileSourceLocked(t *testing.T) { + root := t.TempDir() + if err := InitializeDeployment(root); err != nil { + t.Fatal(err) + } + writeRuntimeFixture(t, root, validRuntimeConfig()) + manager, err := NewConfigManagerWithSecurity(root, ConfigLoaderOptions{Validation: RuntimeValidationOptions{HasSuperAdmin: true}}, ConfigSecurity{Audit: &MemoryAuditWriter{}}) + if err != nil { + t.Fatal(err) + } + source, err := manager.transactions.openState() + if err != nil { + t.Fatal(err) + } + defer source.Close() + if _, err := source.Exec("BEGIN IMMEDIATE"); err != nil { + t.Fatal(err) + } + defer source.Exec("ROLLBACK") + target := manager.Snapshot().Root.Database + target.LocalRef = "next.db" + if err := manager.SwitchDatabase(context.Background(), 1, target, authorizedConfigActor()); err != nil { + t.Fatalf("源库锁定阻止了已完成复制后的配置切换：%v", err) + } + if _, err := manager.transactions.LoadBackup(1); err != nil { + t.Fatalf("切换前历史未写入目标库：%v", err) + } + var count int + if err := source.QueryRow("SELECT COUNT(*) FROM config_records").Scan(&count); err != nil || count != 0 { + t.Fatalf("切换阶段写入了源库：%d，%v", count, err) + } +} + +func TestUnavailableSelectedDatabaseDoesNotCreateSQLite(t *testing.T) { + for _, kind := range []DatabaseType{DatabaseTypeMySQL, DatabaseTypePostgreSQL} { + t.Run(string(kind), func(t *testing.T) { + root := t.TempDir() + if err := InitializeDeployment(root); err != nil { + t.Fatal(err) + } + current := validMySQLRuntimeConfig(t, root) + current.Root.Database.Type = kind + current.Root.Database.Port = 1 + writeRuntimeFixture(t, root, current) + store := NewTransactionStore(root, RuntimeValidationOptions{HasSuperAdmin: true}) + if _, err := store.CreateBackup(current); !errors.Is(err, ErrStateDatabaseUnavailable) { + t.Fatalf("未拒绝不可用数据库：%v", err) + } + candidate := cloneRuntimeConfig(current) + setCandidateGeneration(&candidate, 2) + store.WithHooks(TransactionHooks{BeforePhase: func(phase TransactionPhase) error { + if phase == TransactionSectionsCommitted { + return errors.New("模拟进程中断") + } + return nil + }}) + if err := store.Commit(current, candidate); err == nil { + t.Fatal("未注入配置提交故障") + } + if err := store.Recover(); err != nil { + t.Fatalf("数据库离线时无法恢复配置：%v", err) + } + entries, err := os.ReadDir(filepath.Join(root, "data")) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.Name() != ".config-state.lock" && entry.Name() != "transaction" { + t.Fatalf("远程数据库失败后创建了额外文件：%s", entry.Name()) + } + } + }) + } +} + +func TestLegacyStateImportsAndRecoversIntoSelectedDatabase(t *testing.T) { + for _, phase := range []TransactionPhase{TransactionPrepared, TransactionSectionsCommitted, TransactionRootCommitted, TransactionVerified} { + t.Run(string(phase), func(t *testing.T) { + root := t.TempDir() + if err := InitializeDeployment(root); err != nil { + t.Fatal(err) + } + current := validRuntimeConfig() + writeRuntimeFixture(t, root, current) + filename := filepath.Join(root, "data", "smail-state.db") + db, err := sql.Open("sqlite", filename) + if err != nil { + t.Fatal(err) + } + _, err = db.Exec(configstate.Schema("sqlite")) + db.Close() + if err != nil { + t.Fatal(err) + } + legacy := NewTransactionStore(root, RuntimeValidationOptions{HasSuperAdmin: true}) + legacy.legacyState = true + if _, err := legacy.CreateBackup(current); err != nil { + t.Fatal(err) + } + candidate := cloneRuntimeConfig(current) + setCandidateGeneration(&candidate, 2) + legacy.WithHooks(TransactionHooks{BeforePhase: func(actual TransactionPhase) error { + if actual == phase { + return errors.New("模拟旧版中断") + } + return nil + }}) + if err := legacy.commitState(current, candidate); err == nil { + t.Fatal("未注入旧版故障") + } + store := NewTransactionStore(root, RuntimeValidationOptions{HasSuperAdmin: true}) + for i := 0; i < 2; i++ { + if err := store.Recover(); err != nil { + t.Fatal(err) + } + } + if _, err := os.Stat(filename); !os.IsNotExist(err) { + t.Fatal("旧状态库仍在运行路径中") + } + if _, err := store.LoadBackup(1); err != nil { + t.Fatalf("升级后历史丢失：%v", err) + } + loaded, err := NewConfigLoader(root, ConfigLoaderOptions{Validation: store.validation}).Load() + want := uint64(1) + if phase == TransactionRootCommitted || phase == TransactionVerified { + want = 2 + } + if err != nil || loaded.Generation != want { + t.Fatalf("旧事务恢复代次错误：%d，%v", loaded.Generation, err) + } + }) + } +} + +// 云端测试必须使用预先清空的专用测试库；凭据只从进程环境读取。 +func TestSelectedDatabaseLiveIntegration(t *testing.T) { + kind := DatabaseType(os.Getenv("SMAIL_TEST_STATE_DATABASE")) + if kind == "" { + t.Skip("未配置云端专用数据库") + } + if kind != DatabaseTypeMySQL && kind != DatabaseTypePostgreSQL && kind != DatabaseTypeSQLite { + t.Fatal("测试数据库类型无效") + } + root := t.TempDir() + if err := InitializeDeployment(root); err != nil { + t.Fatal(err) + } + current := validRuntimeConfig() + if kind != DatabaseTypeSQLite { + current = validMySQLRuntimeConfig(t, root) + current.Root.Database.Type = kind + current.Root.Database.Username = os.Getenv("SMAIL_TEST_STATE_USER") + current.Root.Database.Name = os.Getenv("SMAIL_TEST_STATE_NAME") + if kind == DatabaseTypePostgreSQL { + current.Root.Database.Port = DefaultPostgreSQLPort + } + vault, err := newTestStateVault(root, os.Getenv("SMAIL_TEST_STATE_PASSWORD")) + if err != nil { + t.Fatal("测试凭据初始化失败") + } + current.Root.Database.Password = vault + } + writeRuntimeFixture(t, root, current) + store := NewTransactionStore(root, RuntimeValidationOptions{HasSuperAdmin: true}) + database, err := store.openState() + if err != nil { + t.Fatal(err) + } + dialect, err := migration.ParseDialect(database.driver) + if err != nil { + t.Fatal(err) + } + _, err = (migration.Runner{DB: database.DB, Dialect: dialect, Migrations: migration.DefaultMigrations()}).Run(context.Background()) + database.Close() + if err != nil { + t.Fatal(err) + } + for index := 0; index < 6; index++ { + if _, err := store.createBackupAt(current, time.Now().Add(time.Duration(index)*time.Hour)); err != nil { + t.Fatal(err) + } + } + store = NewTransactionStore(root, store.validation) + if err := store.Recover(); err != nil { + t.Fatal(err) + } + items, err := store.stateBackups() + if err != nil || len(items) != 4 { + t.Fatalf("真实数据库历史读写或保留策略失败：%d，%v", len(items), err) + } + if _, err := store.LoadBackup(1); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(filepath.Join(root, "data")) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.Name() == ".config-state.lock" || entry.Name() == "transaction" || kind == DatabaseTypeSQLite && entry.Name() == "smail.db" { + continue + } + t.Fatalf("真实数据库模式产生额外数据库：%s", entry.Name()) + } +} diff --git a/backend/config/state_store.go b/backend/config/state_store.go index e811a75..74bbe00 100644 --- a/backend/config/state_store.go +++ b/backend/config/state_store.go @@ -1,7 +1,6 @@ package config import ( - "database/sql" "encoding/json" "errors" "os" @@ -11,10 +10,9 @@ import ( "git.pcux.cn/xuans/Smail/utils/atomicfile" "git.pcux.cn/xuans/Smail/utils/runtimepath" - _ "modernc.org/sqlite" ) -// 状态数据库只保存配置历史和恢复事务，五个生效配置文件仍是配置读取来源。 +// 配置历史保存在所选业务数据库；事务字段仅用于读取旧版独立状态库。 type stateRecord struct { Transaction *TransactionManifest `json:"transaction,omitempty"` Backup *ConfigBackupManifest `json:"backup,omitempty"` @@ -23,14 +21,17 @@ type stateRecord struct { } func (s *TransactionStore) lockState() (func(), error) { - db, err := s.openState() + directory, err := runtimepath.Directory(s.root) if err != nil { return nil, err } - if err := db.Close(); err != nil { + filename := filepath.Join(directory, ".config-state.lock") + if info, err := os.Lstat(filename); err == nil && !info.Mode().IsRegular() { + return nil, ErrManagerStorage + } else if err != nil && !os.IsNotExist(err) { return nil, err } - file, err := os.OpenFile(filepath.Join(s.root, "data", "smail-state.db"), os.O_RDWR, 0600) + file, err := os.OpenFile(filename, os.O_CREATE|os.O_RDWR, 0600) if err != nil { return nil, err } @@ -42,40 +43,6 @@ func (s *TransactionStore) lockState() (func(), error) { return func() { unlock(); file.Close() }, nil } -func (s *TransactionStore) openState() (*sql.DB, error) { - directory, err := runtimepath.Directory(s.root) - if err != nil { - return nil, err - } - filename := filepath.Join(directory, "smail-state.db") - info, err := os.Lstat(filename) - if err == nil && (!info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0) { - return nil, ErrManagerStorage - } - if err != nil && !os.IsNotExist(err) { - return nil, err - } - db, err := sql.Open("sqlite", filepath.ToSlash(filename)) - if err != nil { - return nil, err - } - db.SetMaxOpenConns(1) - for _, statement := range []string{ - "PRAGMA busy_timeout=5000", "PRAGMA synchronous=FULL", - `CREATE TABLE IF NOT EXISTS config_records (kind TEXT NOT NULL, id TEXT NOT NULL, payload BLOB NOT NULL, digest TEXT NOT NULL, PRIMARY KEY(kind,id))`, - } { - if _, err = db.Exec(statement); err != nil { - db.Close() - return nil, err - } - } - if err = os.Chmod(filename, 0600); err != nil { - db.Close() - return nil, err - } - return db, nil -} - func (s *TransactionStore) saveState(kind, id string, record stateRecord) error { if !transactionIDPattern.MatchString(id) { return ErrTransactionInvalid @@ -89,7 +56,7 @@ func (s *TransactionStore) saveState(kind, id string, record stateRecord) error return err } defer db.Close() - _, err = db.Exec(`INSERT INTO config_records(kind,id,payload,digest) VALUES(?,?,?,?) ON CONFLICT(kind,id) DO UPDATE SET payload=excluded.payload,digest=excluded.digest`, kind, id, data, SectionDigest(data)) + _, err = db.Exec(db.upsert(), kind, id, data, SectionDigest(data)) return err } @@ -289,6 +256,10 @@ func (s *TransactionStore) recoverState() error { return err } defer unlock() + return s.recoverStateLocked() +} + +func (s *TransactionStore) recoverStateLocked() error { if entries, err := os.ReadDir(filepath.Join(s.root, "data", "transaction")); err == nil && len(entries) != 0 { return ErrConfigRecoveryRequired } else if err != nil && !os.IsNotExist(err) { diff --git a/backend/config/state_upgrade.go b/backend/config/state_upgrade.go new file mode 100644 index 0000000..24b4a74 --- /dev/null +++ b/backend/config/state_upgrade.go @@ -0,0 +1,84 @@ +package config + +import ( + "database/sql" + "encoding/json" + "errors" + "os" + "path/filepath" + + "git.pcux.cn/xuans/Smail/utils/atomicfile" + "git.pcux.cn/xuans/Smail/utils/runtimepath" +) + +// 旧独立状态库只用于一次性升级；校验和导入成功后退出运行路径，不覆盖冲突历史。 +func (s *TransactionStore) migrateLegacyState() error { + unlock, err := s.lockState() + if err != nil { + return err + } + defer unlock() + filename := filepath.Join(s.root, "data", "smail-state.db") + info, err := os.Lstat(filename) + if os.IsNotExist(err) { + return nil + } + if err != nil || !info.Mode().IsRegular() { + return errors.Join(ErrManagerStorage, err) + } + legacy := *s + legacy.legacyState = true + if err := legacy.recoverStateLocked(); err != nil { + return err + } + records, err := legacy.stateBackups() + if err != nil { + return err + } + database, err := s.openState() + if err != nil { + return err + } + defer database.Close() + tx, err := database.Begin() + if err != nil { + return err + } + defer tx.Rollback() + for _, record := range records { + data, err := json.Marshal(record) + if err != nil { + return err + } + var digest string + var payload []byte + err = tx.QueryRow(database.bind("SELECT digest,payload FROM config_records WHERE kind=? AND id=?"), "backup", record.Backup.ID).Scan(&digest, &payload) + if err == nil { + if digest != SectionDigest(data) || SectionDigest(payload) != digest { + return ErrConfigDigestMismatch + } + continue + } + if !errors.Is(err, sql.ErrNoRows) { + return err + } + if _, err := tx.Exec(database.bind("INSERT INTO config_records(kind,id,payload,digest) VALUES(?,?,?,?)"), "backup", record.Backup.ID, data, SectionDigest(data)); err != nil { + return err + } + } + if err := tx.Commit(); err != nil { + return err + } + archive, err := runtimepath.SecureDirectory(filepath.Join(s.root, "data"), "migration-history") + if err != nil { + return err + } + target := filepath.Join(archive, "smail-state.retired") + if _, err := os.Lstat(target); !os.IsNotExist(err) { + return ErrManagerStorage + } + if err := os.Rename(filename, target); err != nil { + return err + } + return errors.Join(atomicfile.SyncDirectory(archive), atomicfile.SyncDirectory(filepath.Dir(filename))) +} diff --git a/backend/config/transaction.go b/backend/config/transaction.go index 679c9cc..9e43e83 100644 --- a/backend/config/transaction.go +++ b/backend/config/transaction.go @@ -44,9 +44,11 @@ type TransactionHooks struct { } type TransactionStore struct { - root string - validation RuntimeValidationOptions - hooks TransactionHooks + root string + validation RuntimeValidationOptions + hooks TransactionHooks + database *DatabaseConfig + legacyState bool } var transactionIDPattern = regexp.MustCompile(`^[a-f0-9]{32}$`) @@ -78,7 +80,22 @@ func (s *TransactionStore) Commit(current, candidate RuntimeConfig) error { return ErrManagerStorage } if runtimepath.DeploymentLayout(s.root) { - return s.commitState(current, candidate) + unlock, err := s.lockState() + if err != nil { + return err + } + defer unlock() + live, err := NewConfigLoader(s.root, ConfigLoaderOptions{Validation: s.validation}).Load() + if err != nil || live.Generation != current.Generation { + return errors.Join(ErrGenerationConflict, err) + } + entries, err := os.ReadDir(filepath.Join(s.root, "data", "transaction")) + if err != nil && !os.IsNotExist(err) { + return err + } + if len(entries) != 0 { + return ErrConfigRecoveryRequired + } } if current.Generation == 0 || candidate.Generation != current.Generation+1 { return ErrGenerationConflict @@ -201,7 +218,14 @@ func (s *TransactionStore) Recover() error { return ErrManagerStorage } if runtimepath.DeploymentLayout(s.root) { - return s.recoverState() + if err := s.migrateLegacyState(); err != nil { + return err + } + unlock, err := s.lockState() + if err != nil { + return err + } + defer unlock() } directory, err := secureManagerConfigDirectory(s.root) if err != nil { diff --git a/backend/config/validator.go b/backend/config/validator.go index 226955d..6e4471b 100644 --- a/backend/config/validator.go +++ b/backend/config/validator.go @@ -101,6 +101,9 @@ func validateDatabase(database DatabaseConfig) error { func validSQLiteLocalRef(value string) bool { value = filepath.ToSlash(value) + if strings.EqualFold(value, "smail-state.db") { + return false + } if value == "" || filepath.IsAbs(value) || isWindowsAbsolute(value) || strings.HasPrefix(value, "config/") || strings.ContainsAny(value, `\\:`) { return false } diff --git a/backend/db/migration/registry.go b/backend/db/migration/registry.go index db6201d..94d28a2 100644 --- a/backend/db/migration/registry.go +++ b/backend/db/migration/registry.go @@ -1,6 +1,7 @@ package migration import ( + "git.pcux.cn/xuans/Smail/internal/configstate" "regexp" "strings" ) @@ -33,6 +34,11 @@ func DefaultMigrations() []Migration { DefineCustom(21, "expand_trusted_sender_blacklist", "expand_trusted_sender_blacklist_v1", applyTrustedSenderBlacklistSchema), DefineForDialects(22, "create_client_credentials", clientCredentialStatements()), DefineForDialects(23, "create_outbound_delivery", outboundDeliveryStatements()), + DefineForDialects(24, "unify_configuration_history", map[Dialect][]string{ + DialectSQLite: {configstate.Schema("sqlite")}, + DialectMySQL: {configstate.Schema("mysql")}, + DialectPostgres: {configstate.Schema("postgres")}, + }), } } diff --git a/backend/db/migration/registry_test.go b/backend/db/migration/registry_test.go index 9f29c50..9d5a8f8 100644 --- a/backend/db/migration/registry_test.go +++ b/backend/db/migration/registry_test.go @@ -297,6 +297,7 @@ func TestDefaultMigrationChecksumsAreImmutable(t *testing.T) { 21: "ac1a0b3be90128c0e0512d2cec858f8cd7e70abc5ff4734d7113b9674f08dbb0", 22: "1047cb22f5a32c8a09ba4ccedff8713a5225c14fdb98357db86a6e0723c30d8b", 23: "5d80f2c2c583fc9a033fc320c0852c73bd6701ce6c8f12e8904deedfe0ba44b2", + 24: "36533c34245853580f0e1c67ebb3a4eaeb8dafc8df2f3a85acff6b15d0ea4a6b", } migrations := DefaultMigrations() if len(migrations) != len(expected) { @@ -357,5 +358,3 @@ func TestDefaultMigrationsResumePublishedVersionOne(t *testing.T) { }) } } - - diff --git a/backend/internal/configstate/schema.go b/backend/internal/configstate/schema.go new file mode 100644 index 0000000..fe6c70f --- /dev/null +++ b/backend/internal/configstate/schema.go @@ -0,0 +1,12 @@ +package configstate + +func Schema(driver string) string { + switch driver { + case "mysql": + return "CREATE TABLE IF NOT EXISTS config_records (kind VARCHAR(16) NOT NULL, id VARCHAR(32) NOT NULL, payload LONGBLOB NOT NULL, digest VARCHAR(64) NOT NULL, PRIMARY KEY(kind,id)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" + case "postgres": + return "CREATE TABLE IF NOT EXISTS config_records (kind VARCHAR(16) NOT NULL, id VARCHAR(32) NOT NULL, payload BYTEA NOT NULL, digest VARCHAR(64) NOT NULL, PRIMARY KEY(kind,id))" + default: + return "CREATE TABLE IF NOT EXISTS config_records (kind TEXT NOT NULL, id TEXT NOT NULL, payload BLOB NOT NULL, digest TEXT NOT NULL, PRIMARY KEY(kind,id))" + } +} diff --git a/backend/utils/runtimepath/deployment.go b/backend/utils/runtimepath/deployment.go index 808b277..7bcca35 100644 --- a/backend/utils/runtimepath/deployment.go +++ b/backend/utils/runtimepath/deployment.go @@ -41,7 +41,7 @@ func DeploymentLayout(root string) bool { if _, err := os.Lstat(filepath.Join(root, "data", "credential.key")); err == nil { return false } - for _, name := range []string{"data/smail-state.db", "ssl/credential.key", "ssl/credentials/credential.key"} { + for _, name := range []string{"data/.config-state.lock", "data/smail-state.db", "ssl/credential.key", "ssl/credentials/credential.key"} { info, err := os.Lstat(filepath.Join(root, filepath.FromSlash(name))) if err == nil && info.Mode().IsRegular() { return true diff --git a/docs/单数据库云端测试报告.md b/docs/单数据库云端测试报告.md new file mode 100644 index 0000000..b8eb1a3 --- /dev/null +++ b/docs/单数据库云端测试报告.md @@ -0,0 +1,68 @@ +# 单数据库云端测试报告 + +## 范围与结论 + +测试日期：2026 年 9 月 20 日。 + +本次要求是每套部署在 SQLite、MySQL、PostgreSQL 中三选一，业务数据与配置历史共用同一个数据库，不再附带独立 SQLite 状态库。域名和账户仅作为本次测试参数，不作为产品默认值或验收条件。 + +- [✓] 首次启动不提前创建数据库，远程数据库不可用时不回退 SQLite。 +- [✓] 配置历史使用所选数据库中的 `config_records`，配置提交恢复使用文件事务日志。 +- [✓] 旧双库历史导入、故障阶段恢复、冲突校验与重复启动回归。 +- [✓] 云端三种真实数据库的第 1～24 版迁移、历史读写、保留策略与重新打开检查。 +- [✓] 最新程序编译部署、清理后首次初始化、配置保存、服务重启与浏览器会话恢复。 +- [✓] 专用测试数据库、账号和远程测试目录清理；其他项目保持运行。 + +## 本地自动化验证 + +最终执行 `node test/run.mjs all -p 1 ./...`，退出码为 0。该入口同时执行前端、部署和带测试覆盖层的后端测试；单独执行普通 Go 测试不能替代此入口。 + +证据位于本地忽略目录 `test-log/cloud-clean-20260920`： + +- `single-database-release-final.log`：最终全量测试。 +- `single-database-final-targeted.log`：配置、迁移及路径保护定向测试。 +- `live-three-database.log`：SQLite、MySQL、PostgreSQL 实机测试均为通过。 +- `build-latest.log`：前后端与部署工具构建。 + +通用测试中未提供外部环境的用例仍按自身条件跳过，不能把跳过当成实机通过。三种数据库的实机结论仅来自独立的云端测试日志。 + +## 清理与真实初始化 + +按本次授权只清理 Smail 测试环境，没有额外备份，没有停止共享数据库或其他项目。保留原有证书材料，清空并复用 `/opt/smail/data/smail.db`，文件 inode 清理前后均为 `1185535`。旧 `smail-state.db` 已移除。 + +部署构建版本为 `single-db-20260920`。真实桌面浏览器通过独立 Chrome MCP 完成数据库、管理员、域名、DNS 与证书初始化。首次配置入口通过 SSH 隧道访问；正式登录与刷新访问真实域名，不使用浏览器强制域名解析。 + +初始化完成后使用正常配置接口将本次 Nginx 反向代理的公网 HTTPS 端口设置为 443，内部 HTTPS 端口保持 8002。第一次初始化自动跳转曾指向内部端口；设置公网端口后，正式入口不再依赖该端口。 + +配置保存后 `config_records` 出现一条历史记录。服务重启后检查结果： + +- `PRAGMA integrity_check` 返回 `ok`。 +- 业务表、用户、会话、邮件和配置历史位于同一个 `smail.db`。 +- 运行目录中只有一个数据库文件，没有再次生成 `smail-state.db`。 +- 历史记录保留，浏览器刷新后仍显示已登录的超级管理员。 + +证据：`clean-deploy.log`、`live-final-runtime.log`、`live-restart-mail.log`。 + +## 域名、证书与邮件边界 + +本次主域和后台域配置正确。公共 DNS 查询确认根域、后台及随机子域均指向测试服务器；现有 MX、SPF、DKIM、DMARC 保留，没有为了匹配向导示例覆盖既有邮件路由。DKIM 公钥与初始化页面展示值一致。 + +证书 SAN 包含根域和泛域，TLS 主机名与信任链校验通过。Nginx 上游证书校验保持启用，项目证书同步服务执行成功，监视单元已恢复。 + +本机系统 DNS 曾返回代理使用的地址，命令行 HTTPS 握手失败；独立浏览器通过正常域名成功打开并登录。二者属于不同访问路径，不能据此宣称所有本机网络客户端均已通过。 + +邮件冒烟测试验证了本机 SMTP 接收、邮件与用户关联落库、后台列表展示，以及 SMTP TLS、IMAP TLS、POP3 TLS 的证书验证和协议欢迎响应。测试邮件最终状态为“已删除”，不在收件箱；本次未单独审计该分类策略原因，也没有放宽安全策略来制造成功结果。 + +本次没有验证外部收件服务的最终到达，也没有执行三种数据库各自完整的浏览器初始化、完整真实跨库切换链或 IMAP/POP3 身份认证收信，不将上述内容列为已通过。 + +## 清理与其他项目 + +本轮专用 MySQL、PostgreSQL 测试数据库和账号已删除，查询确认均不存在；远程 `/opt/smail-single-db-test` 已移除。没有清理正式测试部署的唯一业务数据库。 + +核对后 Scert、FDM、MySQL、PostgreSQL、Nginx 主进程 PID 与清理前一致，均处于运行状态。Smail 服务及证书同步监视单元正常。 + +证据：`test-resource-cleanup.log`。原有 `云端全项目测试报告.md` 的未提交修改未覆盖，本报告单独记录本轮范围。 + +## 旧部署升级说明 + +旧双库部署必须停服升级。导入并验证成功后，旧状态文件保留为不再打开的迁移原件 `data/migration-history/smail-state.retired`，不是第二个运行数据库。全新部署不生成该原件。本次未针对旧进程并发运行进行升级测试，不能跳过停服要求。 diff --git a/docs/单数据库部署说明.md b/docs/单数据库部署说明.md new file mode 100644 index 0000000..0fda541 --- /dev/null +++ b/docs/单数据库部署说明.md @@ -0,0 +1,32 @@ +# 单数据库部署说明 + +## 部署原则 + +- 每套部署在 SQLite、MySQL、PostgreSQL 中选择一种，只连接该数据库保存业务数据和配置历史。 +- SQLite 模式默认使用 `data/smail.db`；MySQL 和 PostgreSQL 模式不额外创建 SQLite 文件。 +- 配置历史保存在同一数据库的 `config_records` 表，由第 24 版数据库迁移管理。 +- 首次启动仅准备运行目录和密钥，不在选择数据库前创建状态数据库。 +- 所选数据库不可连接时明确报错，不回退到另一个数据库。 + +## 配置与恢复 + +五个生效 JSON 配置文件仍位于 `config`。配置提交使用 `data/transaction` 中的文件事务日志，跨进程锁位于 `data/.config-state.lock`；两者都不是数据库。这样即使远程数据库离线，也能恢复未完成的配置提交。配置历史继续保留摘要校验、原始加密内容和数量、期限限制。 + +数据库切换必须通过迁移流程：复制并校验所有版本化表（包含配置历史）后才切换配置。源库被迁移锁定期间，切换前配置历史写入已经复制完成的目标库，避免反向写入源库造成阻塞。旧业务数据库按原迁移安全策略保留，不会被新部署继续使用；确认切换成功后才人工清理，不擅自删除。 + +## 从双库版本升级 + +必须停止旧进程后升级，不能让新旧程序同时写入。启动时先恢复旧 `data/smail-state.db` 中未完成的事务，再校验并导入历史到所选数据库；摘要冲突或损坏会阻止升级。完成后旧文件移到 `data/migration-history/smail-state.retired`，仅作为不再打开的迁移原件保留，不是第二个运行数据库。核验成功且获准清理后可移除此原件；全新部署不会生成它。 + +不要把 `smail-state.db` 作为业务 SQLite 文件名，它是旧版升级识别保留名。 + +## 验证入口 + +```powershell +node test/run.mjs backend -p 1 ./config ./db/migration ./services/dbmigration ./utils/runtimepath +node test/run.mjs all -p 1 ./... +``` + +`TestSelectedDatabaseLiveIntegration` 用于预先清空的专用云端数据库。通过进程环境设置 `SMAIL_TEST_STATE_DATABASE`（`sqlite`、`mysql` 或 `postgresql`），远程数据库同时提供 `SMAIL_TEST_STATE_USER`、`SMAIL_TEST_STATE_NAME`、`SMAIL_TEST_STATE_PASSWORD`。凭据不能写入代码、文档或提交记录。未提供连接时测试明确跳过，不能将跳过当作实机验证通过。 + +本次指定的域名、管理员和密码仅是测试参数，不应硬编码为部署默认值或验收条件。 diff --git a/test/backend/deployment_state_test.go b/test/backend/deployment_state_test.go index f4c372a..434f2fe 100644 --- a/test/backend/deployment_state_test.go +++ b/test/backend/deployment_state_test.go @@ -58,7 +58,7 @@ func TestDeploymentStateRecoversEveryCommitPhase(t *testing.T) { } entries, _ = os.ReadDir(filepath.Join(root, "data")) for _, entry := range entries { - if entry.Name() != "smail-state.db" { + if entry.Name() != ".config-state.lock" && entry.Name() != "transaction" { t.Fatalf("data 中产生额外文件：%s", entry.Name()) } } @@ -90,7 +90,9 @@ func TestDeploymentStateImportsHistoryWithoutChangingSecret(t *testing.T) { if err := os.WriteFile(filepath.Join(target, "ssl", "credential.key"), key, 0600); err != nil { t.Fatal(err) } - writeRuntimeFixture(t, target, current) + targetConfig := cloneRuntimeConfig(current) + targetConfig.Root.Database = DatabaseConfig{Type: DatabaseTypeSQLite, LocalRef: "smail.db"} + writeRuntimeFixture(t, target, targetConfig) if err := ImportDeploymentHistory(source, target, validation); err != nil { t.Fatal(err) }