分离五项配置与运行数据并实现无损目录迁移
Security Checks / security (push) Waiting to run

This commit is contained in:
xuans committed 2026-09-08 17:16:09 +08:00
1 parent dc188b1ed8
commit d6cc81ed87
53 files changed
+909 -127

No files matched your search

+128
View File
@@ -0,0 +1,128 @@
package config
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.pcux.cn/xuans/Smail/services/externalemail"
"git.pcux.cn/xuans/Smail/utils/credential"
"git.pcux.cn/xuans/Smail/utils/runtimepath"
)
func TestDataMigrationKeepsSecretsCertificatesAndBackupRestorable(t *testing.T) {
root := t.TempDir()
runtime := validMySQLRuntimeConfig(t, root)
vault, err := credential.NewVault(root)
if err != nil {
t.Fatal(err)
}
external, err := externalemail.NewVault(root)
if err != nil {
t.Fatal(err)
}
emailCipher, _ := external.Encrypt("test@example.com")
fingerprint, _ := external.Fingerprint("test@example.com")
private, certificate := testManagedCertificate(t, "example.com", time.Now().Add(-time.Hour), time.Now().Add(time.Hour), 2048)
reference, err := stageManagedSSLAtRoot(root, ManagedSSLModeUpload, private, certificate, nil, []string{"example.com"})
if err != nil {
t.Fatal(err)
}
runtime.SSL.TLS = SSLConfig{
Mode: ManagedSSLModeUpload, Management: SSLManagementCustom,
DefaultDirectory: "config/ssl", CustomDirectory: "config/ssl",
CertificateRef: strings.Replace(reference.CertificateRef, "data/", "config/", 1),
PrivateKeyRef: strings.Replace(reference.PrivateKeyRef, "data/", "config/", 1),
}
writeRuntimeFixture(t, root, runtime)
validation := RuntimeValidationOptions{HasSuperAdmin: true}
store := NewTransactionStore(root, validation)
if _, err := store.CreateBackup(runtime); err != nil {
t.Fatal(err)
}
writer, err := NewFileAuditWriter(root)
if err != nil {
t.Fatal(err)
}
if err := writer.WriteConfigAudit(context.Background(), ConfigAuditEvent{
Operation: ConfigOperationUpdate, Section: SectionSMTP, ActorID: "1", ActorRole: "super_admin",
FromGeneration: 1, ToGeneration: 2, Result: "success", CreatedAt: time.Now().UTC(),
}); err != nil {
t.Fatal(err)
}
// 构造旧版磁盘布局;文件内容及密文均来自真实实现,而不是手写假密钥。
entries, _ := os.ReadDir(filepath.Join(root, "data"))
for _, entry := range entries {
if err := os.Rename(filepath.Join(root, "data", entry.Name()), filepath.Join(root, "config", entry.Name())); err != nil {
t.Fatal(err)
}
}
before, _ := os.ReadFile(filepath.Join(root, "config", "ssl.json"))
if _, err := credential.NewVault(root); err == nil {
t.Fatal("未迁移的旧密钥必须阻止生成新密钥")
}
if _, err := runtimepath.Migrate(root); err != nil {
t.Fatal(err)
}
after, _ := os.ReadFile(filepath.Join(root, "config", "ssl.json"))
if !bytes.Equal(before, after) {
t.Fatal("目录迁移不得篡改当前配置代次或摘要")
}
newVault, err := credential.NewVault(root)
if err != nil {
t.Fatal(err)
}
secret, err := OpenStoredSecret(newVault, runtime.Root.Database.Password)
expected, _ := OpenStoredSecret(vault, runtime.Root.Database.Password)
if err != nil || secret != expected {
t.Fatal("配置密文无法继续解密")
}
newExternal, err := externalemail.NewVault(root)
if err != nil {
t.Fatal(err)
}
plain, err := newExternal.Decrypt(emailCipher)
newFingerprint, _ := newExternal.Fingerprint("test@example.com")
if err != nil || plain != "test@example.com" || newFingerprint != fingerprint {
t.Fatal("外部邮箱密文或指纹发生改变")
}
loaded, err := NewConfigLoader(root, ConfigLoaderOptions{Validation: validation}).Load()
if err != nil || !strings.HasPrefix(loaded.SSL.TLS.CertificateRef, "data/") {
t.Fatalf("旧配置未兼容新目录:%v", err)
}
resolved, err := resolveManagedSSLReferencesAtRoot(root, loaded.SSL.TLS)
if err != nil {
t.Fatal(err)
}
material, _ := os.ReadFile(resolved.CertificatePath)
if !bytes.Equal(material, certificate) {
t.Fatal("证书发生改变")
}
backup, err := store.LoadBackup(runtime.Generation)
if err != nil {
t.Fatal(err)
}
if _, err := resolveManagedSSLReferencesAtRoot(root, backup.SSL.TLS); err != nil {
t.Fatal("历史备份中的旧证书引用无法使用")
}
setCandidateGeneration(&backup, loaded.Generation+1)
if err := store.Commit(loaded, backup); err != nil {
t.Fatal(err)
}
writer, err = NewFileAuditWriter(root)
if err != nil {
t.Fatal(err)
}
items, err := writer.Query(context.Background(), ConfigAuditQuery{})
if err != nil || len(items) != 1 {
t.Fatal("历史审计丢失")
}
entries, _ = os.ReadDir(filepath.Join(root, "config"))
if len(entries) != 5 {
t.Fatal("保存或备份恢复又向 config 写入了运行文件")
}
}
+156
View File
@@ -0,0 +1,156 @@
package runtimepath
import (
"bytes"
"errors"
"os"
"path/filepath"
"testing"
)
func put(t *testing.T, root, name string, data []byte) {
t.Helper()
path := filepath.Join(root, name)
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, data, 0600); err != nil {
t.Fatal(err)
}
}
func TestRuntimeMigrationPreservesFilesAndIsIdempotent(t *testing.T) {
root := t.TempDir()
for name := range configurationFiles {
put(t, root, "config/"+name, []byte(`{"generation":21}`))
}
for _, name := range []string{"credential.key", "external-email-encryption.key", "external-email-fingerprint.key", "smail.db", "smail.db-wal", "smail.db-shm", "ssl/managed/upload/current/private.key", "dkim/dkim.priv", "backups-v2/old/files/config.json", "transaction/pending/manifest.json", "config-audit-v2.jsonl", "config.dev.json"} {
put(t, root, "config/"+name, []byte("preserved:"+name))
}
if !errors.Is(CheckReady(root), ErrMigrationRequired) {
t.Fatal("旧目录应阻止启动")
}
plan, err := Plan(root)
if err != nil {
t.Fatal(err)
}
original := map[string][]byte{}
for _, item := range plan {
if info, _ := os.Stat(item.Source); info.Mode().IsRegular() {
original[item.Target], _ = os.ReadFile(item.Source)
}
}
if _, err := Migrate(root); err != nil {
t.Fatal(err)
}
for path, expected := range original {
actual, err := os.ReadFile(path)
if err != nil || !bytes.Equal(actual, expected) {
t.Fatal("迁移改变了文件内容")
}
}
entries, _ := os.ReadDir(filepath.Join(root, "config"))
if len(entries) != 5 {
t.Fatal("config 必须恰好保留五个文件")
}
for _, entry := range entries {
data, _ := os.ReadFile(filepath.Join(root, "config", entry.Name()))
if string(data) != `{"generation":21}` {
t.Fatal("当前配置被覆盖")
}
}
if moves, err := Migrate(root); err != nil || len(moves) != 0 {
t.Fatal("重复迁移不应改动文件")
}
}
func TestRuntimeMigrationRejectsConflictAndUnknownFilesBeforeMoving(t *testing.T) {
for _, conflict := range []bool{false, true} {
root := t.TempDir()
put(t, root, "config/credential.key", []byte("old"))
if conflict {
put(t, root, "data/credential.key", []byte("new"))
} else {
put(t, root, "config/unknown", []byte("unknown"))
}
if _, err := Migrate(root); err == nil {
t.Fatal("冲突或未知文件应拒绝自动迁移")
}
data, _ := os.ReadFile(filepath.Join(root, "config", "credential.key"))
if string(data) != "old" {
t.Fatal("检查失败不应移动密钥")
}
}
}
func TestRuntimeMigrationRejectsLinks(t *testing.T) {
root, outside := t.TempDir(), t.TempDir()
if err := os.Symlink(outside, filepath.Join(root, "config")); err != nil {
t.Skipf("当前环境不支持符号链接:%v", err)
}
if _, err := Migrate(root); err == nil {
t.Fatal("符号链接必须被拒绝")
}
}
func TestReadyDeploymentCannotCreateNewDatabaseOrKeys(t *testing.T) {
root := t.TempDir()
put(t, root, "config/config.json", []byte(`{"setupState":"ready","database":{"type":"sqlite","localRef":"mail.db"}}`))
if err := CheckReady(root); err == nil {
t.Fatal("必要文件缺失必须阻止启动")
}
if _, err := os.Stat(filepath.Join(root, "data")); !os.IsNotExist(err) {
t.Fatal("只读启动检查不应生成空数据库或密钥")
}
}
func TestRuntimeMigrationRollsBackOnMoveFailure(t *testing.T) {
root := t.TempDir()
put(t, root, "config/credential.key", []byte("original-key"))
put(t, root, "config/smail.db", []byte("original-database"))
calls := 0
_, err := migrate(root, func(source, target string) error {
calls++
if calls == 2 {
return errors.New("模拟迁移中途失败")
}
return os.Rename(source, target)
})
if err == nil || calls != 3 {
t.Fatal("应执行逆序回滚")
}
for name, expected := range map[string]string{"credential.key": "original-key", "smail.db": "original-database"} {
data, err := os.ReadFile(filepath.Join(root, "config", name))
if err != nil || string(data) != expected {
t.Fatal("回滚后原文件应完整保留")
}
}
if _, err := Migrate(root); err != nil {
t.Fatal("正常回滚后应能重新迁移", err)
}
}
func TestRuntimeMigrationLockBlocksConcurrentMigrationAndStartup(t *testing.T) {
root := t.TempDir()
put(t, root, "data/"+migrationLock, nil)
if _, err := Migrate(root); err == nil {
t.Fatal("并发迁移或中断现场必须拒绝继续")
}
if err := CheckReady(root); err == nil {
t.Fatal("迁移锁必须阻止主服务启动")
}
}
func TestLegacyReferenceOnlyChangesKnownRelativeDirectories(t *testing.T) {
for input, expected := range map[string]string{
"./config/ssl/public.crt": "data/ssl/public.crt",
"config/dkim/dkim.priv": "data/dkim/dkim.priv",
"/config/ssl/public.crt": "/config/ssl/public.crt",
"config/ssl-other/key": "config/ssl-other/key",
"../config/ssl/key": "../config/ssl/key",
} {
if actual := LegacyReference(input); actual != expected {
t.Fatalf("相对目录转换错误:%q", input)
}
}
}
+2 -2
View File
@@ -66,7 +66,7 @@ func customSSLFixture(t *testing.T) (string, *ConfigManager, UpdateRequest, []by
func assertNoCustomSSLVersions(t *testing.T, root string) {
t.Helper()
if _, err := os.Stat(filepath.Join(root, "config", "ssl", "managed", "upload")); !os.IsNotExist(err) {
if _, err := os.Stat(filepath.Join(root, "data", "ssl", "managed", "upload")); !os.IsNotExist(err) {
t.Fatalf("失败或只读操作产生了证书存储副作用: %v", err)
}
}
@@ -101,7 +101,7 @@ func TestCustomSSLManagerValidateSaveApplyReload(t *testing.T) {
if manager.Snapshot().Generation != request.ExpectedGeneration {
t.Fatal("暂存提前切换了当前配置")
}
if !strings.HasPrefix(updated.Candidate.SSL.TLS.CertificateRef, "config/ssl/managed/upload/version-") {
if !strings.HasPrefix(updated.Candidate.SSL.TLS.CertificateRef, "data/ssl/managed/upload/version-") {
t.Fatal("候选未使用独立受管引用")
}
currentBytes, err := os.ReadFile(oldReference.CertificatePath)
+2 -2
View File
@@ -94,8 +94,8 @@ test('初始化模板能编译,实际渲染按反代策略隐藏自动选项
if (tlsMode === 'proxy') {
assert.doesNotMatch(html, /data-label="自动申请证书"|ssl-challenge-select-row|ssl-storage-card|ssl-port-warning/)
assert.match(html, /外部续期后也需重新导入并应用/)
assert.match(html, /modelvalue="\.\/config\/ssl\/private\.key"/)
assert.match(html, /modelvalue="\.\/config\/ssl\/public\.crt"/)
assert.match(html, /modelvalue="\.\/data\/ssl\/private\.key"/)
assert.match(html, /modelvalue="\.\/data\/ssl\/public\.crt"/)
} else {
assert.match(html, /data-label="自动申请证书"/)
assert.match(html, /ssl-challenge-select-row|ssl-storage-card/)
+2
View File
@@ -42,6 +42,8 @@ if (suite !== 'frontend' && result === 0) {
const args = extra[0] === '--' ? extra.slice(1) : extra
// 独立回归源码集中放在 test/backend,通过 overlay 保留同包测试语义。
const replacements = {
'backend/utils/runtimepath/layout_test.go': 'test/backend/runtime_layout_test.go',
'backend/config/data_paths_test.go': 'test/backend/data_paths_test.go',
'backend/config/ssl_custom_regression_test.go': 'test/backend/ssl_custom_test.go',
'backend/config/webauthn_defaults_test.go': 'test/backend/webauthn_config_test.go',
'backend/res_init/webauthn_runtime_test.go': 'test/backend/webauthn_runtime_test.go',
+108
View File
@@ -0,0 +1,108 @@
"""只读核对运行目录迁移前后的文件及 SQLite 内容,不输出用户数据或密钥。"""
import argparse
import hashlib
import json
import sqlite3
from pathlib import Path
CONFIG_FILES = {"config.json", "smtp.json", "email.json", "ssl.json", "plugins.json"}
LEGACY_FILES = {"config.dev.json", "config_mysql.json", "config.json.legacy"}
def digest_file(path):
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def database_snapshot(path):
# 必须停服调用。只读连接不会创建空库,不使用会忽略 WAL 的 immutable 模式。
connection = sqlite3.connect(path.resolve().as_uri() + "?mode=ro", uri=True)
try:
connection.execute("PRAGMA query_only = ON")
connection.execute("BEGIN")
if connection.execute("PRAGMA integrity_check").fetchall() != [("ok",)]:
raise RuntimeError("数据库完整性检查失败")
tables = {}
for (name,) in connection.execute(
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
).fetchall():
quoted = '"' + name.replace('"', '""') + '"'
rows = []
for row in connection.execute(f"SELECT * FROM {quoted}"):
encoded = [
{"blob": value.hex()} if isinstance(value, bytes) else value
for value in row
]
payload = json.dumps(encoded, ensure_ascii=False, separators=(",", ":"))
rows.append(hashlib.sha256(payload.encode("utf-8")).hexdigest())
tables[name] = {
"rows": len(rows),
"sha256": hashlib.sha256("".join(sorted(rows)).encode("ascii")).hexdigest(),
}
return {"integrity": "ok", "tables": tables}
finally:
connection.close()
def snapshot(root, layout):
config = root / "config"
files = {}
databases = {}
source = config if layout == "old" else root / "data"
for path in sorted(source.rglob("*")):
if path.is_symlink():
raise RuntimeError("拒绝跟随运行目录符号链接")
if not path.is_file():
continue
relative = path.relative_to(source).as_posix()
# 完整迁移前备份是新建的独立副本,已单独核对,不属于待迁移的原始集合。
if layout == "new" and path.relative_to(source).parts[0] == "runtime-migration-backups":
continue
if layout == "old":
if relative in CONFIG_FILES:
key = "config/" + relative
elif relative in LEGACY_FILES:
key = "data/legacy-config/" + relative
else:
key = "data/" + relative
else:
key = "data/" + relative
files[key] = {"size": path.stat().st_size, "sha256": digest_file(path)}
if path.suffix in {".db", ".sqlite", ".sqlite3"}:
databases[key] = database_snapshot(path)
if layout == "new":
if {path.name for path in config.iterdir()} != CONFIG_FILES:
raise RuntimeError("config 目录必须恰好包含五个配置文件")
for name in sorted(CONFIG_FILES):
path = config / name
files["config/" + name] = {"size": path.stat().st_size, "sha256": digest_file(path)}
generation = json.loads((config / "config.json").read_text(encoding="utf-8"))["generation"]
return {"generation": generation, "files": files, "databases": databases}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", required=True, type=Path)
parser.add_argument("--layout", required=True, choices=("old", "new"))
parser.add_argument("--output", required=True, type=Path)
parser.add_argument("--baseline", type=Path)
args = parser.parse_args()
result = snapshot(args.root.resolve(strict=True), args.layout)
if args.baseline:
before = json.loads(args.baseline.read_text(encoding="utf-8"))
if before != result:
raise RuntimeError("迁移前后文件或数据库内容不一致,请保留现场")
# 不覆盖既有证据。
with args.output.open("x", encoding="utf-8") as stream:
json.dump(result, stream, ensure_ascii=False, indent=2)
stream.write("\n")
print(f"核对通过:配置代次 {result['generation']},文件 {len(result['files'])} 个,数据库 {len(result['databases'])} 个")
if __name__ == "__main__":
main()