diff --git a/backend/services/runtimelog/migration.go b/backend/services/runtimelog/migration.go new file mode 100644 index 0000000..33267a5 --- /dev/null +++ b/backend/services/runtimelog/migration.go @@ -0,0 +1,155 @@ +package runtimelog + +import ( + "errors" + "fmt" + "io" + "os" +) + +const legacyDirectory = "运行日志" + +// migrateLegacyLogs 在日志写入和维护启动前迁移旧目录;不覆盖同名文件,不处理未知文件。 +func migrateLegacyLogs(root *os.Root) error { + info, err := root.Lstat(legacyDirectory) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("旧运行日志目录必须是实际目录:%w", ErrInvalid) + } + legacy, err := root.OpenRoot(legacyDirectory) + if err != nil { + return err + } + defer legacy.Close() + directory, err := legacy.Open(".") + if err != nil { + return err + } + opened, statErr := directory.Stat() + entries, readErr := directory.ReadDir(-1) + if err := errors.Join(statErr, readErr, directory.Close()); err != nil { + return err + } + if !os.SameFile(info, opened) { + return fmt.Errorf("旧运行日志目录在迁移前发生变化:%w", ErrInvalid) + } + type sourceFile struct { + name string + info os.FileInfo + } + var sources []sourceFile + unknown := false + // 先检查全部源文件和目标,不能迁移到一半才发现已有同名日志。 + for _, entry := range entries { + name := entry.Name() + if _, ok := fileDay(name); !ok { + unknown = true + continue + } + source, err := legacy.Lstat(name) + if err != nil { + return err + } + if !source.Mode().IsRegular() { + return fmt.Errorf("旧运行日志不是普通文件 %s:%w", name, ErrInvalid) + } + if _, err := root.Lstat(name); err == nil { + return fmt.Errorf("运行日志迁移目标已存在,请核对两个目录的同名文件,拒绝覆盖:%s", name) + } else if !os.IsNotExist(err) { + return err + } + sources = append(sources, sourceFile{name: name, info: source}) + } + for _, source := range sources { + if err := moveLegacyLog(legacy, root, source.name, source.info); err != nil { + return fmt.Errorf("运行日志迁移失败,未完成的源文件已保留 %s:%w", source.name, err) + } + } + if unknown { + return nil + } + if err := legacy.Close(); err != nil { + return err + } + current, err := root.Lstat(legacyDirectory) + if err != nil { + return err + } + if !os.SameFile(info, current) || !current.IsDir() || current.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("旧运行日志目录在迁移时发生变化:%w", ErrInvalid) + } + // 只移除迁移后的空目录;新出现的文件会使移除失败,绝不递归删除。 + return root.Remove(legacyDirectory) +} + +// moveLegacyLog 使用排他创建和流式复制,兼容不支持硬链接的文件系统。 +func moveLegacyLog(legacy, root *os.Root, name string, expected os.FileInfo) error { + source, err := legacy.Open(name) + if err != nil { + return err + } + defer source.Close() + opened, err := source.Stat() + if err != nil { + return err + } + if !unchangedLog(expected, opened) { + return ErrInvalid + } + target, err := root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + created, err := target.Stat() + complete := false + defer func() { + _ = target.Close() + // 只清理本次创建且尚未复制完成的目标;中断或异常时源文件仍然存在。 + if !complete && created != nil { + if current, err := root.Lstat(name); err == nil && os.SameFile(created, current) { + _ = root.Remove(name) + } + } + }() + if err != nil { + return err + } + written, copyErr := io.Copy(target, source) + after, statErr := source.Stat() + if err := errors.Join(copyErr, statErr, target.Sync(), target.Close(), source.Close()); err != nil { + return err + } + if written != expected.Size() || !unchangedLog(expected, after) { + return fmt.Errorf("源日志在复制时发生变化:%w", ErrInvalid) + } + if err := root.Chtimes(name, expected.ModTime(), expected.ModTime()); err != nil { + return err + } + current, err := legacy.Lstat(name) + if err != nil { + return err + } + if !unchangedLog(expected, current) { + return fmt.Errorf("源日志在迁移时发生变化:%w", ErrInvalid) + } + destination, err := root.Lstat(name) + if err != nil { + return err + } + if !destination.Mode().IsRegular() || !os.SameFile(created, destination) || destination.Size() != written { + return fmt.Errorf("目标日志在迁移时发生变化:%w", ErrInvalid) + } + // 目标已完整落盘;删除源失败时保留两份,重启明确报冲突,不猜测应保留哪份。 + complete = true + return legacy.Remove(name) +} + +func unchangedLog(expected, current os.FileInfo) bool { + return current.Mode().IsRegular() && os.SameFile(expected, current) && + expected.Size() == current.Size() && expected.ModTime().Equal(current.ModTime()) +} diff --git a/backend/services/runtimelog/writer.go b/backend/services/runtimelog/writer.go index b602027..9b6d2fd 100644 --- a/backend/services/runtimelog/writer.go +++ b/backend/services/runtimelog/writer.go @@ -42,14 +42,13 @@ func New(root string) (*Writer, error) { if err != nil { return nil, err } - directory, err = runtimepath.SecureDirectory(directory, "运行日志") - if err != nil { - return nil, err - } handle, err := os.OpenRoot(directory) if err != nil { return nil, err } + if err := migrateLegacyLogs(handle); err != nil { + return nil, errors.Join(err, handle.Close()) + } return &Writer{root: handle, now: time.Now}, nil } diff --git a/docs/日志改造任务.md b/docs/日志改造任务.md index 3cb7601..270a7ec 100644 --- a/docs/日志改造任务.md +++ b/docs/日志改造任务.md @@ -1,10 +1,21 @@ # 日志存储与可视化改造 +## 2026-09-09 运行日志目录简化 + +- [✓] 已确认:保留顶层 `log/`,取消 `log/运行日志/` 这一层;修改前检查点 `3d8d1c8` 已推送。 +- [x] 已完成,等待复审:新部署直接写入 `log/runtime-YYYY-MM-DD.log`,压缩文件也直接保存在 `log/`。 +- [x] 已完成,等待复审:启动写入和维护之前自动迁移旧目录的受管日志;预检同名冲突,排他创建目标,流式复制完整落盘后才移除源文件,保留内容与修改时间。 +- [x] 已完成,等待复审:仅移除迁移后的空目录;未知文件、旧审计文件和启动诊断不动。迁移期间必须停止旧服务;遇到同名冲突(包括异常中断残留)保留现场并明确报错,需人工核对,不自动覆盖或合并。 +- [✓] 复审通过:新部署、旧文件及压缩文件迁移、重复启动、空目录、同日两种格式、同名冲突、源文件变化和原有日志保留策略回归。 +- [!] 验证限制:Windows 普通权限不能创建符号链接,目标链接、旧目录链接和源文件链接三个测试分支跳过;拒绝链接逻辑已实现,不声明这些分支已实测通过。 +- [✓] 全量回归:`test-log/run-2026-09-08T16-12-55-458Z-37964`,前端 149 项、部署 4 项和后端全包通过;测试使用独立临时目录,不清空现有账号或邮件。 +- [~] 进行中:全量构建部署到 `yunx-test`、重启以及 MCP 日志列表和详情验收。 + ## 已确认需求 - 操作记录、配置审计、收发信记录兼容 SQLite、MySQL、PostgreSQL,默认沿用当前业务数据库连接,不要求额外安装 MySQL,不自动迁移业务数据库。 - 设置导航在插件配置后增加日志入口,支持分类、时间筛选、分页与详情。 -- 运行日志保存在 `log/运行日志`,按天写入;满七天的日文件压缩为独立的 `.log.gz`,启动补做,避免整周归档跨越清理边界。 +- 运行日志直接保存在 `log/`,按天写入;满七天的日文件压缩为独立的 `.log.gz`,启动补做,避免整周归档跨越清理边界。 - 压缩后的运行日志仍可在后台查看;仅在压缩校验通过后替换对应明文文件。 - 配置审计展示操作者、时间、具体字段及修改前后值;秘密只显示配置状态。 - 旧配置审计幂等迁移并保留原文件,不伪造历史缺失的变更详情。 diff --git a/test/backend/app_runtime_log_test.go b/test/backend/app_runtime_log_test.go index c56ac97..0688487 100644 --- a/test/backend/app_runtime_log_test.go +++ b/test/backend/app_runtime_log_test.go @@ -31,7 +31,7 @@ func TestFatalErrorIsLoggedBeforeRuntimeClose(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "fatal-probe") { t.Fatalf("fatal error lost: %v", err) } - files, err := filepath.Glob(filepath.Join(root, "log", "运行日志", "runtime-*.log")) + files, err := filepath.Glob(filepath.Join(root, "log", "runtime-*.log")) if err != nil || len(files) != 1 { t.Fatalf("missing runtime log: %v", err) } diff --git a/test/backend/runtime_log_migration_test.go b/test/backend/runtime_log_migration_test.go new file mode 100644 index 0000000..90fa0e5 --- /dev/null +++ b/test/backend/runtime_log_migration_test.go @@ -0,0 +1,320 @@ +package runtimelog + +import ( + "bytes" + "compress/gzip" + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func writeMigrationFile(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } +} + +func TestRuntimeLogFlatDirectoryAndLegacyMigration(t *testing.T) { + for _, upgrade := range []bool{false, true} { + t.Run(map[bool]string{false: "fresh", true: "upgrade"}[upgrade], func(t *testing.T) { + root := t.TempDir() + directory := filepath.Join(root, "log") + old := filepath.Join(directory, legacyDirectory) + now := time.Now().UTC() + name := prefix + now.Format("2006-01-02") + ".log" + archivedAt := now.Add(-9 * 24 * time.Hour) + archived := prefix + archivedAt.Format("2006-01-02") + ".log.gz" + line := "[" + now.Format(time.RFC3339Nano) + "] 迁移前的运行日志\n" + var compressed bytes.Buffer + archive := gzip.NewWriter(&compressed) + if _, err := archive.Write([]byte("[" + archivedAt.Format(time.RFC3339Nano) + "] 压缩记录\n")); err != nil { + t.Fatal(err) + } + if err := archive.Close(); err != nil { + t.Fatal(err) + } + var modified time.Time + if upgrade { + writeMigrationFile(t, filepath.Join(old, name), []byte(line)) + writeMigrationFile(t, filepath.Join(old, archived), compressed.Bytes()) + if err := os.Chtimes(filepath.Join(old, archived), archivedAt, archivedAt); err != nil { + t.Fatal(err) + } + info, err := os.Stat(filepath.Join(old, archived)) + if err != nil { + t.Fatal(err) + } + modified = info.ModTime() + } + // 重复启动不重新创建旧目录,迁移后的当前日志仍可追加。 + for attempt := 0; attempt < 2; attempt++ { + w, err := New(root) + if err != nil { + t.Fatal(err) + } + defer w.Close() + w.now = func() time.Time { return now } + if _, err := os.Lstat(old); !os.IsNotExist(err) { + t.Fatalf("legacy directory exists: %v", err) + } + if upgrade { + data, err := os.ReadFile(filepath.Join(directory, archived)) + if err != nil || !bytes.Equal(data, compressed.Bytes()) { + t.Fatalf("archive changed: %v", err) + } + info, err := os.Stat(filepath.Join(directory, archived)) + if err != nil || !modified.Equal(info.ModTime()) { + t.Fatalf("archive time changed: %v", err) + } + page, err := w.Read(context.Background(), archived, 0, 100) + if err != nil || len(page.Lines) != 1 || !strings.Contains(page.Lines[0], "压缩记录") { + t.Fatalf("archive query: %+v %v", page, err) + } + } + if err := w.WriteAt(now, "迁移后的日志"); err != nil { + t.Fatal(err) + } + if err := w.Maintain(context.Background(), now); err != nil { + t.Fatal(err) + } + files, err := w.Files() + if err != nil || len(files) != map[bool]int{false: 1, true: 2}[upgrade] { + t.Fatalf("files: %+v %v", files, err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(directory, name)) + if err != nil || (upgrade && !bytes.HasPrefix(data, []byte(line))) { + t.Fatalf("original log prefix lost: %v", err) + } + } + }) + } +} + +func TestRuntimeLogMigrationPreflightPreservesConflicts(t *testing.T) { + for _, kind := range []string{"different", "identical", "directory", "symlink"} { + t.Run(kind, func(t *testing.T) { + root := t.TempDir() + directory := filepath.Join(root, "log") + old := filepath.Join(directory, legacyDirectory) + first, conflict := "runtime-2026-09-07.log", "runtime-2026-09-08.log" + for _, name := range []string{first, conflict} { + writeMigrationFile(t, filepath.Join(old, name), []byte("original")) + } + target := filepath.Join(directory, conflict) + switch kind { + case "directory": + if err := os.Mkdir(target, 0700); err != nil { + t.Fatal(err) + } + case "symlink": + if err := os.Symlink(filepath.Join(old, conflict), target); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + default: + data := "conflicting" + if kind == "identical" { + data = "original" + } + writeMigrationFile(t, target, []byte(data)) + } + before, err := os.Lstat(target) + if err != nil { + t.Fatal(err) + } + if w, err := New(root); err == nil { + w.Close() + t.Fatal("conflict accepted") + } + for _, name := range []string{first, conflict} { + data, err := os.ReadFile(filepath.Join(old, name)) + if err != nil || string(data) != "original" { + t.Fatalf("source changed: %v", err) + } + } + if _, err := os.Lstat(filepath.Join(directory, first)); !os.IsNotExist(err) { + t.Fatal("preflight moved a file before detecting conflict") + } + after, err := os.Lstat(target) + if err != nil || !os.SameFile(before, after) || before.Size() != after.Size() || !before.ModTime().Equal(after.ModTime()) { + t.Fatalf("target changed: %v", err) + } + }) + } +} + +func TestRuntimeLogMigrationPreservesUnknownAndResumes(t *testing.T) { + root := t.TempDir() + directory := filepath.Join(root, "log") + old := filepath.Join(directory, legacyDirectory) + // 顶层文件模拟前一次已完成的迁移,损坏归档只搬运,不在迁移阶段改写或删除。 + files := map[string]string{ + filepath.Join(directory, "runtime-2026-09-07.log"): "already migrated", + filepath.Join(directory, "config-audit-v2.jsonl"): "historical audit", + filepath.Join(old, "runtime-2026-09-08.log.gz"): "damaged archive", + filepath.Join(old, "runtime-2026-99-99.log"): "unknown name", + filepath.Join(old, "manual", "notes.log"): "manual notes", + } + for path, content := range files { + writeMigrationFile(t, path, []byte(content)) + } + for attempt := 0; attempt < 2; attempt++ { + w, err := New(root) + if err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + } + for path, content := range files { + if filepath.Base(path) == "runtime-2026-09-08.log.gz" { + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatal("migrated source remains") + } + path = filepath.Join(directory, filepath.Base(path)) + } + data, err := os.ReadFile(path) + if err != nil || string(data) != content { + t.Fatalf("evidence changed %s: %v", path, err) + } + } +} + +func TestRuntimeLogMigrationRejectsUnsafeSource(t *testing.T) { + for _, kind := range []string{"legacy-file", "legacy-link", "entry-directory", "entry-link"} { + t.Run(kind, func(t *testing.T) { + root := t.TempDir() + directory := filepath.Join(root, "log") + old := filepath.Join(directory, legacyDirectory) + outside := t.TempDir() + if err := os.MkdirAll(directory, 0700); err != nil { + t.Fatal(err) + } + name := "runtime-2026-09-08.log" + writeMigrationFile(t, filepath.Join(outside, name), []byte("outside evidence")) + switch kind { + case "legacy-file": + writeMigrationFile(t, old, []byte("not a directory")) + case "legacy-link": + if err := os.Symlink(outside, old); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + default: + if err := os.Mkdir(old, 0700); err != nil { + t.Fatal(err) + } + if kind == "entry-directory" { + if err := os.Mkdir(filepath.Join(old, name), 0700); err != nil { + t.Fatal(err) + } + } else if err := os.Symlink(filepath.Join(outside, name), filepath.Join(old, name)); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + } + if w, err := New(root); err == nil { + w.Close() + t.Fatal("unsafe source accepted") + } + data, err := os.ReadFile(filepath.Join(outside, name)) + if err != nil || string(data) != "outside evidence" { + t.Fatalf("outside file changed: %v", err) + } + if _, err := os.Lstat(filepath.Join(directory, name)); !os.IsNotExist(err) { + t.Fatal("unsafe source copied") + } + }) + } +} + +func TestRuntimeLogMigrationCopyRechecksTargetAndSource(t *testing.T) { + for _, conflict := range []bool{false, true} { + t.Run(map[bool]string{false: "source-changed", true: "target-created"}[conflict], func(t *testing.T) { + directory := t.TempDir() + old := filepath.Join(directory, legacyDirectory) + name := "runtime-2026-09-08.log" + writeMigrationFile(t, filepath.Join(old, name), []byte("original")) + expected, err := os.Stat(filepath.Join(old, name)) + if err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(directory) + if err != nil { + t.Fatal(err) + } + defer root.Close() + legacy, err := root.OpenRoot(legacyDirectory) + if err != nil { + t.Fatal(err) + } + defer legacy.Close() + if conflict { + writeMigrationFile(t, filepath.Join(directory, name), []byte("target evidence")) + } else { + writeMigrationFile(t, filepath.Join(old, name), []byte("changed source")) + } + if err := moveLegacyLog(legacy, root, name, expected); err == nil { + t.Fatal("changed migration input accepted") + } + data, err := os.ReadFile(filepath.Join(old, name)) + want := map[bool]string{false: "changed source", true: "original"}[conflict] + if err != nil || string(data) != want { + t.Fatalf("source lost: %v", err) + } + if conflict { + data, err := os.ReadFile(filepath.Join(directory, name)) + if err != nil || string(data) != "target evidence" { + t.Fatalf("target overwritten: %v", err) + } + } else if _, err := os.Lstat(filepath.Join(directory, name)); !os.IsNotExist(err) { + t.Fatal("unexpected destination") + } + }) + } +} + +func TestRuntimeLogMigrationEmptyDirectoryAndSameDayFormats(t *testing.T) { + for _, empty := range []bool{true, false} { + t.Run(map[bool]string{true: "empty", false: "same-day-formats"}[empty], func(t *testing.T) { + root := t.TempDir() + directory := filepath.Join(root, "log") + old := filepath.Join(directory, legacyDirectory) + if err := os.MkdirAll(old, 0700); err != nil { + t.Fatal(err) + } + files := map[string]string{} + if !empty { + files["runtime-2026-09-08.log"] = "plain evidence" + files["runtime-2026-09-08.log.gz"] = "compressed evidence" + for name, content := range files { + writeMigrationFile(t, filepath.Join(old, name), []byte(content)) + } + } + w, err := New(root) + if err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(old); !os.IsNotExist(err) { + t.Fatalf("empty legacy directory remains: %v", err) + } + for name, content := range files { + data, err := os.ReadFile(filepath.Join(directory, name)) + if err != nil || string(data) != content { + t.Fatalf("same-day evidence changed: %v", err) + } + } + }) + } +} diff --git a/test/backend/runtime_logs_test.go b/test/backend/runtime_logs_test.go index 41bd0d4..b923651 100644 --- a/test/backend/runtime_logs_test.go +++ b/test/backend/runtime_logs_test.go @@ -76,7 +76,7 @@ func TestRuntimeLogCorruptionAndUnknownFilesArePreserved(t *testing.T) { defer w.Close() now := time.Now().UTC() w.now = func() time.Time { return now } - directory := filepath.Join(root, "log", "运行日志") + directory := filepath.Join(root, "log") name := prefix + now.Add(-40*24*time.Hour).Format("2006-01-02") + ".log.gz" if err := os.WriteFile(filepath.Join(directory, name), []byte("broken"), 0600); err != nil { t.Fatal(err) diff --git a/test/run.mjs b/test/run.mjs index 4e0474e..6af67ed 100644 --- a/test/run.mjs +++ b/test/run.mjs @@ -56,6 +56,7 @@ if ((suite === 'all' || suite === 'backend') && result === 0) { 'backend/services/logstore/log_store_test.go': 'test/backend/log_store_test.go', 'backend/services/logstore/log_store_integration_test.go': 'test/backend/log_store_integration_test.go', 'backend/services/runtimelog/runtime_logs_test.go': 'test/backend/runtime_logs_test.go', + 'backend/services/runtimelog/migration_test.go': 'test/backend/runtime_log_migration_test.go', 'backend/db/migration/log_migration_test.go': 'test/backend/log_migration_test.go', 'backend/config/audit_changes_test.go': 'test/backend/audit_changes_test.go', 'backend/config/json_format_test.go': 'test/backend/json_format_test.go',