运行日志直接存入日志根目录并安全迁移旧文件
Some checks are pending
Security Checks / security (push) Waiting to run

This commit is contained in:
smail 2026-09-09 00:14:43 +08:00
parent 3d8d1c80fb
commit fda4786489
7 changed files with 493 additions and 7 deletions

View File

@ -0,0 +1,155 @@
package runtimelog
import (
"errors"
"fmt"
"io"
"os"
)
const legacyDirectory = "运行日志"
// migrateLegacyLogs 在日志写入和维护启动前迁移旧目录;不覆盖同名文件,不处理未知文件。
func migrateLegacyLogs(root *os.Root) error {
info, err := root.Lstat(legacyDirectory)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("旧运行日志目录必须是实际目录:%w", ErrInvalid)
}
legacy, err := root.OpenRoot(legacyDirectory)
if err != nil {
return err
}
defer legacy.Close()
directory, err := legacy.Open(".")
if err != nil {
return err
}
opened, statErr := directory.Stat()
entries, readErr := directory.ReadDir(-1)
if err := errors.Join(statErr, readErr, directory.Close()); err != nil {
return err
}
if !os.SameFile(info, opened) {
return fmt.Errorf("旧运行日志目录在迁移前发生变化:%w", ErrInvalid)
}
type sourceFile struct {
name string
info os.FileInfo
}
var sources []sourceFile
unknown := false
// 先检查全部源文件和目标,不能迁移到一半才发现已有同名日志。
for _, entry := range entries {
name := entry.Name()
if _, ok := fileDay(name); !ok {
unknown = true
continue
}
source, err := legacy.Lstat(name)
if err != nil {
return err
}
if !source.Mode().IsRegular() {
return fmt.Errorf("旧运行日志不是普通文件 %s:%w", name, ErrInvalid)
}
if _, err := root.Lstat(name); err == nil {
return fmt.Errorf("运行日志迁移目标已存在,请核对两个目录的同名文件,拒绝覆盖:%s", name)
} else if !os.IsNotExist(err) {
return err
}
sources = append(sources, sourceFile{name: name, info: source})
}
for _, source := range sources {
if err := moveLegacyLog(legacy, root, source.name, source.info); err != nil {
return fmt.Errorf("运行日志迁移失败,未完成的源文件已保留 %s:%w", source.name, err)
}
}
if unknown {
return nil
}
if err := legacy.Close(); err != nil {
return err
}
current, err := root.Lstat(legacyDirectory)
if err != nil {
return err
}
if !os.SameFile(info, current) || !current.IsDir() || current.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("旧运行日志目录在迁移时发生变化:%w", ErrInvalid)
}
// 只移除迁移后的空目录;新出现的文件会使移除失败,绝不递归删除。
return root.Remove(legacyDirectory)
}
// moveLegacyLog 使用排他创建和流式复制,兼容不支持硬链接的文件系统。
func moveLegacyLog(legacy, root *os.Root, name string, expected os.FileInfo) error {
source, err := legacy.Open(name)
if err != nil {
return err
}
defer source.Close()
opened, err := source.Stat()
if err != nil {
return err
}
if !unchangedLog(expected, opened) {
return ErrInvalid
}
target, err := root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
if err != nil {
return err
}
created, err := target.Stat()
complete := false
defer func() {
_ = target.Close()
// 只清理本次创建且尚未复制完成的目标;中断或异常时源文件仍然存在。
if !complete && created != nil {
if current, err := root.Lstat(name); err == nil && os.SameFile(created, current) {
_ = root.Remove(name)
}
}
}()
if err != nil {
return err
}
written, copyErr := io.Copy(target, source)
after, statErr := source.Stat()
if err := errors.Join(copyErr, statErr, target.Sync(), target.Close(), source.Close()); err != nil {
return err
}
if written != expected.Size() || !unchangedLog(expected, after) {
return fmt.Errorf("源日志在复制时发生变化:%w", ErrInvalid)
}
if err := root.Chtimes(name, expected.ModTime(), expected.ModTime()); err != nil {
return err
}
current, err := legacy.Lstat(name)
if err != nil {
return err
}
if !unchangedLog(expected, current) {
return fmt.Errorf("源日志在迁移时发生变化:%w", ErrInvalid)
}
destination, err := root.Lstat(name)
if err != nil {
return err
}
if !destination.Mode().IsRegular() || !os.SameFile(created, destination) || destination.Size() != written {
return fmt.Errorf("目标日志在迁移时发生变化:%w", ErrInvalid)
}
// 目标已完整落盘;删除源失败时保留两份,重启明确报冲突,不猜测应保留哪份。
complete = true
return legacy.Remove(name)
}
func unchangedLog(expected, current os.FileInfo) bool {
return current.Mode().IsRegular() && os.SameFile(expected, current) &&
expected.Size() == current.Size() && expected.ModTime().Equal(current.ModTime())
}

View File

@ -42,14 +42,13 @@ func New(root string) (*Writer, error) {
if err != nil {
return nil, err
}
directory, err = runtimepath.SecureDirectory(directory, "运行日志")
if err != nil {
return nil, err
}
handle, err := os.OpenRoot(directory)
if err != nil {
return nil, err
}
if err := migrateLegacyLogs(handle); err != nil {
return nil, errors.Join(err, handle.Close())
}
return &Writer{root: handle, now: time.Now}, nil
}

View File

@ -1,10 +1,21 @@
# 日志存储与可视化改造
## 2026-09-09 运行日志目录简化
- [✓] 已确认:保留顶层 `log/`,取消 `log/运行日志/` 这一层;修改前检查点 `3d8d1c8` 已推送。
- [x] 已完成,等待复审:新部署直接写入 `log/runtime-YYYY-MM-DD.log`,压缩文件也直接保存在 `log/`。
- [x] 已完成,等待复审:启动写入和维护之前自动迁移旧目录的受管日志;预检同名冲突,排他创建目标,流式复制完整落盘后才移除源文件,保留内容与修改时间。
- [x] 已完成,等待复审:仅移除迁移后的空目录;未知文件、旧审计文件和启动诊断不动。迁移期间必须停止旧服务;遇到同名冲突(包括异常中断残留)保留现场并明确报错,需人工核对,不自动覆盖或合并。
- [✓] 复审通过:新部署、旧文件及压缩文件迁移、重复启动、空目录、同日两种格式、同名冲突、源文件变化和原有日志保留策略回归。
- [!] 验证限制:Windows 普通权限不能创建符号链接,目标链接、旧目录链接和源文件链接三个测试分支跳过;拒绝链接逻辑已实现,不声明这些分支已实测通过。
- [✓] 全量回归:`test-log/run-2026-09-08T16-12-55-458Z-37964`,前端 149 项、部署 4 项和后端全包通过;测试使用独立临时目录,不清空现有账号或邮件。
- [~] 进行中:全量构建部署到 `yunx-test`、重启以及 MCP 日志列表和详情验收。
## 已确认需求
- 操作记录、配置审计、收发信记录兼容 SQLite、MySQL、PostgreSQL,默认沿用当前业务数据库连接,不要求额外安装 MySQL,不自动迁移业务数据库。
- 设置导航在插件配置后增加日志入口,支持分类、时间筛选、分页与详情。
- 运行日志保存在 `log/运行日志`,按天写入;满七天的日文件压缩为独立的 `.log.gz`,启动补做,避免整周归档跨越清理边界。
- 运行日志直接保存在 `log/`,按天写入;满七天的日文件压缩为独立的 `.log.gz`,启动补做,避免整周归档跨越清理边界。
- 压缩后的运行日志仍可在后台查看;仅在压缩校验通过后替换对应明文文件。
- 配置审计展示操作者、时间、具体字段及修改前后值;秘密只显示配置状态。
- 旧配置审计幂等迁移并保留原文件,不伪造历史缺失的变更详情。

View File

@ -31,7 +31,7 @@ func TestFatalErrorIsLoggedBeforeRuntimeClose(t *testing.T) {
if err == nil || !strings.Contains(err.Error(), "fatal-probe") {
t.Fatalf("fatal error lost: %v", err)
}
files, err := filepath.Glob(filepath.Join(root, "log", "运行日志", "runtime-*.log"))
files, err := filepath.Glob(filepath.Join(root, "log", "runtime-*.log"))
if err != nil || len(files) != 1 {
t.Fatalf("missing runtime log: %v", err)
}

View File

@ -0,0 +1,320 @@
package runtimelog
import (
"bytes"
"compress/gzip"
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func writeMigrationFile(t *testing.T, path string, data []byte) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, data, 0600); err != nil {
t.Fatal(err)
}
}
func TestRuntimeLogFlatDirectoryAndLegacyMigration(t *testing.T) {
for _, upgrade := range []bool{false, true} {
t.Run(map[bool]string{false: "fresh", true: "upgrade"}[upgrade], func(t *testing.T) {
root := t.TempDir()
directory := filepath.Join(root, "log")
old := filepath.Join(directory, legacyDirectory)
now := time.Now().UTC()
name := prefix + now.Format("2006-01-02") + ".log"
archivedAt := now.Add(-9 * 24 * time.Hour)
archived := prefix + archivedAt.Format("2006-01-02") + ".log.gz"
line := "[" + now.Format(time.RFC3339Nano) + "] 迁移前的运行日志\n"
var compressed bytes.Buffer
archive := gzip.NewWriter(&compressed)
if _, err := archive.Write([]byte("[" + archivedAt.Format(time.RFC3339Nano) + "] 压缩记录\n")); err != nil {
t.Fatal(err)
}
if err := archive.Close(); err != nil {
t.Fatal(err)
}
var modified time.Time
if upgrade {
writeMigrationFile(t, filepath.Join(old, name), []byte(line))
writeMigrationFile(t, filepath.Join(old, archived), compressed.Bytes())
if err := os.Chtimes(filepath.Join(old, archived), archivedAt, archivedAt); err != nil {
t.Fatal(err)
}
info, err := os.Stat(filepath.Join(old, archived))
if err != nil {
t.Fatal(err)
}
modified = info.ModTime()
}
// 重复启动不重新创建旧目录,迁移后的当前日志仍可追加。
for attempt := 0; attempt < 2; attempt++ {
w, err := New(root)
if err != nil {
t.Fatal(err)
}
defer w.Close()
w.now = func() time.Time { return now }
if _, err := os.Lstat(old); !os.IsNotExist(err) {
t.Fatalf("legacy directory exists: %v", err)
}
if upgrade {
data, err := os.ReadFile(filepath.Join(directory, archived))
if err != nil || !bytes.Equal(data, compressed.Bytes()) {
t.Fatalf("archive changed: %v", err)
}
info, err := os.Stat(filepath.Join(directory, archived))
if err != nil || !modified.Equal(info.ModTime()) {
t.Fatalf("archive time changed: %v", err)
}
page, err := w.Read(context.Background(), archived, 0, 100)
if err != nil || len(page.Lines) != 1 || !strings.Contains(page.Lines[0], "压缩记录") {
t.Fatalf("archive query: %+v %v", page, err)
}
}
if err := w.WriteAt(now, "迁移后的日志"); err != nil {
t.Fatal(err)
}
if err := w.Maintain(context.Background(), now); err != nil {
t.Fatal(err)
}
files, err := w.Files()
if err != nil || len(files) != map[bool]int{false: 1, true: 2}[upgrade] {
t.Fatalf("files: %+v %v", files, err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(filepath.Join(directory, name))
if err != nil || (upgrade && !bytes.HasPrefix(data, []byte(line))) {
t.Fatalf("original log prefix lost: %v", err)
}
}
})
}
}
func TestRuntimeLogMigrationPreflightPreservesConflicts(t *testing.T) {
for _, kind := range []string{"different", "identical", "directory", "symlink"} {
t.Run(kind, func(t *testing.T) {
root := t.TempDir()
directory := filepath.Join(root, "log")
old := filepath.Join(directory, legacyDirectory)
first, conflict := "runtime-2026-09-07.log", "runtime-2026-09-08.log"
for _, name := range []string{first, conflict} {
writeMigrationFile(t, filepath.Join(old, name), []byte("original"))
}
target := filepath.Join(directory, conflict)
switch kind {
case "directory":
if err := os.Mkdir(target, 0700); err != nil {
t.Fatal(err)
}
case "symlink":
if err := os.Symlink(filepath.Join(old, conflict), target); err != nil {
t.Skipf("symlink unavailable: %v", err)
}
default:
data := "conflicting"
if kind == "identical" {
data = "original"
}
writeMigrationFile(t, target, []byte(data))
}
before, err := os.Lstat(target)
if err != nil {
t.Fatal(err)
}
if w, err := New(root); err == nil {
w.Close()
t.Fatal("conflict accepted")
}
for _, name := range []string{first, conflict} {
data, err := os.ReadFile(filepath.Join(old, name))
if err != nil || string(data) != "original" {
t.Fatalf("source changed: %v", err)
}
}
if _, err := os.Lstat(filepath.Join(directory, first)); !os.IsNotExist(err) {
t.Fatal("preflight moved a file before detecting conflict")
}
after, err := os.Lstat(target)
if err != nil || !os.SameFile(before, after) || before.Size() != after.Size() || !before.ModTime().Equal(after.ModTime()) {
t.Fatalf("target changed: %v", err)
}
})
}
}
func TestRuntimeLogMigrationPreservesUnknownAndResumes(t *testing.T) {
root := t.TempDir()
directory := filepath.Join(root, "log")
old := filepath.Join(directory, legacyDirectory)
// 顶层文件模拟前一次已完成的迁移,损坏归档只搬运,不在迁移阶段改写或删除。
files := map[string]string{
filepath.Join(directory, "runtime-2026-09-07.log"): "already migrated",
filepath.Join(directory, "config-audit-v2.jsonl"): "historical audit",
filepath.Join(old, "runtime-2026-09-08.log.gz"): "damaged archive",
filepath.Join(old, "runtime-2026-99-99.log"): "unknown name",
filepath.Join(old, "manual", "notes.log"): "manual notes",
}
for path, content := range files {
writeMigrationFile(t, path, []byte(content))
}
for attempt := 0; attempt < 2; attempt++ {
w, err := New(root)
if err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
}
for path, content := range files {
if filepath.Base(path) == "runtime-2026-09-08.log.gz" {
if _, err := os.Lstat(path); !os.IsNotExist(err) {
t.Fatal("migrated source remains")
}
path = filepath.Join(directory, filepath.Base(path))
}
data, err := os.ReadFile(path)
if err != nil || string(data) != content {
t.Fatalf("evidence changed %s: %v", path, err)
}
}
}
func TestRuntimeLogMigrationRejectsUnsafeSource(t *testing.T) {
for _, kind := range []string{"legacy-file", "legacy-link", "entry-directory", "entry-link"} {
t.Run(kind, func(t *testing.T) {
root := t.TempDir()
directory := filepath.Join(root, "log")
old := filepath.Join(directory, legacyDirectory)
outside := t.TempDir()
if err := os.MkdirAll(directory, 0700); err != nil {
t.Fatal(err)
}
name := "runtime-2026-09-08.log"
writeMigrationFile(t, filepath.Join(outside, name), []byte("outside evidence"))
switch kind {
case "legacy-file":
writeMigrationFile(t, old, []byte("not a directory"))
case "legacy-link":
if err := os.Symlink(outside, old); err != nil {
t.Skipf("symlink unavailable: %v", err)
}
default:
if err := os.Mkdir(old, 0700); err != nil {
t.Fatal(err)
}
if kind == "entry-directory" {
if err := os.Mkdir(filepath.Join(old, name), 0700); err != nil {
t.Fatal(err)
}
} else if err := os.Symlink(filepath.Join(outside, name), filepath.Join(old, name)); err != nil {
t.Skipf("symlink unavailable: %v", err)
}
}
if w, err := New(root); err == nil {
w.Close()
t.Fatal("unsafe source accepted")
}
data, err := os.ReadFile(filepath.Join(outside, name))
if err != nil || string(data) != "outside evidence" {
t.Fatalf("outside file changed: %v", err)
}
if _, err := os.Lstat(filepath.Join(directory, name)); !os.IsNotExist(err) {
t.Fatal("unsafe source copied")
}
})
}
}
func TestRuntimeLogMigrationCopyRechecksTargetAndSource(t *testing.T) {
for _, conflict := range []bool{false, true} {
t.Run(map[bool]string{false: "source-changed", true: "target-created"}[conflict], func(t *testing.T) {
directory := t.TempDir()
old := filepath.Join(directory, legacyDirectory)
name := "runtime-2026-09-08.log"
writeMigrationFile(t, filepath.Join(old, name), []byte("original"))
expected, err := os.Stat(filepath.Join(old, name))
if err != nil {
t.Fatal(err)
}
root, err := os.OpenRoot(directory)
if err != nil {
t.Fatal(err)
}
defer root.Close()
legacy, err := root.OpenRoot(legacyDirectory)
if err != nil {
t.Fatal(err)
}
defer legacy.Close()
if conflict {
writeMigrationFile(t, filepath.Join(directory, name), []byte("target evidence"))
} else {
writeMigrationFile(t, filepath.Join(old, name), []byte("changed source"))
}
if err := moveLegacyLog(legacy, root, name, expected); err == nil {
t.Fatal("changed migration input accepted")
}
data, err := os.ReadFile(filepath.Join(old, name))
want := map[bool]string{false: "changed source", true: "original"}[conflict]
if err != nil || string(data) != want {
t.Fatalf("source lost: %v", err)
}
if conflict {
data, err := os.ReadFile(filepath.Join(directory, name))
if err != nil || string(data) != "target evidence" {
t.Fatalf("target overwritten: %v", err)
}
} else if _, err := os.Lstat(filepath.Join(directory, name)); !os.IsNotExist(err) {
t.Fatal("unexpected destination")
}
})
}
}
func TestRuntimeLogMigrationEmptyDirectoryAndSameDayFormats(t *testing.T) {
for _, empty := range []bool{true, false} {
t.Run(map[bool]string{true: "empty", false: "same-day-formats"}[empty], func(t *testing.T) {
root := t.TempDir()
directory := filepath.Join(root, "log")
old := filepath.Join(directory, legacyDirectory)
if err := os.MkdirAll(old, 0700); err != nil {
t.Fatal(err)
}
files := map[string]string{}
if !empty {
files["runtime-2026-09-08.log"] = "plain evidence"
files["runtime-2026-09-08.log.gz"] = "compressed evidence"
for name, content := range files {
writeMigrationFile(t, filepath.Join(old, name), []byte(content))
}
}
w, err := New(root)
if err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
if _, err := os.Lstat(old); !os.IsNotExist(err) {
t.Fatalf("empty legacy directory remains: %v", err)
}
for name, content := range files {
data, err := os.ReadFile(filepath.Join(directory, name))
if err != nil || string(data) != content {
t.Fatalf("same-day evidence changed: %v", err)
}
}
})
}
}

View File

@ -76,7 +76,7 @@ func TestRuntimeLogCorruptionAndUnknownFilesArePreserved(t *testing.T) {
defer w.Close()
now := time.Now().UTC()
w.now = func() time.Time { return now }
directory := filepath.Join(root, "log", "运行日志")
directory := filepath.Join(root, "log")
name := prefix + now.Add(-40*24*time.Hour).Format("2006-01-02") + ".log.gz"
if err := os.WriteFile(filepath.Join(directory, name), []byte("broken"), 0600); err != nil {
t.Fatal(err)

View File

@ -56,6 +56,7 @@ if ((suite === 'all' || suite === 'backend') && result === 0) {
'backend/services/logstore/log_store_test.go': 'test/backend/log_store_test.go',
'backend/services/logstore/log_store_integration_test.go': 'test/backend/log_store_integration_test.go',
'backend/services/runtimelog/runtime_logs_test.go': 'test/backend/runtime_logs_test.go',
'backend/services/runtimelog/migration_test.go': 'test/backend/runtime_log_migration_test.go',
'backend/db/migration/log_migration_test.go': 'test/backend/log_migration_test.go',
'backend/config/audit_changes_test.go': 'test/backend/audit_changes_test.go',
'backend/config/json_format_test.go': 'test/backend/json_format_test.go',