过滤无价值日志并实现最近24小时倒序分页查询
Some checks are pending
Security Checks / security (push) Waiting to run

This commit is contained in:
smail 2026-09-09 01:06:45 +08:00
parent 161bc3a345
commit bb4ac4d5f5
20 changed files with 765 additions and 132 deletions

View File

@ -156,6 +156,25 @@ func (l *logFormatter) Format(entry *log.Entry) ([]byte, error) {
}
b.WriteString(fmt.Sprintf("[%s:%d]", entry.Caller.File, entry.Caller.Line))
b.WriteString(entry.Message)
// 错误字段也是有效内容,不能在消息为空时被格式器丢弃。
if len(entry.Data) > 0 {
fields := make(map[string]any, len(entry.Data))
for key, value := range entry.Data {
if cause, ok := value.(error); ok {
fields[key] = cause.Error()
} else if _, err := json.Marshal(value); err != nil {
fields[key] = fmt.Sprintf("无法序列化的字段(%T)", value)
} else {
fields[key] = value
}
}
payload, err := json.Marshal(fields)
if err != nil {
return nil, err
}
b.WriteByte(' ')
b.Write(payload)
}
b.WriteString("\n")
return b.Bytes(), nil

View File

@ -33,7 +33,7 @@ func GetLogs(ctx *appcontext.Context, w http.ResponseWriter, req *http.Request)
return
}
values := req.URL.Query()
limit, offset := 50, 0
limit, offset := 10, 0
var err error
if values.Has("limit") {
limit, err = strconv.Atoi(values.Get("limit"))
@ -47,13 +47,37 @@ func GetLogs(ctx *appcontext.Context, w http.ResponseWriter, req *http.Request)
}
bounded, cancel := context.WithTimeout(req.Context(), 10*time.Second)
defer cancel()
query := logstore.Query{Kind: values.Get("kind"), ActorID: values.Get("actorId"), Limit: limit + 1, Offset: offset}
for _, item := range []struct {
name string
target *time.Time
}{{"from", &query.From}, {"to", &query.To}} {
if values.Has(item.name) {
*item.target, err = time.Parse(time.RFC3339Nano, values.Get(item.name))
if err != nil {
writeConfigError(w, config.ErrInvalidAuditQuery)
return
}
}
}
if query.To.IsZero() {
query.To = time.Now()
}
if query.From.IsZero() {
query.From = query.To.Add(-24 * time.Hour)
}
if query.From.After(query.To) {
writeConfigError(w, config.ErrInvalidAuditQuery)
return
}
if values.Get("kind") == "runtime" {
writer := runtimelog.Current()
if writer == nil {
logQueryError(w, logstore.ErrUnavailable)
return
}
if values.Get("file") == "" {
// 显式保留旧文件列表入口,新页面默认跨文件查询。
if values.Get("listFiles") == "true" {
files, err := writer.Files()
if err != nil {
logQueryError(w, err)
@ -62,7 +86,7 @@ func GetLogs(ctx *appcontext.Context, w http.ResponseWriter, req *http.Request)
response.NewSuccessResponse(map[string]any{"files": files}).FPrint(w)
return
}
page, err := writer.Read(bounded, values.Get("file"), offset, limit)
page, err := writer.Query(bounded, runtimelog.Query{File: values.Get("file"), From: query.From, To: query.To, Offset: offset, Limit: limit})
if err != nil {
logQueryError(w, err)
return
@ -70,19 +94,6 @@ func GetLogs(ctx *appcontext.Context, w http.ResponseWriter, req *http.Request)
response.NewSuccessResponse(page).FPrint(w)
return
}
query := logstore.Query{Kind: values.Get("kind"), ActorID: values.Get("actorId"), Limit: limit + 1, Offset: offset}
for _, item := range []struct {
name string
target *time.Time
}{{"from", &query.From}, {"to", &query.To}} {
if values.Get(item.name) != "" {
*item.target, err = time.Parse(time.RFC3339Nano, values.Get(item.name))
if err != nil {
writeConfigError(w, config.ErrInvalidAuditQuery)
return
}
}
}
if query.Kind != "config" && query.Kind != "operation" && query.Kind != "mail" {
writeConfigError(w, config.ErrInvalidAuditQuery)
return

View File

@ -0,0 +1,38 @@
package logstore
import (
"encoding/json"
"strings"
"git.pcux.cn/xuans/Smail/utils/logpolicy"
)
// emptyEntry 只丢弃没有事件、对象、结果或有效载荷的空记录。
// 已知业务动作(包括无字段变化的配置审计)、失败和未知结果均不去重或省略。
func emptyEntry(entry Entry) bool {
if !logpolicy.EmptyMessage(entry.Action) || strings.TrimSpace(entry.ActorID) != "" || strings.TrimSpace(entry.Section) != "" || strings.TrimSpace(entry.Result) != "" {
return false
}
var payload any
if json.Unmarshal(entry.Payload, &payload) != nil {
return false
}
switch value := payload.(type) {
case nil:
return true
case string:
return logpolicy.EmptyMessage(value)
case []any:
return len(value) == 0
case map[string]any:
if len(value) == 0 {
return true
}
// 只理解单一消息字段,其他键即使值为空也可能是状态证据。
if len(value) == 1 {
message, ok := value["message"].(string)
return ok && logpolicy.EmptyMessage(message)
}
}
return false
}

View File

@ -106,6 +106,9 @@ func (s *Store) Write(ctx context.Context, entry Entry) error {
if entry.CreatedAt.Before(logpolicy.Cutoff(s.now())) {
return nil
}
if emptyEntry(entry) {
return nil
}
if entry.ID == "" {
var token [16]byte
if _, err := rand.Read(token[:]); err != nil {

View File

@ -43,6 +43,13 @@ func lineTime(line []byte) (time.Time, error) {
// scan 严格解析每条记录及压缩校验和,损坏归档不得触发删除源文件。
func (w *Writer) scan(ctx context.Context, name string, consume func([]byte, time.Time) error) error {
return w.scanSize(ctx, name, -1, consume)
}
func (w *Writer) scanSize(ctx context.Context, name string, size int64, consume func([]byte, time.Time) error) error {
if err := ctx.Err(); err != nil {
return err
}
if err := w.regular(name, false); err != nil {
return err
}
@ -52,8 +59,11 @@ func (w *Writer) scan(ctx context.Context, name string, consume func([]byte, tim
}
defer file.Close()
var reader io.Reader = file
if size >= 0 {
reader = io.LimitReader(reader, size)
}
if strings.HasSuffix(name, ".gz") {
archive, err := gzip.NewReader(file)
archive, err := gzip.NewReader(reader)
if err != nil {
return err
}

View File

@ -3,32 +3,41 @@ package runtimelog
import (
"regexp"
"strings"
"git.pcux.cn/xuans/Smail/utils/logpolicy"
)
var legacyStartupLine = regexp.MustCompile(`^(\[[^\]\r\n]+\]\s+\[info\]\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\](?:\[[^\]\r\n]*\])?\[([^\]\r\n]+):\d+\])([^\r\n]*)$`)
var contentLine = regexp.MustCompile(`^\[[^\]\r\n]+\]\s+\[(info|debug|trace)\]\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\](?:\[[^\]\r\n]*\])?\[([^\]\r\n]+):\d+\]([^\r\n]*)$`)
// presentLine 只精简旧版主程序的启动横幅,不修改磁盘原文或隐藏其他来源和级别的记录。
// presentLine 先过滤明确无内容的历史记录;警告、错误和未知格式均保留原文。
func presentLine(line string) (string, bool) {
parts := legacyStartupLine.FindStringSubmatch(line)
parts := contentLine.FindStringSubmatch(line)
if parts == nil {
if end := strings.Index(line, "] "); end >= 0 && logpolicy.EmptyMessage(line[end+2:]) {
return "", false
}
return line, true
}
source := strings.ReplaceAll(parts[2], `\`, "/")
if source != "internal/app/app.go" && !strings.HasSuffix(source, "/internal/app/app.go") {
return line, true
}
message := strings.TrimSpace(parts[3])
if len(message) >= 3 && strings.Trim(message, "*") == "" {
if logpolicy.EmptyMessage(parts[3]) || (parts[1] == "info" && legacyStartupNoise(parts[2], parts[3])) {
return "", false
}
return line, true
}
func legacyStartupNoise(source, message string) bool {
source = strings.ReplaceAll(source, `\`, "/")
if source != "internal/app/app.go" && !strings.HasSuffix(source, "/internal/app/app.go") {
return false
}
message = strings.TrimSpace(message)
if !strings.HasPrefix(message, "***\t") && !strings.HasPrefix(message, "*** ") {
return line, true
return false
}
message = strings.TrimSpace(message[3:])
for _, prefix := range []string{"Server Version:", "Git Commit Hash:", "Build Date:", "Build GoLang Version:"} {
if strings.HasPrefix(message, prefix) {
return "", false
return true
}
}
return line, true
return false
}

View File

@ -24,6 +24,10 @@ type Page struct {
func (w *Writer) Files() ([]File, error) {
w.archiveMu.Lock()
defer w.archiveMu.Unlock()
return w.files()
}
func (w *Writer) files() ([]File, error) {
if w.closed {
return nil, ErrInvalid
}
@ -45,10 +49,15 @@ func (w *Writer) Files() ([]File, error) {
if err := w.regular(entry.Name(), false); err != nil {
return nil, err
}
info, err := entry.Info()
// Windows 目录枚举可能缓存仍打开文件的旧长度,必须通过文件句柄读取。
handle, err := w.root.Open(entry.Name())
if err != nil {
return nil, err
}
info, statErr := handle.Stat()
if err := errors.Join(statErr, handle.Close()); err != nil {
return nil, err
}
files = append(files, File{Name: entry.Name(), Bytes: info.Size(), Compressed: strings.HasSuffix(entry.Name(), ".gz")})
}
sort.Slice(files, func(i, j int) bool { return files[i].Name > files[j].Name })

View File

@ -0,0 +1,189 @@
package runtimelog
import (
"container/heap"
"context"
"errors"
"sort"
"time"
"git.pcux.cn/xuans/Smail/utils/logpolicy"
)
type Query struct {
File string
From time.Time
To time.Time
Offset int
Limit int
}
type lineRef struct {
at time.Time
file int
position int
bytes int
}
// 索引只保存时间与行位置,不把历史日志全文装入内存。
type recentLines []lineRef
func (h recentLines) Len() int { return len(h) }
func (h recentLines) Less(i, j int) bool {
if !h[i].at.Equal(h[j].at) {
return h[i].at.Before(h[j].at)
}
if h[i].file != h[j].file {
return h[i].file > h[j].file
}
return h[i].position < h[j].position
}
func (h recentLines) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h *recentLines) Push(value any) { *h = append(*h, value.(lineRef)) }
func (h *recentLines) Pop() any {
last := len(*h) - 1
value := (*h)[last]
*h = (*h)[:last]
return value
}
// Query 跨日流式扫描并按真实时间倒序,兼容压缩文件和同一文件中的乱序写入。
// 额外内存由最大偏移限制的行索引和两兆响应组成,不随日志文件体积增长。
func (w *Writer) Query(ctx context.Context, query Query) (Page, error) {
if query.File != "" {
if _, ok := fileDay(query.File); !ok {
return Page{}, ErrInvalid
}
}
if query.Offset < 0 || query.Offset > 100000 || query.Limit < 1 || query.Limit > 500 {
return Page{}, ErrInvalid
}
if query.To.IsZero() {
query.To = w.now()
}
if query.From.IsZero() {
query.From = query.To.Add(-24 * time.Hour)
}
if query.From.After(query.To) {
return Page{}, ErrInvalid
}
query.From = maxTime(query.From, logpolicy.Cutoff(w.now()))
// 等待归档时也响应取消,避免查询超时后继续占用 HTTP 工作协程。
for !w.archiveMu.TryLock() {
select {
case <-ctx.Done():
return Page{}, ctx.Err()
case <-time.After(10 * time.Millisecond):
}
}
defer w.archiveMu.Unlock()
if err := ctx.Err(); err != nil {
return Page{}, err
}
// 在短写锁内冻结文件长度;两次扫描不会读到新追加的半条记录。
w.mu.Lock()
files, err := w.files()
w.mu.Unlock()
if err != nil {
return Page{}, err
}
page := Page{Lines: make([]string, 0, query.Limit)}
if query.From.After(query.To) {
return page, nil
}
capacity := query.Offset + query.Limit + 1
refs := make(recentLines, 0, capacity)
for index, file := range files {
if query.File != "" && query.File != file.Name {
continue
}
day, _ := fileDay(file.Name)
if day.After(query.To) || !day.Add(24*time.Hour).After(query.From) {
continue
}
position := 0
err := w.scanSize(ctx, file.Name, file.Bytes, func(line []byte, at time.Time) error {
position++
if !at.UTC().Truncate(24 * time.Hour).Equal(day) {
return ErrInvalid
}
if at.Before(query.From) || at.After(query.To) {
return nil
}
visible, keep := presentLine(string(line))
if !keep {
return nil
}
candidate := lineRef{at: at, file: index, position: position, bytes: len(visible)}
if len(refs) < capacity {
heap.Push(&refs, candidate)
} else if (recentLines{refs[0], candidate}).Less(0, 1) {
refs[0] = candidate
heap.Fix(&refs, 0)
}
return nil
})
if err != nil {
return Page{}, err
}
}
sort.Sort(sort.Reverse(refs))
if query.Offset >= len(refs) {
return page, nil
}
refs = refs[query.Offset:]
page.HasMore = len(refs) > query.Limit
refs = refs[:min(len(refs), query.Limit)]
bytes := 0
for index, ref := range refs {
if bytes+ref.bytes > 2*1024*1024 {
refs = refs[:index]
page.HasMore = true
break
}
bytes += ref.bytes
}
page.Lines = make([]string, len(refs))
selected := make(map[int]map[int]int)
for index, ref := range refs {
if selected[ref.file] == nil {
selected[ref.file] = make(map[int]int)
}
selected[ref.file][ref.position] = index
}
finished := errors.New("本页内容读取完成")
for fileIndex, positions := range selected {
position, remaining := 0, len(positions)
file := files[fileIndex]
err := w.scanSize(ctx, file.Name, file.Bytes, func(line []byte, at time.Time) error {
position++
index, ok := positions[position]
if !ok {
return nil
}
if !at.Equal(refs[index].at) || len(line) != refs[index].bytes {
return ErrInvalid
}
page.Lines[index] = string(line)
remaining--
if remaining == 0 {
return finished
}
return nil
})
if err != nil && !errors.Is(err, finished) {
return Page{}, err
}
if remaining != 0 {
return Page{}, ErrInvalid
}
}
return page, nil
}
func maxTime(left, right time.Time) time.Time {
if left.After(right) {
return left
}
return right
}

View File

@ -86,15 +86,34 @@ func Start(root string) (*Writer, error) {
func (w *Writer) Levels() []log.Level { return log.AllLevels }
func (w *Writer) Fire(entry *log.Entry) error {
// 结构化字段和警告以上级别本身具有排错价值,不能只看消息是否为空。
if entry.Level >= log.InfoLevel && len(entry.Data) == 0 {
if logpolicy.EmptyMessage(entry.Message) || (entry.Level == log.InfoLevel && entry.Caller != nil && legacyStartupNoise(entry.Caller.File, entry.Message)) {
return nil
}
}
// 复用已有格式器保留请求标识、错误字段和源位置;转义换行保证一条记录一行。
data, err := entry.Logger.Formatter.Format(entry)
if err != nil {
return err
}
return w.WriteAt(entry.Time, strings.TrimRight(string(data), "\r\n"))
return w.writeAt(entry.Time, strings.TrimRight(string(data), "\r\n"))
}
func (w *Writer) WriteAt(at time.Time, message string) error {
if at.IsZero() {
return ErrInvalid
}
if logpolicy.EmptyMessage(message) {
return nil
}
if _, keep := presentLine("[" + at.UTC().Format(time.RFC3339Nano) + "] " + message); !keep {
return nil
}
return w.writeAt(at, message)
}
func (w *Writer) writeAt(at time.Time, message string) error {
if at.IsZero() {
return ErrInvalid
}

View File

@ -0,0 +1,12 @@
package logpolicy
import "strings"
// EmptyMessage 仅识别空白和纯装饰线,不猜测业务文案、状态或未知格式的价值。
func EmptyMessage(message string) bool {
message = strings.TrimSpace(strings.ReplaceAll(message, "\x00", ""))
if message == "" {
return true
}
return len([]rune(message)) >= 3 && strings.Trim(message, "*-=─━_~ \t\r\n") == ""
}

View File

@ -1,5 +1,18 @@
# 日志存储与可视化改造
## 2026-09-09 倒序查询与写入前过滤
- [✓] 需求确认:四类日志统一倒序、每页十条、支持起止时间,默认最近二十四小时;翻页固定查询时段。
- [✓] 修改前检查点 `161bc3a` 已推送;不清空现有业务库、不删除历史日志文件。
- [✓] 无价值记录在写入前过滤:空内容、纯装饰分隔线、已识别的旧启动横幅;警告、错误、结构化信息、安全操作及未知事件保留。数据库只丢弃没有动作、对象、结果及有效载荷的空壳,不猜测未知事件价值。
- [✓] 运行日志跨日和压缩文件按真实时间倒序查询;相同时间按文件与行位置稳定排序,过滤在分页前完成。只保存有上限的行位置索引和最多两兆本页内容,不加载全部历史原文;等待归档和扫描均支持取消。
- [✓] 修复 Windows 目录枚举返回正在写入文件旧长度的问题,改为文件句柄取长度;读取前在短锁内冻结长度,避免两次扫描读到新增的半条记录。
- [✓] 文本格式器保留结构化错误和业务字段,不再只留下消息标题;无法序列化字段只保留类型提示,不导致整条有效记录丢失。
- [x] 前端统一时间筛选、十条分页,保留详情、短页回退和请求竞态保护;查询及分类切换才采纳新筛选,翻页固定上一次查询时段。
- [✓] 全量回归 `test-log/run-2026-09-08T17-03-28-009Z-39800`:前端 151 项、部署 4 项及后端全包通过。并发检测 `test-log/run-2026-09-08T17-05-13-325Z-35604`:运行日志、数据库日志、程序启动和配置包通过。
- [~] 完整编译并部署 `yunx-test`、浏览器 MCP 真实验收。
- [!] 三十天滚动留存、七天压缩不变;三库保持通用设计,MySQL 和 PostgreSQL 实库验收仍需专用测试连接。
## 2026-09-09 精简启动日志
- [✓] 范围已核对:不再显示无业务价值的启动分隔线,以及单独占行的版本、提交号、编译时间、Go 版本;修改前检查点 `0dc459d` 已推送。

View File

@ -92,29 +92,15 @@
max-width: 190px;
}
.log-filters .el-select {
flex: 1;
min-width: 0;
}
.log-filters-runtime {
flex-wrap: nowrap;
}
.log-filters :deep(.el-select__wrapper),
.log-filters :deep(.el-button) {
border-radius: 8px;
}
.log-file-option {
.log-filter-actions {
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
flex-shrink: 0;
margin-left: auto;
}
.log-file-option small {
color: var(--el-text-color-secondary);
.log-filters :deep(.el-button) {
border-radius: 8px;
}
.log-content {

View File

@ -1,25 +1,28 @@
import { computed, onBeforeUnmount, onMounted, ref, watch } from 'vue'
import { http } from '@/utils/axios'
import { formatFileSize, parseRuntimeLine, summarizeRuntime, timeParts } from './RuntimeLog.js'
import { parseRuntimeLine, summarizeRuntime, timeParts } from './RuntimeLog.js'
export function useLogs() {
const category = ref('config')
const loading = ref(false)
const error = ref('')
const items = ref([])
const files = ref([])
const selectedFile = ref('')
const lines = ref([])
const offset = ref(0)
const hasMore = ref(false)
const actorId = ref('')
const dateRange = ref([])
const recentRange = () => {
const end = new Date()
return [new Date(end.getTime() - 24 * 60 * 60 * 1000), end]
}
const dateRange = ref(recentRange())
const detail = ref(null)
const runtimeDetail = ref(null)
const pageSize = 50
const pageSize = 10
let requestId = 0
let aborter
const previousOffsets = []
let appliedFilters = {}
const categories = [
{ label: '配置审计', value: 'config' },
@ -36,7 +39,6 @@ export function useLogs() {
const changes = computed(() => details.value.changes || [])
const runtimeRows = computed(() => lines.value.map((line, index) => parseRuntimeLine(line, offset.value + index + 1)))
const runtimeSummary = computed(() => summarizeRuntime(runtimeRows.value))
const currentFile = computed(() => files.value.find(file => file.name === selectedFile.value))
const visibleCount = computed(() => isRuntime.value ? lines.value.length : items.value.length)
const pageRange = computed(() => visibleCount.value ? `第 ${offset.value + 1}–${offset.value + visibleCount.value} 条` : '暂无记录')
const currentCategory = computed(() => categories.find(item => item.value === category.value)?.label)
@ -55,32 +57,7 @@ export function useLogs() {
loading.value = true
error.value = ''
try {
const params = { kind: category.value, offset: offset.value, limit: pageSize }
if (isRuntime.value) {
const listing = requireData(await http.get('/api/v1/logs', {
params: { kind: 'runtime' }, signal: aborter.signal
}))
if (id !== requestId) return
files.value = listing.files || []
if (!files.value.some(file => file.name === selectedFile.value)) {
selectedFile.value = files.value[0]?.name || ''
offset.value = 0
params.offset = 0
previousOffsets.length = 0
}
if (!selectedFile.value) {
lines.value = []
hasMore.value = false
return
}
params.file = selectedFile.value
} else {
if (actorId.value.trim()) params.actorId = actorId.value.trim()
if (dateRange.value?.length === 2) {
params.from = new Date(dateRange.value[0]).toISOString()
params.to = new Date(dateRange.value[1]).toISOString()
}
}
const params = { ...appliedFilters, offset: offset.value, limit: pageSize }
const data = requireData(await http.get('/api/v1/logs', { params, signal: aborter.signal }))
if (id !== requestId) return
items.value = data.items || []
@ -99,10 +76,31 @@ export function useLogs() {
}
function search() {
// 只在查询时采纳筛选条件,翻页沿用快照,不随当前时间或未提交输入漂移。
if (!dateRange.value?.length) dateRange.value = recentRange()
const [from, to] = (dateRange.value || []).map(value => new Date(value))
if (!from || !to || !Number.isFinite(from.getTime()) || !Number.isFinite(to.getTime()) || from > to) {
++requestId
aborter?.abort()
loading.value = false
error.value = '请选择有效的起止时间'
items.value = []
lines.value = []
hasMore.value = false
return
}
appliedFilters = { kind: category.value, from: from.toISOString(), to: to.toISOString() }
if (!isRuntime.value && actorId.value.trim()) appliedFilters.actorId = actorId.value.trim()
offset.value = 0
previousOffsets.length = 0
detail.value = null
runtimeDetail.value = null
return load()
}
function recent() {
dateRange.value = recentRange()
return search()
}
function next() {
if (loading.value || !hasMore.value) return
previousOffsets.push(offset.value)
@ -119,17 +117,17 @@ export function useLogs() {
runtimeDetail.value = null
search()
})
onMounted(load)
onMounted(search)
onBeforeUnmount(() => {
++requestId
aborter?.abort()
})
return {
category, categories, loading, error, items, files, selectedFile, lines, offset,
category, categories, loading, error, items, lines, offset,
hasMore, actorId, dateRange, detail, details, changes, isRuntime,
runtimeDetail, runtimeRows, runtimeSummary, currentFile, visibleCount, pageRange,
currentCategory, resultType, runtimeRowClass, timeParts, formatFileSize,
formatTime, resultLabel, actionLabel, displayActor, search, next, previous
runtimeDetail, runtimeRows, runtimeSummary, visibleCount, pageRange,
currentCategory, resultType, runtimeRowClass, timeParts,
formatTime, resultLabel, actionLabel, displayActor, search, recent, next, previous
}
}

View File

@ -2,51 +2,34 @@
<section class="log-settings">
<header class="log-heading">
<h2>日志记录</h2>
<span class="log-retention">最近 30 天</span>
<span class="log-retention">保留 30 天</span>
</header>
<el-radio-group v-model="category" class="log-categories" aria-label="日志分类">
<el-radio-button v-for="item in categories" :key="item.value" :value="item.value">
{{ item.label }}
</el-radio-button>
</el-radio-group>
<div class="log-filters" :class="{ 'log-filters-runtime': isRuntime }">
<template v-if="isRuntime">
<el-select v-model="selectedFile" placeholder="选择日志文件" aria-label="日志文件" @change="search">
<el-option
v-for="file in files"
:key="file.name"
:value="file.name"
:label="`${file.name}${file.compressed ? '(已压缩)' : ''}`"
>
<div class="log-file-option">
<span>{{ file.name }}</span>
<small>{{ file.compressed ? '已压缩 · ' : '' }}{{ formatFileSize(file.bytes) }}</small>
</div>
</el-option>
</el-select>
</template>
<template v-else>
<el-date-picker
v-model="dateRange"
type="datetimerange"
start-placeholder="开始时间"
end-placeholder="结束时间"
format="YYYY-MM-DD HH:mm"
aria-label="日志时间范围"
/>
<el-input v-model="actorId" clearable placeholder="操作者编号" aria-label="操作者编号" @keyup.enter="search" />
</template>
<el-button type="primary" :loading="loading" @click="search">{{ isRuntime ? '刷新' : '查询' }}</el-button>
<div class="log-filters">
<el-date-picker
v-model="dateRange"
type="datetimerange"
start-placeholder="开始时间"
end-placeholder="结束时间"
format="YYYY-MM-DD HH:mm"
aria-label="日志时间范围"
/>
<el-input v-if="!isRuntime" v-model="actorId" clearable placeholder="操作者编号" aria-label="操作者编号" @keyup.enter="search" />
<div class="log-filter-actions">
<el-button :disabled="loading" @click="recent">最近 24 小时</el-button>
<el-button type="primary" :loading="loading" @click="search">查询</el-button>
</div>
</div>
<el-alert v-if="error" :title="error" type="error" :closable="false" show-icon />
<div v-loading="loading" class="log-content" :aria-busy="loading">
<div class="log-table-heading">
<div class="log-table-title">
<strong>{{ currentCategory }}</strong>
<span v-if="isRuntime && currentFile" class="log-file-meta">
{{ formatFileSize(currentFile.bytes) }} · {{ currentFile.compressed ? '压缩归档' : '原始文件' }}
</span>
<span v-else class="log-file-meta">本页 {{ visibleCount }} 条</span>
<span class="log-file-meta">最新在前 · 每页 10 条</span>
</div>
<div v-if="isRuntime" class="log-summary" aria-label="本页日志统计">
<span>本页 {{ runtimeSummary.total }} 条</span>
@ -173,11 +156,11 @@
import { useLogs } from './LogSettings.js'
const {
category, categories, loading, error, items, files, selectedFile, lines, offset,
category, categories, loading, error, items, offset,
hasMore, actorId, dateRange, detail, details, changes, isRuntime,
runtimeDetail, runtimeRows, runtimeSummary, currentFile, visibleCount, pageRange,
currentCategory, resultType, runtimeRowClass, timeParts, formatFileSize,
formatTime, resultLabel, actionLabel, displayActor, search, next, previous
runtimeDetail, runtimeRows, runtimeSummary, pageRange,
currentCategory, resultType, runtimeRowClass, timeParts,
formatTime, resultLabel, actionLabel, displayActor, search, recent, next, previous
} = useLogs()
</script>

View File

@ -0,0 +1,35 @@
package config
import (
"errors"
"runtime"
"strings"
"testing"
"time"
log "github.com/sirupsen/logrus"
)
func TestLogFormatterPreservesStructuredEvidence(t *testing.T) {
formatter := &logFormatter{}
for _, message := range []string{"", "连接失败"} {
entry := &log.Entry{
Time: time.Now(), Level: log.InfoLevel, Message: message,
Caller: &runtime.Frame{File: "test.go", Line: 12},
Data: log.Fields{"error": errors.New("测试错误\n下一行"), "code": 0, "unsupported": make(chan int)},
}
data, err := formatter.Format(entry)
if err != nil {
t.Fatal(err)
}
text := string(data)
for _, expected := range []string{`"error":"测试错误\n下一行"`, `"code":0`, "无法序列化的字段", message} {
if !strings.Contains(text, expected) {
t.Fatalf("字段丢失:%s", text)
}
}
if strings.Count(text, "\n") != 1 {
t.Fatal("结构化错误中的换行不应生成伪记录")
}
}
}

View File

@ -36,6 +36,44 @@ func testStore(t *testing.T) *Store {
return store
}
func TestLogStoreEmptyEntriesAreNotInserted(t *testing.T) {
s := testStore(t)
ctx := context.Background()
for _, kind := range []string{"config", "operation", "mail"} {
for _, payload := range []string{`{}`, `null`, `[]`, `" "`, `{"message":"***"}`} {
if err := s.Write(ctx, Entry{Kind: kind, CreatedAt: time.Now(), Action: "---", Payload: []byte(payload)}); err != nil {
t.Fatal(err)
}
}
var count int
table, _ := logTable(kind)
if err := s.db.QueryRow("SELECT COUNT(*) FROM "+table+" WHERE kind = ?", kind).Scan(&count); err != nil || count != 0 {
t.Fatalf("空记录已落库:%s %d %v", kind, count, err)
}
for _, entry := range []Entry{
{Action: "登录验证", Result: "success", Payload: []byte(`{}`)},
{Action: "update", Result: "success", Payload: []byte(`{"changes":[]}`)},
{Result: "failure", Payload: []byte(`{}`)},
{Result: "unknown", Payload: []byte(`{}`)},
{Payload: []byte(`{"errorCode":0}`)},
{Payload: []byte(`{"message":"投递成功"}`)},
{Payload: []byte(`{"unknown":null}`)},
} {
entry.Kind, entry.CreatedAt = kind, time.Now()
if err := s.Write(ctx, entry); err != nil {
t.Fatal(err)
}
}
items, err := s.Query(ctx, Query{Kind: kind})
if err != nil || len(items) != 7 {
t.Fatalf("有效证据丢失:%s %d %v", kind, len(items), err)
}
}
if err := s.Write(ctx, Entry{Kind: "mail", CreatedAt: time.Now(), Payload: []byte(`invalid`)}); err != ErrInvalid {
t.Fatal("过滤不能掩盖非法载荷")
}
}
func TestLogRetentionBoundariesAndBusinessSafety(t *testing.T) {
s := testStore(t)
now := time.Date(2026, 9, 8, 9, 37, 0, 0, time.UTC)

View File

@ -4,7 +4,9 @@ import (
"context"
"database/sql"
"encoding/json"
"fmt"
"net/http/httptest"
"net/url"
"path/filepath"
"testing"
"time"
@ -13,6 +15,7 @@ import (
"git.pcux.cn/xuans/Smail/dto/response"
"git.pcux.cn/xuans/Smail/services/access"
"git.pcux.cn/xuans/Smail/services/logstore"
"git.pcux.cn/xuans/Smail/services/runtimelog"
appcontext "git.pcux.cn/xuans/Smail/utils/context"
_ "modernc.org/sqlite"
)
@ -37,6 +40,11 @@ func TestLogsAPIRejectsUnscopedAccessAndValidatesQueries(t *testing.T) {
}
runtime := logstore.Start(store)
defer runtime.Stop()
writer, err := runtimelog.Start(t.TempDir())
if err != nil {
t.Fatal(err)
}
defer writer.Close()
if err := store.Write(context.Background(), logstore.Entry{Kind: "operation", CreatedAt: time.Now(), ActorID: "42", Action: "probe", Result: "success", Payload: []byte(`{}`)}); err != nil {
t.Fatal(err)
}
@ -66,6 +74,11 @@ func TestLogsAPIRejectsUnscopedAccessAndValidatesQueries(t *testing.T) {
t.Fatalf("invalid query accepted: %s => %d", query, got.ErrorNo)
}
}
for _, query := range []string{"kind=runtime&from=invalid", "kind=runtime&to=", "kind=runtime&file=../config/config.json", "kind=runtime&from=2026-09-08T00:00:00Z&to=2026-09-01T00:00:00Z"} {
if got := call(access.RoleSuperAdmin, query); got.ErrorNo != response.ParamsError {
t.Fatalf("运行日志未校验参数:%s => %d", query, got.ErrorNo)
}
}
result := call(access.RoleSuperAdmin, "kind=operation&actorId=42&limit=1")
if result.ErrorNo != 0 {
t.Fatalf("administrator query failed: %d", result.ErrorNo)
@ -74,4 +87,38 @@ func TestLogsAPIRejectsUnscopedAccessAndValidatesQueries(t *testing.T) {
if len(data["items"].([]any)) != 1 || data["retentionDays"] != float64(30) {
t.Fatal("log query lost records or retention policy")
}
now := time.Now().UTC().Add(-time.Minute)
for index := 0; index < 14; index++ {
at := now.Add(time.Duration(index) * time.Second)
if index == 13 {
at = now.Add(-25 * time.Hour)
}
if err := writer.WriteAt(at, fmt.Sprintf("运行事件%d", index)); err != nil {
t.Fatal(err)
}
if err := store.Write(context.Background(), logstore.Entry{Kind: "mail", CreatedAt: at, Action: "投递邮件", Result: "success", Payload: []byte(`{}`)}); err != nil {
t.Fatal(err)
}
}
for _, kind := range []string{"mail", "runtime"} {
key := "items"
if kind == "runtime" {
key = "lines"
}
first := call(access.RoleSuperAdmin, "kind="+kind)
data := first.Data.(map[string]any)
if first.ErrorNo != 0 || len(data[key].([]any)) != 10 || data["hasMore"] != true {
t.Fatalf("默认十条:%+v", first)
}
second := call(access.RoleSuperAdmin, "kind="+kind+"&offset=10")
data = second.Data.(map[string]any)
if len(data[key].([]any)) != 3 || data["hasMore"] != false {
t.Fatalf("默认时段不是最近二十四小时:%+v", second)
}
values := url.Values{"kind": {kind}, "from": {now.Add(-26 * time.Hour).Format(time.RFC3339Nano)}, "to": {now.Add(-24 * time.Hour).Format(time.RFC3339Nano)}}
history := call(access.RoleSuperAdmin, values.Encode())
if history.ErrorNo != 0 || len(history.Data.(map[string]any)[key].([]any)) != 1 {
t.Fatalf("指定时段:%+v", history)
}
}
}

View File

@ -4,15 +4,19 @@ import (
"bytes"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"reflect"
"runtime"
"strings"
"testing"
"time"
"git.pcux.cn/xuans/Smail/utils/logpolicy"
log "github.com/sirupsen/logrus"
)
func TestRuntimeLogStartupPresentation(t *testing.T) {
@ -22,6 +26,7 @@ func TestRuntimeLogStartupPresentation(t *testing.T) {
for _, request := range []string{"", "[request-42]"} {
prefix := header + request + "[" + location + ":39]"
for _, message := range []string{
"", " ", "--------",
"*******************************************************************", "***\tServer Version: dev-test ",
"***\tGit Commit Hash: abc123 ", "***\tBuild Date: 2026-09-08T14:55:12Z ",
"***\tBuild GoLang Version: go version go1.26.6 windows/amd64 ",
@ -43,7 +48,6 @@ func TestRuntimeLogStartupPresentation(t *testing.T) {
header + "[" + source + ":39]***\tServer Start Success 后续内容",
header + "[" + source + ":39]***\t未知启动消息",
header + "[other/app.go:39]***\tBuild Date: 保留业务消息",
header + "[other/app.go:39]********",
header + "[" + source + ":39]Build Date: 不属于横幅",
strings.Replace(header, "[info]", "[warning]", 1) + "[" + source + ":39]***\tBuild Date: 保留警告",
strings.Replace(header, "[info]", "[error]", 1) + "[" + source + ":39]********",
@ -55,6 +59,158 @@ func TestRuntimeLogStartupPresentation(t *testing.T) {
}
}
func TestRuntimeLogNoiseIsNotWritten(t *testing.T) {
w, err := New(t.TempDir())
if err != nil {
t.Fatal(err)
}
defer w.Close()
at := time.Now().UTC()
logger := log.New()
logger.SetOutput(io.Discard)
logger.SetFormatter(&log.JSONFormatter{})
for _, message := range []string{"", " \n\t", "\x00", "********", "---===---"} {
if err := w.WriteAt(at, message); err != nil {
t.Fatal(err)
}
if err := w.Fire(&log.Entry{Logger: logger, Time: at, Level: log.InfoLevel, Message: message}); err != nil {
t.Fatal(err)
}
}
if err := w.Fire(&log.Entry{Logger: logger, Time: at, Level: log.InfoLevel, Message: "***\tBuild Date: old", Caller: &runtime.Frame{File: "internal/app/app.go"}}); err != nil {
t.Fatal(err)
}
files, err := w.Files()
if err != nil || len(files) != 0 {
t.Fatalf("无内容记录不应创建日志文件:%+v %v", files, err)
}
for _, entry := range []*log.Entry{
{Level: log.WarnLevel, Message: ""},
{Level: log.ErrorLevel, Message: "***"},
{Level: log.InfoLevel, Data: log.Fields{"error": "连接失败"}},
{Level: log.InfoLevel, Data: log.Fields{"error": errors.New("真实错误")}},
{Level: log.InfoLevel, Message: "服务启动,版本:test"},
{Level: log.InfoLevel, Message: "投递成功"},
{Level: log.DebugLevel, Message: "连接耗时:123ms"},
} {
entry.Logger, entry.Time = logger, at
if err := w.Fire(entry); err != nil {
t.Fatal(err)
}
}
page, err := w.Query(context.Background(), Query{From: at.Add(-time.Hour), To: at, Limit: 10})
if err != nil || len(page.Lines) != 7 || !strings.Contains(strings.Join(page.Lines, ""), "真实错误") {
t.Fatalf("重要事件被误丢弃:%+v %v", page, err)
}
}
func TestRuntimeLogRangeDescendingAndStablePagination(t *testing.T) {
w, err := New(t.TempDir())
if err != nil {
t.Fatal(err)
}
defer w.Close()
now := time.Date(2026, 9, 9, 12, 0, 0, 0, time.UTC)
w.now = func() time.Time { return now }
old := now.Add(-8 * 24 * time.Hour)
for _, record := range []struct {
at time.Time
message string
}{
{now.Add(-time.Hour), "同秒第一条"}, {now.Add(-24 * time.Hour), "最近二十四小时边界"},
{old, "压缩历史"}, {now.Add(-2 * time.Hour), "乱序较早写入"},
{now.Add(-time.Hour), "同秒第二条"}, {now, "时段上边界"},
{now.Add(time.Nanosecond), "上边界之外"},
} {
if err := w.WriteAt(record.at, record.message); err != nil {
t.Fatal(err)
}
}
if err := w.Maintain(context.Background(), now); err != nil {
t.Fatal(err)
}
if _, err := w.root.Stat(prefix + old.Format("2006-01-02") + ".log.gz"); err != nil {
t.Fatal(err)
}
query := Query{Limit: 2}
var got []string
for {
page, err := w.Query(context.Background(), query)
if err != nil {
t.Fatal(err)
}
for _, line := range page.Lines {
got = append(got, line[strings.Index(line, "] ")+2:])
}
if !page.HasMore {
break
}
query.Offset += len(page.Lines)
if query.Offset > 10 {
t.Fatal("分页重复")
}
}
want := []string{"时段上边界", "同秒第二条", "同秒第一条", "乱序较早写入", "最近二十四小时边界"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("倒序与边界:%v", got)
}
page, err := w.Query(context.Background(), Query{From: old.In(time.FixedZone("east", 8*3600)), To: old, Limit: 10})
if err != nil || len(page.Lines) != 1 || !strings.Contains(page.Lines[0], "压缩历史") {
t.Fatalf("压缩查询:%+v %v", page, err)
}
for _, query := range []Query{{From: now, To: old, Limit: 10}, {Limit: 501}, {Limit: 10, Offset: 100001}} {
if _, err := w.Query(context.Background(), query); !errors.Is(err, ErrInvalid) {
t.Fatalf("非法参数:%v", err)
}
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
w.archiveMu.Lock()
_, err = w.Query(ctx, Query{Limit: 10})
w.archiveMu.Unlock()
if !errors.Is(err, context.Canceled) {
t.Fatalf("取消查询仍等待归档:%v", err)
}
}
func TestRuntimeLogRangeResponseBudget(t *testing.T) {
w, err := New(t.TempDir())
if err != nil {
t.Fatal(err)
}
defer w.Close()
now := time.Now().UTC()
for index := 0; index < 40; index++ {
if err := w.WriteAt(now, fmt.Sprintf("%02d-", index)+strings.Repeat("x", maxLineBytes-200)); err != nil {
t.Fatal(err)
}
}
query := Query{From: now, To: now, Limit: 10}
count := 0
for {
page, err := w.Query(context.Background(), query)
if err != nil {
t.Fatal(err)
}
if len(page.Lines) == 0 || len(page.Lines) > 10 || len(strings.Join(page.Lines, "")) > 2*1024*1024 {
t.Fatal("响应上限或空短页错误")
}
for _, line := range page.Lines {
if !strings.Contains(line, fmt.Sprintf("] %02d-", 39-count)) {
t.Fatal("短页重复或丢失")
}
count++
}
if !page.HasMore {
break
}
query.Offset += len(page.Lines)
}
if count != 40 {
t.Fatalf("短页记录数:%d", count)
}
}
func TestRuntimeLogStartupFilteringBeforePagination(t *testing.T) {
for _, compressed := range []bool{false, true} {
for _, onlyNoise := range []bool{false, true} {
@ -119,6 +275,29 @@ func TestRuntimeLogStartupFilteringBeforePagination(t *testing.T) {
if err != nil || len(page.Lines) != 0 || page.HasMore {
t.Fatalf("尾部隐藏记录触发多余下一页:%+v %v", page, err)
}
var descending []string
for offset := 0; ; {
page, err := w.Query(context.Background(), Query{Limit: 2, Offset: offset})
if err != nil {
t.Fatal(err)
}
descending = append(descending, page.Lines...)
if !page.HasMore {
break
}
if len(page.Lines) != 2 || offset > 4 {
t.Fatal("历史过滤产生空页或重复页")
}
offset += len(page.Lines)
}
if len(descending) != len(expected) {
t.Fatal("倒序过滤丢失记录")
}
for index := range descending {
if descending[index] != expected[len(expected)-1-index] {
t.Fatal("历史记录倒序错误")
}
}
after, err := w.root.ReadFile(name)
if err != nil || !bytes.Equal(after, data) {
t.Fatalf("只读过滤修改了原文件:%v", err)

View File

@ -55,24 +55,58 @@ test('日志查询传递时间与操作者筛选,并将业务失败显示为
assert.equal(page.displayActor({ actorId: '0' }), '系统')
})
test('压缩日志短页正确返回上一页,文件过期后重置偏移', async () => {
let filename = 'runtime-2026-08-15.log.gz'
test('跨文件运行日志短页正确返回上一页,翻页固定时间和筛选条件', async () => {
const offsets = []
const ranges = []
const { page } = fixture(async (_, { params }) => {
if (!params.file) return { errorNo: 0, data: { files: [{ name: filename, compressed: true }] } }
assert.equal(params.limit, 10)
assert.equal(params.file, undefined)
assert.equal(params.actorId, undefined)
offsets.push(params.offset)
return { errorNo: 0, data: { lines: Array(params.offset === 0 ? 8 : 12).fill('日志'), hasMore: true } }
ranges.push([params.from, params.to])
return { errorNo: 0, data: { lines: Array(params.offset === 0 ? 8 : 10).fill('日志'), hasMore: true } }
})
page.category.value = 'runtime'
page.actorId.value = '42'
await page.search()
page.dateRange.value = ['2026-09-01T00:00:00Z', '2026-09-02T00:00:00Z']
await page.next()
await page.next()
await page.previous()
assert.deepEqual(offsets, [0, 8, 20, 8])
filename = 'runtime-2026-08-16.log.gz'
await page.next()
assert.deepEqual(offsets, [0, 8, 18, 8])
for (const range of ranges) assert.deepEqual(range, ranges[0])
await page.search()
assert.equal(page.offset.value, 0)
assert.equal(offsets.at(-1), 0)
assert.equal(ranges.at(-1)[0], '2026-09-01T00:00:00.000Z')
})
test('默认最近二十四小时和十条分页,清空时恢复默认,非法范围不请求', async () => {
const requests = []
const { page } = fixture(async (_, { params }) => {
requests.push(params)
return { errorNo: 0, data: { items: Array(10).fill({ id: 'entry' }), hasMore: true } }
})
await page.search()
assert.equal(requests[0].limit, 10)
assert.equal(new Date(requests[0].to) - new Date(requests[0].from), 24 * 60 * 60 * 1000)
await page.next()
assert.equal(requests[1].offset, 10)
page.dateRange.value = null
await page.search()
assert.equal(requests[2].offset, 0)
assert.equal(page.dateRange.value.length, 2)
page.dateRange.value = ['invalid', 'invalid']
await page.search()
assert.equal(requests.length, 3)
assert.equal(page.error.value, '请选择有效的起止时间')
assert.equal(page.items.value.length, 0)
assert.equal(page.hasMore.value, false)
page.dateRange.value = ['2026-09-02', '2026-09-01']
await page.search()
assert.equal(requests.length, 3)
await page.recent()
assert.equal(requests.length, 4)
})
test('慢响应及卸载后的响应不得覆盖最新日志', async () => {

View File

@ -60,6 +60,7 @@ if ((suite === 'all' || suite === 'backend') && result === 0) {
'backend/db/migration/log_migration_test.go': 'test/backend/log_migration_test.go',
'backend/config/audit_changes_test.go': 'test/backend/audit_changes_test.go',
'backend/config/json_format_test.go': 'test/backend/json_format_test.go',
'backend/config/log_formatter_test.go': 'test/backend/log_formatter_test.go',
'backend/config/deployment_state_test.go': 'test/backend/deployment_state_test.go',
'backend/config/ssl_retention_test.go': 'test/backend/ssl_retention_test.go',
'backend/utils/runtimepath/layout_test.go': 'test/backend/runtime_layout_test.go',