过滤无价值日志并实现最近24小时倒序分页查询
Security Checks / security (push) Waiting to run

This commit is contained in:
xuans committed 2026-09-09 01:06:45 +08:00
1 parent 161bc3a345
commit bb4ac4d5f5
20 files changed
+765 -132

No files matched your search

+38
View File
@@ -0,0 +1,38 @@
package logstore
import (
"encoding/json"
"strings"
"git.pcux.cn/xuans/Smail/utils/logpolicy"
)
// emptyEntry 只丢弃没有事件、对象、结果或有效载荷的空记录。
// 已知业务动作(包括无字段变化的配置审计)、失败和未知结果均不去重或省略。
func emptyEntry(entry Entry) bool {
if !logpolicy.EmptyMessage(entry.Action) || strings.TrimSpace(entry.ActorID) != "" || strings.TrimSpace(entry.Section) != "" || strings.TrimSpace(entry.Result) != "" {
return false
}
var payload any
if json.Unmarshal(entry.Payload, &payload) != nil {
return false
}
switch value := payload.(type) {
case nil:
return true
case string:
return logpolicy.EmptyMessage(value)
case []any:
return len(value) == 0
case map[string]any:
if len(value) == 0 {
return true
}
// 只理解单一消息字段,其他键即使值为空也可能是状态证据。
if len(value) == 1 {
message, ok := value["message"].(string)
return ok && logpolicy.EmptyMessage(message)
}
}
return false
}
+3
View File
@@ -106,6 +106,9 @@ func (s *Store) Write(ctx context.Context, entry Entry) error {
if entry.CreatedAt.Before(logpolicy.Cutoff(s.now())) {
return nil
}
if emptyEntry(entry) {
return nil
}
if entry.ID == "" {
var token [16]byte
if _, err := rand.Read(token[:]); err != nil {
+11 -1
View File
@@ -43,6 +43,13 @@ func lineTime(line []byte) (time.Time, error) {
// scan 严格解析每条记录及压缩校验和,损坏归档不得触发删除源文件。
func (w *Writer) scan(ctx context.Context, name string, consume func([]byte, time.Time) error) error {
return w.scanSize(ctx, name, -1, consume)
}
func (w *Writer) scanSize(ctx context.Context, name string, size int64, consume func([]byte, time.Time) error) error {
if err := ctx.Err(); err != nil {
return err
}
if err := w.regular(name, false); err != nil {
return err
}
@@ -52,8 +59,11 @@ func (w *Writer) scan(ctx context.Context, name string, consume func([]byte, tim
}
defer file.Close()
var reader io.Reader = file
if size >= 0 {
reader = io.LimitReader(reader, size)
}
if strings.HasSuffix(name, ".gz") {
archive, err := gzip.NewReader(file)
archive, err := gzip.NewReader(reader)
if err != nil {
return err
}
+21 -12
View File
@@ -3,32 +3,41 @@ package runtimelog
import (
"regexp"
"strings"
"git.pcux.cn/xuans/Smail/utils/logpolicy"
)
var legacyStartupLine = regexp.MustCompile(`^(\[[^\]\r\n]+\]\s+\[info\]\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\](?:\[[^\]\r\n]*\])?\[([^\]\r\n]+):\d+\])([^\r\n]*)$`)
var contentLine = regexp.MustCompile(`^\[[^\]\r\n]+\]\s+\[(info|debug|trace)\]\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\](?:\[[^\]\r\n]*\])?\[([^\]\r\n]+):\d+\]([^\r\n]*)$`)
// presentLine 只精简旧版主程序的启动横幅,不修改磁盘原文或隐藏其他来源和级别的记录。
// presentLine 先过滤明确无内容的历史记录;警告、错误和未知格式均保留原文。
func presentLine(line string) (string, bool) {
parts := legacyStartupLine.FindStringSubmatch(line)
parts := contentLine.FindStringSubmatch(line)
if parts == nil {
if end := strings.Index(line, "] "); end >= 0 && logpolicy.EmptyMessage(line[end+2:]) {
return "", false
}
return line, true
}
source := strings.ReplaceAll(parts[2], `\`, "/")
if source != "internal/app/app.go" && !strings.HasSuffix(source, "/internal/app/app.go") {
return line, true
}
message := strings.TrimSpace(parts[3])
if len(message) >= 3 && strings.Trim(message, "*") == "" {
if logpolicy.EmptyMessage(parts[3]) || (parts[1] == "info" && legacyStartupNoise(parts[2], parts[3])) {
return "", false
}
return line, true
}
func legacyStartupNoise(source, message string) bool {
source = strings.ReplaceAll(source, `\`, "/")
if source != "internal/app/app.go" && !strings.HasSuffix(source, "/internal/app/app.go") {
return false
}
message = strings.TrimSpace(message)
if !strings.HasPrefix(message, "***\t") && !strings.HasPrefix(message, "*** ") {
return line, true
return false
}
message = strings.TrimSpace(message[3:])
for _, prefix := range []string{"Server Version:", "Git Commit Hash:", "Build Date:", "Build GoLang Version:"} {
if strings.HasPrefix(message, prefix) {
return "", false
return true
}
}
return line, true
return false
}
+10 -1
View File
@@ -24,6 +24,10 @@ type Page struct {
func (w *Writer) Files() ([]File, error) {
w.archiveMu.Lock()
defer w.archiveMu.Unlock()
return w.files()
}
func (w *Writer) files() ([]File, error) {
if w.closed {
return nil, ErrInvalid
}
@@ -45,10 +49,15 @@ func (w *Writer) Files() ([]File, error) {
if err := w.regular(entry.Name(), false); err != nil {
return nil, err
}
info, err := entry.Info()
// Windows 目录枚举可能缓存仍打开文件的旧长度,必须通过文件句柄读取。
handle, err := w.root.Open(entry.Name())
if err != nil {
return nil, err
}
info, statErr := handle.Stat()
if err := errors.Join(statErr, handle.Close()); err != nil {
return nil, err
}
files = append(files, File{Name: entry.Name(), Bytes: info.Size(), Compressed: strings.HasSuffix(entry.Name(), ".gz")})
}
sort.Slice(files, func(i, j int) bool { return files[i].Name > files[j].Name })
+189
View File
@@ -0,0 +1,189 @@
package runtimelog
import (
"container/heap"
"context"
"errors"
"sort"
"time"
"git.pcux.cn/xuans/Smail/utils/logpolicy"
)
type Query struct {
File string
From time.Time
To time.Time
Offset int
Limit int
}
type lineRef struct {
at time.Time
file int
position int
bytes int
}
// 索引只保存时间与行位置,不把历史日志全文装入内存。
type recentLines []lineRef
func (h recentLines) Len() int { return len(h) }
func (h recentLines) Less(i, j int) bool {
if !h[i].at.Equal(h[j].at) {
return h[i].at.Before(h[j].at)
}
if h[i].file != h[j].file {
return h[i].file > h[j].file
}
return h[i].position < h[j].position
}
func (h recentLines) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h *recentLines) Push(value any) { *h = append(*h, value.(lineRef)) }
func (h *recentLines) Pop() any {
last := len(*h) - 1
value := (*h)[last]
*h = (*h)[:last]
return value
}
// Query 跨日流式扫描并按真实时间倒序,兼容压缩文件和同一文件中的乱序写入。
// 额外内存由最大偏移限制的行索引和两兆响应组成,不随日志文件体积增长。
func (w *Writer) Query(ctx context.Context, query Query) (Page, error) {
if query.File != "" {
if _, ok := fileDay(query.File); !ok {
return Page{}, ErrInvalid
}
}
if query.Offset < 0 || query.Offset > 100000 || query.Limit < 1 || query.Limit > 500 {
return Page{}, ErrInvalid
}
if query.To.IsZero() {
query.To = w.now()
}
if query.From.IsZero() {
query.From = query.To.Add(-24 * time.Hour)
}
if query.From.After(query.To) {
return Page{}, ErrInvalid
}
query.From = maxTime(query.From, logpolicy.Cutoff(w.now()))
// 等待归档时也响应取消,避免查询超时后继续占用 HTTP 工作协程。
for !w.archiveMu.TryLock() {
select {
case <-ctx.Done():
return Page{}, ctx.Err()
case <-time.After(10 * time.Millisecond):
}
}
defer w.archiveMu.Unlock()
if err := ctx.Err(); err != nil {
return Page{}, err
}
// 在短写锁内冻结文件长度;两次扫描不会读到新追加的半条记录。
w.mu.Lock()
files, err := w.files()
w.mu.Unlock()
if err != nil {
return Page{}, err
}
page := Page{Lines: make([]string, 0, query.Limit)}
if query.From.After(query.To) {
return page, nil
}
capacity := query.Offset + query.Limit + 1
refs := make(recentLines, 0, capacity)
for index, file := range files {
if query.File != "" && query.File != file.Name {
continue
}
day, _ := fileDay(file.Name)
if day.After(query.To) || !day.Add(24*time.Hour).After(query.From) {
continue
}
position := 0
err := w.scanSize(ctx, file.Name, file.Bytes, func(line []byte, at time.Time) error {
position++
if !at.UTC().Truncate(24 * time.Hour).Equal(day) {
return ErrInvalid
}
if at.Before(query.From) || at.After(query.To) {
return nil
}
visible, keep := presentLine(string(line))
if !keep {
return nil
}
candidate := lineRef{at: at, file: index, position: position, bytes: len(visible)}
if len(refs) < capacity {
heap.Push(&refs, candidate)
} else if (recentLines{refs[0], candidate}).Less(0, 1) {
refs[0] = candidate
heap.Fix(&refs, 0)
}
return nil
})
if err != nil {
return Page{}, err
}
}
sort.Sort(sort.Reverse(refs))
if query.Offset >= len(refs) {
return page, nil
}
refs = refs[query.Offset:]
page.HasMore = len(refs) > query.Limit
refs = refs[:min(len(refs), query.Limit)]
bytes := 0
for index, ref := range refs {
if bytes+ref.bytes > 2*1024*1024 {
refs = refs[:index]
page.HasMore = true
break
}
bytes += ref.bytes
}
page.Lines = make([]string, len(refs))
selected := make(map[int]map[int]int)
for index, ref := range refs {
if selected[ref.file] == nil {
selected[ref.file] = make(map[int]int)
}
selected[ref.file][ref.position] = index
}
finished := errors.New("本页内容读取完成")
for fileIndex, positions := range selected {
position, remaining := 0, len(positions)
file := files[fileIndex]
err := w.scanSize(ctx, file.Name, file.Bytes, func(line []byte, at time.Time) error {
position++
index, ok := positions[position]
if !ok {
return nil
}
if !at.Equal(refs[index].at) || len(line) != refs[index].bytes {
return ErrInvalid
}
page.Lines[index] = string(line)
remaining--
if remaining == 0 {
return finished
}
return nil
})
if err != nil && !errors.Is(err, finished) {
return Page{}, err
}
if remaining != 0 {
return Page{}, ErrInvalid
}
}
return page, nil
}
func maxTime(left, right time.Time) time.Time {
if left.After(right) {
return left
}
return right
}
+20 -1
View File
@@ -86,15 +86,34 @@ func Start(root string) (*Writer, error) {
func (w *Writer) Levels() []log.Level { return log.AllLevels }
func (w *Writer) Fire(entry *log.Entry) error {
// 结构化字段和警告以上级别本身具有排错价值,不能只看消息是否为空。
if entry.Level >= log.InfoLevel && len(entry.Data) == 0 {
if logpolicy.EmptyMessage(entry.Message) || (entry.Level == log.InfoLevel && entry.Caller != nil && legacyStartupNoise(entry.Caller.File, entry.Message)) {
return nil
}
}
// 复用已有格式器保留请求标识、错误字段和源位置;转义换行保证一条记录一行。
data, err := entry.Logger.Formatter.Format(entry)
if err != nil {
return err
}
return w.WriteAt(entry.Time, strings.TrimRight(string(data), "\r\n"))
return w.writeAt(entry.Time, strings.TrimRight(string(data), "\r\n"))
}
func (w *Writer) WriteAt(at time.Time, message string) error {
if at.IsZero() {
return ErrInvalid
}
if logpolicy.EmptyMessage(message) {
return nil
}
if _, keep := presentLine("[" + at.UTC().Format(time.RFC3339Nano) + "] " + message); !keep {
return nil
}
return w.writeAt(at, message)
}
func (w *Writer) writeAt(at time.Time, message string) error {
if at.IsZero() {
return ErrInvalid
}