This commit is contained in:
1 parent
161bc3a345
commit
bb4ac4d5f5
20 files changed
+765
-132
No files matched your search
@@ -43,6 +43,13 @@ func lineTime(line []byte) (time.Time, error) {
|
||||
|
||||
// scan 严格解析每条记录及压缩校验和,损坏归档不得触发删除源文件。
|
||||
func (w *Writer) scan(ctx context.Context, name string, consume func([]byte, time.Time) error) error {
|
||||
return w.scanSize(ctx, name, -1, consume)
|
||||
}
|
||||
|
||||
func (w *Writer) scanSize(ctx context.Context, name string, size int64, consume func([]byte, time.Time) error) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.regular(name, false); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -52,8 +59,11 @@ func (w *Writer) scan(ctx context.Context, name string, consume func([]byte, tim
|
||||
}
|
||||
defer file.Close()
|
||||
var reader io.Reader = file
|
||||
if size >= 0 {
|
||||
reader = io.LimitReader(reader, size)
|
||||
}
|
||||
if strings.HasSuffix(name, ".gz") {
|
||||
archive, err := gzip.NewReader(file)
|
||||
archive, err := gzip.NewReader(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -3,32 +3,41 @@ package runtimelog
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/utils/logpolicy"
|
||||
)
|
||||
|
||||
var legacyStartupLine = regexp.MustCompile(`^(\[[^\]\r\n]+\]\s+\[info\]\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\](?:\[[^\]\r\n]*\])?\[([^\]\r\n]+):\d+\])([^\r\n]*)$`)
|
||||
var contentLine = regexp.MustCompile(`^\[[^\]\r\n]+\]\s+\[(info|debug|trace)\]\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\](?:\[[^\]\r\n]*\])?\[([^\]\r\n]+):\d+\]([^\r\n]*)$`)
|
||||
|
||||
// presentLine 只精简旧版主程序的启动横幅,不修改磁盘原文或隐藏其他来源和级别的记录。
|
||||
// presentLine 先过滤明确无内容的历史记录;警告、错误和未知格式均保留原文。
|
||||
func presentLine(line string) (string, bool) {
|
||||
parts := legacyStartupLine.FindStringSubmatch(line)
|
||||
parts := contentLine.FindStringSubmatch(line)
|
||||
if parts == nil {
|
||||
if end := strings.Index(line, "] "); end >= 0 && logpolicy.EmptyMessage(line[end+2:]) {
|
||||
return "", false
|
||||
}
|
||||
return line, true
|
||||
}
|
||||
source := strings.ReplaceAll(parts[2], `\`, "/")
|
||||
if source != "internal/app/app.go" && !strings.HasSuffix(source, "/internal/app/app.go") {
|
||||
return line, true
|
||||
}
|
||||
message := strings.TrimSpace(parts[3])
|
||||
if len(message) >= 3 && strings.Trim(message, "*") == "" {
|
||||
if logpolicy.EmptyMessage(parts[3]) || (parts[1] == "info" && legacyStartupNoise(parts[2], parts[3])) {
|
||||
return "", false
|
||||
}
|
||||
return line, true
|
||||
}
|
||||
|
||||
func legacyStartupNoise(source, message string) bool {
|
||||
source = strings.ReplaceAll(source, `\`, "/")
|
||||
if source != "internal/app/app.go" && !strings.HasSuffix(source, "/internal/app/app.go") {
|
||||
return false
|
||||
}
|
||||
message = strings.TrimSpace(message)
|
||||
if !strings.HasPrefix(message, "***\t") && !strings.HasPrefix(message, "*** ") {
|
||||
return line, true
|
||||
return false
|
||||
}
|
||||
message = strings.TrimSpace(message[3:])
|
||||
for _, prefix := range []string{"Server Version:", "Git Commit Hash:", "Build Date:", "Build GoLang Version:"} {
|
||||
if strings.HasPrefix(message, prefix) {
|
||||
return "", false
|
||||
return true
|
||||
}
|
||||
}
|
||||
return line, true
|
||||
return false
|
||||
}
|
||||
@@ -24,6 +24,10 @@ type Page struct {
|
||||
func (w *Writer) Files() ([]File, error) {
|
||||
w.archiveMu.Lock()
|
||||
defer w.archiveMu.Unlock()
|
||||
return w.files()
|
||||
}
|
||||
|
||||
func (w *Writer) files() ([]File, error) {
|
||||
if w.closed {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
@@ -45,10 +49,15 @@ func (w *Writer) Files() ([]File, error) {
|
||||
if err := w.regular(entry.Name(), false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := entry.Info()
|
||||
// Windows 目录枚举可能缓存仍打开文件的旧长度,必须通过文件句柄读取。
|
||||
handle, err := w.root.Open(entry.Name())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, statErr := handle.Stat()
|
||||
if err := errors.Join(statErr, handle.Close()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, File{Name: entry.Name(), Bytes: info.Size(), Compressed: strings.HasSuffix(entry.Name(), ".gz")})
|
||||
}
|
||||
sort.Slice(files, func(i, j int) bool { return files[i].Name > files[j].Name })
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
package runtimelog
|
||||
|
||||
import (
|
||||
"container/heap"
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/utils/logpolicy"
|
||||
)
|
||||
|
||||
type Query struct {
|
||||
File string
|
||||
From time.Time
|
||||
To time.Time
|
||||
Offset int
|
||||
Limit int
|
||||
}
|
||||
|
||||
type lineRef struct {
|
||||
at time.Time
|
||||
file int
|
||||
position int
|
||||
bytes int
|
||||
}
|
||||
|
||||
// 索引只保存时间与行位置,不把历史日志全文装入内存。
|
||||
type recentLines []lineRef
|
||||
|
||||
func (h recentLines) Len() int { return len(h) }
|
||||
func (h recentLines) Less(i, j int) bool {
|
||||
if !h[i].at.Equal(h[j].at) {
|
||||
return h[i].at.Before(h[j].at)
|
||||
}
|
||||
if h[i].file != h[j].file {
|
||||
return h[i].file > h[j].file
|
||||
}
|
||||
return h[i].position < h[j].position
|
||||
}
|
||||
func (h recentLines) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
|
||||
func (h *recentLines) Push(value any) { *h = append(*h, value.(lineRef)) }
|
||||
func (h *recentLines) Pop() any {
|
||||
last := len(*h) - 1
|
||||
value := (*h)[last]
|
||||
*h = (*h)[:last]
|
||||
return value
|
||||
}
|
||||
|
||||
// Query 跨日流式扫描并按真实时间倒序,兼容压缩文件和同一文件中的乱序写入。
|
||||
// 额外内存由最大偏移限制的行索引和两兆响应组成,不随日志文件体积增长。
|
||||
func (w *Writer) Query(ctx context.Context, query Query) (Page, error) {
|
||||
if query.File != "" {
|
||||
if _, ok := fileDay(query.File); !ok {
|
||||
return Page{}, ErrInvalid
|
||||
}
|
||||
}
|
||||
if query.Offset < 0 || query.Offset > 100000 || query.Limit < 1 || query.Limit > 500 {
|
||||
return Page{}, ErrInvalid
|
||||
}
|
||||
if query.To.IsZero() {
|
||||
query.To = w.now()
|
||||
}
|
||||
if query.From.IsZero() {
|
||||
query.From = query.To.Add(-24 * time.Hour)
|
||||
}
|
||||
if query.From.After(query.To) {
|
||||
return Page{}, ErrInvalid
|
||||
}
|
||||
query.From = maxTime(query.From, logpolicy.Cutoff(w.now()))
|
||||
// 等待归档时也响应取消,避免查询超时后继续占用 HTTP 工作协程。
|
||||
for !w.archiveMu.TryLock() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return Page{}, ctx.Err()
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
defer w.archiveMu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Page{}, err
|
||||
}
|
||||
// 在短写锁内冻结文件长度;两次扫描不会读到新追加的半条记录。
|
||||
w.mu.Lock()
|
||||
files, err := w.files()
|
||||
w.mu.Unlock()
|
||||
if err != nil {
|
||||
return Page{}, err
|
||||
}
|
||||
page := Page{Lines: make([]string, 0, query.Limit)}
|
||||
if query.From.After(query.To) {
|
||||
return page, nil
|
||||
}
|
||||
capacity := query.Offset + query.Limit + 1
|
||||
refs := make(recentLines, 0, capacity)
|
||||
for index, file := range files {
|
||||
if query.File != "" && query.File != file.Name {
|
||||
continue
|
||||
}
|
||||
day, _ := fileDay(file.Name)
|
||||
if day.After(query.To) || !day.Add(24*time.Hour).After(query.From) {
|
||||
continue
|
||||
}
|
||||
position := 0
|
||||
err := w.scanSize(ctx, file.Name, file.Bytes, func(line []byte, at time.Time) error {
|
||||
position++
|
||||
if !at.UTC().Truncate(24 * time.Hour).Equal(day) {
|
||||
return ErrInvalid
|
||||
}
|
||||
if at.Before(query.From) || at.After(query.To) {
|
||||
return nil
|
||||
}
|
||||
visible, keep := presentLine(string(line))
|
||||
if !keep {
|
||||
return nil
|
||||
}
|
||||
candidate := lineRef{at: at, file: index, position: position, bytes: len(visible)}
|
||||
if len(refs) < capacity {
|
||||
heap.Push(&refs, candidate)
|
||||
} else if (recentLines{refs[0], candidate}).Less(0, 1) {
|
||||
refs[0] = candidate
|
||||
heap.Fix(&refs, 0)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return Page{}, err
|
||||
}
|
||||
}
|
||||
sort.Sort(sort.Reverse(refs))
|
||||
if query.Offset >= len(refs) {
|
||||
return page, nil
|
||||
}
|
||||
refs = refs[query.Offset:]
|
||||
page.HasMore = len(refs) > query.Limit
|
||||
refs = refs[:min(len(refs), query.Limit)]
|
||||
bytes := 0
|
||||
for index, ref := range refs {
|
||||
if bytes+ref.bytes > 2*1024*1024 {
|
||||
refs = refs[:index]
|
||||
page.HasMore = true
|
||||
break
|
||||
}
|
||||
bytes += ref.bytes
|
||||
}
|
||||
page.Lines = make([]string, len(refs))
|
||||
selected := make(map[int]map[int]int)
|
||||
for index, ref := range refs {
|
||||
if selected[ref.file] == nil {
|
||||
selected[ref.file] = make(map[int]int)
|
||||
}
|
||||
selected[ref.file][ref.position] = index
|
||||
}
|
||||
finished := errors.New("本页内容读取完成")
|
||||
for fileIndex, positions := range selected {
|
||||
position, remaining := 0, len(positions)
|
||||
file := files[fileIndex]
|
||||
err := w.scanSize(ctx, file.Name, file.Bytes, func(line []byte, at time.Time) error {
|
||||
position++
|
||||
index, ok := positions[position]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if !at.Equal(refs[index].at) || len(line) != refs[index].bytes {
|
||||
return ErrInvalid
|
||||
}
|
||||
page.Lines[index] = string(line)
|
||||
remaining--
|
||||
if remaining == 0 {
|
||||
return finished
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil && !errors.Is(err, finished) {
|
||||
return Page{}, err
|
||||
}
|
||||
if remaining != 0 {
|
||||
return Page{}, ErrInvalid
|
||||
}
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func maxTime(left, right time.Time) time.Time {
|
||||
if left.After(right) {
|
||||
return left
|
||||
}
|
||||
return right
|
||||
}
|
||||
@@ -86,15 +86,34 @@ func Start(root string) (*Writer, error) {
|
||||
func (w *Writer) Levels() []log.Level { return log.AllLevels }
|
||||
|
||||
func (w *Writer) Fire(entry *log.Entry) error {
|
||||
// 结构化字段和警告以上级别本身具有排错价值,不能只看消息是否为空。
|
||||
if entry.Level >= log.InfoLevel && len(entry.Data) == 0 {
|
||||
if logpolicy.EmptyMessage(entry.Message) || (entry.Level == log.InfoLevel && entry.Caller != nil && legacyStartupNoise(entry.Caller.File, entry.Message)) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// 复用已有格式器保留请求标识、错误字段和源位置;转义换行保证一条记录一行。
|
||||
data, err := entry.Logger.Formatter.Format(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return w.WriteAt(entry.Time, strings.TrimRight(string(data), "\r\n"))
|
||||
return w.writeAt(entry.Time, strings.TrimRight(string(data), "\r\n"))
|
||||
}
|
||||
|
||||
func (w *Writer) WriteAt(at time.Time, message string) error {
|
||||
if at.IsZero() {
|
||||
return ErrInvalid
|
||||
}
|
||||
if logpolicy.EmptyMessage(message) {
|
||||
return nil
|
||||
}
|
||||
if _, keep := presentLine("[" + at.UTC().Format(time.RFC3339Nano) + "] " + message); !keep {
|
||||
return nil
|
||||
}
|
||||
return w.writeAt(at, message)
|
||||
}
|
||||
|
||||
func (w *Writer) writeAt(at time.Time, message string) error {
|
||||
if at.IsZero() {
|
||||
return ErrInvalid
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user