This commit is contained in:
1 parent
161bc3a345
commit
bb4ac4d5f5
20 files changed
+765
-132
No files matched your search
@@ -156,6 +156,25 @@ func (l *logFormatter) Format(entry *log.Entry) ([]byte, error) {
|
||||
}
|
||||
b.WriteString(fmt.Sprintf("[%s:%d]", entry.Caller.File, entry.Caller.Line))
|
||||
b.WriteString(entry.Message)
|
||||
// 错误字段也是有效内容,不能在消息为空时被格式器丢弃。
|
||||
if len(entry.Data) > 0 {
|
||||
fields := make(map[string]any, len(entry.Data))
|
||||
for key, value := range entry.Data {
|
||||
if cause, ok := value.(error); ok {
|
||||
fields[key] = cause.Error()
|
||||
} else if _, err := json.Marshal(value); err != nil {
|
||||
fields[key] = fmt.Sprintf("无法序列化的字段(%T)", value)
|
||||
} else {
|
||||
fields[key] = value
|
||||
}
|
||||
}
|
||||
payload, err := json.Marshal(fields)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b.WriteByte(' ')
|
||||
b.Write(payload)
|
||||
}
|
||||
|
||||
b.WriteString("\n")
|
||||
return b.Bytes(), nil
|
||||
|
||||
+27
-16
@@ -33,7 +33,7 @@ func GetLogs(ctx *appcontext.Context, w http.ResponseWriter, req *http.Request)
|
||||
return
|
||||
}
|
||||
values := req.URL.Query()
|
||||
limit, offset := 50, 0
|
||||
limit, offset := 10, 0
|
||||
var err error
|
||||
if values.Has("limit") {
|
||||
limit, err = strconv.Atoi(values.Get("limit"))
|
||||
@@ -47,13 +47,37 @@ func GetLogs(ctx *appcontext.Context, w http.ResponseWriter, req *http.Request)
|
||||
}
|
||||
bounded, cancel := context.WithTimeout(req.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
query := logstore.Query{Kind: values.Get("kind"), ActorID: values.Get("actorId"), Limit: limit + 1, Offset: offset}
|
||||
for _, item := range []struct {
|
||||
name string
|
||||
target *time.Time
|
||||
}{{"from", &query.From}, {"to", &query.To}} {
|
||||
if values.Has(item.name) {
|
||||
*item.target, err = time.Parse(time.RFC3339Nano, values.Get(item.name))
|
||||
if err != nil {
|
||||
writeConfigError(w, config.ErrInvalidAuditQuery)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if query.To.IsZero() {
|
||||
query.To = time.Now()
|
||||
}
|
||||
if query.From.IsZero() {
|
||||
query.From = query.To.Add(-24 * time.Hour)
|
||||
}
|
||||
if query.From.After(query.To) {
|
||||
writeConfigError(w, config.ErrInvalidAuditQuery)
|
||||
return
|
||||
}
|
||||
if values.Get("kind") == "runtime" {
|
||||
writer := runtimelog.Current()
|
||||
if writer == nil {
|
||||
logQueryError(w, logstore.ErrUnavailable)
|
||||
return
|
||||
}
|
||||
if values.Get("file") == "" {
|
||||
// 显式保留旧文件列表入口,新页面默认跨文件查询。
|
||||
if values.Get("listFiles") == "true" {
|
||||
files, err := writer.Files()
|
||||
if err != nil {
|
||||
logQueryError(w, err)
|
||||
@@ -62,7 +86,7 @@ func GetLogs(ctx *appcontext.Context, w http.ResponseWriter, req *http.Request)
|
||||
response.NewSuccessResponse(map[string]any{"files": files}).FPrint(w)
|
||||
return
|
||||
}
|
||||
page, err := writer.Read(bounded, values.Get("file"), offset, limit)
|
||||
page, err := writer.Query(bounded, runtimelog.Query{File: values.Get("file"), From: query.From, To: query.To, Offset: offset, Limit: limit})
|
||||
if err != nil {
|
||||
logQueryError(w, err)
|
||||
return
|
||||
@@ -70,19 +94,6 @@ func GetLogs(ctx *appcontext.Context, w http.ResponseWriter, req *http.Request)
|
||||
response.NewSuccessResponse(page).FPrint(w)
|
||||
return
|
||||
}
|
||||
query := logstore.Query{Kind: values.Get("kind"), ActorID: values.Get("actorId"), Limit: limit + 1, Offset: offset}
|
||||
for _, item := range []struct {
|
||||
name string
|
||||
target *time.Time
|
||||
}{{"from", &query.From}, {"to", &query.To}} {
|
||||
if values.Get(item.name) != "" {
|
||||
*item.target, err = time.Parse(time.RFC3339Nano, values.Get(item.name))
|
||||
if err != nil {
|
||||
writeConfigError(w, config.ErrInvalidAuditQuery)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if query.Kind != "config" && query.Kind != "operation" && query.Kind != "mail" {
|
||||
writeConfigError(w, config.ErrInvalidAuditQuery)
|
||||
return
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/utils/logpolicy"
|
||||
)
|
||||
|
||||
// emptyEntry 只丢弃没有事件、对象、结果或有效载荷的空记录。
|
||||
// 已知业务动作(包括无字段变化的配置审计)、失败和未知结果均不去重或省略。
|
||||
func emptyEntry(entry Entry) bool {
|
||||
if !logpolicy.EmptyMessage(entry.Action) || strings.TrimSpace(entry.ActorID) != "" || strings.TrimSpace(entry.Section) != "" || strings.TrimSpace(entry.Result) != "" {
|
||||
return false
|
||||
}
|
||||
var payload any
|
||||
if json.Unmarshal(entry.Payload, &payload) != nil {
|
||||
return false
|
||||
}
|
||||
switch value := payload.(type) {
|
||||
case nil:
|
||||
return true
|
||||
case string:
|
||||
return logpolicy.EmptyMessage(value)
|
||||
case []any:
|
||||
return len(value) == 0
|
||||
case map[string]any:
|
||||
if len(value) == 0 {
|
||||
return true
|
||||
}
|
||||
// 只理解单一消息字段,其他键即使值为空也可能是状态证据。
|
||||
if len(value) == 1 {
|
||||
message, ok := value["message"].(string)
|
||||
return ok && logpolicy.EmptyMessage(message)
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -106,6 +106,9 @@ func (s *Store) Write(ctx context.Context, entry Entry) error {
|
||||
if entry.CreatedAt.Before(logpolicy.Cutoff(s.now())) {
|
||||
return nil
|
||||
}
|
||||
if emptyEntry(entry) {
|
||||
return nil
|
||||
}
|
||||
if entry.ID == "" {
|
||||
var token [16]byte
|
||||
if _, err := rand.Read(token[:]); err != nil {
|
||||
|
||||
@@ -43,6 +43,13 @@ func lineTime(line []byte) (time.Time, error) {
|
||||
|
||||
// scan 严格解析每条记录及压缩校验和,损坏归档不得触发删除源文件。
|
||||
func (w *Writer) scan(ctx context.Context, name string, consume func([]byte, time.Time) error) error {
|
||||
return w.scanSize(ctx, name, -1, consume)
|
||||
}
|
||||
|
||||
func (w *Writer) scanSize(ctx context.Context, name string, size int64, consume func([]byte, time.Time) error) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.regular(name, false); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -52,8 +59,11 @@ func (w *Writer) scan(ctx context.Context, name string, consume func([]byte, tim
|
||||
}
|
||||
defer file.Close()
|
||||
var reader io.Reader = file
|
||||
if size >= 0 {
|
||||
reader = io.LimitReader(reader, size)
|
||||
}
|
||||
if strings.HasSuffix(name, ".gz") {
|
||||
archive, err := gzip.NewReader(file)
|
||||
archive, err := gzip.NewReader(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -3,32 +3,41 @@ package runtimelog
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/utils/logpolicy"
|
||||
)
|
||||
|
||||
var legacyStartupLine = regexp.MustCompile(`^(\[[^\]\r\n]+\]\s+\[info\]\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\](?:\[[^\]\r\n]*\])?\[([^\]\r\n]+):\d+\])([^\r\n]*)$`)
|
||||
var contentLine = regexp.MustCompile(`^\[[^\]\r\n]+\]\s+\[(info|debug|trace)\]\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\](?:\[[^\]\r\n]*\])?\[([^\]\r\n]+):\d+\]([^\r\n]*)$`)
|
||||
|
||||
// presentLine 只精简旧版主程序的启动横幅,不修改磁盘原文或隐藏其他来源和级别的记录。
|
||||
// presentLine 先过滤明确无内容的历史记录;警告、错误和未知格式均保留原文。
|
||||
func presentLine(line string) (string, bool) {
|
||||
parts := legacyStartupLine.FindStringSubmatch(line)
|
||||
parts := contentLine.FindStringSubmatch(line)
|
||||
if parts == nil {
|
||||
if end := strings.Index(line, "] "); end >= 0 && logpolicy.EmptyMessage(line[end+2:]) {
|
||||
return "", false
|
||||
}
|
||||
return line, true
|
||||
}
|
||||
source := strings.ReplaceAll(parts[2], `\`, "/")
|
||||
if source != "internal/app/app.go" && !strings.HasSuffix(source, "/internal/app/app.go") {
|
||||
return line, true
|
||||
}
|
||||
message := strings.TrimSpace(parts[3])
|
||||
if len(message) >= 3 && strings.Trim(message, "*") == "" {
|
||||
if logpolicy.EmptyMessage(parts[3]) || (parts[1] == "info" && legacyStartupNoise(parts[2], parts[3])) {
|
||||
return "", false
|
||||
}
|
||||
return line, true
|
||||
}
|
||||
|
||||
func legacyStartupNoise(source, message string) bool {
|
||||
source = strings.ReplaceAll(source, `\`, "/")
|
||||
if source != "internal/app/app.go" && !strings.HasSuffix(source, "/internal/app/app.go") {
|
||||
return false
|
||||
}
|
||||
message = strings.TrimSpace(message)
|
||||
if !strings.HasPrefix(message, "***\t") && !strings.HasPrefix(message, "*** ") {
|
||||
return line, true
|
||||
return false
|
||||
}
|
||||
message = strings.TrimSpace(message[3:])
|
||||
for _, prefix := range []string{"Server Version:", "Git Commit Hash:", "Build Date:", "Build GoLang Version:"} {
|
||||
if strings.HasPrefix(message, prefix) {
|
||||
return "", false
|
||||
return true
|
||||
}
|
||||
}
|
||||
return line, true
|
||||
return false
|
||||
}
|
||||
@@ -24,6 +24,10 @@ type Page struct {
|
||||
func (w *Writer) Files() ([]File, error) {
|
||||
w.archiveMu.Lock()
|
||||
defer w.archiveMu.Unlock()
|
||||
return w.files()
|
||||
}
|
||||
|
||||
func (w *Writer) files() ([]File, error) {
|
||||
if w.closed {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
@@ -45,10 +49,15 @@ func (w *Writer) Files() ([]File, error) {
|
||||
if err := w.regular(entry.Name(), false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := entry.Info()
|
||||
// Windows 目录枚举可能缓存仍打开文件的旧长度,必须通过文件句柄读取。
|
||||
handle, err := w.root.Open(entry.Name())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, statErr := handle.Stat()
|
||||
if err := errors.Join(statErr, handle.Close()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, File{Name: entry.Name(), Bytes: info.Size(), Compressed: strings.HasSuffix(entry.Name(), ".gz")})
|
||||
}
|
||||
sort.Slice(files, func(i, j int) bool { return files[i].Name > files[j].Name })
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
package runtimelog
|
||||
|
||||
import (
|
||||
"container/heap"
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/utils/logpolicy"
|
||||
)
|
||||
|
||||
type Query struct {
|
||||
File string
|
||||
From time.Time
|
||||
To time.Time
|
||||
Offset int
|
||||
Limit int
|
||||
}
|
||||
|
||||
type lineRef struct {
|
||||
at time.Time
|
||||
file int
|
||||
position int
|
||||
bytes int
|
||||
}
|
||||
|
||||
// 索引只保存时间与行位置,不把历史日志全文装入内存。
|
||||
type recentLines []lineRef
|
||||
|
||||
func (h recentLines) Len() int { return len(h) }
|
||||
func (h recentLines) Less(i, j int) bool {
|
||||
if !h[i].at.Equal(h[j].at) {
|
||||
return h[i].at.Before(h[j].at)
|
||||
}
|
||||
if h[i].file != h[j].file {
|
||||
return h[i].file > h[j].file
|
||||
}
|
||||
return h[i].position < h[j].position
|
||||
}
|
||||
func (h recentLines) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
|
||||
func (h *recentLines) Push(value any) { *h = append(*h, value.(lineRef)) }
|
||||
func (h *recentLines) Pop() any {
|
||||
last := len(*h) - 1
|
||||
value := (*h)[last]
|
||||
*h = (*h)[:last]
|
||||
return value
|
||||
}
|
||||
|
||||
// Query 跨日流式扫描并按真实时间倒序,兼容压缩文件和同一文件中的乱序写入。
|
||||
// 额外内存由最大偏移限制的行索引和两兆响应组成,不随日志文件体积增长。
|
||||
func (w *Writer) Query(ctx context.Context, query Query) (Page, error) {
|
||||
if query.File != "" {
|
||||
if _, ok := fileDay(query.File); !ok {
|
||||
return Page{}, ErrInvalid
|
||||
}
|
||||
}
|
||||
if query.Offset < 0 || query.Offset > 100000 || query.Limit < 1 || query.Limit > 500 {
|
||||
return Page{}, ErrInvalid
|
||||
}
|
||||
if query.To.IsZero() {
|
||||
query.To = w.now()
|
||||
}
|
||||
if query.From.IsZero() {
|
||||
query.From = query.To.Add(-24 * time.Hour)
|
||||
}
|
||||
if query.From.After(query.To) {
|
||||
return Page{}, ErrInvalid
|
||||
}
|
||||
query.From = maxTime(query.From, logpolicy.Cutoff(w.now()))
|
||||
// 等待归档时也响应取消,避免查询超时后继续占用 HTTP 工作协程。
|
||||
for !w.archiveMu.TryLock() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return Page{}, ctx.Err()
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
defer w.archiveMu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Page{}, err
|
||||
}
|
||||
// 在短写锁内冻结文件长度;两次扫描不会读到新追加的半条记录。
|
||||
w.mu.Lock()
|
||||
files, err := w.files()
|
||||
w.mu.Unlock()
|
||||
if err != nil {
|
||||
return Page{}, err
|
||||
}
|
||||
page := Page{Lines: make([]string, 0, query.Limit)}
|
||||
if query.From.After(query.To) {
|
||||
return page, nil
|
||||
}
|
||||
capacity := query.Offset + query.Limit + 1
|
||||
refs := make(recentLines, 0, capacity)
|
||||
for index, file := range files {
|
||||
if query.File != "" && query.File != file.Name {
|
||||
continue
|
||||
}
|
||||
day, _ := fileDay(file.Name)
|
||||
if day.After(query.To) || !day.Add(24*time.Hour).After(query.From) {
|
||||
continue
|
||||
}
|
||||
position := 0
|
||||
err := w.scanSize(ctx, file.Name, file.Bytes, func(line []byte, at time.Time) error {
|
||||
position++
|
||||
if !at.UTC().Truncate(24 * time.Hour).Equal(day) {
|
||||
return ErrInvalid
|
||||
}
|
||||
if at.Before(query.From) || at.After(query.To) {
|
||||
return nil
|
||||
}
|
||||
visible, keep := presentLine(string(line))
|
||||
if !keep {
|
||||
return nil
|
||||
}
|
||||
candidate := lineRef{at: at, file: index, position: position, bytes: len(visible)}
|
||||
if len(refs) < capacity {
|
||||
heap.Push(&refs, candidate)
|
||||
} else if (recentLines{refs[0], candidate}).Less(0, 1) {
|
||||
refs[0] = candidate
|
||||
heap.Fix(&refs, 0)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return Page{}, err
|
||||
}
|
||||
}
|
||||
sort.Sort(sort.Reverse(refs))
|
||||
if query.Offset >= len(refs) {
|
||||
return page, nil
|
||||
}
|
||||
refs = refs[query.Offset:]
|
||||
page.HasMore = len(refs) > query.Limit
|
||||
refs = refs[:min(len(refs), query.Limit)]
|
||||
bytes := 0
|
||||
for index, ref := range refs {
|
||||
if bytes+ref.bytes > 2*1024*1024 {
|
||||
refs = refs[:index]
|
||||
page.HasMore = true
|
||||
break
|
||||
}
|
||||
bytes += ref.bytes
|
||||
}
|
||||
page.Lines = make([]string, len(refs))
|
||||
selected := make(map[int]map[int]int)
|
||||
for index, ref := range refs {
|
||||
if selected[ref.file] == nil {
|
||||
selected[ref.file] = make(map[int]int)
|
||||
}
|
||||
selected[ref.file][ref.position] = index
|
||||
}
|
||||
finished := errors.New("本页内容读取完成")
|
||||
for fileIndex, positions := range selected {
|
||||
position, remaining := 0, len(positions)
|
||||
file := files[fileIndex]
|
||||
err := w.scanSize(ctx, file.Name, file.Bytes, func(line []byte, at time.Time) error {
|
||||
position++
|
||||
index, ok := positions[position]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if !at.Equal(refs[index].at) || len(line) != refs[index].bytes {
|
||||
return ErrInvalid
|
||||
}
|
||||
page.Lines[index] = string(line)
|
||||
remaining--
|
||||
if remaining == 0 {
|
||||
return finished
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil && !errors.Is(err, finished) {
|
||||
return Page{}, err
|
||||
}
|
||||
if remaining != 0 {
|
||||
return Page{}, ErrInvalid
|
||||
}
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func maxTime(left, right time.Time) time.Time {
|
||||
if left.After(right) {
|
||||
return left
|
||||
}
|
||||
return right
|
||||
}
|
||||
@@ -86,15 +86,34 @@ func Start(root string) (*Writer, error) {
|
||||
func (w *Writer) Levels() []log.Level { return log.AllLevels }
|
||||
|
||||
func (w *Writer) Fire(entry *log.Entry) error {
|
||||
// 结构化字段和警告以上级别本身具有排错价值,不能只看消息是否为空。
|
||||
if entry.Level >= log.InfoLevel && len(entry.Data) == 0 {
|
||||
if logpolicy.EmptyMessage(entry.Message) || (entry.Level == log.InfoLevel && entry.Caller != nil && legacyStartupNoise(entry.Caller.File, entry.Message)) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// 复用已有格式器保留请求标识、错误字段和源位置;转义换行保证一条记录一行。
|
||||
data, err := entry.Logger.Formatter.Format(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return w.WriteAt(entry.Time, strings.TrimRight(string(data), "\r\n"))
|
||||
return w.writeAt(entry.Time, strings.TrimRight(string(data), "\r\n"))
|
||||
}
|
||||
|
||||
func (w *Writer) WriteAt(at time.Time, message string) error {
|
||||
if at.IsZero() {
|
||||
return ErrInvalid
|
||||
}
|
||||
if logpolicy.EmptyMessage(message) {
|
||||
return nil
|
||||
}
|
||||
if _, keep := presentLine("[" + at.UTC().Format(time.RFC3339Nano) + "] " + message); !keep {
|
||||
return nil
|
||||
}
|
||||
return w.writeAt(at, message)
|
||||
}
|
||||
|
||||
func (w *Writer) writeAt(at time.Time, message string) error {
|
||||
if at.IsZero() {
|
||||
return ErrInvalid
|
||||
}
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
package logpolicy
|
||||
|
||||
import "strings"
|
||||
|
||||
// EmptyMessage 仅识别空白和纯装饰线,不猜测业务文案、状态或未知格式的价值。
|
||||
func EmptyMessage(message string) bool {
|
||||
message = strings.TrimSpace(strings.ReplaceAll(message, "\x00", ""))
|
||||
if message == "" {
|
||||
return true
|
||||
}
|
||||
return len([]rune(message)) >= 3 && strings.Trim(message, "*-=─━_~ \t\r\n") == ""
|
||||
}
|
||||
Reference in new issue
Block a user