This commit is contained in:
1 parent
9bdf866a87
commit
5faadcd6eb
41 files changed
+1994
-59
No files matched your search
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"git.pcux.cn/xuans/Smail/config"
|
||||
"git.pcux.cn/xuans/Smail/db/migration"
|
||||
"git.pcux.cn/xuans/Smail/services/logstore"
|
||||
)
|
||||
|
||||
// PortableExecutor 使用版本化目标结构复制业务数据,不拼接用户提供的 SQL。
|
||||
@@ -220,7 +221,13 @@ func (e *PortableExecutor) Run(ctx context.Context, plan Plan, update func(Statu
|
||||
if err := update(StatusSwitching); err != nil {
|
||||
return err
|
||||
}
|
||||
err = e.switchConfig(ctx, plan.expectedGeneration, plan.targetConfig, plan.actor)
|
||||
// 源库此时仍持有写锁,切换审计必须写入已复制且校验完成的目标库。
|
||||
logStore, err := logstore.New(target, dialect)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switchContext := config.WithMigrationAuditWriter(ctx, &logstore.AuditWriter{Store: logStore})
|
||||
err = e.switchConfig(switchContext, plan.expectedGeneration, plan.targetConfig, plan.actor)
|
||||
if err != nil && isKnownSwitchRejection(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/config"
|
||||
"git.pcux.cn/xuans/Smail/utils/logpolicy"
|
||||
)
|
||||
|
||||
type AuditWriter struct{ Store *Store }
|
||||
|
||||
func (w *AuditWriter) WriteConfigAudit(ctx context.Context, event config.ConfigAuditEvent) error {
|
||||
if w == nil || w.Store == nil {
|
||||
return ErrUnavailable
|
||||
}
|
||||
normalized, err := config.NormalizeConfigAuditEvent(event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
payload, err := json.Marshal(normalized)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return w.Store.Write(ctx, Entry{
|
||||
ID: normalized.ID, CreatedAt: normalized.CreatedAt, Kind: "config",
|
||||
ActorID: normalized.ActorID, Action: string(normalized.Operation),
|
||||
Section: string(normalized.Section), Result: normalized.Result, Payload: payload,
|
||||
})
|
||||
}
|
||||
|
||||
func (w *AuditWriter) Query(ctx context.Context, query config.ConfigAuditQuery) ([]config.ConfigAuditEvent, error) {
|
||||
if w == nil || w.Store == nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
if err := config.ValidateConfigAuditQuery(query); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := w.Store.Query(ctx, Query{
|
||||
Kind: "config", ID: query.ID, From: query.From, To: query.To, ActorID: query.ActorID,
|
||||
Action: string(query.Operation), Section: string(query.Section), Limit: query.Limit,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items := make([]config.ConfigAuditEvent, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
var event config.ConfigAuditEvent
|
||||
if json.Unmarshal(entry.Payload, &event) != nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
items = append(items, event)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (w *AuditWriter) ExportJSON(ctx context.Context, query config.ConfigAuditQuery) ([]byte, error) {
|
||||
items, err := w.Query(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.MarshalIndent(items, "", " ")
|
||||
}
|
||||
|
||||
func (w *AuditWriter) ExportCSV(ctx context.Context, query config.ConfigAuditQuery) ([]byte, error) {
|
||||
items, err := w.Query(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return config.ExportConfigAuditCSV(items)
|
||||
}
|
||||
|
||||
func (w *AuditWriter) ImportLegacy(ctx context.Context, root string) error {
|
||||
if w == nil || w.Store == nil {
|
||||
return ErrUnavailable
|
||||
}
|
||||
legacy, err := config.NewFileAuditWriter(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return legacy.Visit(ctx, logpolicy.Cutoff(w.Store.now()), func(event config.ConfigAuditEvent) error {
|
||||
return w.WriteConfigAudit(ctx, event)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MailResult 记录投递尝试元数据,不复制正文、附件、认证信息或服务器错误原文。
|
||||
func MailResult(userID int, direction string, sender string, recipientCount, size int, result string) {
|
||||
if len(sender) > 320 {
|
||||
sender = "地址超出显示长度"
|
||||
}
|
||||
payload, _ := json.Marshal(struct {
|
||||
Direction string `json:"direction"`
|
||||
Sender string `json:"sender"`
|
||||
RecipientCount int `json:"recipientCount"`
|
||||
Bytes int `json:"bytes"`
|
||||
}{direction, sender, recipientCount, size})
|
||||
action := "接收邮件"
|
||||
if direction == "outbound" {
|
||||
action = "投递邮件"
|
||||
}
|
||||
if direction == "submission" {
|
||||
action = "客户端提交邮件"
|
||||
}
|
||||
Record(context.Background(), Entry{
|
||||
Kind: "mail", ActorID: strconv.Itoa(userID), CreatedAt: time.Now(), Action: action, Result: result, Payload: payload,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/utils/logpolicy"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var active atomic.Pointer[Store]
|
||||
|
||||
func Current() *Store { return active.Load() }
|
||||
|
||||
// Record 同步落库,不建立无限内存队列;故障必须进入运行日志。
|
||||
func Record(ctx context.Context, entry Entry) {
|
||||
store := Current()
|
||||
if store == nil {
|
||||
return
|
||||
}
|
||||
bounded, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
if err := store.Write(bounded, entry); err != nil {
|
||||
log.WithError(err).Error("业务日志写入失败")
|
||||
}
|
||||
}
|
||||
|
||||
type Runtime struct {
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
once sync.Once
|
||||
store *Store
|
||||
}
|
||||
|
||||
func Start(store *Store) *Runtime {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
runtime := &Runtime{cancel: cancel, done: make(chan struct{}), store: store}
|
||||
active.Store(store)
|
||||
go func() {
|
||||
defer close(runtime.done)
|
||||
ticker := time.NewTicker(logpolicy.MaintenanceInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
bounded, stop := context.WithTimeout(ctx, 2*time.Minute)
|
||||
_, err := store.Cleanup(bounded, time.Now())
|
||||
stop()
|
||||
if err != nil && ctx.Err() == nil {
|
||||
log.WithError(err).Error("过期业务日志清理失败,将在下一轮重试")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}()
|
||||
return runtime
|
||||
}
|
||||
|
||||
func (r *Runtime) Stop() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.once.Do(func() {
|
||||
active.CompareAndSwap(r.store, nil)
|
||||
r.cancel()
|
||||
<-r.done
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
// Package logstore 在当前业务数据库中保存操作与收发信日志。
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/db/migration"
|
||||
"git.pcux.cn/xuans/Smail/utils/logpolicy"
|
||||
)
|
||||
|
||||
var ErrUnavailable = errors.New("日志数据库不可用")
|
||||
var ErrInvalid = errors.New("日志参数无效")
|
||||
|
||||
type Entry struct {
|
||||
ID string `json:"id"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
Kind string `json:"kind"`
|
||||
ActorID string `json:"actorId"`
|
||||
Action string `json:"action"`
|
||||
Section string `json:"section"`
|
||||
Result string `json:"result"`
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
}
|
||||
|
||||
type Query struct {
|
||||
ID string
|
||||
Kind string
|
||||
ActorID string
|
||||
Action string
|
||||
Section string
|
||||
From time.Time
|
||||
To time.Time
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
dialect migration.Dialect
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func New(database *sql.DB, dialect migration.Dialect) (*Store, error) {
|
||||
if database == nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
if _, err := migration.ParseDialect(string(dialect)); err != nil {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
return &Store{db: database, dialect: dialect, now: time.Now}, nil
|
||||
}
|
||||
|
||||
func logTable(kind string) (string, error) {
|
||||
switch kind {
|
||||
case "config", "operation":
|
||||
return "operation_log", nil
|
||||
case "mail":
|
||||
return "mail_log", nil
|
||||
default:
|
||||
return "", ErrInvalid
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) bind(statement string) string {
|
||||
if s.dialect != migration.DialectPostgres {
|
||||
return statement
|
||||
}
|
||||
index := 0
|
||||
return replacePlaceholders(statement, &index)
|
||||
}
|
||||
|
||||
func replacePlaceholders(statement string, index *int) string {
|
||||
parts := strings.Split(statement, "?")
|
||||
var builder strings.Builder
|
||||
for position, part := range parts {
|
||||
if position > 0 {
|
||||
*index++
|
||||
fmt.Fprintf(&builder, "$%d", *index)
|
||||
}
|
||||
builder.WriteString(part)
|
||||
}
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
// Write 只接受业务点构造的脱敏载荷,禁止直接传入 HTTP 请求或邮件正文。
|
||||
func (s *Store) Write(ctx context.Context, entry Entry) error {
|
||||
if s == nil || s.db == nil {
|
||||
return ErrUnavailable
|
||||
}
|
||||
table, err := logTable(entry.Kind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.CreatedAt.IsZero() || len(entry.ID) > 80 || len(entry.ActorID) > 128 || len(entry.Action) > 64 || len(entry.Section) > 32 || len(entry.Result) > 16 || len(entry.Payload) > 256*1024 || !json.Valid(entry.Payload) {
|
||||
return ErrInvalid
|
||||
}
|
||||
// 迁移、重放都不能让过期记录重新进入保留窗口。
|
||||
if entry.CreatedAt.Before(logpolicy.Cutoff(s.now())) {
|
||||
return nil
|
||||
}
|
||||
if entry.ID == "" {
|
||||
var token [16]byte
|
||||
if _, err := rand.Read(token[:]); err != nil {
|
||||
return ErrUnavailable
|
||||
}
|
||||
entry.ID = "log-" + hex.EncodeToString(token[:])
|
||||
}
|
||||
statement := "INSERT INTO " + table + " (id, created_at, kind, actor_id, action, section, result, payload) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
|
||||
// 仅忽略同一稳定标识的重放,不用 INSERT IGNORE 掩盖截断、缺字段等错误。
|
||||
if s.dialect == migration.DialectMySQL {
|
||||
statement += " ON DUPLICATE KEY UPDATE id = id"
|
||||
} else {
|
||||
statement += " ON CONFLICT (id) DO NOTHING"
|
||||
}
|
||||
_, err = s.db.ExecContext(ctx, s.bind(statement), entry.ID, entry.CreatedAt.UTC().UnixNano(), entry.Kind, entry.ActorID, entry.Action, entry.Section, entry.Result, string(entry.Payload))
|
||||
if err != nil {
|
||||
return ErrUnavailable
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) Query(ctx context.Context, query Query) ([]Entry, error) {
|
||||
if s == nil || s.db == nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
table, err := logTable(query.Kind)
|
||||
if err != nil || query.Limit < 0 || query.Limit > 1000 || query.Offset < 0 || query.Offset > 100000 || (!query.From.IsZero() && !query.To.IsZero() && query.From.After(query.To)) {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
from := logpolicy.Cutoff(s.now())
|
||||
if query.From.After(from) {
|
||||
from = query.From
|
||||
}
|
||||
statement := "SELECT id, created_at, kind, actor_id, action, section, result, payload FROM " + table + " WHERE created_at >= ? AND kind = ?"
|
||||
args := []any{from.UnixNano(), query.Kind}
|
||||
if !query.To.IsZero() {
|
||||
statement += " AND created_at <= ?"
|
||||
args = append(args, query.To.UnixNano())
|
||||
}
|
||||
for _, filter := range []struct{ column, value string }{
|
||||
{"id", query.ID}, {"actor_id", query.ActorID}, {"action", query.Action}, {"section", query.Section},
|
||||
} {
|
||||
if filter.value != "" {
|
||||
statement += " AND " + filter.column + " = ?"
|
||||
args = append(args, filter.value)
|
||||
}
|
||||
}
|
||||
limit := query.Limit
|
||||
if limit == 0 {
|
||||
limit = 100
|
||||
}
|
||||
statement += " ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?"
|
||||
args = append(args, limit, query.Offset)
|
||||
rows, err := s.db.QueryContext(ctx, s.bind(statement), args...)
|
||||
if err != nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]Entry, 0)
|
||||
for rows.Next() {
|
||||
var entry Entry
|
||||
var timestamp int64
|
||||
var payload string
|
||||
if err := rows.Scan(&entry.ID, ×tamp, &entry.Kind, &entry.ActorID, &entry.Action, &entry.Section, &entry.Result, &payload); err != nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
entry.CreatedAt = time.Unix(0, timestamp).UTC()
|
||||
entry.Payload = json.RawMessage(payload)
|
||||
items = append(items, entry)
|
||||
}
|
||||
if rows.Err() != nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// Cleanup 每批先按索引选取标识再删除,避免 MySQL 专属 DELETE LIMIT 和大事务锁表。
|
||||
func (s *Store) Cleanup(ctx context.Context, now time.Time) (int64, error) {
|
||||
if s == nil || s.db == nil {
|
||||
return 0, ErrUnavailable
|
||||
}
|
||||
if now.IsZero() {
|
||||
return 0, ErrInvalid
|
||||
}
|
||||
cutoff := logpolicy.Cutoff(now).UnixNano()
|
||||
var deleted int64
|
||||
for _, table := range []string{"operation_log", "mail_log"} {
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return deleted, err
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, s.bind("SELECT id FROM "+table+" WHERE created_at < ? ORDER BY created_at, id LIMIT ?"), cutoff, logpolicy.DeleteBatchSize)
|
||||
if err != nil {
|
||||
return deleted, ErrUnavailable
|
||||
}
|
||||
ids := make([]any, 0, logpolicy.DeleteBatchSize)
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err = rows.Scan(&id); err != nil {
|
||||
break
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
rowErr := rows.Err()
|
||||
closeErr := rows.Close()
|
||||
if err != nil || rowErr != nil || closeErr != nil {
|
||||
return deleted, ErrUnavailable
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
break
|
||||
}
|
||||
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",")
|
||||
args := append(ids, cutoff)
|
||||
result, err := s.db.ExecContext(ctx, s.bind("DELETE FROM "+table+" WHERE id IN ("+placeholders+") AND created_at < ?"), args...)
|
||||
if err != nil {
|
||||
return deleted, ErrUnavailable
|
||||
}
|
||||
count, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return deleted, ErrUnavailable
|
||||
}
|
||||
deleted += count
|
||||
}
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package runtimelog
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/utils/logpolicy"
|
||||
)
|
||||
|
||||
func fileDay(name string) (time.Time, bool) {
|
||||
base := strings.TrimSuffix(name, ".gz")
|
||||
if !strings.HasPrefix(base, prefix) || !strings.HasSuffix(base, ".log") {
|
||||
return time.Time{}, false
|
||||
}
|
||||
value := strings.TrimSuffix(strings.TrimPrefix(base, prefix), ".log")
|
||||
day, err := time.Parse("2006-01-02", value)
|
||||
return day, err == nil && (name == prefix+value+".log" || name == prefix+value+".log.gz")
|
||||
}
|
||||
|
||||
func lineTime(line []byte) (time.Time, error) {
|
||||
if len(line) < 3 || line[0] != '[' {
|
||||
return time.Time{}, ErrInvalid
|
||||
}
|
||||
end := strings.IndexByte(string(line), ']')
|
||||
if end < 0 {
|
||||
return time.Time{}, ErrInvalid
|
||||
}
|
||||
at, err := time.Parse(time.RFC3339Nano, string(line[1:end]))
|
||||
if err != nil {
|
||||
return time.Time{}, ErrInvalid
|
||||
}
|
||||
return at, nil
|
||||
}
|
||||
|
||||
// scan 严格解析每条记录及压缩校验和,损坏归档不得触发删除源文件。
|
||||
func (w *Writer) scan(ctx context.Context, name string, consume func([]byte, time.Time) error) error {
|
||||
if err := w.regular(name, false); err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := w.root.Open(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
var reader io.Reader = file
|
||||
if strings.HasSuffix(name, ".gz") {
|
||||
archive, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer archive.Close()
|
||||
reader = archive
|
||||
}
|
||||
scanner := bufio.NewScanner(reader)
|
||||
scanner.Buffer(make([]byte, 4096), maxLineBytes)
|
||||
for scanner.Scan() {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
at, err := lineTime(scanner.Bytes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := consume(scanner.Bytes(), at); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return scanner.Err()
|
||||
}
|
||||
|
||||
func (w *Writer) Maintain(ctx context.Context, now time.Time) error {
|
||||
if now.IsZero() {
|
||||
return ErrInvalid
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.closed {
|
||||
return nil
|
||||
}
|
||||
// 关闭当前文件后维护,避免 Windows 无法替换句柄;下一条日志会重新打开。
|
||||
if w.file != nil {
|
||||
err := errors.Join(w.file.Sync(), w.file.Close())
|
||||
w.file = nil
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
directory, err := w.root.Open(".")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries, err := directory.ReadDir(-1)
|
||||
closeErr := directory.Close()
|
||||
if err != nil || closeErr != nil {
|
||||
return errors.Join(err, closeErr)
|
||||
}
|
||||
cutoff := logpolicy.Cutoff(now)
|
||||
var failures []error
|
||||
for _, entry := range entries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
day, ok := fileDay(entry.Name())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
compress := !strings.HasSuffix(entry.Name(), ".gz") && !day.Add(24*time.Hour).After(now.Add(-logpolicy.CompressionAge))
|
||||
if !day.After(cutoff) || compress {
|
||||
if err := w.transform(ctx, entry.Name(), day, cutoff, compress); err != nil {
|
||||
failures = append(failures, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors.Join(failures...)
|
||||
}
|
||||
|
||||
// transform 流式过滤边界日;额外内存不随一天的日志量增长。
|
||||
func (w *Writer) transform(ctx context.Context, source string, day, cutoff time.Time, compress bool) error {
|
||||
target := source
|
||||
if compress {
|
||||
target += ".gz"
|
||||
}
|
||||
var token [16]byte
|
||||
if _, err := rand.Read(token[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
temporary := ".pending-" + hex.EncodeToString(token[:]) + ".log"
|
||||
if strings.HasSuffix(target, ".gz") {
|
||||
temporary += ".gz"
|
||||
}
|
||||
file, err := w.root.OpenFile(temporary, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close(); _ = w.root.Remove(temporary) }()
|
||||
var output io.Writer = file
|
||||
var archive *gzip.Writer
|
||||
if strings.HasSuffix(target, ".gz") {
|
||||
archive = gzip.NewWriter(file)
|
||||
output = archive
|
||||
}
|
||||
hash := sha256.New()
|
||||
output = io.MultiWriter(output, hash)
|
||||
kept, removed := 0, 0
|
||||
err = w.scan(ctx, source, func(line []byte, at time.Time) error {
|
||||
// 文件名和记录日期不一致时保留现场,不按文件名误删。
|
||||
if at.UTC().Format("2006-01-02") != day.Format("2006-01-02") {
|
||||
return ErrInvalid
|
||||
}
|
||||
if at.Before(cutoff) {
|
||||
removed++
|
||||
return nil
|
||||
}
|
||||
kept++
|
||||
_, err := output.Write(append(append([]byte(nil), line...), '\n'))
|
||||
return err
|
||||
})
|
||||
if archive != nil {
|
||||
err = errors.Join(err, archive.Close())
|
||||
}
|
||||
if err = errors.Join(err, file.Sync(), file.Close()); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if removed == 0 && !compress {
|
||||
return nil
|
||||
}
|
||||
if kept == 0 {
|
||||
if err := w.regular(source, false); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.root.Remove(source)
|
||||
}
|
||||
verified := sha256.New()
|
||||
if err := w.scan(ctx, temporary, func(line []byte, _ time.Time) error {
|
||||
_, err := verified.Write(append(append([]byte(nil), line...), '\n'))
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if !strings.EqualFold(hex.EncodeToString(hash.Sum(nil)), hex.EncodeToString(verified.Sum(nil))) {
|
||||
return ErrInvalid
|
||||
}
|
||||
if source != target {
|
||||
if _, err := w.root.Lstat(target); err == nil {
|
||||
// 压缩成功但删除前中断时,只有目标与本轮内容完全一致才允许补做删除。
|
||||
existing := sha256.New()
|
||||
if err := w.scan(ctx, target, func(line []byte, _ time.Time) error {
|
||||
_, err := existing.Write(append(append([]byte(nil), line...), '\n'))
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if hex.EncodeToString(existing.Sum(nil)) != hex.EncodeToString(hash.Sum(nil)) {
|
||||
return ErrInvalid
|
||||
}
|
||||
return w.root.Remove(source)
|
||||
} else if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := w.regular(target, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.root.Rename(temporary, target); err != nil {
|
||||
return err
|
||||
}
|
||||
if source != target {
|
||||
return w.root.Remove(source)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package runtimelog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/utils/logpolicy"
|
||||
)
|
||||
|
||||
type File struct {
|
||||
Name string `json:"name"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Compressed bool `json:"compressed"`
|
||||
}
|
||||
|
||||
type Page struct {
|
||||
Lines []string `json:"lines"`
|
||||
HasMore bool `json:"hasMore"`
|
||||
}
|
||||
|
||||
func (w *Writer) Files() ([]File, error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.closed {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
directory, err := w.root.Open(".")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer directory.Close()
|
||||
entries, err := directory.ReadDir(-1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files := make([]File, 0)
|
||||
for _, entry := range entries {
|
||||
day, ok := fileDay(entry.Name())
|
||||
if !ok || !day.Add(24*time.Hour).After(logpolicy.Cutoff(w.now())) {
|
||||
continue
|
||||
}
|
||||
if err := w.regular(entry.Name(), false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, File{Name: entry.Name(), Bytes: info.Size(), Compressed: strings.HasSuffix(entry.Name(), ".gz")})
|
||||
}
|
||||
sort.Slice(files, func(i, j int) bool { return files[i].Name > files[j].Name })
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// Read 仅允许固定名称的受管日志,不接受任意路径;压缩内容在服务端流式读取。
|
||||
func (w *Writer) Read(ctx context.Context, name string, offset, limit int) (Page, error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if _, ok := fileDay(name); !ok || offset < 0 || offset > 1000000 || limit < 1 || limit > 500 || w.closed {
|
||||
return Page{}, ErrInvalid
|
||||
}
|
||||
page := Page{Lines: make([]string, 0, limit)}
|
||||
cutoff := logpolicy.Cutoff(w.now())
|
||||
position := 0
|
||||
const maxResponseBytes = 2 * 1024 * 1024
|
||||
bytes := 0
|
||||
finished := errors.New("当前页读取完成")
|
||||
err := w.scan(ctx, name, func(line []byte, at time.Time) error {
|
||||
if at.Before(cutoff) {
|
||||
return nil
|
||||
}
|
||||
position++
|
||||
if position <= offset {
|
||||
return nil
|
||||
}
|
||||
if len(page.Lines) >= limit || bytes+len(line) > maxResponseBytes {
|
||||
page.HasMore = true
|
||||
return finished
|
||||
}
|
||||
page.Lines = append(page.Lines, string(line))
|
||||
bytes += len(line)
|
||||
return nil
|
||||
})
|
||||
if errors.Is(err, finished) {
|
||||
err = nil
|
||||
}
|
||||
return page, err
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
// Package runtimelog 管理按天写入、七天压缩和三十天滚动清理的运行日志。
|
||||
package runtimelog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/utils/logpolicy"
|
||||
"git.pcux.cn/xuans/Smail/utils/runtimepath"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const prefix = "runtime-"
|
||||
const maxLineBytes = 256 * 1024
|
||||
|
||||
var ErrInvalid = errors.New("运行日志文件或查询参数无效")
|
||||
var active atomic.Pointer[Writer]
|
||||
|
||||
type Writer struct {
|
||||
mu sync.Mutex
|
||||
root *os.Root
|
||||
file *os.File
|
||||
day string
|
||||
now func() time.Time
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
closed bool
|
||||
}
|
||||
|
||||
func Current() *Writer { return active.Load() }
|
||||
|
||||
func New(root string) (*Writer, error) {
|
||||
directory, err := runtimepath.SecureDirectory(root, "log")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
directory, err = runtimepath.SecureDirectory(directory, "运行日志")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
handle, err := os.OpenRoot(directory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Writer{root: handle, now: time.Now}, nil
|
||||
}
|
||||
|
||||
// Start 使用日志钩子,配置重载重设 stdout 时不会丢失文件输出。
|
||||
func Start(root string) (*Writer, error) {
|
||||
w, err := New(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
w.cancel, w.done = cancel, make(chan struct{})
|
||||
active.Store(w)
|
||||
log.AddHook(w)
|
||||
go func() {
|
||||
defer close(w.done)
|
||||
ticker := time.NewTicker(logpolicy.MaintenanceInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
bounded, stop := context.WithTimeout(ctx, 2*time.Minute)
|
||||
err := w.Maintain(bounded, w.now())
|
||||
stop()
|
||||
if err != nil && ctx.Err() == nil {
|
||||
log.WithError(err).Error("运行日志维护失败,源文件保留并在下一轮重试")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}()
|
||||
return w, nil
|
||||
}
|
||||
|
||||
func (w *Writer) Levels() []log.Level { return log.AllLevels }
|
||||
|
||||
func (w *Writer) Fire(entry *log.Entry) error {
|
||||
// 复用已有格式器保留请求标识、错误字段和源位置;转义换行保证一条记录一行。
|
||||
data, err := entry.Logger.Formatter.Format(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return w.WriteAt(entry.Time, strings.TrimRight(string(data), "\r\n"))
|
||||
}
|
||||
|
||||
func (w *Writer) WriteAt(at time.Time, message string) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.closed {
|
||||
return nil
|
||||
}
|
||||
if at.IsZero() {
|
||||
return ErrInvalid
|
||||
}
|
||||
if at.Before(logpolicy.Cutoff(w.now())) {
|
||||
return nil
|
||||
}
|
||||
message = strings.NewReplacer("\r", "\\r", "\n", "\\n", "\x00", "").Replace(message)
|
||||
if len(message) > maxLineBytes-128 {
|
||||
message = string([]rune(message)[:min(len([]rune(message)), (maxLineBytes-128)/4)]) + "…"
|
||||
}
|
||||
day := at.UTC().Format("2006-01-02")
|
||||
if w.file == nil || w.day != day {
|
||||
if w.file != nil {
|
||||
if err := errors.Join(w.file.Sync(), w.file.Close()); err != nil {
|
||||
w.file = nil
|
||||
return err
|
||||
}
|
||||
w.file = nil
|
||||
}
|
||||
name := prefix + day + ".log"
|
||||
if err := w.regular(name, true); err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := w.root.OpenFile(name, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.file, w.day = file, day
|
||||
}
|
||||
_, err := fmt.Fprintf(w.file, "[%s] %s\n", at.UTC().Format(time.RFC3339Nano), message)
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) regular(name string, allowMissing bool) error {
|
||||
if !filepath.IsLocal(name) || filepath.Base(name) != name {
|
||||
return ErrInvalid
|
||||
}
|
||||
info, err := w.root.Lstat(name)
|
||||
if allowMissing && os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return ErrInvalid
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) Close() error {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
if w.cancel != nil {
|
||||
w.cancel()
|
||||
<-w.done
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.closed {
|
||||
return nil
|
||||
}
|
||||
w.closed = true
|
||||
active.CompareAndSwap(w, nil)
|
||||
var err error
|
||||
if w.file != nil {
|
||||
err = errors.Join(w.file.Sync(), w.file.Close())
|
||||
}
|
||||
return errors.Join(err, w.root.Close())
|
||||
}
|
||||
Reference in new issue
Block a user