保存三数据库日志与三十天清理实现,继续补充验收
Security Checks / security (push) Waiting to run

This commit is contained in:
xuans committed 2026-09-08 22:30:50 +08:00
1 parent 9bdf866a87
commit 5faadcd6eb
41 files changed
+1994 -59

No files matched your search

+66 -20
View File
@@ -4,15 +4,16 @@ import (
"encoding/json"
"reflect"
"sort"
"strconv"
"strings"
)
// ConfigAuditChange 只保存安全的显示值,不保存密文或请求原文。
type ConfigAuditChange struct {
Field string `json:"field"`
Before string `json:"before"`
After string `json:"after"`
Sensitive bool `json:"sensitive,omitempty"`
Field string `json:"field"`
Before string `json:"before"`
After string `json:"after"`
Sensitive bool `json:"sensitive,omitempty"`
}
// AuditChanges 对比业务字段,排除代次、摘要和内部持久化元数据。
@@ -20,20 +21,32 @@ func AuditChanges(before, after RuntimeConfig) []ConfigAuditChange {
left := auditFields(before)
right := auditFields(after)
keys := make(map[string]bool)
for key := range left { keys[key] = true }
for key := range right { keys[key] = true }
for key := range left {
keys[key] = true
}
for key := range right {
keys[key] = true
}
ordered := make([]string, 0, len(keys))
for key := range keys { ordered = append(ordered, key) }
for key := range keys {
ordered = append(ordered, key)
}
sort.Strings(ordered)
changes := make([]ConfigAuditChange, 0)
for _, key := range ordered {
if reflect.DeepEqual(left[key], right[key]) { continue }
if reflect.DeepEqual(left[key], right[key]) {
continue
}
change := ConfigAuditChange{Field: key, Before: auditDisplay(left[key]), After: auditDisplay(right[key])}
if auditSensitive(key) {
change.Sensitive = true
change.Before, change.After = "未配置", "未配置"
if auditConfigured(left[key]) { change.Before = "已配置(隐藏)" }
if auditConfigured(right[key]) { change.After = "已配置(隐藏,已更新)" }
if auditConfigured(left[key]) {
change.Before = "已配置(隐藏)"
}
if auditConfigured(right[key]) {
change.After = "已配置(隐藏,已更新)"
}
}
changes = append(changes, change)
}
@@ -60,10 +73,23 @@ func flattenAudit(result map[string]any, path string, value any) {
case "meta", "sections", "generation", "updatedAt", "schemaVersion", "configVersion", "version":
return
}
if path == "root.smtp" || path == "root.email" || path == "root.ssl" || path == "root.plugins" { return }
if auditSensitive(path) { result[path] = value; return }
if path == "root.smtp" || path == "root.email" || path == "root.ssl" || path == "root.plugins" {
return
}
if auditSensitive(path) {
result[path] = value
return
}
if items, ok := value.([]any); ok {
for index, item := range items {
flattenAudit(result, path+"."+strconv.Itoa(index), item)
}
return
}
if object, ok := value.(map[string]any); ok {
for key, item := range object { flattenAudit(result, path+"."+key, item) }
for key, item := range object {
flattenAudit(result, path+"."+key, item)
}
return
}
result[path] = value
@@ -72,30 +98,50 @@ func flattenAudit(result map[string]any, path string, value any) {
func auditSensitive(path string) bool {
name := strings.ToLower(path[strings.LastIndex(path, ".")+1:])
for _, word := range []string{"password", "secret", "token", "apikey", "api_key", "credential", "ciphertext", "authorization", "cookie"} {
if strings.Contains(name, word) { return true }
if strings.Contains(name, word) {
return true
}
}
// 插件设置是自由结构,未知键也不能作为明文审计输出。
return strings.HasSuffix(path, ".settings") || strings.HasSuffix(path, ".encryptionKeyRef")
}
func auditConfigured(value any) bool {
if value == nil || value == "" { return false }
if value == nil {
return false
}
if text, ok := value.(string); ok {
return text != ""
}
if object, ok := value.(map[string]any); ok {
if ciphertext, exists := object["ciphertext"]; exists { return ciphertext != "" }
if ciphertext, exists := object["ciphertext"]; exists {
return ciphertext != ""
}
return len(object) > 0
}
return true
}
func auditDisplay(value any) string {
if value == nil { return "未设置" }
if value == nil {
return "未设置"
}
if text, ok := value.(string); ok {
if len([]rune(text)) > 512 { return string([]rune(text)[:512])+"…" }
if len([]rune(text)) > 512 {
return string([]rune(text)[:512]) + "…"
}
return text
}
if flag, ok := value.(bool); ok { if flag { return "开启" }; return "关闭" }
if flag, ok := value.(bool); ok {
if flag {
return "开启"
}
return "关闭"
}
data, _ := json.Marshal(value)
text := []rune(string(data))
if len(text) > 512 { return string(text[:512])+"…" }
if len(text) > 512 {
return string(text[:512]) + "…"
}
return string(text)
}
+21 -3
View File
@@ -2,6 +2,7 @@ package config
import (
"encoding/csv"
"encoding/json"
"fmt"
"strings"
"time"
@@ -10,11 +11,15 @@ import (
func exportAuditCSV(events []ConfigAuditEvent) ([]byte, error) {
var builder strings.Builder
writer := csv.NewWriter(&builder)
if err := writer.Write([]string{"id", "operation", "section", "actor_id", "actor_role", "from_generation", "to_generation", "target_generation", "changed_fields", "confirmation_method", "result", "created_at"}); err != nil {
if err := writer.Write([]string{"id", "operation", "section", "actor_id", "actor_role", "from_generation", "to_generation", "target_generation", "changed_fields", "confirmation_method", "result", "created_at", "actor_name", "changes"}); err != nil {
return nil, err
}
for _, event := range events {
if err := writer.Write([]string{
changes, err := json.Marshal(event.Changes)
if err != nil {
return nil, err
}
cells := []string{
event.ID,
string(event.Operation),
string(event.Section),
@@ -27,7 +32,15 @@ func exportAuditCSV(events []ConfigAuditEvent) ([]byte, error) {
event.ConfirmationMethod,
event.Result,
event.CreatedAt.UTC().Format(time.RFC3339Nano),
}); err != nil {
event.ActorName,
string(changes),
}
for index, cell := range cells {
if strings.HasPrefix(cell, "=") || strings.HasPrefix(cell, "+") || strings.HasPrefix(cell, "-") || strings.HasPrefix(cell, "@") || strings.HasPrefix(cell, "\t") || strings.HasPrefix(cell, "\r") {
cells[index] = "'" + cell
}
}
if err := writer.Write(cells); err != nil {
return nil, err
}
}
@@ -37,3 +50,8 @@ func exportAuditCSV(events []ConfigAuditEvent) ([]byte, error) {
}
return []byte(builder.String()), nil
}
// ExportConfigAuditCSV 让文件、内存和数据库导出共享字段及表格公式保护。
func ExportConfigAuditCSV(events []ConfigAuditEvent) ([]byte, error) {
return exportAuditCSV(events)
}
+15 -3
View File
@@ -17,10 +17,11 @@ import (
"time"
"git.pcux.cn/xuans/Smail/utils/atomicfile"
"git.pcux.cn/xuans/Smail/utils/logpolicy"
"git.pcux.cn/xuans/Smail/utils/runtimepath"
)
const ConfigAuditRetention = 365 * 24 * time.Hour
const ConfigAuditRetention = logpolicy.Retention
var (
ErrAuditStorageUnavailable = errors.New("configuration audit storage unavailable")
@@ -296,6 +297,7 @@ func normalizeAuditEvent(event ConfigAuditEvent) (ConfigAuditEvent, error) {
event.ActorRole = strings.TrimSpace(event.ActorRole)
event.ConfirmationMethod = strings.ToLower(strings.TrimSpace(event.ConfirmationMethod))
event.ChangedFields = append([]string(nil), event.ChangedFields...)
event.Changes = append([]ConfigAuditChange(nil), event.Changes...)
if err := validateAuditEvent(event); err != nil {
return ConfigAuditEvent{}, err
}
@@ -326,17 +328,27 @@ func validateAuditEvent(event ConfigAuditEvent) error {
if event.Result != "success" && event.Result != "failure" {
return ErrAuditEventInvalid
}
if strings.ContainsAny(event.ActorID+event.ActorRole+event.ConfirmationMethod, "\r\n") || len(event.ActorID) > 128 || len(event.ActorRole) > 64 {
if strings.ContainsAny(event.ActorID+event.ActorRole+event.ActorName+event.ConfirmationMethod, "\r\n") || len(event.ActorID) > 128 || len(event.ActorRole) > 64 || len(event.ActorName) > 256 {
return ErrAuditEventInvalid
}
for _, field := range event.ChangedFields {
if field == "" || len(field) > 64 || strings.ContainsAny(field, "\r\n") {
if field == "" || len(field) > 1024 || strings.ContainsAny(field, "\r\n") {
return ErrAuditEventInvalid
}
}
for _, change := range event.Changes {
if change.Field == "" || len(change.Field) > 1024 || strings.ContainsAny(change.Field, "\r\n") || len([]rune(change.Before)) > 513 || len([]rune(change.After)) > 513 {
return ErrAuditEventInvalid
}
}
return nil
}
// NormalizeConfigAuditEvent 供数据库审计复用同一套标识、校验和复制规则。
func NormalizeConfigAuditEvent(event ConfigAuditEvent) (ConfigAuditEvent, error) {
return normalizeAuditEvent(event)
}
func auditEventMatches(event ConfigAuditEvent, query ConfigAuditQuery) bool {
if query.ID != "" && event.ID != query.ID {
return false
+57
View File
@@ -0,0 +1,57 @@
package config
import (
"bufio"
"context"
"encoding/json"
"os"
"time"
)
// Visit 从旧文件流式迁移,不受管理界面的查询条数上限影响,不修改源文件。
func (w *FileAuditWriter) Visit(ctx context.Context, from time.Time, visit func(ConfigAuditEvent) error) error {
if w == nil || visit == nil {
return ErrAuditStorageUnavailable
}
w.mu.Lock()
defer w.mu.Unlock()
if info, err := os.Lstat(w.path); err != nil {
if os.IsNotExist(err) {
return nil
}
return ErrAuditStorageUnavailable
} else if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
return ErrAuditStorageUnavailable
}
file, err := os.Open(w.path)
if err != nil {
return ErrAuditStorageUnavailable
}
defer file.Close()
scanner := bufio.NewScanner(file)
scanner.Buffer(make([]byte, 4096), 256*1024)
line := 0
for scanner.Scan() {
if err := ctx.Err(); err != nil {
return err
}
line++
var event ConfigAuditEvent
if json.Unmarshal(scanner.Bytes(), &event) != nil {
return ErrAuditStorageUnavailable
}
if event.ID == "" {
event.ID = legacyAuditEventID(scanner.Bytes(), line)
}
if err := validateAuditEvent(event); err != nil {
return err
}
if event.CreatedAt.Before(from) {
continue
}
if err := visit(event); err != nil {
return err
}
}
return scanner.Err()
}
+17 -2
View File
@@ -9,6 +9,14 @@ import (
var ErrDatabaseMigrationTarget = errors.New("database migration target is invalid")
type migrationAuditKey struct{}
// WithMigrationAuditWriter 仅供持有迁移目标连接的执行器设置迁移审计去向。
// 普通配置更新不读取该上下文值,HTTP 请求不能指定审计数据库。
func WithMigrationAuditWriter(ctx context.Context, writer ConfigAuditWriter) context.Context {
return context.WithValue(ctx, migrationAuditKey{}, writer)
}
// SwitchDatabase 提交已完成复制与校验的目标连接,保留旧库并标记重启。
// 普通配置编辑不能绕过本迁移入口切换数据库类型。
func (m *ConfigManager) SwitchDatabase(ctx context.Context, expectedGeneration uint64, target DatabaseConfig, actor AuditActor) error {
@@ -91,7 +99,7 @@ func (m *ConfigManager) SwitchDatabase(ctx context.Context, expectedGeneration u
if !sameDatabaseConnection(current.Root.Database, target) {
changedField = "root.database"
}
return m.writeAudit(ctx, ConfigAuditEvent{
event := ConfigAuditEvent{
Operation: ConfigOperationMigration,
Section: SectionRoot,
ActorID: actor.UserID,
@@ -104,7 +112,14 @@ func (m *ConfigManager) SwitchDatabase(ctx context.Context, expectedGeneration u
ChangedFields: []string{changedField},
Result: "success",
CreatedAt: now,
})
}
if writer, ok := ctx.Value(migrationAuditKey{}).(ConfigAuditWriter); ok && writer != nil {
if err := writer.WriteConfigAudit(ctx, event); err != nil {
return errors.Join(ErrAuditWrite, err)
}
return nil
}
return m.writeAudit(ctx, event)
}
// SameDatabaseLocation 不把用户名或密码不同误认为不同数据库。
+3 -1
View File
@@ -777,7 +777,9 @@ func (m *ConfigManager) writeAudit(ctx context.Context, event ConfigAuditEvent)
func changedFields(current, candidate RuntimeConfig, section Section) []string {
var fields []string
for _, change := range AuditChanges(current, candidate) { fields = append(fields, change.Field) }
for _, change := range AuditChanges(current, candidate) {
fields = append(fields, change.Field)
}
return fields
}
+2 -1
View File
@@ -1771,7 +1771,8 @@ func TestConfigManagerWritesRedactedAuditAndDegradesOnAuditFailure(t *testing.T)
t.Fatal(err)
}
events := writer.Events()
if len(events) != 1 || events[0].Section != SectionEmail || len(events[0].ChangedFields) != 1 || events[0].ChangedFields[0] != string(SectionEmail) {
// 原样保存只记录操作,不能伪造业务字段发生变更。
if len(events) != 1 || events[0].Section != SectionEmail || len(events[0].ChangedFields) != 0 || len(events[0].Changes) != 0 {
t.Fatalf("unexpected audit events: %+v", events)
}
if events[0].Result != "success" || events[0].ToGeneration != 2 {
+15 -13
View File
@@ -26,20 +26,20 @@ const (
)
type ConfigAuditEvent struct {
ID string `json:"id"`
Operation ConfigOperation `json:"operation"`
Section Section `json:"section,omitempty"`
ActorID string `json:"actorId"`
ActorRole string `json:"actorRole"`
ActorName string `json:"actorName,omitempty"`
ID string `json:"id"`
Operation ConfigOperation `json:"operation"`
Section Section `json:"section,omitempty"`
ActorID string `json:"actorId"`
ActorRole string `json:"actorRole"`
ActorName string `json:"actorName,omitempty"`
Changes []ConfigAuditChange `json:"changes,omitempty"`
FromGeneration uint64 `json:"fromGeneration"`
ToGeneration uint64 `json:"toGeneration"`
TargetGeneration uint64 `json:"targetGeneration,omitempty"`
ChangedFields []string `json:"changedFields,omitempty"`
ConfirmationMethod string `json:"confirmationMethod,omitempty"`
Result string `json:"result"`
CreatedAt time.Time `json:"createdAt"`
FromGeneration uint64 `json:"fromGeneration"`
ToGeneration uint64 `json:"toGeneration"`
TargetGeneration uint64 `json:"targetGeneration,omitempty"`
ChangedFields []string `json:"changedFields,omitempty"`
ConfirmationMethod string `json:"confirmationMethod,omitempty"`
Result string `json:"result"`
CreatedAt time.Time `json:"createdAt"`
}
type ConfigAuditWriter interface {
WriteConfigAudit(context.Context, ConfigAuditEvent) error
@@ -107,6 +107,7 @@ func (w *MemoryAuditWriter) Query(ctx context.Context, query ConfigAuditQuery) (
}
copyEvent := event
copyEvent.ChangedFields = append([]string(nil), event.ChangedFields...)
copyEvent.Changes = append([]ConfigAuditChange(nil), event.Changes...)
result = append(result, copyEvent)
if len(result) >= limit {
break
@@ -141,6 +142,7 @@ func (w *MemoryAuditWriter) Events() []ConfigAuditEvent {
copy(result, w.events)
for index := range result {
result[index].ChangedFields = append([]string(nil), result[index].ChangedFields...)
result[index].Changes = append([]ConfigAuditChange(nil), result[index].Changes...)
}
return result
}
+93
View File
@@ -0,0 +1,93 @@
package controllers
import (
"context"
"net/http"
"strconv"
"time"
"git.pcux.cn/xuans/Smail/config"
"git.pcux.cn/xuans/Smail/dto/response"
"git.pcux.cn/xuans/Smail/services/access"
"git.pcux.cn/xuans/Smail/services/logstore"
"git.pcux.cn/xuans/Smail/services/runtimelog"
appcontext "git.pcux.cn/xuans/Smail/utils/context"
)
func logQueryError(w http.ResponseWriter, err error) {
response.NewErrorResponse(response.ServerError, "日志查询失败,请检查日志存储状态", "LOG_QUERY_FAILED").FPrint(w)
}
// GetLogs 只读日志入口,授权在服务端检查,不依赖导航隐藏。
func GetLogs(ctx *appcontext.Context, w http.ResponseWriter, req *http.Request) {
if err := authorizeCapability(ctx, access.CapabilityAuditRead, "", false); err != nil {
writeConfigAccessError(w, err)
return
}
values := req.URL.Query()
limit, offset := 50, 0
var err error
if values.Has("limit") {
limit, err = strconv.Atoi(values.Get("limit"))
}
if err == nil && values.Has("offset") {
offset, err = strconv.Atoi(values.Get("offset"))
}
if err != nil || limit < 1 || limit > 500 || offset < 0 || offset > 100000 {
writeConfigError(w, config.ErrInvalidAuditQuery)
return
}
bounded, cancel := context.WithTimeout(req.Context(), 10*time.Second)
defer cancel()
if values.Get("kind") == "runtime" {
writer := runtimelog.Current()
if writer == nil {
logQueryError(w, runtimelog.ErrInvalid)
return
}
if values.Get("file") == "" {
files, err := writer.Files()
if err != nil {
logQueryError(w, err)
return
}
response.NewSuccessResponse(map[string]any{"files": files}).FPrint(w)
return
}
page, err := writer.Read(bounded, values.Get("file"), offset, limit)
if err != nil {
logQueryError(w, err)
return
}
response.NewSuccessResponse(page).FPrint(w)
return
}
query := logstore.Query{Kind: values.Get("kind"), ActorID: values.Get("actorId"), Limit: limit + 1, Offset: offset}
for _, item := range []struct {
name string
target *time.Time
}{{"from", &query.From}, {"to", &query.To}} {
if values.Get(item.name) != "" {
*item.target, err = time.Parse(time.RFC3339Nano, values.Get(item.name))
if err != nil {
writeConfigError(w, config.ErrInvalidAuditQuery)
return
}
}
}
if query.Kind != "config" && query.Kind != "operation" && query.Kind != "mail" {
writeConfigError(w, config.ErrInvalidAuditQuery)
return
}
store := logstore.Current()
items, err := store.Query(bounded, query)
if err != nil {
logQueryError(w, err)
return
}
hasMore := len(items) > limit
if hasMore {
items = items[:limit]
}
response.NewSuccessResponse(map[string]any{"items": items, "hasMore": hasMore, "retentionDays": 30}).FPrint(w)
}
+27
View File
@@ -0,0 +1,27 @@
package migration
// 日志使用可移植的数值时间和文本载荷,不依赖数据库专属 JSON、分区或事件调度器。
func logStatements() map[Dialect][]string {
result := make(map[Dialect][]string)
for _, dialect := range []Dialect{DialectSQLite, DialectMySQL, DialectPostgres} {
for _, table := range []string{"operation_log", "mail_log"} {
payloadType := "TEXT"
if dialect == DialectMySQL {
payloadType = "LONGTEXT"
}
statement := "CREATE TABLE IF NOT EXISTS " + table + " (id VARCHAR(80) PRIMARY KEY, created_at BIGINT NOT NULL, kind VARCHAR(32) NOT NULL, actor_id VARCHAR(128) NOT NULL, action VARCHAR(64) NOT NULL, section VARCHAR(32) NOT NULL, result VARCHAR(16) NOT NULL, payload " + payloadType + " NOT NULL"
if dialect == DialectMySQL {
statement += ", KEY idx_" + table + "_time (created_at, id), KEY idx_" + table + "_kind_time (kind, created_at, id)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4"
} else {
statement += ")"
}
result[dialect] = append(result[dialect], statement)
if dialect != DialectMySQL {
result[dialect] = append(result[dialect],
"CREATE INDEX IF NOT EXISTS \"idx_"+table+"_time\" ON "+table+" (created_at, id)",
"CREATE INDEX IF NOT EXISTS \"idx_"+table+"_kind_time\" ON "+table+" (kind, created_at, id)")
}
}
}
return result
}
+1
View File
@@ -26,6 +26,7 @@ func DefaultMigrations() []Migration {
DefineCustom(14, "expand_imap_deleted_flag", "expand_imap_deleted_flag_v1", applyIMAPDeletedFlagSchema),
DefineCustom(15, "expand_security_setup", "expand_security_setup_v1", applySecuritySetupSchema),
DefineCustom(16, "expand_user_security_preferences", "expand_user_security_preferences_v1", applyUserSecurityPreferencesSchema),
DefineForDialects(17, "create_managed_logs", logStatements()),
}
}
+1
View File
@@ -220,6 +220,7 @@ func TestDefaultMigrationChecksumsAreImmutable(t *testing.T) {
14: "531398c2b8fc83a65753fae08666946bb253ac3da6596829e71cc8ccc4ddf62f",
15: "59f0ff9acfed3619c17f078d21a1362ddd5f1feacb9a2d77626f69040d25de65",
16: "00cc275529e522d93b41e608692fb6e2271f1aa9300ebe020fcaed4d68d0ff76",
17: "1dad66acb5e7fecf6c39f4a78526b994748d90045220f446a2e80c1b5fd881e7",
}
migrations := DefaultMigrations()
if len(migrations) != len(expected) {
+3
View File
@@ -28,6 +28,9 @@ func (p *Response) FPrint(w http.ResponseWriter) {
http.Error(w, "\u54cd\u5e94\u5e8f\u5217\u5316\u5931\u8d25", http.StatusInternalServerError)
return
}
if observer, ok := w.(interface{ ObserveResult(int) }); ok {
observer.ObserveResult(p.ErrorNo)
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
if _, err := w.Write(bytesData); err != nil {
log.WithError(err).Error("write HTTP response failed")
+6
View File
@@ -8,6 +8,7 @@ import (
"git.pcux.cn/xuans/Smail/internal/buildinfo"
"git.pcux.cn/xuans/Smail/listen/cron_server"
"git.pcux.cn/xuans/Smail/res_init"
"git.pcux.cn/xuans/Smail/services/runtimelog"
"git.pcux.cn/xuans/Smail/utils/runtimepath"
log "github.com/sirupsen/logrus"
)
@@ -23,6 +24,11 @@ func Run() error {
if err := config.Init(); err != nil {
return err
}
runtimeLogs, err := runtimelog.Start(config.ROOT_PATH)
if err != nil {
return fmt.Errorf("启动运行日志失败: %w", err)
}
defer runtimeLogs.Close()
config.MaintainManagedSSLHistory()
build := buildinfo.Current()
@@ -110,6 +110,7 @@ func router(mux *http.ServeMux) error {
mux.HandleFunc("/api/v1/config/", limitBody(4<<20, contextIterceptor(configRoute)))
mux.HandleFunc("/api/v1/audits/config", limitBody(1<<20, contextIterceptor(configAuditRoute)))
mux.HandleFunc("/api/v1/audits/config/", limitBody(1<<20, contextIterceptor(configAuditSubRoute)))
mux.HandleFunc("/api/v1/logs", onlyMethod(http.MethodGet, contextIterceptor(controllers.GetLogs)))
mux.HandleFunc("/api/v1/database/migrations/preview", onlyMethod(http.MethodPost, limitBody(16<<10, contextIterceptor(controllers.PreviewDatabaseMigration))))
mux.HandleFunc("/api/v1/database/migrations", onlyMethod(http.MethodPost, limitBody(16<<10, contextIterceptor(controllers.ExecuteDatabaseMigration))))
mux.HandleFunc("/api/v1/database/migrations/", contextIterceptor(databaseMigrationSubRoute))
+3 -1
View File
@@ -177,7 +177,9 @@ func contextIterceptor(h controllers.HandlerFunc) http.HandlerFunc {
response.NewErrorResponse(response.NeedSetup, "", "").FPrint(w)
return
}
h(ctx, w, r)
observed, finish := operationLog(ctx, r, w)
defer finish()
h(ctx, observed, r)
})).ServeHTTP(w, r)
}
}
@@ -0,0 +1,71 @@
package http_server
import (
"context"
"encoding/json"
"net/http"
"strconv"
"time"
"git.pcux.cn/xuans/Smail/services/logstore"
appcontext "git.pcux.cn/xuans/Smail/utils/context"
)
// 仅记录明确的变更接口,不将 POST 列表查询误认为操作,也不复制请求或响应载荷。
var loggedOperations = map[string]string{
"/api/login": "处理账号登录", "/api/logout": "退出登录",
"/api/user/create": "创建用户", "/api/user/edit": "编辑用户",
"/api/group/add": "创建分组", "/api/group/del": "删除分组",
"/api/email/del": "删除邮件", "/api/email/move": "移动邮件", "/api/email/read": "修改已读状态",
"/api/rule/add": "创建邮件规则", "/api/rule/update": "修改邮件规则", "/api/rule/del": "删除邮件规则",
"/api/settings/modify_password": "修改账号密码",
"/api/security/credentials/recovery-codes/generate": "生成安全码",
"/api/security/credentials/recovery-codes/revoke": "撤销安全码",
"/api/security/external-email/bind": "申请绑定外部邮箱",
"/api/security/external-email/verify": "验证外部邮箱",
"/api/security/external-email/change": "申请修改外部邮箱",
"/api/security/external-email/change/verify": "验证外部邮箱修改",
"/api/security/external-email/disable": "解绑外部邮箱",
"/api/security/credentials/protocols/enabled": "设置邮件客户端授权",
"/api/security/credentials/protocols/generate": "生成邮件客户端安全码",
"/api/security/credentials/protocols/revoke": "撤销邮件客户端安全码",
"/api/security/recovery/complete": "完成账号恢复",
"/api/user/security/recovery-codes/generate": "管理用户安全码",
"/api/user/security/recovery-codes/revoke": "撤销用户安全码",
"/api/user/security/external-email/change": "申请修改用户外部邮箱",
"/api/user/security/external-email/verify": "验证用户外部邮箱修改",
}
type operationResponse struct {
http.ResponseWriter
code int
known bool
}
func (w *operationResponse) Unwrap() http.ResponseWriter { return w.ResponseWriter }
func (w *operationResponse) ObserveResult(code int) { w.code, w.known = code, true }
func operationLog(ctx *appcontext.Context, req *http.Request, w http.ResponseWriter) (http.ResponseWriter, func()) {
action := loggedOperations[normalizedAPIPath(req.URL.Path)]
if action == "" || !stateChangingMethod(req.Method) {
return w, func() {}
}
observed := &operationResponse{ResponseWriter: w}
return observed, func() {
result := "unknown"
if observed.known {
result = "success"
if observed.code != 0 {
result = "failure"
}
}
payload, _ := json.Marshal(struct {
ActorName string `json:"actorName"`
ActorRole string `json:"actorRole"`
ErrorCode int `json:"errorCode"`
}{ctx.UserAccount, string(ctx.EffectiveRole()), observed.code})
logstore.Record(context.Background(), logstore.Entry{
Kind: "operation", CreatedAt: time.Now(), ActorID: strconv.Itoa(ctx.UserID), Action: action, Result: result, Payload: payload,
})
}
}
+14 -1
View File
@@ -20,6 +20,7 @@ import (
"git.pcux.cn/xuans/Smail/hooks/framework"
"git.pcux.cn/xuans/Smail/listen/imap_server"
"git.pcux.cn/xuans/Smail/models"
"git.pcux.cn/xuans/Smail/services/logstore"
"git.pcux.cn/xuans/Smail/services/rule"
"git.pcux.cn/xuans/Smail/utils/array"
"git.pcux.cn/xuans/Smail/utils/async"
@@ -38,13 +39,25 @@ import (
// 注意:此开关只能通过修改代码来更改,不暴露给用户配置
const DropUnknownRecipientEmails = true
func (s *Session) Data(r io.Reader) error {
func (s *Session) Data(r io.Reader) (resultErr error) {
messageBytes := 0
defer func() {
result, direction := "success", "inbound"
if resultErr != nil {
result = "failure"
}
if s.Ctx.UserID > 0 {
direction = "submission"
}
logstore.MailResult(s.Ctx.UserID, direction, s.From, len(s.To), messageBytes, result)
}()
ctx := s.Ctx
log.WithContext(ctx).Debugf("收到邮件")
emailData, err := io.ReadAll(io.LimitReader(r, maxSMTPMessageBytes+1))
messageBytes = len(emailData)
if err != nil {
log.WithContext(ctx).Error("邮件内容无法读取", err)
return err
+30 -2
View File
@@ -12,6 +12,7 @@ import (
"git.pcux.cn/xuans/Smail/config"
"git.pcux.cn/xuans/Smail/controllers"
"git.pcux.cn/xuans/Smail/db"
"git.pcux.cn/xuans/Smail/db/migration"
"git.pcux.cn/xuans/Smail/dto/parsemail"
"git.pcux.cn/xuans/Smail/hooks"
"git.pcux.cn/xuans/Smail/listen/http_server"
@@ -23,6 +24,7 @@ import (
"git.pcux.cn/xuans/Smail/services/credentials"
"git.pcux.cn/xuans/Smail/services/dbmigration"
"git.pcux.cn/xuans/Smail/services/externalemail"
"git.pcux.cn/xuans/Smail/services/logstore"
pluginservice "git.pcux.cn/xuans/Smail/services/plugin"
"git.pcux.cn/xuans/Smail/services/recovery"
"git.pcux.cn/xuans/Smail/services/restart"
@@ -44,6 +46,7 @@ var adminResetServeErrors chan error
var verificationRuntime *verification.Runtime
var credentialService *credentials.Service
var webAuthnRuntimeService *webauthnservice.Service
var logRuntime *logstore.Runtime
func startConfigManager() error {
root := filepath.Join(config.ROOT_PATH, "config")
@@ -72,10 +75,20 @@ func startConfigManager() error {
dbmigration.SetCurrent(nil)
return nil
}
manager, err := config.NewConfigManager(config.ROOT_PATH, config.ConfigLoaderOptions{Validation: config.RuntimeValidationOptions{
security := config.ConfigSecurity{}
if store := logstore.Current(); store != nil {
audit := &logstore.AuditWriter{Store: store}
importCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
if err := audit.ImportLegacy(importCtx, config.ROOT_PATH); err != nil {
return fmt.Errorf("迁移旧配置审计失败,源文件已保留: %w", err)
}
security.Audit = audit
}
manager, err := config.NewConfigManagerWithSecurity(config.ROOT_PATH, config.ConfigLoaderOptions{Validation: config.RuntimeValidationOptions{
HasSuperAdmin: true,
Production: !config.DevelopmentMode,
}})
}}, security)
if err != nil {
return fmt.Errorf("load partitioned configuration: %w", err)
}
@@ -97,6 +110,8 @@ func startConfigManager() error {
}
func stopConfigManager() {
logRuntime.Stop()
logRuntime = nil
dbmigration.Current().Close()
config.SetRuntimeManager(nil)
dbmigration.SetCurrent(nil)
@@ -166,7 +181,18 @@ func Init(serverVersion string) error {
if err := db.Init(serverVersion); err != nil {
return err
}
dialect, err := migration.ParseDialect(cfg.DbType)
if err != nil {
return err
}
store, err := logstore.New(db.Instance.DB().DB, dialect)
if err != nil {
return err
}
logRuntime = logstore.Start(store)
if err := startConfigManager(); err != nil {
logRuntime.Stop()
logRuntime = nil
return err
}
if err := startPluginConfigMigration(); err != nil {
@@ -273,6 +299,8 @@ func stopServices() error {
// 先关闭业务入口,再撤下安全服务及会话清理器,避免新请求读取停止中的依赖。
stopVerificationRuntime()
session.Stop()
logRuntime.Stop()
logRuntime = nil
// 关闭连接池后还要等待借出的连接归还,不能提前解除迁移源锁。
if db.Instance != nil {
_ = db.Instance.Close()
@@ -12,6 +12,7 @@ import (
"git.pcux.cn/xuans/Smail/config"
"git.pcux.cn/xuans/Smail/db/migration"
"git.pcux.cn/xuans/Smail/services/logstore"
)
// PortableExecutor 使用版本化目标结构复制业务数据,不拼接用户提供的 SQL。
@@ -220,7 +221,13 @@ func (e *PortableExecutor) Run(ctx context.Context, plan Plan, update func(Statu
if err := update(StatusSwitching); err != nil {
return err
}
err = e.switchConfig(ctx, plan.expectedGeneration, plan.targetConfig, plan.actor)
// 源库此时仍持有写锁,切换审计必须写入已复制且校验完成的目标库。
logStore, err := logstore.New(target, dialect)
if err != nil {
return err
}
switchContext := config.WithMigrationAuditWriter(ctx, &logstore.AuditWriter{Store: logStore})
err = e.switchConfig(switchContext, plan.expectedGeneration, plan.targetConfig, plan.actor)
if err != nil && isKnownSwitchRejection(err) {
return err
}
+84
View File
@@ -0,0 +1,84 @@
package logstore
import (
"context"
"encoding/json"
"git.pcux.cn/xuans/Smail/config"
"git.pcux.cn/xuans/Smail/utils/logpolicy"
)
type AuditWriter struct{ Store *Store }
func (w *AuditWriter) WriteConfigAudit(ctx context.Context, event config.ConfigAuditEvent) error {
if w == nil || w.Store == nil {
return ErrUnavailable
}
normalized, err := config.NormalizeConfigAuditEvent(event)
if err != nil {
return err
}
payload, err := json.Marshal(normalized)
if err != nil {
return err
}
return w.Store.Write(ctx, Entry{
ID: normalized.ID, CreatedAt: normalized.CreatedAt, Kind: "config",
ActorID: normalized.ActorID, Action: string(normalized.Operation),
Section: string(normalized.Section), Result: normalized.Result, Payload: payload,
})
}
func (w *AuditWriter) Query(ctx context.Context, query config.ConfigAuditQuery) ([]config.ConfigAuditEvent, error) {
if w == nil || w.Store == nil {
return nil, ErrUnavailable
}
if err := config.ValidateConfigAuditQuery(query); err != nil {
return nil, err
}
entries, err := w.Store.Query(ctx, Query{
Kind: "config", ID: query.ID, From: query.From, To: query.To, ActorID: query.ActorID,
Action: string(query.Operation), Section: string(query.Section), Limit: query.Limit,
})
if err != nil {
return nil, err
}
items := make([]config.ConfigAuditEvent, 0, len(entries))
for _, entry := range entries {
var event config.ConfigAuditEvent
if json.Unmarshal(entry.Payload, &event) != nil {
return nil, ErrUnavailable
}
items = append(items, event)
}
return items, nil
}
func (w *AuditWriter) ExportJSON(ctx context.Context, query config.ConfigAuditQuery) ([]byte, error) {
items, err := w.Query(ctx, query)
if err != nil {
return nil, err
}
return json.MarshalIndent(items, "", " ")
}
func (w *AuditWriter) ExportCSV(ctx context.Context, query config.ConfigAuditQuery) ([]byte, error) {
items, err := w.Query(ctx, query)
if err != nil {
return nil, err
}
return config.ExportConfigAuditCSV(items)
}
func (w *AuditWriter) ImportLegacy(ctx context.Context, root string) error {
if w == nil || w.Store == nil {
return ErrUnavailable
}
legacy, err := config.NewFileAuditWriter(root)
if err != nil {
return err
}
return legacy.Visit(ctx, logpolicy.Cutoff(w.Store.now()), func(event config.ConfigAuditEvent) error {
return w.WriteConfigAudit(ctx, event)
})
}
+31
View File
@@ -0,0 +1,31 @@
package logstore
import (
"context"
"encoding/json"
"strconv"
"time"
)
// MailResult 记录投递尝试元数据,不复制正文、附件、认证信息或服务器错误原文。
func MailResult(userID int, direction string, sender string, recipientCount, size int, result string) {
if len(sender) > 320 {
sender = "地址超出显示长度"
}
payload, _ := json.Marshal(struct {
Direction string `json:"direction"`
Sender string `json:"sender"`
RecipientCount int `json:"recipientCount"`
Bytes int `json:"bytes"`
}{direction, sender, recipientCount, size})
action := "接收邮件"
if direction == "outbound" {
action = "投递邮件"
}
if direction == "submission" {
action = "客户端提交邮件"
}
Record(context.Background(), Entry{
Kind: "mail", ActorID: strconv.Itoa(userID), CreatedAt: time.Now(), Action: action, Result: result, Payload: payload,
})
}
+71
View File
@@ -0,0 +1,71 @@
package logstore
import (
"context"
"sync"
"sync/atomic"
"time"
"git.pcux.cn/xuans/Smail/utils/logpolicy"
log "github.com/sirupsen/logrus"
)
var active atomic.Pointer[Store]
func Current() *Store { return active.Load() }
// Record 同步落库,不建立无限内存队列;故障必须进入运行日志。
func Record(ctx context.Context, entry Entry) {
store := Current()
if store == nil {
return
}
bounded, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
if err := store.Write(bounded, entry); err != nil {
log.WithError(err).Error("业务日志写入失败")
}
}
type Runtime struct {
cancel context.CancelFunc
done chan struct{}
once sync.Once
store *Store
}
func Start(store *Store) *Runtime {
ctx, cancel := context.WithCancel(context.Background())
runtime := &Runtime{cancel: cancel, done: make(chan struct{}), store: store}
active.Store(store)
go func() {
defer close(runtime.done)
ticker := time.NewTicker(logpolicy.MaintenanceInterval)
defer ticker.Stop()
for {
bounded, stop := context.WithTimeout(ctx, 2*time.Minute)
_, err := store.Cleanup(bounded, time.Now())
stop()
if err != nil && ctx.Err() == nil {
log.WithError(err).Error("过期业务日志清理失败,将在下一轮重试")
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}()
return runtime
}
func (r *Runtime) Stop() {
if r == nil {
return
}
r.once.Do(func() {
active.CompareAndSwap(r.store, nil)
r.cancel()
<-r.done
})
}
+234
View File
@@ -0,0 +1,234 @@
// Package logstore 在当前业务数据库中保存操作与收发信日志。
package logstore
import (
"context"
"crypto/rand"
"database/sql"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"git.pcux.cn/xuans/Smail/db/migration"
"git.pcux.cn/xuans/Smail/utils/logpolicy"
)
var ErrUnavailable = errors.New("日志数据库不可用")
var ErrInvalid = errors.New("日志参数无效")
type Entry struct {
ID string `json:"id"`
CreatedAt time.Time `json:"createdAt"`
Kind string `json:"kind"`
ActorID string `json:"actorId"`
Action string `json:"action"`
Section string `json:"section"`
Result string `json:"result"`
Payload json.RawMessage `json:"payload"`
}
type Query struct {
ID string
Kind string
ActorID string
Action string
Section string
From time.Time
To time.Time
Limit int
Offset int
}
type Store struct {
db *sql.DB
dialect migration.Dialect
now func() time.Time
}
func New(database *sql.DB, dialect migration.Dialect) (*Store, error) {
if database == nil {
return nil, ErrUnavailable
}
if _, err := migration.ParseDialect(string(dialect)); err != nil {
return nil, ErrInvalid
}
return &Store{db: database, dialect: dialect, now: time.Now}, nil
}
func logTable(kind string) (string, error) {
switch kind {
case "config", "operation":
return "operation_log", nil
case "mail":
return "mail_log", nil
default:
return "", ErrInvalid
}
}
func (s *Store) bind(statement string) string {
if s.dialect != migration.DialectPostgres {
return statement
}
index := 0
return replacePlaceholders(statement, &index)
}
func replacePlaceholders(statement string, index *int) string {
parts := strings.Split(statement, "?")
var builder strings.Builder
for position, part := range parts {
if position > 0 {
*index++
fmt.Fprintf(&builder, "$%d", *index)
}
builder.WriteString(part)
}
return builder.String()
}
// Write 只接受业务点构造的脱敏载荷,禁止直接传入 HTTP 请求或邮件正文。
func (s *Store) Write(ctx context.Context, entry Entry) error {
if s == nil || s.db == nil {
return ErrUnavailable
}
table, err := logTable(entry.Kind)
if err != nil {
return err
}
if entry.CreatedAt.IsZero() || len(entry.ID) > 80 || len(entry.ActorID) > 128 || len(entry.Action) > 64 || len(entry.Section) > 32 || len(entry.Result) > 16 || len(entry.Payload) > 256*1024 || !json.Valid(entry.Payload) {
return ErrInvalid
}
// 迁移、重放都不能让过期记录重新进入保留窗口。
if entry.CreatedAt.Before(logpolicy.Cutoff(s.now())) {
return nil
}
if entry.ID == "" {
var token [16]byte
if _, err := rand.Read(token[:]); err != nil {
return ErrUnavailable
}
entry.ID = "log-" + hex.EncodeToString(token[:])
}
statement := "INSERT INTO " + table + " (id, created_at, kind, actor_id, action, section, result, payload) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
// 仅忽略同一稳定标识的重放,不用 INSERT IGNORE 掩盖截断、缺字段等错误。
if s.dialect == migration.DialectMySQL {
statement += " ON DUPLICATE KEY UPDATE id = id"
} else {
statement += " ON CONFLICT (id) DO NOTHING"
}
_, err = s.db.ExecContext(ctx, s.bind(statement), entry.ID, entry.CreatedAt.UTC().UnixNano(), entry.Kind, entry.ActorID, entry.Action, entry.Section, entry.Result, string(entry.Payload))
if err != nil {
return ErrUnavailable
}
return nil
}
func (s *Store) Query(ctx context.Context, query Query) ([]Entry, error) {
if s == nil || s.db == nil {
return nil, ErrUnavailable
}
table, err := logTable(query.Kind)
if err != nil || query.Limit < 0 || query.Limit > 1000 || query.Offset < 0 || query.Offset > 100000 || (!query.From.IsZero() && !query.To.IsZero() && query.From.After(query.To)) {
return nil, ErrInvalid
}
from := logpolicy.Cutoff(s.now())
if query.From.After(from) {
from = query.From
}
statement := "SELECT id, created_at, kind, actor_id, action, section, result, payload FROM " + table + " WHERE created_at >= ? AND kind = ?"
args := []any{from.UnixNano(), query.Kind}
if !query.To.IsZero() {
statement += " AND created_at <= ?"
args = append(args, query.To.UnixNano())
}
for _, filter := range []struct{ column, value string }{
{"id", query.ID}, {"actor_id", query.ActorID}, {"action", query.Action}, {"section", query.Section},
} {
if filter.value != "" {
statement += " AND " + filter.column + " = ?"
args = append(args, filter.value)
}
}
limit := query.Limit
if limit == 0 {
limit = 100
}
statement += " ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?"
args = append(args, limit, query.Offset)
rows, err := s.db.QueryContext(ctx, s.bind(statement), args...)
if err != nil {
return nil, ErrUnavailable
}
defer rows.Close()
items := make([]Entry, 0)
for rows.Next() {
var entry Entry
var timestamp int64
var payload string
if err := rows.Scan(&entry.ID, &timestamp, &entry.Kind, &entry.ActorID, &entry.Action, &entry.Section, &entry.Result, &payload); err != nil {
return nil, ErrUnavailable
}
entry.CreatedAt = time.Unix(0, timestamp).UTC()
entry.Payload = json.RawMessage(payload)
items = append(items, entry)
}
if rows.Err() != nil {
return nil, ErrUnavailable
}
return items, nil
}
// Cleanup 每批先按索引选取标识再删除,避免 MySQL 专属 DELETE LIMIT 和大事务锁表。
func (s *Store) Cleanup(ctx context.Context, now time.Time) (int64, error) {
if s == nil || s.db == nil {
return 0, ErrUnavailable
}
if now.IsZero() {
return 0, ErrInvalid
}
cutoff := logpolicy.Cutoff(now).UnixNano()
var deleted int64
for _, table := range []string{"operation_log", "mail_log"} {
for {
if err := ctx.Err(); err != nil {
return deleted, err
}
rows, err := s.db.QueryContext(ctx, s.bind("SELECT id FROM "+table+" WHERE created_at < ? ORDER BY created_at, id LIMIT ?"), cutoff, logpolicy.DeleteBatchSize)
if err != nil {
return deleted, ErrUnavailable
}
ids := make([]any, 0, logpolicy.DeleteBatchSize)
for rows.Next() {
var id string
if err = rows.Scan(&id); err != nil {
break
}
ids = append(ids, id)
}
rowErr := rows.Err()
closeErr := rows.Close()
if err != nil || rowErr != nil || closeErr != nil {
return deleted, ErrUnavailable
}
if len(ids) == 0 {
break
}
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",")
args := append(ids, cutoff)
result, err := s.db.ExecContext(ctx, s.bind("DELETE FROM "+table+" WHERE id IN ("+placeholders+") AND created_at < ?"), args...)
if err != nil {
return deleted, ErrUnavailable
}
count, err := result.RowsAffected()
if err != nil {
return deleted, ErrUnavailable
}
deleted += count
}
}
return deleted, nil
}
+223
View File
@@ -0,0 +1,223 @@
package runtimelog
import (
"bufio"
"compress/gzip"
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"os"
"strings"
"time"
"git.pcux.cn/xuans/Smail/utils/logpolicy"
)
func fileDay(name string) (time.Time, bool) {
base := strings.TrimSuffix(name, ".gz")
if !strings.HasPrefix(base, prefix) || !strings.HasSuffix(base, ".log") {
return time.Time{}, false
}
value := strings.TrimSuffix(strings.TrimPrefix(base, prefix), ".log")
day, err := time.Parse("2006-01-02", value)
return day, err == nil && (name == prefix+value+".log" || name == prefix+value+".log.gz")
}
func lineTime(line []byte) (time.Time, error) {
if len(line) < 3 || line[0] != '[' {
return time.Time{}, ErrInvalid
}
end := strings.IndexByte(string(line), ']')
if end < 0 {
return time.Time{}, ErrInvalid
}
at, err := time.Parse(time.RFC3339Nano, string(line[1:end]))
if err != nil {
return time.Time{}, ErrInvalid
}
return at, nil
}
// scan 严格解析每条记录及压缩校验和,损坏归档不得触发删除源文件。
func (w *Writer) scan(ctx context.Context, name string, consume func([]byte, time.Time) error) error {
if err := w.regular(name, false); err != nil {
return err
}
file, err := w.root.Open(name)
if err != nil {
return err
}
defer file.Close()
var reader io.Reader = file
if strings.HasSuffix(name, ".gz") {
archive, err := gzip.NewReader(file)
if err != nil {
return err
}
defer archive.Close()
reader = archive
}
scanner := bufio.NewScanner(reader)
scanner.Buffer(make([]byte, 4096), maxLineBytes)
for scanner.Scan() {
if err := ctx.Err(); err != nil {
return err
}
at, err := lineTime(scanner.Bytes())
if err != nil {
return err
}
if err := consume(scanner.Bytes(), at); err != nil {
return err
}
}
return scanner.Err()
}
func (w *Writer) Maintain(ctx context.Context, now time.Time) error {
if now.IsZero() {
return ErrInvalid
}
w.mu.Lock()
defer w.mu.Unlock()
if w.closed {
return nil
}
// 关闭当前文件后维护,避免 Windows 无法替换句柄;下一条日志会重新打开。
if w.file != nil {
err := errors.Join(w.file.Sync(), w.file.Close())
w.file = nil
if err != nil {
return err
}
}
directory, err := w.root.Open(".")
if err != nil {
return err
}
entries, err := directory.ReadDir(-1)
closeErr := directory.Close()
if err != nil || closeErr != nil {
return errors.Join(err, closeErr)
}
cutoff := logpolicy.Cutoff(now)
var failures []error
for _, entry := range entries {
if err := ctx.Err(); err != nil {
return err
}
day, ok := fileDay(entry.Name())
if !ok {
continue
}
compress := !strings.HasSuffix(entry.Name(), ".gz") && !day.Add(24*time.Hour).After(now.Add(-logpolicy.CompressionAge))
if !day.After(cutoff) || compress {
if err := w.transform(ctx, entry.Name(), day, cutoff, compress); err != nil {
failures = append(failures, err)
}
}
}
return errors.Join(failures...)
}
// transform 流式过滤边界日;额外内存不随一天的日志量增长。
func (w *Writer) transform(ctx context.Context, source string, day, cutoff time.Time, compress bool) error {
target := source
if compress {
target += ".gz"
}
var token [16]byte
if _, err := rand.Read(token[:]); err != nil {
return err
}
temporary := ".pending-" + hex.EncodeToString(token[:]) + ".log"
if strings.HasSuffix(target, ".gz") {
temporary += ".gz"
}
file, err := w.root.OpenFile(temporary, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
if err != nil {
return err
}
defer func() { _ = file.Close(); _ = w.root.Remove(temporary) }()
var output io.Writer = file
var archive *gzip.Writer
if strings.HasSuffix(target, ".gz") {
archive = gzip.NewWriter(file)
output = archive
}
hash := sha256.New()
output = io.MultiWriter(output, hash)
kept, removed := 0, 0
err = w.scan(ctx, source, func(line []byte, at time.Time) error {
// 文件名和记录日期不一致时保留现场,不按文件名误删。
if at.UTC().Format("2006-01-02") != day.Format("2006-01-02") {
return ErrInvalid
}
if at.Before(cutoff) {
removed++
return nil
}
kept++
_, err := output.Write(append(append([]byte(nil), line...), '\n'))
return err
})
if archive != nil {
err = errors.Join(err, archive.Close())
}
if err = errors.Join(err, file.Sync(), file.Close()); err != nil {
return err
}
if err := ctx.Err(); err != nil {
return err
}
if removed == 0 && !compress {
return nil
}
if kept == 0 {
if err := w.regular(source, false); err != nil {
return err
}
return w.root.Remove(source)
}
verified := sha256.New()
if err := w.scan(ctx, temporary, func(line []byte, _ time.Time) error {
_, err := verified.Write(append(append([]byte(nil), line...), '\n'))
return err
}); err != nil {
return err
}
if !strings.EqualFold(hex.EncodeToString(hash.Sum(nil)), hex.EncodeToString(verified.Sum(nil))) {
return ErrInvalid
}
if source != target {
if _, err := w.root.Lstat(target); err == nil {
// 压缩成功但删除前中断时,只有目标与本轮内容完全一致才允许补做删除。
existing := sha256.New()
if err := w.scan(ctx, target, func(line []byte, _ time.Time) error {
_, err := existing.Write(append(append([]byte(nil), line...), '\n'))
return err
}); err != nil {
return err
}
if hex.EncodeToString(existing.Sum(nil)) != hex.EncodeToString(hash.Sum(nil)) {
return ErrInvalid
}
return w.root.Remove(source)
} else if !os.IsNotExist(err) {
return err
}
}
if err := w.regular(target, true); err != nil {
return err
}
if err := w.root.Rename(temporary, target); err != nil {
return err
}
if source != target {
return w.root.Remove(source)
}
return nil
}
+91
View File
@@ -0,0 +1,91 @@
package runtimelog
import (
"context"
"errors"
"sort"
"strings"
"time"
"git.pcux.cn/xuans/Smail/utils/logpolicy"
)
type File struct {
Name string `json:"name"`
Bytes int64 `json:"bytes"`
Compressed bool `json:"compressed"`
}
type Page struct {
Lines []string `json:"lines"`
HasMore bool `json:"hasMore"`
}
func (w *Writer) Files() ([]File, error) {
w.mu.Lock()
defer w.mu.Unlock()
if w.closed {
return nil, ErrInvalid
}
directory, err := w.root.Open(".")
if err != nil {
return nil, err
}
defer directory.Close()
entries, err := directory.ReadDir(-1)
if err != nil {
return nil, err
}
files := make([]File, 0)
for _, entry := range entries {
day, ok := fileDay(entry.Name())
if !ok || !day.Add(24*time.Hour).After(logpolicy.Cutoff(w.now())) {
continue
}
if err := w.regular(entry.Name(), false); err != nil {
return nil, err
}
info, err := entry.Info()
if err != nil {
return nil, err
}
files = append(files, File{Name: entry.Name(), Bytes: info.Size(), Compressed: strings.HasSuffix(entry.Name(), ".gz")})
}
sort.Slice(files, func(i, j int) bool { return files[i].Name > files[j].Name })
return files, nil
}
// Read 仅允许固定名称的受管日志,不接受任意路径;压缩内容在服务端流式读取。
func (w *Writer) Read(ctx context.Context, name string, offset, limit int) (Page, error) {
w.mu.Lock()
defer w.mu.Unlock()
if _, ok := fileDay(name); !ok || offset < 0 || offset > 1000000 || limit < 1 || limit > 500 || w.closed {
return Page{}, ErrInvalid
}
page := Page{Lines: make([]string, 0, limit)}
cutoff := logpolicy.Cutoff(w.now())
position := 0
const maxResponseBytes = 2 * 1024 * 1024
bytes := 0
finished := errors.New("当前页读取完成")
err := w.scan(ctx, name, func(line []byte, at time.Time) error {
if at.Before(cutoff) {
return nil
}
position++
if position <= offset {
return nil
}
if len(page.Lines) >= limit || bytes+len(line) > maxResponseBytes {
page.HasMore = true
return finished
}
page.Lines = append(page.Lines, string(line))
bytes += len(line)
return nil
})
if errors.Is(err, finished) {
err = nil
}
return page, err
}
+173
View File
@@ -0,0 +1,173 @@
// Package runtimelog 管理按天写入、七天压缩和三十天滚动清理的运行日志。
package runtimelog
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"git.pcux.cn/xuans/Smail/utils/logpolicy"
"git.pcux.cn/xuans/Smail/utils/runtimepath"
log "github.com/sirupsen/logrus"
)
const prefix = "runtime-"
const maxLineBytes = 256 * 1024
var ErrInvalid = errors.New("运行日志文件或查询参数无效")
var active atomic.Pointer[Writer]
type Writer struct {
mu sync.Mutex
root *os.Root
file *os.File
day string
now func() time.Time
cancel context.CancelFunc
done chan struct{}
closed bool
}
func Current() *Writer { return active.Load() }
func New(root string) (*Writer, error) {
directory, err := runtimepath.SecureDirectory(root, "log")
if err != nil {
return nil, err
}
directory, err = runtimepath.SecureDirectory(directory, "运行日志")
if err != nil {
return nil, err
}
handle, err := os.OpenRoot(directory)
if err != nil {
return nil, err
}
return &Writer{root: handle, now: time.Now}, nil
}
// Start 使用日志钩子,配置重载重设 stdout 时不会丢失文件输出。
func Start(root string) (*Writer, error) {
w, err := New(root)
if err != nil {
return nil, err
}
ctx, cancel := context.WithCancel(context.Background())
w.cancel, w.done = cancel, make(chan struct{})
active.Store(w)
log.AddHook(w)
go func() {
defer close(w.done)
ticker := time.NewTicker(logpolicy.MaintenanceInterval)
defer ticker.Stop()
for {
bounded, stop := context.WithTimeout(ctx, 2*time.Minute)
err := w.Maintain(bounded, w.now())
stop()
if err != nil && ctx.Err() == nil {
log.WithError(err).Error("运行日志维护失败,源文件保留并在下一轮重试")
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}()
return w, nil
}
func (w *Writer) Levels() []log.Level { return log.AllLevels }
func (w *Writer) Fire(entry *log.Entry) error {
// 复用已有格式器保留请求标识、错误字段和源位置;转义换行保证一条记录一行。
data, err := entry.Logger.Formatter.Format(entry)
if err != nil {
return err
}
return w.WriteAt(entry.Time, strings.TrimRight(string(data), "\r\n"))
}
func (w *Writer) WriteAt(at time.Time, message string) error {
w.mu.Lock()
defer w.mu.Unlock()
if w.closed {
return nil
}
if at.IsZero() {
return ErrInvalid
}
if at.Before(logpolicy.Cutoff(w.now())) {
return nil
}
message = strings.NewReplacer("\r", "\\r", "\n", "\\n", "\x00", "").Replace(message)
if len(message) > maxLineBytes-128 {
message = string([]rune(message)[:min(len([]rune(message)), (maxLineBytes-128)/4)]) + "…"
}
day := at.UTC().Format("2006-01-02")
if w.file == nil || w.day != day {
if w.file != nil {
if err := errors.Join(w.file.Sync(), w.file.Close()); err != nil {
w.file = nil
return err
}
w.file = nil
}
name := prefix + day + ".log"
if err := w.regular(name, true); err != nil {
return err
}
file, err := w.root.OpenFile(name, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
return err
}
w.file, w.day = file, day
}
_, err := fmt.Fprintf(w.file, "[%s] %s\n", at.UTC().Format(time.RFC3339Nano), message)
return err
}
func (w *Writer) regular(name string, allowMissing bool) error {
if !filepath.IsLocal(name) || filepath.Base(name) != name {
return ErrInvalid
}
info, err := w.root.Lstat(name)
if allowMissing && os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
return ErrInvalid
}
return nil
}
func (w *Writer) Close() error {
if w == nil {
return nil
}
if w.cancel != nil {
w.cancel()
<-w.done
}
w.mu.Lock()
defer w.mu.Unlock()
if w.closed {
return nil
}
w.closed = true
active.CompareAndSwap(w, nil)
var err error
if w.file != nil {
err = errors.Join(w.file.Sync(), w.file.Close())
}
return errors.Join(err, w.root.Close())
}
+16
View File
@@ -0,0 +1,16 @@
// Package logpolicy 统一三类日志的滚动保留期限。
package logpolicy
import "time"
const (
Retention = 30 * 24 * time.Hour
CompressionAge = 7 * 24 * time.Hour
MaintenanceInterval = time.Hour
DeleteBatchSize = 1000
)
// Cutoff 使用绝对时长,避免月份长度、时区或夏令时改变保留期。
func Cutoff(now time.Time) time.Time {
return now.UTC().Add(-Retention)
}
+13 -1
View File
@@ -7,6 +7,7 @@ import (
"git.pcux.cn/xuans/Smail/config"
"git.pcux.cn/xuans/Smail/dto/parsemail"
"git.pcux.cn/xuans/Smail/models"
"git.pcux.cn/xuans/Smail/services/logstore"
"git.pcux.cn/xuans/Smail/utils/array"
"git.pcux.cn/xuans/Smail/utils/async"
"git.pcux.cn/xuans/Smail/utils/consts"
@@ -119,7 +120,18 @@ func parseRecipientAddress(user *parsemail.User) (string, string, error) {
return address, domain, nil
}
func doSend(ctx *context.Context, fromDomain string, data []byte, to []*parsemail.User, from string) (error, map[string]error) {
func doSend(ctx *context.Context, fromDomain string, data []byte, to []*parsemail.User, from string) (sendErr error, recipientErrors map[string]error) {
defer func() {
result := "success"
if sendErr != nil || len(recipientErrors) > 0 {
result = "failure"
}
userID := 0
if ctx != nil {
userID = ctx.UserID
}
logstore.MailResult(userID, "outbound", from, len(to), len(data), result)
}()
if len(to) == 0 {
err := errors.New("no recipients")
return err, map[string]error{"recipients": err}