diff --git a/backend/config/audit_changes.go b/backend/config/audit_changes.go index a0de1570..3fad1244 100644 --- a/backend/config/audit_changes.go +++ b/backend/config/audit_changes.go @@ -4,15 +4,16 @@ import ( "encoding/json" "reflect" "sort" + "strconv" "strings" ) // ConfigAuditChange 只保存安全的显示值，不保存密文或请求原文。 type ConfigAuditChange struct { - Field string `json:"field"` - Before string `json:"before"` - After string `json:"after"` - Sensitive bool `json:"sensitive,omitempty"` + Field string `json:"field"` + Before string `json:"before"` + After string `json:"after"` + Sensitive bool `json:"sensitive,omitempty"` } // AuditChanges 对比业务字段，排除代次、摘要和内部持久化元数据。 @@ -20,20 +21,32 @@ func AuditChanges(before, after RuntimeConfig) []ConfigAuditChange { left := auditFields(before) right := auditFields(after) keys := make(map[string]bool) - for key := range left { keys[key] = true } - for key := range right { keys[key] = true } + for key := range left { + keys[key] = true + } + for key := range right { + keys[key] = true + } ordered := make([]string, 0, len(keys)) - for key := range keys { ordered = append(ordered, key) } + for key := range keys { + ordered = append(ordered, key) + } sort.Strings(ordered) changes := make([]ConfigAuditChange, 0) for _, key := range ordered { - if reflect.DeepEqual(left[key], right[key]) { continue } + if reflect.DeepEqual(left[key], right[key]) { + continue + } change := ConfigAuditChange{Field: key, Before: auditDisplay(left[key]), After: auditDisplay(right[key])} if auditSensitive(key) { change.Sensitive = true change.Before, change.After = "未配置", "未配置" - if auditConfigured(left[key]) { change.Before = "已配置（隐藏）" } - if auditConfigured(right[key]) { change.After = "已配置（隐藏，已更新）" } + if auditConfigured(left[key]) { + change.Before = "已配置（隐藏）" + } + if auditConfigured(right[key]) { + change.After = "已配置（隐藏，已更新）" + } } changes = append(changes, change) } @@ -60,10 +73,23 @@ func flattenAudit(result map[string]any, path string, value any) { case "meta", "sections", "generation", "updatedAt", "schemaVersion", "configVersion", "version": return } - if path == "root.smtp" || path == "root.email" || path == "root.ssl" || path == "root.plugins" { return } - if auditSensitive(path) { result[path] = value; return } + if path == "root.smtp" || path == "root.email" || path == "root.ssl" || path == "root.plugins" { + return + } + if auditSensitive(path) { + result[path] = value + return + } + if items, ok := value.([]any); ok { + for index, item := range items { + flattenAudit(result, path+"."+strconv.Itoa(index), item) + } + return + } if object, ok := value.(map[string]any); ok { - for key, item := range object { flattenAudit(result, path+"."+key, item) } + for key, item := range object { + flattenAudit(result, path+"."+key, item) + } return } result[path] = value @@ -72,30 +98,50 @@ func flattenAudit(result map[string]any, path string, value any) { func auditSensitive(path string) bool { name := strings.ToLower(path[strings.LastIndex(path, ".")+1:]) for _, word := range []string{"password", "secret", "token", "apikey", "api_key", "credential", "ciphertext", "authorization", "cookie"} { - if strings.Contains(name, word) { return true } + if strings.Contains(name, word) { + return true + } } // 插件设置是自由结构，未知键也不能作为明文审计输出。 return strings.HasSuffix(path, ".settings") || strings.HasSuffix(path, ".encryptionKeyRef") } func auditConfigured(value any) bool { - if value == nil || value == "" { return false } + if value == nil { + return false + } + if text, ok := value.(string); ok { + return text != "" + } if object, ok := value.(map[string]any); ok { - if ciphertext, exists := object["ciphertext"]; exists { return ciphertext != "" } + if ciphertext, exists := object["ciphertext"]; exists { + return ciphertext != "" + } return len(object) > 0 } return true } func auditDisplay(value any) string { - if value == nil { return "未设置" } + if value == nil { + return "未设置" + } if text, ok := value.(string); ok { - if len([]rune(text)) > 512 { return string([]rune(text)[:512])+"…" } + if len([]rune(text)) > 512 { + return string([]rune(text)[:512]) + "…" + } return text } - if flag, ok := value.(bool); ok { if flag { return "开启" }; return "关闭" } + if flag, ok := value.(bool); ok { + if flag { + return "开启" + } + return "关闭" + } data, _ := json.Marshal(value) text := []rune(string(data)) - if len(text) > 512 { return string(text[:512])+"…" } + if len(text) > 512 { + return string(text[:512]) + "…" + } return string(text) } diff --git a/backend/config/audit_export.go b/backend/config/audit_export.go index a96b7750..92011c01 100644 --- a/backend/config/audit_export.go +++ b/backend/config/audit_export.go @@ -2,6 +2,7 @@ package config import ( "encoding/csv" + "encoding/json" "fmt" "strings" "time" @@ -10,11 +11,15 @@ import ( func exportAuditCSV(events []ConfigAuditEvent) ([]byte, error) { var builder strings.Builder writer := csv.NewWriter(&builder) - if err := writer.Write([]string{"id", "operation", "section", "actor_id", "actor_role", "from_generation", "to_generation", "target_generation", "changed_fields", "confirmation_method", "result", "created_at"}); err != nil { + if err := writer.Write([]string{"id", "operation", "section", "actor_id", "actor_role", "from_generation", "to_generation", "target_generation", "changed_fields", "confirmation_method", "result", "created_at", "actor_name", "changes"}); err != nil { return nil, err } for _, event := range events { - if err := writer.Write([]string{ + changes, err := json.Marshal(event.Changes) + if err != nil { + return nil, err + } + cells := []string{ event.ID, string(event.Operation), string(event.Section), @@ -27,7 +32,15 @@ func exportAuditCSV(events []ConfigAuditEvent) ([]byte, error) { event.ConfirmationMethod, event.Result, event.CreatedAt.UTC().Format(time.RFC3339Nano), - }); err != nil { + event.ActorName, + string(changes), + } + for index, cell := range cells { + if strings.HasPrefix(cell, "=") || strings.HasPrefix(cell, "+") || strings.HasPrefix(cell, "-") || strings.HasPrefix(cell, "@") || strings.HasPrefix(cell, "\t") || strings.HasPrefix(cell, "\r") { + cells[index] = "'" + cell + } + } + if err := writer.Write(cells); err != nil { return nil, err } } @@ -37,3 +50,8 @@ func exportAuditCSV(events []ConfigAuditEvent) ([]byte, error) { } return []byte(builder.String()), nil } + +// ExportConfigAuditCSV 让文件、内存和数据库导出共享字段及表格公式保护。 +func ExportConfigAuditCSV(events []ConfigAuditEvent) ([]byte, error) { + return exportAuditCSV(events) +} diff --git a/backend/config/audit_store.go b/backend/config/audit_store.go index 6c5d10e0..b16c41e6 100644 --- a/backend/config/audit_store.go +++ b/backend/config/audit_store.go @@ -17,10 +17,11 @@ import ( "time" "git.pcux.cn/xuans/Smail/utils/atomicfile" + "git.pcux.cn/xuans/Smail/utils/logpolicy" "git.pcux.cn/xuans/Smail/utils/runtimepath" ) -const ConfigAuditRetention = 365 * 24 * time.Hour +const ConfigAuditRetention = logpolicy.Retention var ( ErrAuditStorageUnavailable = errors.New("configuration audit storage unavailable") @@ -296,6 +297,7 @@ func normalizeAuditEvent(event ConfigAuditEvent) (ConfigAuditEvent, error) { event.ActorRole = strings.TrimSpace(event.ActorRole) event.ConfirmationMethod = strings.ToLower(strings.TrimSpace(event.ConfirmationMethod)) event.ChangedFields = append([]string(nil), event.ChangedFields...) + event.Changes = append([]ConfigAuditChange(nil), event.Changes...) if err := validateAuditEvent(event); err != nil { return ConfigAuditEvent{}, err } @@ -326,17 +328,27 @@ func validateAuditEvent(event ConfigAuditEvent) error { if event.Result != "success" && event.Result != "failure" { return ErrAuditEventInvalid } - if strings.ContainsAny(event.ActorID+event.ActorRole+event.ConfirmationMethod, "\r\n") || len(event.ActorID) > 128 || len(event.ActorRole) > 64 { + if strings.ContainsAny(event.ActorID+event.ActorRole+event.ActorName+event.ConfirmationMethod, "\r\n") || len(event.ActorID) > 128 || len(event.ActorRole) > 64 || len(event.ActorName) > 256 { return ErrAuditEventInvalid } for _, field := range event.ChangedFields { - if field == "" || len(field) > 64 || strings.ContainsAny(field, "\r\n") { + if field == "" || len(field) > 1024 || strings.ContainsAny(field, "\r\n") { + return ErrAuditEventInvalid + } + } + for _, change := range event.Changes { + if change.Field == "" || len(change.Field) > 1024 || strings.ContainsAny(change.Field, "\r\n") || len([]rune(change.Before)) > 513 || len([]rune(change.After)) > 513 { return ErrAuditEventInvalid } } return nil } +// NormalizeConfigAuditEvent 供数据库审计复用同一套标识、校验和复制规则。 +func NormalizeConfigAuditEvent(event ConfigAuditEvent) (ConfigAuditEvent, error) { + return normalizeAuditEvent(event) +} + func auditEventMatches(event ConfigAuditEvent, query ConfigAuditQuery) bool { if query.ID != "" && event.ID != query.ID { return false diff --git a/backend/config/audit_visit.go b/backend/config/audit_visit.go new file mode 100644 index 00000000..1a8b4693 --- /dev/null +++ b/backend/config/audit_visit.go @@ -0,0 +1,57 @@ +package config + +import ( + "bufio" + "context" + "encoding/json" + "os" + "time" +) + +// Visit 从旧文件流式迁移，不受管理界面的查询条数上限影响，不修改源文件。 +func (w *FileAuditWriter) Visit(ctx context.Context, from time.Time, visit func(ConfigAuditEvent) error) error { + if w == nil || visit == nil { + return ErrAuditStorageUnavailable + } + w.mu.Lock() + defer w.mu.Unlock() + if info, err := os.Lstat(w.path); err != nil { + if os.IsNotExist(err) { + return nil + } + return ErrAuditStorageUnavailable + } else if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return ErrAuditStorageUnavailable + } + file, err := os.Open(w.path) + if err != nil { + return ErrAuditStorageUnavailable + } + defer file.Close() + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 4096), 256*1024) + line := 0 + for scanner.Scan() { + if err := ctx.Err(); err != nil { + return err + } + line++ + var event ConfigAuditEvent + if json.Unmarshal(scanner.Bytes(), &event) != nil { + return ErrAuditStorageUnavailable + } + if event.ID == "" { + event.ID = legacyAuditEventID(scanner.Bytes(), line) + } + if err := validateAuditEvent(event); err != nil { + return err + } + if event.CreatedAt.Before(from) { + continue + } + if err := visit(event); err != nil { + return err + } + } + return scanner.Err() +} diff --git a/backend/config/database_migration.go b/backend/config/database_migration.go index b4527b29..24aba771 100644 --- a/backend/config/database_migration.go +++ b/backend/config/database_migration.go @@ -9,6 +9,14 @@ import ( var ErrDatabaseMigrationTarget = errors.New("database migration target is invalid") +type migrationAuditKey struct{} + +// WithMigrationAuditWriter 仅供持有迁移目标连接的执行器设置迁移审计去向。 +// 普通配置更新不读取该上下文值，HTTP 请求不能指定审计数据库。 +func WithMigrationAuditWriter(ctx context.Context, writer ConfigAuditWriter) context.Context { + return context.WithValue(ctx, migrationAuditKey{}, writer) +} + // SwitchDatabase 提交已完成复制与校验的目标连接，保留旧库并标记重启。 // 普通配置编辑不能绕过本迁移入口切换数据库类型。 func (m *ConfigManager) SwitchDatabase(ctx context.Context, expectedGeneration uint64, target DatabaseConfig, actor AuditActor) error { @@ -91,7 +99,7 @@ func (m *ConfigManager) SwitchDatabase(ctx context.Context, expectedGeneration u if !sameDatabaseConnection(current.Root.Database, target) { changedField = "root.database" } - return m.writeAudit(ctx, ConfigAuditEvent{ + event := ConfigAuditEvent{ Operation: ConfigOperationMigration, Section: SectionRoot, ActorID: actor.UserID, @@ -104,7 +112,14 @@ func (m *ConfigManager) SwitchDatabase(ctx context.Context, expectedGeneration u ChangedFields: []string{changedField}, Result: "success", CreatedAt: now, - }) + } + if writer, ok := ctx.Value(migrationAuditKey{}).(ConfigAuditWriter); ok && writer != nil { + if err := writer.WriteConfigAudit(ctx, event); err != nil { + return errors.Join(ErrAuditWrite, err) + } + return nil + } + return m.writeAudit(ctx, event) } // SameDatabaseLocation 不把用户名或密码不同误认为不同数据库。 diff --git a/backend/config/manager_v2.go b/backend/config/manager_v2.go index 0d3e97ba..ebcda20f 100644 --- a/backend/config/manager_v2.go +++ b/backend/config/manager_v2.go @@ -777,7 +777,9 @@ func (m *ConfigManager) writeAudit(ctx context.Context, event ConfigAuditEvent) func changedFields(current, candidate RuntimeConfig, section Section) []string { var fields []string - for _, change := range AuditChanges(current, candidate) { fields = append(fields, change.Field) } + for _, change := range AuditChanges(current, candidate) { + fields = append(fields, change.Field) + } return fields } diff --git a/backend/config/manager_v2_test.go b/backend/config/manager_v2_test.go index c2b2954f..b95239c3 100644 --- a/backend/config/manager_v2_test.go +++ b/backend/config/manager_v2_test.go @@ -1771,7 +1771,8 @@ func TestConfigManagerWritesRedactedAuditAndDegradesOnAuditFailure(t *testing.T) t.Fatal(err) } events := writer.Events() - if len(events) != 1 || events[0].Section != SectionEmail || len(events[0].ChangedFields) != 1 || events[0].ChangedFields[0] != string(SectionEmail) { + // 原样保存只记录操作，不能伪造业务字段发生变更。 + if len(events) != 1 || events[0].Section != SectionEmail || len(events[0].ChangedFields) != 0 || len(events[0].Changes) != 0 { t.Fatalf("unexpected audit events: %+v", events) } if events[0].Result != "success" || events[0].ToGeneration != 2 { diff --git a/backend/config/security.go b/backend/config/security.go index 7ed554fa..d87354a7 100644 --- a/backend/config/security.go +++ b/backend/config/security.go @@ -26,20 +26,20 @@ const ( ) type ConfigAuditEvent struct { - ID string `json:"id"` - Operation ConfigOperation `json:"operation"` - Section Section `json:"section,omitempty"` - ActorID string `json:"actorId"` - ActorRole string `json:"actorRole"` - ActorName string `json:"actorName,omitempty"` + ID string `json:"id"` + Operation ConfigOperation `json:"operation"` + Section Section `json:"section,omitempty"` + ActorID string `json:"actorId"` + ActorRole string `json:"actorRole"` + ActorName string `json:"actorName,omitempty"` Changes []ConfigAuditChange `json:"changes,omitempty"` - FromGeneration uint64 `json:"fromGeneration"` - ToGeneration uint64 `json:"toGeneration"` - TargetGeneration uint64 `json:"targetGeneration,omitempty"` - ChangedFields []string `json:"changedFields,omitempty"` - ConfirmationMethod string `json:"confirmationMethod,omitempty"` - Result string `json:"result"` - CreatedAt time.Time `json:"createdAt"` + FromGeneration uint64 `json:"fromGeneration"` + ToGeneration uint64 `json:"toGeneration"` + TargetGeneration uint64 `json:"targetGeneration,omitempty"` + ChangedFields []string `json:"changedFields,omitempty"` + ConfirmationMethod string `json:"confirmationMethod,omitempty"` + Result string `json:"result"` + CreatedAt time.Time `json:"createdAt"` } type ConfigAuditWriter interface { WriteConfigAudit(context.Context, ConfigAuditEvent) error @@ -107,6 +107,7 @@ func (w *MemoryAuditWriter) Query(ctx context.Context, query ConfigAuditQuery) ( } copyEvent := event copyEvent.ChangedFields = append([]string(nil), event.ChangedFields...) + copyEvent.Changes = append([]ConfigAuditChange(nil), event.Changes...) result = append(result, copyEvent) if len(result) >= limit { break @@ -141,6 +142,7 @@ func (w *MemoryAuditWriter) Events() []ConfigAuditEvent { copy(result, w.events) for index := range result { result[index].ChangedFields = append([]string(nil), result[index].ChangedFields...) + result[index].Changes = append([]ConfigAuditChange(nil), result[index].Changes...) } return result } diff --git a/backend/controllers/logs.go b/backend/controllers/logs.go new file mode 100644 index 00000000..09236904 --- /dev/null +++ b/backend/controllers/logs.go @@ -0,0 +1,93 @@ +package controllers + +import ( + "context" + "net/http" + "strconv" + "time" + + "git.pcux.cn/xuans/Smail/config" + "git.pcux.cn/xuans/Smail/dto/response" + "git.pcux.cn/xuans/Smail/services/access" + "git.pcux.cn/xuans/Smail/services/logstore" + "git.pcux.cn/xuans/Smail/services/runtimelog" + appcontext "git.pcux.cn/xuans/Smail/utils/context" +) + +func logQueryError(w http.ResponseWriter, err error) { + response.NewErrorResponse(response.ServerError, "日志查询失败，请检查日志存储状态", "LOG_QUERY_FAILED").FPrint(w) +} + +// GetLogs 只读日志入口，授权在服务端检查，不依赖导航隐藏。 +func GetLogs(ctx *appcontext.Context, w http.ResponseWriter, req *http.Request) { + if err := authorizeCapability(ctx, access.CapabilityAuditRead, "", false); err != nil { + writeConfigAccessError(w, err) + return + } + values := req.URL.Query() + limit, offset := 50, 0 + var err error + if values.Has("limit") { + limit, err = strconv.Atoi(values.Get("limit")) + } + if err == nil && values.Has("offset") { + offset, err = strconv.Atoi(values.Get("offset")) + } + if err != nil || limit < 1 || limit > 500 || offset < 0 || offset > 100000 { + writeConfigError(w, config.ErrInvalidAuditQuery) + return + } + bounded, cancel := context.WithTimeout(req.Context(), 10*time.Second) + defer cancel() + if values.Get("kind") == "runtime" { + writer := runtimelog.Current() + if writer == nil { + logQueryError(w, runtimelog.ErrInvalid) + return + } + if values.Get("file") == "" { + files, err := writer.Files() + if err != nil { + logQueryError(w, err) + return + } + response.NewSuccessResponse(map[string]any{"files": files}).FPrint(w) + return + } + page, err := writer.Read(bounded, values.Get("file"), offset, limit) + if err != nil { + logQueryError(w, err) + return + } + response.NewSuccessResponse(page).FPrint(w) + return + } + query := logstore.Query{Kind: values.Get("kind"), ActorID: values.Get("actorId"), Limit: limit + 1, Offset: offset} + for _, item := range []struct { + name string + target *time.Time + }{{"from", &query.From}, {"to", &query.To}} { + if values.Get(item.name) != "" { + *item.target, err = time.Parse(time.RFC3339Nano, values.Get(item.name)) + if err != nil { + writeConfigError(w, config.ErrInvalidAuditQuery) + return + } + } + } + if query.Kind != "config" && query.Kind != "operation" && query.Kind != "mail" { + writeConfigError(w, config.ErrInvalidAuditQuery) + return + } + store := logstore.Current() + items, err := store.Query(bounded, query) + if err != nil { + logQueryError(w, err) + return + } + hasMore := len(items) > limit + if hasMore { + items = items[:limit] + } + response.NewSuccessResponse(map[string]any{"items": items, "hasMore": hasMore, "retentionDays": 30}).FPrint(w) +} diff --git a/backend/db/migration/logs.go b/backend/db/migration/logs.go new file mode 100644 index 00000000..fcdcdf53 --- /dev/null +++ b/backend/db/migration/logs.go @@ -0,0 +1,27 @@ +package migration + +// 日志使用可移植的数值时间和文本载荷，不依赖数据库专属 JSON、分区或事件调度器。 +func logStatements() map[Dialect][]string { + result := make(map[Dialect][]string) + for _, dialect := range []Dialect{DialectSQLite, DialectMySQL, DialectPostgres} { + for _, table := range []string{"operation_log", "mail_log"} { + payloadType := "TEXT" + if dialect == DialectMySQL { + payloadType = "LONGTEXT" + } + statement := "CREATE TABLE IF NOT EXISTS " + table + " (id VARCHAR(80) PRIMARY KEY, created_at BIGINT NOT NULL, kind VARCHAR(32) NOT NULL, actor_id VARCHAR(128) NOT NULL, action VARCHAR(64) NOT NULL, section VARCHAR(32) NOT NULL, result VARCHAR(16) NOT NULL, payload " + payloadType + " NOT NULL" + if dialect == DialectMySQL { + statement += ", KEY idx_" + table + "_time (created_at, id), KEY idx_" + table + "_kind_time (kind, created_at, id)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4" + } else { + statement += ")" + } + result[dialect] = append(result[dialect], statement) + if dialect != DialectMySQL { + result[dialect] = append(result[dialect], + "CREATE INDEX IF NOT EXISTS \"idx_"+table+"_time\" ON "+table+" (created_at, id)", + "CREATE INDEX IF NOT EXISTS \"idx_"+table+"_kind_time\" ON "+table+" (kind, created_at, id)") + } + } + } + return result +} diff --git a/backend/db/migration/registry.go b/backend/db/migration/registry.go index 2ded224e..eea537fe 100644 --- a/backend/db/migration/registry.go +++ b/backend/db/migration/registry.go @@ -26,6 +26,7 @@ func DefaultMigrations() []Migration { DefineCustom(14, "expand_imap_deleted_flag", "expand_imap_deleted_flag_v1", applyIMAPDeletedFlagSchema), DefineCustom(15, "expand_security_setup", "expand_security_setup_v1", applySecuritySetupSchema), DefineCustom(16, "expand_user_security_preferences", "expand_user_security_preferences_v1", applyUserSecurityPreferencesSchema), + DefineForDialects(17, "create_managed_logs", logStatements()), } } diff --git a/backend/db/migration/registry_test.go b/backend/db/migration/registry_test.go index 5721a47d..9285b008 100644 --- a/backend/db/migration/registry_test.go +++ b/backend/db/migration/registry_test.go @@ -220,6 +220,7 @@ func TestDefaultMigrationChecksumsAreImmutable(t *testing.T) { 14: "531398c2b8fc83a65753fae08666946bb253ac3da6596829e71cc8ccc4ddf62f", 15: "59f0ff9acfed3619c17f078d21a1362ddd5f1feacb9a2d77626f69040d25de65", 16: "00cc275529e522d93b41e608692fb6e2271f1aa9300ebe020fcaed4d68d0ff76", + 17: "1dad66acb5e7fecf6c39f4a78526b994748d90045220f446a2e80c1b5fd881e7", } migrations := DefaultMigrations() if len(migrations) != len(expected) { diff --git a/backend/dto/response/response.go b/backend/dto/response/response.go index 023f355a..8b599d93 100644 --- a/backend/dto/response/response.go +++ b/backend/dto/response/response.go @@ -28,6 +28,9 @@ func (p *Response) FPrint(w http.ResponseWriter) { http.Error(w, "\u54cd\u5e94\u5e8f\u5217\u5316\u5931\u8d25", http.StatusInternalServerError) return } + if observer, ok := w.(interface{ ObserveResult(int) }); ok { + observer.ObserveResult(p.ErrorNo) + } w.Header().Set("Content-Type", "application/json; charset=utf-8") if _, err := w.Write(bytesData); err != nil { log.WithError(err).Error("write HTTP response failed") diff --git a/backend/internal/app/app.go b/backend/internal/app/app.go index e92f7262..9c56fe3c 100644 --- a/backend/internal/app/app.go +++ b/backend/internal/app/app.go @@ -8,6 +8,7 @@ import ( "git.pcux.cn/xuans/Smail/internal/buildinfo" "git.pcux.cn/xuans/Smail/listen/cron_server" "git.pcux.cn/xuans/Smail/res_init" + "git.pcux.cn/xuans/Smail/services/runtimelog" "git.pcux.cn/xuans/Smail/utils/runtimepath" log "github.com/sirupsen/logrus" ) @@ -23,6 +24,11 @@ func Run() error { if err := config.Init(); err != nil { return err } + runtimeLogs, err := runtimelog.Start(config.ROOT_PATH) + if err != nil { + return fmt.Errorf("启动运行日志失败: %w", err) + } + defer runtimeLogs.Close() config.MaintainManagedSSLHistory() build := buildinfo.Current() diff --git a/backend/listen/http_server/http_server.go b/backend/listen/http_server/http_server.go index 31948758..7807a899 100644 --- a/backend/listen/http_server/http_server.go +++ b/backend/listen/http_server/http_server.go @@ -110,6 +110,7 @@ func router(mux *http.ServeMux) error { mux.HandleFunc("/api/v1/config/", limitBody(4<<20, contextIterceptor(configRoute))) mux.HandleFunc("/api/v1/audits/config", limitBody(1<<20, contextIterceptor(configAuditRoute))) mux.HandleFunc("/api/v1/audits/config/", limitBody(1<<20, contextIterceptor(configAuditSubRoute))) + mux.HandleFunc("/api/v1/logs", onlyMethod(http.MethodGet, contextIterceptor(controllers.GetLogs))) mux.HandleFunc("/api/v1/database/migrations/preview", onlyMethod(http.MethodPost, limitBody(16<<10, contextIterceptor(controllers.PreviewDatabaseMigration)))) mux.HandleFunc("/api/v1/database/migrations", onlyMethod(http.MethodPost, limitBody(16<<10, contextIterceptor(controllers.ExecuteDatabaseMigration)))) mux.HandleFunc("/api/v1/database/migrations/", contextIterceptor(databaseMigrationSubRoute)) diff --git a/backend/listen/http_server/https_server.go b/backend/listen/http_server/https_server.go index a7c4069b..34afe971 100644 --- a/backend/listen/http_server/https_server.go +++ b/backend/listen/http_server/https_server.go @@ -177,7 +177,9 @@ func contextIterceptor(h controllers.HandlerFunc) http.HandlerFunc { response.NewErrorResponse(response.NeedSetup, "", "").FPrint(w) return } - h(ctx, w, r) + observed, finish := operationLog(ctx, r, w) + defer finish() + h(ctx, observed, r) })).ServeHTTP(w, r) } } diff --git a/backend/listen/http_server/operation_log.go b/backend/listen/http_server/operation_log.go new file mode 100644 index 00000000..b8581799 --- /dev/null +++ b/backend/listen/http_server/operation_log.go @@ -0,0 +1,71 @@ +package http_server + +import ( + "context" + "encoding/json" + "net/http" + "strconv" + "time" + + "git.pcux.cn/xuans/Smail/services/logstore" + appcontext "git.pcux.cn/xuans/Smail/utils/context" +) + +// 仅记录明确的变更接口，不将 POST 列表查询误认为操作，也不复制请求或响应载荷。 +var loggedOperations = map[string]string{ + "/api/login": "处理账号登录", "/api/logout": "退出登录", + "/api/user/create": "创建用户", "/api/user/edit": "编辑用户", + "/api/group/add": "创建分组", "/api/group/del": "删除分组", + "/api/email/del": "删除邮件", "/api/email/move": "移动邮件", "/api/email/read": "修改已读状态", + "/api/rule/add": "创建邮件规则", "/api/rule/update": "修改邮件规则", "/api/rule/del": "删除邮件规则", + "/api/settings/modify_password": "修改账号密码", + "/api/security/credentials/recovery-codes/generate": "生成安全码", + "/api/security/credentials/recovery-codes/revoke": "撤销安全码", + "/api/security/external-email/bind": "申请绑定外部邮箱", + "/api/security/external-email/verify": "验证外部邮箱", + "/api/security/external-email/change": "申请修改外部邮箱", + "/api/security/external-email/change/verify": "验证外部邮箱修改", + "/api/security/external-email/disable": "解绑外部邮箱", + "/api/security/credentials/protocols/enabled": "设置邮件客户端授权", + "/api/security/credentials/protocols/generate": "生成邮件客户端安全码", + "/api/security/credentials/protocols/revoke": "撤销邮件客户端安全码", + "/api/security/recovery/complete": "完成账号恢复", + "/api/user/security/recovery-codes/generate": "管理用户安全码", + "/api/user/security/recovery-codes/revoke": "撤销用户安全码", + "/api/user/security/external-email/change": "申请修改用户外部邮箱", + "/api/user/security/external-email/verify": "验证用户外部邮箱修改", +} + +type operationResponse struct { + http.ResponseWriter + code int + known bool +} + +func (w *operationResponse) Unwrap() http.ResponseWriter { return w.ResponseWriter } +func (w *operationResponse) ObserveResult(code int) { w.code, w.known = code, true } + +func operationLog(ctx *appcontext.Context, req *http.Request, w http.ResponseWriter) (http.ResponseWriter, func()) { + action := loggedOperations[normalizedAPIPath(req.URL.Path)] + if action == "" || !stateChangingMethod(req.Method) { + return w, func() {} + } + observed := &operationResponse{ResponseWriter: w} + return observed, func() { + result := "unknown" + if observed.known { + result = "success" + if observed.code != 0 { + result = "failure" + } + } + payload, _ := json.Marshal(struct { + ActorName string `json:"actorName"` + ActorRole string `json:"actorRole"` + ErrorCode int `json:"errorCode"` + }{ctx.UserAccount, string(ctx.EffectiveRole()), observed.code}) + logstore.Record(context.Background(), logstore.Entry{ + Kind: "operation", CreatedAt: time.Now(), ActorID: strconv.Itoa(ctx.UserID), Action: action, Result: result, Payload: payload, + }) + } +} diff --git a/backend/listen/smtp_server/read_content.go b/backend/listen/smtp_server/read_content.go index 11d0a39f..925e7603 100644 --- a/backend/listen/smtp_server/read_content.go +++ b/backend/listen/smtp_server/read_content.go @@ -20,6 +20,7 @@ import ( "git.pcux.cn/xuans/Smail/hooks/framework" "git.pcux.cn/xuans/Smail/listen/imap_server" "git.pcux.cn/xuans/Smail/models" + "git.pcux.cn/xuans/Smail/services/logstore" "git.pcux.cn/xuans/Smail/services/rule" "git.pcux.cn/xuans/Smail/utils/array" "git.pcux.cn/xuans/Smail/utils/async" @@ -38,13 +39,25 @@ import ( // 注意：此开关只能通过修改代码来更改，不暴露给用户配置 const DropUnknownRecipientEmails = true -func (s *Session) Data(r io.Reader) error { +func (s *Session) Data(r io.Reader) (resultErr error) { + messageBytes := 0 + defer func() { + result, direction := "success", "inbound" + if resultErr != nil { + result = "failure" + } + if s.Ctx.UserID > 0 { + direction = "submission" + } + logstore.MailResult(s.Ctx.UserID, direction, s.From, len(s.To), messageBytes, result) + }() ctx := s.Ctx log.WithContext(ctx).Debugf("收到邮件") emailData, err := io.ReadAll(io.LimitReader(r, maxSMTPMessageBytes+1)) + messageBytes = len(emailData) if err != nil { log.WithContext(ctx).Error("邮件内容无法读取", err) return err diff --git a/backend/res_init/init.go b/backend/res_init/init.go index edf6001d..4eb99d91 100644 --- a/backend/res_init/init.go +++ b/backend/res_init/init.go @@ -12,6 +12,7 @@ import ( "git.pcux.cn/xuans/Smail/config" "git.pcux.cn/xuans/Smail/controllers" "git.pcux.cn/xuans/Smail/db" + "git.pcux.cn/xuans/Smail/db/migration" "git.pcux.cn/xuans/Smail/dto/parsemail" "git.pcux.cn/xuans/Smail/hooks" "git.pcux.cn/xuans/Smail/listen/http_server" @@ -23,6 +24,7 @@ import ( "git.pcux.cn/xuans/Smail/services/credentials" "git.pcux.cn/xuans/Smail/services/dbmigration" "git.pcux.cn/xuans/Smail/services/externalemail" + "git.pcux.cn/xuans/Smail/services/logstore" pluginservice "git.pcux.cn/xuans/Smail/services/plugin" "git.pcux.cn/xuans/Smail/services/recovery" "git.pcux.cn/xuans/Smail/services/restart" @@ -44,6 +46,7 @@ var adminResetServeErrors chan error var verificationRuntime *verification.Runtime var credentialService *credentials.Service var webAuthnRuntimeService *webauthnservice.Service +var logRuntime *logstore.Runtime func startConfigManager() error { root := filepath.Join(config.ROOT_PATH, "config") @@ -72,10 +75,20 @@ func startConfigManager() error { dbmigration.SetCurrent(nil) return nil } - manager, err := config.NewConfigManager(config.ROOT_PATH, config.ConfigLoaderOptions{Validation: config.RuntimeValidationOptions{ + security := config.ConfigSecurity{} + if store := logstore.Current(); store != nil { + audit := &logstore.AuditWriter{Store: store} + importCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if err := audit.ImportLegacy(importCtx, config.ROOT_PATH); err != nil { + return fmt.Errorf("迁移旧配置审计失败，源文件已保留: %w", err) + } + security.Audit = audit + } + manager, err := config.NewConfigManagerWithSecurity(config.ROOT_PATH, config.ConfigLoaderOptions{Validation: config.RuntimeValidationOptions{ HasSuperAdmin: true, Production: !config.DevelopmentMode, - }}) + }}, security) if err != nil { return fmt.Errorf("load partitioned configuration: %w", err) } @@ -97,6 +110,8 @@ func startConfigManager() error { } func stopConfigManager() { + logRuntime.Stop() + logRuntime = nil dbmigration.Current().Close() config.SetRuntimeManager(nil) dbmigration.SetCurrent(nil) @@ -166,7 +181,18 @@ func Init(serverVersion string) error { if err := db.Init(serverVersion); err != nil { return err } + dialect, err := migration.ParseDialect(cfg.DbType) + if err != nil { + return err + } + store, err := logstore.New(db.Instance.DB().DB, dialect) + if err != nil { + return err + } + logRuntime = logstore.Start(store) if err := startConfigManager(); err != nil { + logRuntime.Stop() + logRuntime = nil return err } if err := startPluginConfigMigration(); err != nil { @@ -273,6 +299,8 @@ func stopServices() error { // 先关闭业务入口，再撤下安全服务及会话清理器，避免新请求读取停止中的依赖。 stopVerificationRuntime() session.Stop() + logRuntime.Stop() + logRuntime = nil // 关闭连接池后还要等待借出的连接归还，不能提前解除迁移源锁。 if db.Instance != nil { _ = db.Instance.Close() diff --git a/backend/services/dbmigration/portable_executor.go b/backend/services/dbmigration/portable_executor.go index dd21bfdf..67459714 100644 --- a/backend/services/dbmigration/portable_executor.go +++ b/backend/services/dbmigration/portable_executor.go @@ -12,6 +12,7 @@ import ( "git.pcux.cn/xuans/Smail/config" "git.pcux.cn/xuans/Smail/db/migration" + "git.pcux.cn/xuans/Smail/services/logstore" ) // PortableExecutor 使用版本化目标结构复制业务数据，不拼接用户提供的 SQL。 @@ -220,7 +221,13 @@ func (e *PortableExecutor) Run(ctx context.Context, plan Plan, update func(Statu if err := update(StatusSwitching); err != nil { return err } - err = e.switchConfig(ctx, plan.expectedGeneration, plan.targetConfig, plan.actor) + // 源库此时仍持有写锁，切换审计必须写入已复制且校验完成的目标库。 + logStore, err := logstore.New(target, dialect) + if err != nil { + return err + } + switchContext := config.WithMigrationAuditWriter(ctx, &logstore.AuditWriter{Store: logStore}) + err = e.switchConfig(switchContext, plan.expectedGeneration, plan.targetConfig, plan.actor) if err != nil && isKnownSwitchRejection(err) { return err } diff --git a/backend/services/logstore/audit.go b/backend/services/logstore/audit.go new file mode 100644 index 00000000..417413c0 --- /dev/null +++ b/backend/services/logstore/audit.go @@ -0,0 +1,84 @@ +package logstore + +import ( + "context" + "encoding/json" + + "git.pcux.cn/xuans/Smail/config" + "git.pcux.cn/xuans/Smail/utils/logpolicy" +) + +type AuditWriter struct{ Store *Store } + +func (w *AuditWriter) WriteConfigAudit(ctx context.Context, event config.ConfigAuditEvent) error { + if w == nil || w.Store == nil { + return ErrUnavailable + } + normalized, err := config.NormalizeConfigAuditEvent(event) + if err != nil { + return err + } + payload, err := json.Marshal(normalized) + if err != nil { + return err + } + return w.Store.Write(ctx, Entry{ + ID: normalized.ID, CreatedAt: normalized.CreatedAt, Kind: "config", + ActorID: normalized.ActorID, Action: string(normalized.Operation), + Section: string(normalized.Section), Result: normalized.Result, Payload: payload, + }) +} + +func (w *AuditWriter) Query(ctx context.Context, query config.ConfigAuditQuery) ([]config.ConfigAuditEvent, error) { + if w == nil || w.Store == nil { + return nil, ErrUnavailable + } + if err := config.ValidateConfigAuditQuery(query); err != nil { + return nil, err + } + entries, err := w.Store.Query(ctx, Query{ + Kind: "config", ID: query.ID, From: query.From, To: query.To, ActorID: query.ActorID, + Action: string(query.Operation), Section: string(query.Section), Limit: query.Limit, + }) + if err != nil { + return nil, err + } + items := make([]config.ConfigAuditEvent, 0, len(entries)) + for _, entry := range entries { + var event config.ConfigAuditEvent + if json.Unmarshal(entry.Payload, &event) != nil { + return nil, ErrUnavailable + } + items = append(items, event) + } + return items, nil +} + +func (w *AuditWriter) ExportJSON(ctx context.Context, query config.ConfigAuditQuery) ([]byte, error) { + items, err := w.Query(ctx, query) + if err != nil { + return nil, err + } + return json.MarshalIndent(items, "", " ") +} + +func (w *AuditWriter) ExportCSV(ctx context.Context, query config.ConfigAuditQuery) ([]byte, error) { + items, err := w.Query(ctx, query) + if err != nil { + return nil, err + } + return config.ExportConfigAuditCSV(items) +} + +func (w *AuditWriter) ImportLegacy(ctx context.Context, root string) error { + if w == nil || w.Store == nil { + return ErrUnavailable + } + legacy, err := config.NewFileAuditWriter(root) + if err != nil { + return err + } + return legacy.Visit(ctx, logpolicy.Cutoff(w.Store.now()), func(event config.ConfigAuditEvent) error { + return w.WriteConfigAudit(ctx, event) + }) +} diff --git a/backend/services/logstore/mail.go b/backend/services/logstore/mail.go new file mode 100644 index 00000000..36b628d1 --- /dev/null +++ b/backend/services/logstore/mail.go @@ -0,0 +1,31 @@ +package logstore + +import ( + "context" + "encoding/json" + "strconv" + "time" +) + +// MailResult 记录投递尝试元数据，不复制正文、附件、认证信息或服务器错误原文。 +func MailResult(userID int, direction string, sender string, recipientCount, size int, result string) { + if len(sender) > 320 { + sender = "地址超出显示长度" + } + payload, _ := json.Marshal(struct { + Direction string `json:"direction"` + Sender string `json:"sender"` + RecipientCount int `json:"recipientCount"` + Bytes int `json:"bytes"` + }{direction, sender, recipientCount, size}) + action := "接收邮件" + if direction == "outbound" { + action = "投递邮件" + } + if direction == "submission" { + action = "客户端提交邮件" + } + Record(context.Background(), Entry{ + Kind: "mail", ActorID: strconv.Itoa(userID), CreatedAt: time.Now(), Action: action, Result: result, Payload: payload, + }) +} diff --git a/backend/services/logstore/runtime.go b/backend/services/logstore/runtime.go new file mode 100644 index 00000000..54bfdc25 --- /dev/null +++ b/backend/services/logstore/runtime.go @@ -0,0 +1,71 @@ +package logstore + +import ( + "context" + "sync" + "sync/atomic" + "time" + + "git.pcux.cn/xuans/Smail/utils/logpolicy" + log "github.com/sirupsen/logrus" +) + +var active atomic.Pointer[Store] + +func Current() *Store { return active.Load() } + +// Record 同步落库，不建立无限内存队列；故障必须进入运行日志。 +func Record(ctx context.Context, entry Entry) { + store := Current() + if store == nil { + return + } + bounded, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := store.Write(bounded, entry); err != nil { + log.WithError(err).Error("业务日志写入失败") + } +} + +type Runtime struct { + cancel context.CancelFunc + done chan struct{} + once sync.Once + store *Store +} + +func Start(store *Store) *Runtime { + ctx, cancel := context.WithCancel(context.Background()) + runtime := &Runtime{cancel: cancel, done: make(chan struct{}), store: store} + active.Store(store) + go func() { + defer close(runtime.done) + ticker := time.NewTicker(logpolicy.MaintenanceInterval) + defer ticker.Stop() + for { + bounded, stop := context.WithTimeout(ctx, 2*time.Minute) + _, err := store.Cleanup(bounded, time.Now()) + stop() + if err != nil && ctx.Err() == nil { + log.WithError(err).Error("过期业务日志清理失败，将在下一轮重试") + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } + }() + return runtime +} + +func (r *Runtime) Stop() { + if r == nil { + return + } + r.once.Do(func() { + active.CompareAndSwap(r.store, nil) + r.cancel() + <-r.done + }) +} diff --git a/backend/services/logstore/store.go b/backend/services/logstore/store.go new file mode 100644 index 00000000..b2b734bc --- /dev/null +++ b/backend/services/logstore/store.go @@ -0,0 +1,234 @@ +// Package logstore 在当前业务数据库中保存操作与收发信日志。 +package logstore + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "git.pcux.cn/xuans/Smail/db/migration" + "git.pcux.cn/xuans/Smail/utils/logpolicy" +) + +var ErrUnavailable = errors.New("日志数据库不可用") +var ErrInvalid = errors.New("日志参数无效") + +type Entry struct { + ID string `json:"id"` + CreatedAt time.Time `json:"createdAt"` + Kind string `json:"kind"` + ActorID string `json:"actorId"` + Action string `json:"action"` + Section string `json:"section"` + Result string `json:"result"` + Payload json.RawMessage `json:"payload"` +} + +type Query struct { + ID string + Kind string + ActorID string + Action string + Section string + From time.Time + To time.Time + Limit int + Offset int +} + +type Store struct { + db *sql.DB + dialect migration.Dialect + now func() time.Time +} + +func New(database *sql.DB, dialect migration.Dialect) (*Store, error) { + if database == nil { + return nil, ErrUnavailable + } + if _, err := migration.ParseDialect(string(dialect)); err != nil { + return nil, ErrInvalid + } + return &Store{db: database, dialect: dialect, now: time.Now}, nil +} + +func logTable(kind string) (string, error) { + switch kind { + case "config", "operation": + return "operation_log", nil + case "mail": + return "mail_log", nil + default: + return "", ErrInvalid + } +} + +func (s *Store) bind(statement string) string { + if s.dialect != migration.DialectPostgres { + return statement + } + index := 0 + return replacePlaceholders(statement, &index) +} + +func replacePlaceholders(statement string, index *int) string { + parts := strings.Split(statement, "?") + var builder strings.Builder + for position, part := range parts { + if position > 0 { + *index++ + fmt.Fprintf(&builder, "$%d", *index) + } + builder.WriteString(part) + } + return builder.String() +} + +// Write 只接受业务点构造的脱敏载荷，禁止直接传入 HTTP 请求或邮件正文。 +func (s *Store) Write(ctx context.Context, entry Entry) error { + if s == nil || s.db == nil { + return ErrUnavailable + } + table, err := logTable(entry.Kind) + if err != nil { + return err + } + if entry.CreatedAt.IsZero() || len(entry.ID) > 80 || len(entry.ActorID) > 128 || len(entry.Action) > 64 || len(entry.Section) > 32 || len(entry.Result) > 16 || len(entry.Payload) > 256*1024 || !json.Valid(entry.Payload) { + return ErrInvalid + } + // 迁移、重放都不能让过期记录重新进入保留窗口。 + if entry.CreatedAt.Before(logpolicy.Cutoff(s.now())) { + return nil + } + if entry.ID == "" { + var token [16]byte + if _, err := rand.Read(token[:]); err != nil { + return ErrUnavailable + } + entry.ID = "log-" + hex.EncodeToString(token[:]) + } + statement := "INSERT INTO " + table + " (id, created_at, kind, actor_id, action, section, result, payload) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + // 仅忽略同一稳定标识的重放，不用 INSERT IGNORE 掩盖截断、缺字段等错误。 + if s.dialect == migration.DialectMySQL { + statement += " ON DUPLICATE KEY UPDATE id = id" + } else { + statement += " ON CONFLICT (id) DO NOTHING" + } + _, err = s.db.ExecContext(ctx, s.bind(statement), entry.ID, entry.CreatedAt.UTC().UnixNano(), entry.Kind, entry.ActorID, entry.Action, entry.Section, entry.Result, string(entry.Payload)) + if err != nil { + return ErrUnavailable + } + return nil +} + +func (s *Store) Query(ctx context.Context, query Query) ([]Entry, error) { + if s == nil || s.db == nil { + return nil, ErrUnavailable + } + table, err := logTable(query.Kind) + if err != nil || query.Limit < 0 || query.Limit > 1000 || query.Offset < 0 || query.Offset > 100000 || (!query.From.IsZero() && !query.To.IsZero() && query.From.After(query.To)) { + return nil, ErrInvalid + } + from := logpolicy.Cutoff(s.now()) + if query.From.After(from) { + from = query.From + } + statement := "SELECT id, created_at, kind, actor_id, action, section, result, payload FROM " + table + " WHERE created_at >= ? AND kind = ?" + args := []any{from.UnixNano(), query.Kind} + if !query.To.IsZero() { + statement += " AND created_at <= ?" + args = append(args, query.To.UnixNano()) + } + for _, filter := range []struct{ column, value string }{ + {"id", query.ID}, {"actor_id", query.ActorID}, {"action", query.Action}, {"section", query.Section}, + } { + if filter.value != "" { + statement += " AND " + filter.column + " = ?" + args = append(args, filter.value) + } + } + limit := query.Limit + if limit == 0 { + limit = 100 + } + statement += " ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?" + args = append(args, limit, query.Offset) + rows, err := s.db.QueryContext(ctx, s.bind(statement), args...) + if err != nil { + return nil, ErrUnavailable + } + defer rows.Close() + items := make([]Entry, 0) + for rows.Next() { + var entry Entry + var timestamp int64 + var payload string + if err := rows.Scan(&entry.ID, ×tamp, &entry.Kind, &entry.ActorID, &entry.Action, &entry.Section, &entry.Result, &payload); err != nil { + return nil, ErrUnavailable + } + entry.CreatedAt = time.Unix(0, timestamp).UTC() + entry.Payload = json.RawMessage(payload) + items = append(items, entry) + } + if rows.Err() != nil { + return nil, ErrUnavailable + } + return items, nil +} + +// Cleanup 每批先按索引选取标识再删除，避免 MySQL 专属 DELETE LIMIT 和大事务锁表。 +func (s *Store) Cleanup(ctx context.Context, now time.Time) (int64, error) { + if s == nil || s.db == nil { + return 0, ErrUnavailable + } + if now.IsZero() { + return 0, ErrInvalid + } + cutoff := logpolicy.Cutoff(now).UnixNano() + var deleted int64 + for _, table := range []string{"operation_log", "mail_log"} { + for { + if err := ctx.Err(); err != nil { + return deleted, err + } + rows, err := s.db.QueryContext(ctx, s.bind("SELECT id FROM "+table+" WHERE created_at < ? ORDER BY created_at, id LIMIT ?"), cutoff, logpolicy.DeleteBatchSize) + if err != nil { + return deleted, ErrUnavailable + } + ids := make([]any, 0, logpolicy.DeleteBatchSize) + for rows.Next() { + var id string + if err = rows.Scan(&id); err != nil { + break + } + ids = append(ids, id) + } + rowErr := rows.Err() + closeErr := rows.Close() + if err != nil || rowErr != nil || closeErr != nil { + return deleted, ErrUnavailable + } + if len(ids) == 0 { + break + } + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",") + args := append(ids, cutoff) + result, err := s.db.ExecContext(ctx, s.bind("DELETE FROM "+table+" WHERE id IN ("+placeholders+") AND created_at < ?"), args...) + if err != nil { + return deleted, ErrUnavailable + } + count, err := result.RowsAffected() + if err != nil { + return deleted, ErrUnavailable + } + deleted += count + } + } + return deleted, nil +} diff --git a/backend/services/runtimelog/archive.go b/backend/services/runtimelog/archive.go new file mode 100644 index 00000000..b274e425 --- /dev/null +++ b/backend/services/runtimelog/archive.go @@ -0,0 +1,223 @@ +package runtimelog + +import ( + "bufio" + "compress/gzip" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "strings" + "time" + + "git.pcux.cn/xuans/Smail/utils/logpolicy" +) + +func fileDay(name string) (time.Time, bool) { + base := strings.TrimSuffix(name, ".gz") + if !strings.HasPrefix(base, prefix) || !strings.HasSuffix(base, ".log") { + return time.Time{}, false + } + value := strings.TrimSuffix(strings.TrimPrefix(base, prefix), ".log") + day, err := time.Parse("2006-01-02", value) + return day, err == nil && (name == prefix+value+".log" || name == prefix+value+".log.gz") +} + +func lineTime(line []byte) (time.Time, error) { + if len(line) < 3 || line[0] != '[' { + return time.Time{}, ErrInvalid + } + end := strings.IndexByte(string(line), ']') + if end < 0 { + return time.Time{}, ErrInvalid + } + at, err := time.Parse(time.RFC3339Nano, string(line[1:end])) + if err != nil { + return time.Time{}, ErrInvalid + } + return at, nil +} + +// scan 严格解析每条记录及压缩校验和，损坏归档不得触发删除源文件。 +func (w *Writer) scan(ctx context.Context, name string, consume func([]byte, time.Time) error) error { + if err := w.regular(name, false); err != nil { + return err + } + file, err := w.root.Open(name) + if err != nil { + return err + } + defer file.Close() + var reader io.Reader = file + if strings.HasSuffix(name, ".gz") { + archive, err := gzip.NewReader(file) + if err != nil { + return err + } + defer archive.Close() + reader = archive + } + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 4096), maxLineBytes) + for scanner.Scan() { + if err := ctx.Err(); err != nil { + return err + } + at, err := lineTime(scanner.Bytes()) + if err != nil { + return err + } + if err := consume(scanner.Bytes(), at); err != nil { + return err + } + } + return scanner.Err() +} + +func (w *Writer) Maintain(ctx context.Context, now time.Time) error { + if now.IsZero() { + return ErrInvalid + } + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return nil + } + // 关闭当前文件后维护，避免 Windows 无法替换句柄；下一条日志会重新打开。 + if w.file != nil { + err := errors.Join(w.file.Sync(), w.file.Close()) + w.file = nil + if err != nil { + return err + } + } + directory, err := w.root.Open(".") + if err != nil { + return err + } + entries, err := directory.ReadDir(-1) + closeErr := directory.Close() + if err != nil || closeErr != nil { + return errors.Join(err, closeErr) + } + cutoff := logpolicy.Cutoff(now) + var failures []error + for _, entry := range entries { + if err := ctx.Err(); err != nil { + return err + } + day, ok := fileDay(entry.Name()) + if !ok { + continue + } + compress := !strings.HasSuffix(entry.Name(), ".gz") && !day.Add(24*time.Hour).After(now.Add(-logpolicy.CompressionAge)) + if !day.After(cutoff) || compress { + if err := w.transform(ctx, entry.Name(), day, cutoff, compress); err != nil { + failures = append(failures, err) + } + } + } + return errors.Join(failures...) +} + +// transform 流式过滤边界日；额外内存不随一天的日志量增长。 +func (w *Writer) transform(ctx context.Context, source string, day, cutoff time.Time, compress bool) error { + target := source + if compress { + target += ".gz" + } + var token [16]byte + if _, err := rand.Read(token[:]); err != nil { + return err + } + temporary := ".pending-" + hex.EncodeToString(token[:]) + ".log" + if strings.HasSuffix(target, ".gz") { + temporary += ".gz" + } + file, err := w.root.OpenFile(temporary, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + defer func() { _ = file.Close(); _ = w.root.Remove(temporary) }() + var output io.Writer = file + var archive *gzip.Writer + if strings.HasSuffix(target, ".gz") { + archive = gzip.NewWriter(file) + output = archive + } + hash := sha256.New() + output = io.MultiWriter(output, hash) + kept, removed := 0, 0 + err = w.scan(ctx, source, func(line []byte, at time.Time) error { + // 文件名和记录日期不一致时保留现场，不按文件名误删。 + if at.UTC().Format("2006-01-02") != day.Format("2006-01-02") { + return ErrInvalid + } + if at.Before(cutoff) { + removed++ + return nil + } + kept++ + _, err := output.Write(append(append([]byte(nil), line...), '\n')) + return err + }) + if archive != nil { + err = errors.Join(err, archive.Close()) + } + if err = errors.Join(err, file.Sync(), file.Close()); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + if removed == 0 && !compress { + return nil + } + if kept == 0 { + if err := w.regular(source, false); err != nil { + return err + } + return w.root.Remove(source) + } + verified := sha256.New() + if err := w.scan(ctx, temporary, func(line []byte, _ time.Time) error { + _, err := verified.Write(append(append([]byte(nil), line...), '\n')) + return err + }); err != nil { + return err + } + if !strings.EqualFold(hex.EncodeToString(hash.Sum(nil)), hex.EncodeToString(verified.Sum(nil))) { + return ErrInvalid + } + if source != target { + if _, err := w.root.Lstat(target); err == nil { + // 压缩成功但删除前中断时，只有目标与本轮内容完全一致才允许补做删除。 + existing := sha256.New() + if err := w.scan(ctx, target, func(line []byte, _ time.Time) error { + _, err := existing.Write(append(append([]byte(nil), line...), '\n')) + return err + }); err != nil { + return err + } + if hex.EncodeToString(existing.Sum(nil)) != hex.EncodeToString(hash.Sum(nil)) { + return ErrInvalid + } + return w.root.Remove(source) + } else if !os.IsNotExist(err) { + return err + } + } + if err := w.regular(target, true); err != nil { + return err + } + if err := w.root.Rename(temporary, target); err != nil { + return err + } + if source != target { + return w.root.Remove(source) + } + return nil +} diff --git a/backend/services/runtimelog/query.go b/backend/services/runtimelog/query.go new file mode 100644 index 00000000..c06bbc38 --- /dev/null +++ b/backend/services/runtimelog/query.go @@ -0,0 +1,91 @@ +package runtimelog + +import ( + "context" + "errors" + "sort" + "strings" + "time" + + "git.pcux.cn/xuans/Smail/utils/logpolicy" +) + +type File struct { + Name string `json:"name"` + Bytes int64 `json:"bytes"` + Compressed bool `json:"compressed"` +} + +type Page struct { + Lines []string `json:"lines"` + HasMore bool `json:"hasMore"` +} + +func (w *Writer) Files() ([]File, error) { + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return nil, ErrInvalid + } + directory, err := w.root.Open(".") + if err != nil { + return nil, err + } + defer directory.Close() + entries, err := directory.ReadDir(-1) + if err != nil { + return nil, err + } + files := make([]File, 0) + for _, entry := range entries { + day, ok := fileDay(entry.Name()) + if !ok || !day.Add(24*time.Hour).After(logpolicy.Cutoff(w.now())) { + continue + } + if err := w.regular(entry.Name(), false); err != nil { + return nil, err + } + info, err := entry.Info() + if err != nil { + return nil, err + } + files = append(files, File{Name: entry.Name(), Bytes: info.Size(), Compressed: strings.HasSuffix(entry.Name(), ".gz")}) + } + sort.Slice(files, func(i, j int) bool { return files[i].Name > files[j].Name }) + return files, nil +} + +// Read 仅允许固定名称的受管日志，不接受任意路径；压缩内容在服务端流式读取。 +func (w *Writer) Read(ctx context.Context, name string, offset, limit int) (Page, error) { + w.mu.Lock() + defer w.mu.Unlock() + if _, ok := fileDay(name); !ok || offset < 0 || offset > 1000000 || limit < 1 || limit > 500 || w.closed { + return Page{}, ErrInvalid + } + page := Page{Lines: make([]string, 0, limit)} + cutoff := logpolicy.Cutoff(w.now()) + position := 0 + const maxResponseBytes = 2 * 1024 * 1024 + bytes := 0 + finished := errors.New("当前页读取完成") + err := w.scan(ctx, name, func(line []byte, at time.Time) error { + if at.Before(cutoff) { + return nil + } + position++ + if position <= offset { + return nil + } + if len(page.Lines) >= limit || bytes+len(line) > maxResponseBytes { + page.HasMore = true + return finished + } + page.Lines = append(page.Lines, string(line)) + bytes += len(line) + return nil + }) + if errors.Is(err, finished) { + err = nil + } + return page, err +} diff --git a/backend/services/runtimelog/writer.go b/backend/services/runtimelog/writer.go new file mode 100644 index 00000000..766206cb --- /dev/null +++ b/backend/services/runtimelog/writer.go @@ -0,0 +1,173 @@ +// Package runtimelog 管理按天写入、七天压缩和三十天滚动清理的运行日志。 +package runtimelog + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" + + "git.pcux.cn/xuans/Smail/utils/logpolicy" + "git.pcux.cn/xuans/Smail/utils/runtimepath" + log "github.com/sirupsen/logrus" +) + +const prefix = "runtime-" +const maxLineBytes = 256 * 1024 + +var ErrInvalid = errors.New("运行日志文件或查询参数无效") +var active atomic.Pointer[Writer] + +type Writer struct { + mu sync.Mutex + root *os.Root + file *os.File + day string + now func() time.Time + cancel context.CancelFunc + done chan struct{} + closed bool +} + +func Current() *Writer { return active.Load() } + +func New(root string) (*Writer, error) { + directory, err := runtimepath.SecureDirectory(root, "log") + if err != nil { + return nil, err + } + directory, err = runtimepath.SecureDirectory(directory, "运行日志") + if err != nil { + return nil, err + } + handle, err := os.OpenRoot(directory) + if err != nil { + return nil, err + } + return &Writer{root: handle, now: time.Now}, nil +} + +// Start 使用日志钩子，配置重载重设 stdout 时不会丢失文件输出。 +func Start(root string) (*Writer, error) { + w, err := New(root) + if err != nil { + return nil, err + } + ctx, cancel := context.WithCancel(context.Background()) + w.cancel, w.done = cancel, make(chan struct{}) + active.Store(w) + log.AddHook(w) + go func() { + defer close(w.done) + ticker := time.NewTicker(logpolicy.MaintenanceInterval) + defer ticker.Stop() + for { + bounded, stop := context.WithTimeout(ctx, 2*time.Minute) + err := w.Maintain(bounded, w.now()) + stop() + if err != nil && ctx.Err() == nil { + log.WithError(err).Error("运行日志维护失败，源文件保留并在下一轮重试") + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } + }() + return w, nil +} + +func (w *Writer) Levels() []log.Level { return log.AllLevels } + +func (w *Writer) Fire(entry *log.Entry) error { + // 复用已有格式器保留请求标识、错误字段和源位置；转义换行保证一条记录一行。 + data, err := entry.Logger.Formatter.Format(entry) + if err != nil { + return err + } + return w.WriteAt(entry.Time, strings.TrimRight(string(data), "\r\n")) +} + +func (w *Writer) WriteAt(at time.Time, message string) error { + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return nil + } + if at.IsZero() { + return ErrInvalid + } + if at.Before(logpolicy.Cutoff(w.now())) { + return nil + } + message = strings.NewReplacer("\r", "\\r", "\n", "\\n", "\x00", "").Replace(message) + if len(message) > maxLineBytes-128 { + message = string([]rune(message)[:min(len([]rune(message)), (maxLineBytes-128)/4)]) + "…" + } + day := at.UTC().Format("2006-01-02") + if w.file == nil || w.day != day { + if w.file != nil { + if err := errors.Join(w.file.Sync(), w.file.Close()); err != nil { + w.file = nil + return err + } + w.file = nil + } + name := prefix + day + ".log" + if err := w.regular(name, true); err != nil { + return err + } + file, err := w.root.OpenFile(name, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + return err + } + w.file, w.day = file, day + } + _, err := fmt.Fprintf(w.file, "[%s] %s\n", at.UTC().Format(time.RFC3339Nano), message) + return err +} + +func (w *Writer) regular(name string, allowMissing bool) error { + if !filepath.IsLocal(name) || filepath.Base(name) != name { + return ErrInvalid + } + info, err := w.root.Lstat(name) + if allowMissing && os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return ErrInvalid + } + return nil +} + +func (w *Writer) Close() error { + if w == nil { + return nil + } + if w.cancel != nil { + w.cancel() + <-w.done + } + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return nil + } + w.closed = true + active.CompareAndSwap(w, nil) + var err error + if w.file != nil { + err = errors.Join(w.file.Sync(), w.file.Close()) + } + return errors.Join(err, w.root.Close()) +} diff --git a/backend/utils/logpolicy/policy.go b/backend/utils/logpolicy/policy.go new file mode 100644 index 00000000..6aeac32a --- /dev/null +++ b/backend/utils/logpolicy/policy.go @@ -0,0 +1,16 @@ +// Package logpolicy 统一三类日志的滚动保留期限。 +package logpolicy + +import "time" + +const ( + Retention = 30 * 24 * time.Hour + CompressionAge = 7 * 24 * time.Hour + MaintenanceInterval = time.Hour + DeleteBatchSize = 1000 +) + +// Cutoff 使用绝对时长，避免月份长度、时区或夏令时改变保留期。 +func Cutoff(now time.Time) time.Time { + return now.UTC().Add(-Retention) +} diff --git a/backend/utils/send/send.go b/backend/utils/send/send.go index 864272d7..8f94fdb4 100644 --- a/backend/utils/send/send.go +++ b/backend/utils/send/send.go @@ -7,6 +7,7 @@ import ( "git.pcux.cn/xuans/Smail/config" "git.pcux.cn/xuans/Smail/dto/parsemail" "git.pcux.cn/xuans/Smail/models" + "git.pcux.cn/xuans/Smail/services/logstore" "git.pcux.cn/xuans/Smail/utils/array" "git.pcux.cn/xuans/Smail/utils/async" "git.pcux.cn/xuans/Smail/utils/consts" @@ -119,7 +120,18 @@ func parseRecipientAddress(user *parsemail.User) (string, string, error) { return address, domain, nil } -func doSend(ctx *context.Context, fromDomain string, data []byte, to []*parsemail.User, from string) (error, map[string]error) { +func doSend(ctx *context.Context, fromDomain string, data []byte, to []*parsemail.User, from string) (sendErr error, recipientErrors map[string]error) { + defer func() { + result := "success" + if sendErr != nil || len(recipientErrors) > 0 { + result = "failure" + } + userID := 0 + if ctx != nil { + userID = ctx.UserID + } + logstore.MailResult(userID, "outbound", from, len(to), len(data), result) + }() if len(to) == 0 { err := errors.New("no recipients") return err, map[string]error{"recipients": err} diff --git a/docs/优化方案.md b/docs/优化方案.md index b5ba3286..ea59a3a1 100644 --- a/docs/优化方案.md +++ b/docs/优化方案.md @@ -62,4 +62,6 @@ xxx/(部署目录) ├──log/（日志文件） | └── xxx.log ├──smail-admin（管理员重置密码程序） -└──smail(运行程序) \ No newline at end of file +└──smail(运行程序) + +进行全面审查，然后使用mcp工具进行全面测试，每个功能都测试 \ No newline at end of file diff --git a/docs/日志改造任务.md b/docs/日志改造任务.md index a17d372d..4056c27e 100644 --- a/docs/日志改造任务.md +++ b/docs/日志改造任务.md @@ -2,25 +2,29 @@ ## 已确认需求 -- 操作记录、配置审计、收发信记录写入独立 MySQL，邮件业务数据库不随之迁移。 +- 操作记录、配置审计、收发信记录兼容 SQLite、MySQL、PostgreSQL，默认沿用当前业务数据库连接，不要求额外安装 MySQL，不自动迁移业务数据库。 - 设置导航在插件配置后增加日志入口，支持分类、时间筛选、分页与详情。 -- 运行日志保存在 `log/运行日志`，按天写入；每周一压缩已经结束的自然周，停机后启动补做。 +- 运行日志保存在 `log/运行日志`，按天写入；满七天的日文件压缩为独立的 `.log.gz`，启动补做，避免整周归档跨越清理边界。 - 压缩后的运行日志仍可在后台查看；仅在压缩校验通过后替换对应明文文件。 - 配置审计展示操作者、时间、具体字段及修改前后值；秘密只显示配置状态。 - 旧配置审计幂等迁移并保留原文件，不伪造历史缺失的变更详情。 -- 不自动删除历史记录；15 GB 是容量预算而不是容量保证。 -- 配置仍在原有五个 JSON 文件中，日志连接保存在 `config.json`，密码加密保存。 +- 用户于 2026-09-08 补充授权：三类日志均只保留最近 30×24 小时，启动补清理，运行期间定时滚动清理；压缩不重置保留时间。 +- 数据库按记录时间分批删除；运行日志同时覆盖明文、压缩归档和边界日内的过期记录。查询也应用同一截止时间，不能展示等待下一次清理的过期记录。 +- 清理仅作用于受管日志表和运行日志，绝不清理账号、邮件正文、附件、验证码业务表或其他项目数据库。15 GB 是容量预算而不是容量保证。 +- 配置仍在原有五个 JSON 文件中，复用既有数据库配置，不新增 MySQL 专用日志连接。 ## 实施状态 - [x] 已完成，等待复审：需求确认与修改前 Git 检查点；推送认证失败，未声称远程更新。 -- [~] 进行中：MySQL 存储、连接配置及旧配置审计迁移。 +- [~] 进行中：三数据库兼容存储、30 天滚动清理及旧配置审计迁移。 - [ ] 未开始：操作及收发信记录接入。 -- [ ] 未开始：运行日志轮转、每周压缩及只读查询。 +- [~] 进行中：运行日志轮转、七天压缩、30 天清理及只读查询。 - [ ] 未开始：具体配置变更明细与脱敏。 - [ ] 未开始：日志管理界面及原审计界面优化。 - [ ] 未开始：隔离测试、最新构建部署及浏览器验收。 ## 验收边界 -生产 MySQL 连接必须由部署者配置，不能猜测或复用其他项目的密码。未配置时保留旧审计路径并明确显示未启用，不能把 SQLite 或内存存储称为 MySQL 已启用。已配置后的写入故障必须可观察，不能无限排队或静默丢弃。 +数据库连接沿用部署者当前配置，不能猜测或复用其他项目的密码。SQLite 实测通过不等于 MySQL、PostgreSQL 真实连接验收通过；方言检查与真实数据库测试必须分别报告。写入或清理失败必须可观察、可重试，不能无限排队或静默丢弃。 + +过期删除让数据库页可重用，但不保证数据库文件立即缩小；不在在线清理中执行全库 VACUUM、OPTIMIZE 或重建表，以免长时间锁住邮件业务。数据库保留期以记录创建时间为准，不因升级、迁移、压缩而延长。旧审计源文件仅用于迁移核对，不继续双写，暂不擅自删除。 diff --git a/frontend/src/views/components/ConfigSettings/ConfigMaintenance/ConfigMaintenance.vue b/frontend/src/views/components/ConfigSettings/ConfigMaintenance/ConfigMaintenance.vue index 92379d6a..a61b4bae 100644 --- a/frontend/src/views/components/ConfigSettings/ConfigMaintenance/ConfigMaintenance.vue +++ b/frontend/src/views/components/ConfigSettings/ConfigMaintenance/ConfigMaintenance.vue @@ -164,7 +164,13 @@
{{ operationLabel(item.operation) }} - {{ sectionLabel(item.section) }} · {{ item.actorId || '系统' }} + {{ sectionLabel(item.section) }} · {{ item.actorName || item.actorId || '系统' }} +
+ {{ item.changes.length }} 个字段变更 +

+ {{ change.field }}：{{ change.before }} → {{ change.after }} +

+
diff --git a/frontend/src/views/components/HomeHeader/HomeHeader.js b/frontend/src/views/components/HomeHeader/HomeHeader.js index 2552ccaa..703fe610 100644 --- a/frontend/src/views/components/HomeHeader/HomeHeader.js +++ b/frontend/src/views/components/HomeHeader/HomeHeader.js @@ -1,5 +1,5 @@ import {EpMenu} from "vue-icons-plus/ep"; -import { Lock, Connection, Setting, FolderOpened, Filter, User, Grid } from '@element-plus/icons-vue'; +import { Lock, Connection, Setting, FolderOpened, Filter, User, Grid, Document } from '@element-plus/icons-vue'; import {computed, ref, onMounted, onUnmounted, defineAsyncComponent} from 'vue' import SettingsPageStatus from '../SettingsPageStatus/SettingsPageStatus.vue' @@ -25,6 +25,7 @@ const GroupSettings = lazySettings(() => import('../GroupSettings/GroupSettings. const RuleSettings = lazySettings(() => import('../RuleSettings/RuleSettings.vue')) const UserManagement = lazySettings(() => import('../UserManagement/UserManagement.vue')) const PluginSettings = lazySettings(() => import('../PluginSettings/PluginSettings.vue')) +const LogSettings = lazySettings(() => import('../LogSettings/LogSettings.vue')) export function usePage() { const globalStatus = useGlobalStatusStore(); @@ -53,6 +54,7 @@ export function usePage() { { key: 'rule', label: lang.rule_setting, hint: '邮件处理规则', icon: Filter, component: RuleSettings, capability: 'user.read' }, { key: 'users', label: lang.user_management, hint: '账号与权限管理', icon: User, component: UserManagement, capability: 'user.manage' }, { key: 'plugins', label: lang.plugin_settings, hint: '扩展功能管理', icon: Grid, component: PluginSettings, capability: 'plugin.read' }, + { key: 'logs', label: '日志', hint: '操作、收发信与运行记录', icon: Document, component: LogSettings, capability: 'audit.read' }, ].filter(item => globalStatus.hasCapability(item.capability) || (item.key === 'plugins' && globalStatus.hasCapability('config.read')))) diff --git a/frontend/src/views/components/LogSettings/LogSettings.css b/frontend/src/views/components/LogSettings/LogSettings.css new file mode 100644 index 00000000..53181f86 --- /dev/null +++ b/frontend/src/views/components/LogSettings/LogSettings.css @@ -0,0 +1,94 @@ +.log-settings { + height: 100%; + min-height: 0; + min-width: 0; + display: flex; + flex-direction: column; + gap: 14px; + padding: 18px; + box-sizing: border-box; +} + +.log-heading { + display: flex; + align-items: baseline; + justify-content: space-between; + flex-wrap: wrap; + gap: 8px; +} + +.log-heading h2 { + margin: 0; + font-size: 20px; +} + +.log-heading span, +.log-pagination span { + color: var(--el-text-color-secondary); + font-size: 12px; +} + +.log-categories { + flex-wrap: wrap; +} + +.log-filters { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 10px; +} + +.log-filters :deep(.el-date-editor) { + flex: 2 1 290px; + min-width: 0; + width: auto; +} + +.log-filters .el-input { + flex: 1 1 130px; + max-width: 190px; +} + +.log-filters .el-select { + flex: 1; + min-width: 200px; +} + +.log-content { + flex: 1; + min-height: 160px; + overflow: hidden; + border: 1px solid var(--el-border-color-lighter); + border-radius: 8px; +} + +.runtime-lines { + height: 100%; + overflow: auto; + box-sizing: border-box; + margin: 0; + padding: 12px; + font: 12px/1.7 Consolas, monospace; + white-space: pre-wrap; + overflow-wrap: anywhere; + background: var(--el-fill-color-light); +} + +.log-pagination { + display: flex; + align-items: center; + justify-content: flex-end; + gap: 8px; +} + +.log-detail-grid { + display: grid; + grid-template-columns: 110px minmax(0, 1fr); + gap: 12px; +} + +.log-detail-grid dd { + margin: 0; + overflow-wrap: anywhere; +} diff --git a/frontend/src/views/components/LogSettings/LogSettings.js b/frontend/src/views/components/LogSettings/LogSettings.js new file mode 100644 index 00000000..975c0bf2 --- /dev/null +++ b/frontend/src/views/components/LogSettings/LogSettings.js @@ -0,0 +1,114 @@ +import { computed, onBeforeUnmount, onMounted, ref, watch } from 'vue' +import http from '@/utils/axios' + +export function useLogs() { + const category = ref('config') + const loading = ref(false) + const error = ref('') + const items = ref([]) + const files = ref([]) + const selectedFile = ref('') + const lines = ref([]) + const offset = ref(0) + const hasMore = ref(false) + const actorId = ref('') + const dateRange = ref([]) + const detail = ref(null) + const pageSize = 50 + let requestId = 0 + let aborter + + const categories = [ + { label: '配置审计', value: 'config' }, + { label: '操作记录', value: 'operation' }, + { label: '收发信记录', value: 'mail' }, + { label: '运行日志', value: 'runtime' } + ] + const isRuntime = computed(() => category.value === 'runtime') + const formatTime = value => value ? new Date(value).toLocaleString('zh-CN', { hour12: false }) : '—' + const resultLabel = value => ({ success: '成功', failure: '失败', unknown: '结果未知', accepted: '已接收' }[value] || value) + const actionLabel = value => ({ update: '保存配置', rollback: '回滚配置', migration: '迁移配置' }[value] || value) + const displayActor = item => item.payload?.actorName || item.actorId || '系统' + const details = computed(() => detail.value?.payload || {}) + const changes = computed(() => details.value.changes || []) + + function requireData(result) { + if (result?.errorNo) throw new Error(result.errorMsg || '日志查询失败') + return result?.data || {} + } + + async function load() { + const id = ++requestId + aborter?.abort() + aborter = new AbortController() + loading.value = true + error.value = '' + try { + const params = { kind: category.value, offset: offset.value, limit: pageSize } + if (isRuntime.value) { + const listing = requireData(await http.get('/api/v1/logs', { + params: { kind: 'runtime' }, signal: aborter.signal + })) + if (id !== requestId) return + files.value = listing.files || [] + if (!files.value.some(file => file.name === selectedFile.value)) { + selectedFile.value = files.value[0]?.name || '' + } + if (!selectedFile.value) { + lines.value = [] + hasMore.value = false + return + } + params.file = selectedFile.value + } else { + if (actorId.value.trim()) params.actorId = actorId.value.trim() + if (dateRange.value?.length === 2) { + params.from = new Date(dateRange.value[0]).toISOString() + params.to = new Date(dateRange.value[1]).toISOString() + } + } + const data = requireData(await http.get('/api/v1/logs', { params, signal: aborter.signal })) + if (id !== requestId) return + items.value = data.items || [] + lines.value = data.lines || [] + hasMore.value = Boolean(data.hasMore) + } catch (cause) { + if (id === requestId && cause?.code !== 'ERR_CANCELED') { + error.value = cause.message || '日志查询失败' + items.value = [] + lines.value = [] + hasMore.value = false + } + } finally { + if (id === requestId) loading.value = false + } + } + + function search() { + offset.value = 0 + load() + } + function next() { + offset.value += isRuntime.value ? lines.value.length : pageSize + load() + } + function previous() { + offset.value = Math.max(0, offset.value - pageSize) + load() + } + watch(category, () => { + detail.value = null + search() + }) + onMounted(load) + onBeforeUnmount(() => { + ++requestId + aborter?.abort() + }) + + return { + category, categories, loading, error, items, files, selectedFile, lines, offset, + hasMore, actorId, dateRange, detail, details, changes, isRuntime, + formatTime, resultLabel, actionLabel, displayActor, search, next, previous + } +} diff --git a/frontend/src/views/components/LogSettings/LogSettings.vue b/frontend/src/views/components/LogSettings/LogSettings.vue new file mode 100644 index 00000000..17379bcf --- /dev/null +++ b/frontend/src/views/components/LogSettings/LogSettings.vue @@ -0,0 +1,103 @@ + + + + + diff --git a/test/backend/audit_changes_test.go b/test/backend/audit_changes_test.go new file mode 100644 index 00000000..9d88a1da --- /dev/null +++ b/test/backend/audit_changes_test.go @@ -0,0 +1,30 @@ +package config + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestAuditChangesRedactPluginArraysAndSettings(t *testing.T) { + before := RuntimeConfig{} + after := RuntimeConfig{ + Root: RootConfig{Database: DatabaseConfig{Password: StoredSecret{Ciphertext: "hidden-cipher"}}}, + Plugins: PluginsSectionConfig{Plugins: map[string]PluginInstance{ + "sample": {Settings: json.RawMessage(`{"items":[{"password":"hidden-password"}]}`)}, + }}, + } + changes := AuditChanges(before, after) + data, err := json.Marshal(changes) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(data), "hidden-") { + t.Fatalf("secret leaked: %s", data) + } + fields := map[string]any{} + flattenAudit(fields, "array", []any{map[string]any{"password": "hidden-password"}}) + if !auditSensitive("array.0.password") || fields["array.0.password"] == nil { + t.Fatal("array traversal failed") + } +} diff --git a/test/backend/log_migration_test.go b/test/backend/log_migration_test.go new file mode 100644 index 00000000..e9656d20 --- /dev/null +++ b/test/backend/log_migration_test.go @@ -0,0 +1,10 @@ +package migration + +import "testing" + +func TestLogMigrationChecksum(t *testing.T) { + items := DefaultMigrations() + if got := items[len(items)-1].Checksum(); got != "1dad66acb5e7fecf6c39f4a78526b994748d90045220f446a2e80c1b5fd881e7" { + t.Fatalf("unexpected log migration checksum: %s", got) + } +} diff --git a/test/backend/log_store_test.go b/test/backend/log_store_test.go new file mode 100644 index 00000000..72415ac2 --- /dev/null +++ b/test/backend/log_store_test.go @@ -0,0 +1,130 @@ +package logstore + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "git.pcux.cn/xuans/Smail/config" + "git.pcux.cn/xuans/Smail/db/migration" + "git.pcux.cn/xuans/Smail/utils/logpolicy" + _ "modernc.org/sqlite" +) + +func testStore(t *testing.T) *Store { + t.Helper() + database, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "logs.db")) + if err != nil { + t.Fatal(err) + } + database.SetMaxOpenConns(1) + t.Cleanup(func() { _ = database.Close() }) + items := migration.DefaultMigrations() + for _, statement := range items[len(items)-1].StatementsFor(migration.DialectSQLite) { + if _, err := database.Exec(statement); err != nil { + t.Fatal(err) + } + } + store, err := New(database, migration.DialectSQLite) + if err != nil { + t.Fatal(err) + } + return store +} + +func TestLogRetentionBoundariesAndBusinessSafety(t *testing.T) { + s := testStore(t) + now := time.Date(2026, 9, 8, 9, 37, 0, 0, time.UTC) + s.now = func() time.Time { return now.Add(-logpolicy.Retention) } + cutoff := logpolicy.Cutoff(now) + ctx := context.Background() + if _, err := s.db.Exec("CREATE TABLE business_probe (value TEXT)"); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec("INSERT INTO business_probe VALUES ('keep')"); err != nil { + t.Fatal(err) + } + for _, kind := range []string{"operation", "mail"} { + for index := 0; index < 2005; index++ { + at := cutoff.Add(-time.Nanosecond) + if index >= 2002 { + at = cutoff.Add(time.Duration(index - 2002)) + } + if err := s.Write(ctx, Entry{ID: fmt.Sprintf("%s-%d", kind, index), Kind: kind, CreatedAt: at, Result: "success", Payload: []byte(`{"message":"test"}`)}); err != nil { + t.Fatal(err) + } + } + } + s.now = func() time.Time { return now } + for _, kind := range []string{"operation", "mail"} { + items, err := s.Query(ctx, Query{Kind: kind}) + if err != nil || len(items) != 3 { + t.Fatalf("visible %s = %d, %v", kind, len(items), err) + } + } + deleted, err := s.Cleanup(ctx, now) + if err != nil || deleted != 4004 { + t.Fatalf("cleanup = %d, %v", deleted, err) + } + deleted, err = s.Cleanup(ctx, now) + if err != nil || deleted != 0 { + t.Fatalf("repeated cleanup = %d, %v", deleted, err) + } + var count int + if err := s.db.QueryRow("SELECT COUNT(*) FROM business_probe").Scan(&count); err != nil || count != 1 { + t.Fatal("business data changed") + } + if err := s.Write(ctx, Entry{Kind: "mail", CreatedAt: cutoff.Add(-1), Payload: []byte(`{}`)}); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT COUNT(*) FROM mail_log").Scan(&count); err != nil || count != 3 { + t.Fatal("expired replay was inserted") + } + ctx, cancel := context.WithCancel(ctx) + cancel() + if _, err := s.Cleanup(ctx, now); err == nil { + t.Fatal("cancelled cleanup succeeded") + } +} + +func TestLogDialectStatementsAndAuditReplay(t *testing.T) { + s := testStore(t) + for _, dialect := range []migration.Dialect{migration.DialectSQLite, migration.DialectMySQL, migration.DialectPostgres} { + s.dialect = dialect + got := s.bind("SELECT id FROM mail_log WHERE created_at < ? LIMIT ?") + if dialect == migration.DialectPostgres { + if !strings.Contains(got, "$1 LIMIT $2") { + t.Fatal(got) + } + } else if strings.Count(got, "?") != 2 { + t.Fatal(got) + } + } + s.dialect = migration.DialectSQLite + audit := &AuditWriter{Store: s} + event, err := config.NormalizeConfigAuditEvent(config.ConfigAuditEvent{ + Operation: config.ConfigOperationUpdate, Section: config.SectionRoot, ActorID: "1", ActorRole: "super_admin", ActorName: "tester", + FromGeneration: 1, ToGeneration: 2, CreatedAt: time.Now(), Result: "success", + Changes: []config.ConfigAuditChange{{Field: "root.logLevel", Before: "info", After: "debug"}}, + }) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + if err := audit.WriteConfigAudit(context.Background(), event); err != nil { + t.Fatal(err) + } + } + items, err := audit.Query(context.Background(), config.ConfigAuditQuery{}) + if err != nil || len(items) != 1 || items[0].Changes[0].After != "debug" { + t.Fatalf("audit: %+v, %v", items, err) + } + data, err := audit.ExportCSV(context.Background(), config.ConfigAuditQuery{}) + if err != nil || !strings.Contains(string(data), "tester") || !strings.Contains(string(data), "debug") { + t.Fatalf("csv: %s, %v", data, err) + } +} diff --git a/test/backend/runtime_logs_test.go b/test/backend/runtime_logs_test.go new file mode 100644 index 00000000..7c0e4a46 --- /dev/null +++ b/test/backend/runtime_logs_test.go @@ -0,0 +1,95 @@ +package runtimelog + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.pcux.cn/xuans/Smail/utils/logpolicy" +) + +func TestRuntimeLogCompressionAndExactRetention(t *testing.T) { + root := t.TempDir() + w, err := New(root) + if err != nil { + t.Fatal(err) + } + defer w.Close() + now := time.Date(2026, 9, 8, 12, 0, 0, 0, time.UTC) + cutoff := logpolicy.Cutoff(now) + w.now = func() time.Time { return cutoff } + for _, at := range []time.Time{cutoff.Add(-time.Hour), cutoff, cutoff.Add(time.Hour), now.Add(-8 * 24 * time.Hour), now} { + if err := w.WriteAt(at, "sample\nsecond line"); err != nil { + t.Fatal(err) + } + } + w.now = func() time.Time { return now } + ctx := context.Background() + if err := w.Maintain(ctx, now); err != nil { + t.Fatal(err) + } + files, err := w.Files() + if err != nil || len(files) != 3 { + t.Fatalf("files: %+v %v", files, err) + } + name := prefix + cutoff.Format("2006-01-02") + ".log.gz" + page, err := w.Read(ctx, name, 0, 100) + if err != nil || len(page.Lines) != 2 { + t.Fatalf("boundary: %+v %v", page, err) + } + if !strings.Contains(page.Lines[0], `\n`) { + t.Fatal("multiline event split") + } + // 压缩后仍须逐条裁剪，不能按整个归档日期删除有效记录。 + now = now.Add(30 * time.Minute) + if err := w.Maintain(ctx, now); err != nil { + t.Fatal(err) + } + page, err = w.Read(ctx, name, 0, 100) + if err != nil || len(page.Lines) != 1 { + t.Fatalf("compressed boundary: %+v %v", page, err) + } + now = now.Add(24 * time.Hour) + if err := w.Maintain(ctx, now); err != nil { + t.Fatal(err) + } + if _, err := w.root.Stat(name); !os.IsNotExist(err) { + t.Fatalf("expired archive still exists: %v", err) + } + if _, err := w.Read(ctx, "../config/config.json", 0, 100); err == nil { + t.Fatal("path traversal accepted") + } + if err := w.Maintain(ctx, now); err != nil { + t.Fatal(err) + } +} + +func TestRuntimeLogCorruptionAndUnknownFilesArePreserved(t *testing.T) { + root := t.TempDir() + w, err := New(root) + if err != nil { + t.Fatal(err) + } + defer w.Close() + now := time.Now().UTC() + w.now = func() time.Time { return now } + directory := filepath.Join(root, "log", "运行日志") + name := prefix + now.Add(-40*24*time.Hour).Format("2006-01-02") + ".log.gz" + if err := os.WriteFile(filepath.Join(directory, name), []byte("broken"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(directory, "manual.log"), []byte("keep"), 0600); err != nil { + t.Fatal(err) + } + if err := w.Maintain(context.Background(), now); err == nil { + t.Fatal("corruption not reported") + } + for _, file := range []string{name, "manual.log"} { + if _, err := os.Stat(filepath.Join(directory, file)); err != nil { + t.Fatal("unmanaged or damaged evidence deleted") + } + } +} diff --git a/test/run.mjs b/test/run.mjs index cc82d33f..7b1a7b73 100644 --- a/test/run.mjs +++ b/test/run.mjs @@ -49,6 +49,10 @@ if ((suite === 'all' || suite === 'backend') && result === 0) { const args = extra[0] === '--' ? extra.slice(1) : extra // 独立回归源码集中放在 test/backend，通过 overlay 保留同包测试语义。 const replacements = { + 'backend/services/logstore/log_store_test.go': 'test/backend/log_store_test.go', + 'backend/services/runtimelog/runtime_logs_test.go': 'test/backend/runtime_logs_test.go', + 'backend/db/migration/log_migration_test.go': 'test/backend/log_migration_test.go', + 'backend/config/audit_changes_test.go': 'test/backend/audit_changes_test.go', 'backend/config/json_format_test.go': 'test/backend/json_format_test.go', 'backend/config/deployment_state_test.go': 'test/backend/deployment_state_test.go', 'backend/config/ssl_retention_test.go': 'test/backend/ssl_retention_test.go',