保存三数据库日志与三十天清理实现,继续补充验收
Security Checks / security (push) Waiting to run

This commit is contained in:
xuans committed 2026-09-08 22:30:50 +08:00
1 parent 9bdf866a87
commit 5faadcd6eb
41 files changed
+1994 -59

No files matched your search

+66 -20
View File
@@ -4,15 +4,16 @@ import (
"encoding/json"
"reflect"
"sort"
"strconv"
"strings"
)
// ConfigAuditChange 只保存安全的显示值,不保存密文或请求原文。
type ConfigAuditChange struct {
Field string `json:"field"`
Before string `json:"before"`
After string `json:"after"`
Sensitive bool `json:"sensitive,omitempty"`
Field string `json:"field"`
Before string `json:"before"`
After string `json:"after"`
Sensitive bool `json:"sensitive,omitempty"`
}
// AuditChanges 对比业务字段,排除代次、摘要和内部持久化元数据。
@@ -20,20 +21,32 @@ func AuditChanges(before, after RuntimeConfig) []ConfigAuditChange {
left := auditFields(before)
right := auditFields(after)
keys := make(map[string]bool)
for key := range left { keys[key] = true }
for key := range right { keys[key] = true }
for key := range left {
keys[key] = true
}
for key := range right {
keys[key] = true
}
ordered := make([]string, 0, len(keys))
for key := range keys { ordered = append(ordered, key) }
for key := range keys {
ordered = append(ordered, key)
}
sort.Strings(ordered)
changes := make([]ConfigAuditChange, 0)
for _, key := range ordered {
if reflect.DeepEqual(left[key], right[key]) { continue }
if reflect.DeepEqual(left[key], right[key]) {
continue
}
change := ConfigAuditChange{Field: key, Before: auditDisplay(left[key]), After: auditDisplay(right[key])}
if auditSensitive(key) {
change.Sensitive = true
change.Before, change.After = "未配置", "未配置"
if auditConfigured(left[key]) { change.Before = "已配置(隐藏)" }
if auditConfigured(right[key]) { change.After = "已配置(隐藏,已更新)" }
if auditConfigured(left[key]) {
change.Before = "已配置(隐藏)"
}
if auditConfigured(right[key]) {
change.After = "已配置(隐藏,已更新)"
}
}
changes = append(changes, change)
}
@@ -60,10 +73,23 @@ func flattenAudit(result map[string]any, path string, value any) {
case "meta", "sections", "generation", "updatedAt", "schemaVersion", "configVersion", "version":
return
}
if path == "root.smtp" || path == "root.email" || path == "root.ssl" || path == "root.plugins" { return }
if auditSensitive(path) { result[path] = value; return }
if path == "root.smtp" || path == "root.email" || path == "root.ssl" || path == "root.plugins" {
return
}
if auditSensitive(path) {
result[path] = value
return
}
if items, ok := value.([]any); ok {
for index, item := range items {
flattenAudit(result, path+"."+strconv.Itoa(index), item)
}
return
}
if object, ok := value.(map[string]any); ok {
for key, item := range object { flattenAudit(result, path+"."+key, item) }
for key, item := range object {
flattenAudit(result, path+"."+key, item)
}
return
}
result[path] = value
@@ -72,30 +98,50 @@ func flattenAudit(result map[string]any, path string, value any) {
func auditSensitive(path string) bool {
name := strings.ToLower(path[strings.LastIndex(path, ".")+1:])
for _, word := range []string{"password", "secret", "token", "apikey", "api_key", "credential", "ciphertext", "authorization", "cookie"} {
if strings.Contains(name, word) { return true }
if strings.Contains(name, word) {
return true
}
}
// 插件设置是自由结构,未知键也不能作为明文审计输出。
return strings.HasSuffix(path, ".settings") || strings.HasSuffix(path, ".encryptionKeyRef")
}
func auditConfigured(value any) bool {
if value == nil || value == "" { return false }
if value == nil {
return false
}
if text, ok := value.(string); ok {
return text != ""
}
if object, ok := value.(map[string]any); ok {
if ciphertext, exists := object["ciphertext"]; exists { return ciphertext != "" }
if ciphertext, exists := object["ciphertext"]; exists {
return ciphertext != ""
}
return len(object) > 0
}
return true
}
func auditDisplay(value any) string {
if value == nil { return "未设置" }
if value == nil {
return "未设置"
}
if text, ok := value.(string); ok {
if len([]rune(text)) > 512 { return string([]rune(text)[:512])+"…" }
if len([]rune(text)) > 512 {
return string([]rune(text)[:512]) + "…"
}
return text
}
if flag, ok := value.(bool); ok { if flag { return "开启" }; return "关闭" }
if flag, ok := value.(bool); ok {
if flag {
return "开启"
}
return "关闭"
}
data, _ := json.Marshal(value)
text := []rune(string(data))
if len(text) > 512 { return string(text[:512])+"…" }
if len(text) > 512 {
return string(text[:512]) + "…"
}
return string(text)
}
+21 -3
View File
@@ -2,6 +2,7 @@ package config
import (
"encoding/csv"
"encoding/json"
"fmt"
"strings"
"time"
@@ -10,11 +11,15 @@ import (
func exportAuditCSV(events []ConfigAuditEvent) ([]byte, error) {
var builder strings.Builder
writer := csv.NewWriter(&builder)
if err := writer.Write([]string{"id", "operation", "section", "actor_id", "actor_role", "from_generation", "to_generation", "target_generation", "changed_fields", "confirmation_method", "result", "created_at"}); err != nil {
if err := writer.Write([]string{"id", "operation", "section", "actor_id", "actor_role", "from_generation", "to_generation", "target_generation", "changed_fields", "confirmation_method", "result", "created_at", "actor_name", "changes"}); err != nil {
return nil, err
}
for _, event := range events {
if err := writer.Write([]string{
changes, err := json.Marshal(event.Changes)
if err != nil {
return nil, err
}
cells := []string{
event.ID,
string(event.Operation),
string(event.Section),
@@ -27,7 +32,15 @@ func exportAuditCSV(events []ConfigAuditEvent) ([]byte, error) {
event.ConfirmationMethod,
event.Result,
event.CreatedAt.UTC().Format(time.RFC3339Nano),
}); err != nil {
event.ActorName,
string(changes),
}
for index, cell := range cells {
if strings.HasPrefix(cell, "=") || strings.HasPrefix(cell, "+") || strings.HasPrefix(cell, "-") || strings.HasPrefix(cell, "@") || strings.HasPrefix(cell, "\t") || strings.HasPrefix(cell, "\r") {
cells[index] = "'" + cell
}
}
if err := writer.Write(cells); err != nil {
return nil, err
}
}
@@ -37,3 +50,8 @@ func exportAuditCSV(events []ConfigAuditEvent) ([]byte, error) {
}
return []byte(builder.String()), nil
}
// ExportConfigAuditCSV 让文件、内存和数据库导出共享字段及表格公式保护。
func ExportConfigAuditCSV(events []ConfigAuditEvent) ([]byte, error) {
return exportAuditCSV(events)
}
+15 -3
View File
@@ -17,10 +17,11 @@ import (
"time"
"git.pcux.cn/xuans/Smail/utils/atomicfile"
"git.pcux.cn/xuans/Smail/utils/logpolicy"
"git.pcux.cn/xuans/Smail/utils/runtimepath"
)
const ConfigAuditRetention = 365 * 24 * time.Hour
const ConfigAuditRetention = logpolicy.Retention
var (
ErrAuditStorageUnavailable = errors.New("configuration audit storage unavailable")
@@ -296,6 +297,7 @@ func normalizeAuditEvent(event ConfigAuditEvent) (ConfigAuditEvent, error) {
event.ActorRole = strings.TrimSpace(event.ActorRole)
event.ConfirmationMethod = strings.ToLower(strings.TrimSpace(event.ConfirmationMethod))
event.ChangedFields = append([]string(nil), event.ChangedFields...)
event.Changes = append([]ConfigAuditChange(nil), event.Changes...)
if err := validateAuditEvent(event); err != nil {
return ConfigAuditEvent{}, err
}
@@ -326,17 +328,27 @@ func validateAuditEvent(event ConfigAuditEvent) error {
if event.Result != "success" && event.Result != "failure" {
return ErrAuditEventInvalid
}
if strings.ContainsAny(event.ActorID+event.ActorRole+event.ConfirmationMethod, "\r\n") || len(event.ActorID) > 128 || len(event.ActorRole) > 64 {
if strings.ContainsAny(event.ActorID+event.ActorRole+event.ActorName+event.ConfirmationMethod, "\r\n") || len(event.ActorID) > 128 || len(event.ActorRole) > 64 || len(event.ActorName) > 256 {
return ErrAuditEventInvalid
}
for _, field := range event.ChangedFields {
if field == "" || len(field) > 64 || strings.ContainsAny(field, "\r\n") {
if field == "" || len(field) > 1024 || strings.ContainsAny(field, "\r\n") {
return ErrAuditEventInvalid
}
}
for _, change := range event.Changes {
if change.Field == "" || len(change.Field) > 1024 || strings.ContainsAny(change.Field, "\r\n") || len([]rune(change.Before)) > 513 || len([]rune(change.After)) > 513 {
return ErrAuditEventInvalid
}
}
return nil
}
// NormalizeConfigAuditEvent 供数据库审计复用同一套标识、校验和复制规则。
func NormalizeConfigAuditEvent(event ConfigAuditEvent) (ConfigAuditEvent, error) {
return normalizeAuditEvent(event)
}
func auditEventMatches(event ConfigAuditEvent, query ConfigAuditQuery) bool {
if query.ID != "" && event.ID != query.ID {
return false
+57
View File
@@ -0,0 +1,57 @@
package config
import (
"bufio"
"context"
"encoding/json"
"os"
"time"
)
// Visit 从旧文件流式迁移,不受管理界面的查询条数上限影响,不修改源文件。
func (w *FileAuditWriter) Visit(ctx context.Context, from time.Time, visit func(ConfigAuditEvent) error) error {
if w == nil || visit == nil {
return ErrAuditStorageUnavailable
}
w.mu.Lock()
defer w.mu.Unlock()
if info, err := os.Lstat(w.path); err != nil {
if os.IsNotExist(err) {
return nil
}
return ErrAuditStorageUnavailable
} else if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
return ErrAuditStorageUnavailable
}
file, err := os.Open(w.path)
if err != nil {
return ErrAuditStorageUnavailable
}
defer file.Close()
scanner := bufio.NewScanner(file)
scanner.Buffer(make([]byte, 4096), 256*1024)
line := 0
for scanner.Scan() {
if err := ctx.Err(); err != nil {
return err
}
line++
var event ConfigAuditEvent
if json.Unmarshal(scanner.Bytes(), &event) != nil {
return ErrAuditStorageUnavailable
}
if event.ID == "" {
event.ID = legacyAuditEventID(scanner.Bytes(), line)
}
if err := validateAuditEvent(event); err != nil {
return err
}
if event.CreatedAt.Before(from) {
continue
}
if err := visit(event); err != nil {
return err
}
}
return scanner.Err()
}
+17 -2
View File
@@ -9,6 +9,14 @@ import (
var ErrDatabaseMigrationTarget = errors.New("database migration target is invalid")
type migrationAuditKey struct{}
// WithMigrationAuditWriter 仅供持有迁移目标连接的执行器设置迁移审计去向。
// 普通配置更新不读取该上下文值,HTTP 请求不能指定审计数据库。
func WithMigrationAuditWriter(ctx context.Context, writer ConfigAuditWriter) context.Context {
return context.WithValue(ctx, migrationAuditKey{}, writer)
}
// SwitchDatabase 提交已完成复制与校验的目标连接,保留旧库并标记重启。
// 普通配置编辑不能绕过本迁移入口切换数据库类型。
func (m *ConfigManager) SwitchDatabase(ctx context.Context, expectedGeneration uint64, target DatabaseConfig, actor AuditActor) error {
@@ -91,7 +99,7 @@ func (m *ConfigManager) SwitchDatabase(ctx context.Context, expectedGeneration u
if !sameDatabaseConnection(current.Root.Database, target) {
changedField = "root.database"
}
return m.writeAudit(ctx, ConfigAuditEvent{
event := ConfigAuditEvent{
Operation: ConfigOperationMigration,
Section: SectionRoot,
ActorID: actor.UserID,
@@ -104,7 +112,14 @@ func (m *ConfigManager) SwitchDatabase(ctx context.Context, expectedGeneration u
ChangedFields: []string{changedField},
Result: "success",
CreatedAt: now,
})
}
if writer, ok := ctx.Value(migrationAuditKey{}).(ConfigAuditWriter); ok && writer != nil {
if err := writer.WriteConfigAudit(ctx, event); err != nil {
return errors.Join(ErrAuditWrite, err)
}
return nil
}
return m.writeAudit(ctx, event)
}
// SameDatabaseLocation 不把用户名或密码不同误认为不同数据库。
+3 -1
View File
@@ -777,7 +777,9 @@ func (m *ConfigManager) writeAudit(ctx context.Context, event ConfigAuditEvent)
func changedFields(current, candidate RuntimeConfig, section Section) []string {
var fields []string
for _, change := range AuditChanges(current, candidate) { fields = append(fields, change.Field) }
for _, change := range AuditChanges(current, candidate) {
fields = append(fields, change.Field)
}
return fields
}
+2 -1
View File
@@ -1771,7 +1771,8 @@ func TestConfigManagerWritesRedactedAuditAndDegradesOnAuditFailure(t *testing.T)
t.Fatal(err)
}
events := writer.Events()
if len(events) != 1 || events[0].Section != SectionEmail || len(events[0].ChangedFields) != 1 || events[0].ChangedFields[0] != string(SectionEmail) {
// 原样保存只记录操作,不能伪造业务字段发生变更。
if len(events) != 1 || events[0].Section != SectionEmail || len(events[0].ChangedFields) != 0 || len(events[0].Changes) != 0 {
t.Fatalf("unexpected audit events: %+v", events)
}
if events[0].Result != "success" || events[0].ToGeneration != 2 {
+15 -13
View File
@@ -26,20 +26,20 @@ const (
)
type ConfigAuditEvent struct {
ID string `json:"id"`
Operation ConfigOperation `json:"operation"`
Section Section `json:"section,omitempty"`
ActorID string `json:"actorId"`
ActorRole string `json:"actorRole"`
ActorName string `json:"actorName,omitempty"`
ID string `json:"id"`
Operation ConfigOperation `json:"operation"`
Section Section `json:"section,omitempty"`
ActorID string `json:"actorId"`
ActorRole string `json:"actorRole"`
ActorName string `json:"actorName,omitempty"`
Changes []ConfigAuditChange `json:"changes,omitempty"`
FromGeneration uint64 `json:"fromGeneration"`
ToGeneration uint64 `json:"toGeneration"`
TargetGeneration uint64 `json:"targetGeneration,omitempty"`
ChangedFields []string `json:"changedFields,omitempty"`
ConfirmationMethod string `json:"confirmationMethod,omitempty"`
Result string `json:"result"`
CreatedAt time.Time `json:"createdAt"`
FromGeneration uint64 `json:"fromGeneration"`
ToGeneration uint64 `json:"toGeneration"`
TargetGeneration uint64 `json:"targetGeneration,omitempty"`
ChangedFields []string `json:"changedFields,omitempty"`
ConfirmationMethod string `json:"confirmationMethod,omitempty"`
Result string `json:"result"`
CreatedAt time.Time `json:"createdAt"`
}
type ConfigAuditWriter interface {
WriteConfigAudit(context.Context, ConfigAuditEvent) error
@@ -107,6 +107,7 @@ func (w *MemoryAuditWriter) Query(ctx context.Context, query ConfigAuditQuery) (
}
copyEvent := event
copyEvent.ChangedFields = append([]string(nil), event.ChangedFields...)
copyEvent.Changes = append([]ConfigAuditChange(nil), event.Changes...)
result = append(result, copyEvent)
if len(result) >= limit {
break
@@ -141,6 +142,7 @@ func (w *MemoryAuditWriter) Events() []ConfigAuditEvent {
copy(result, w.events)
for index := range result {
result[index].ChangedFields = append([]string(nil), result[index].ChangedFields...)
result[index].Changes = append([]ConfigAuditChange(nil), result[index].Changes...)
}
return result
}