This commit is contained in:
parent
7f719ab79e
commit
1b2ba5a6a1
@ -96,6 +96,13 @@ func TestPersistPluginConfigReplacesFileAtomically(t *testing.T) {
|
||||
if !RegisterHook("spam_block", &HookSender{name: "spam_block.exe"}) {
|
||||
t.Fatal("plugin hook registration failed")
|
||||
}
|
||||
if runtime.GOOS == "linux" && os.Geteuid() != 0 {
|
||||
// 普通用户创建的插件目录不满足生产所有者要求，必须拒绝写入。
|
||||
if err := PersistPluginConfig("spam_block", `{"url":"https://model.example.com"}`); err == nil {
|
||||
t.Fatal("non-root-owned plugin directory was accepted")
|
||||
}
|
||||
return
|
||||
}
|
||||
for _, payload := range []string{`{"url":"https://model.example.com"}`, `{"url":"https://model.example.com/v2"}`} {
|
||||
if err = PersistPluginConfig("spam_block", payload); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@ -958,6 +958,9 @@ func TestIDLE(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if err := conn.SetDeadline(time.Now().Add(15 * time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
reader := bufio.NewReader(conn)
|
||||
line, err := reader.ReadString('\n')
|
||||
|
||||
@ -24,6 +24,13 @@ type Runtime struct {
|
||||
done chan struct{}
|
||||
once sync.Once
|
||||
deliver func(*context.Context, *parsemail.Email, string) error
|
||||
pending *pendingResult
|
||||
}
|
||||
|
||||
type pendingResult struct {
|
||||
job models.OutboundDelivery
|
||||
ctx *context.Context
|
||||
message *parsemail.Email
|
||||
}
|
||||
|
||||
func Start(engine *xorm.Engine) (*Runtime, error) {
|
||||
@ -58,20 +65,8 @@ func (r *Runtime) loop() {
|
||||
return
|
||||
default:
|
||||
}
|
||||
var jobs []models.OutboundDelivery
|
||||
err := r.engine.Where("state = ? AND next_attempt <= ?", queued, time.Now().Unix()).Asc("id").Limit(20).Find(&jobs)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("读取持久投递队列失败")
|
||||
}
|
||||
for _, job := range jobs {
|
||||
select {
|
||||
case <-r.stop:
|
||||
return
|
||||
default:
|
||||
}
|
||||
if err := r.process(job); err != nil {
|
||||
log.WithError(err).WithField("delivery_id", job.ID).Error("处理持久投递队列失败")
|
||||
}
|
||||
if err := r.tick(); err != nil {
|
||||
log.WithError(err).Error("处理持久投递队列失败")
|
||||
}
|
||||
select {
|
||||
case <-r.stop:
|
||||
@ -81,12 +76,37 @@ func (r *Runtime) loop() {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runtime) tick() error {
|
||||
// 单工作器最多保留一个未落库结果，恢复前不领取新任务，避免内存积压和重复投递。
|
||||
if r.pending != nil {
|
||||
return dbmigration.WhileWritable(r.flushResult)
|
||||
}
|
||||
var jobs []models.OutboundDelivery
|
||||
if err := r.engine.Where("state = ? AND next_attempt <= ?", queued, time.Now().Unix()).Asc("id").Limit(20).Find(&jobs); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, job := range jobs {
|
||||
select {
|
||||
case <-r.stop:
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
if err := r.process(job); err != nil {
|
||||
return fmt.Errorf("投递任务 %d: %w", job.ID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runtime) process(job models.OutboundDelivery) error {
|
||||
// 迁移必须等待 SMTP 及结果落库结束，冻结源库后不再领取任务。
|
||||
return dbmigration.WhileWritable(func() error { return r.processWritable(job) })
|
||||
}
|
||||
|
||||
func (r *Runtime) processWritable(job models.OutboundDelivery) error {
|
||||
if r.pending != nil {
|
||||
return r.flushResult()
|
||||
}
|
||||
// 读取失败时尚未开始投递，保留排队状态供数据库恢复后继续处理。
|
||||
var stored models.Email
|
||||
emailFound, err := r.engine.ID(job.EmailID).Get(&stored)
|
||||
@ -126,20 +146,23 @@ func (r *Runtime) processWritable(job models.OutboundDelivery) error {
|
||||
}
|
||||
return r.deliver(ctx, message, job.Recipient)
|
||||
}()
|
||||
// 只重试结果落库，绝不因数据库故障再次调用 SMTP。
|
||||
var terminal bool
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
terminal, err = finish(r.engine, job, deliveryErr)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
// 固定首次结果及退避时间，后续只重试落库，绝不再次调用 SMTP。
|
||||
r.pending = &pendingResult{job: deliveryResult(job, deliveryErr), ctx: ctx, message: message}
|
||||
return r.flushResult()
|
||||
}
|
||||
|
||||
func (r *Runtime) flushResult() error {
|
||||
result := r.pending
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
terminal, err := persistResult(r.engine, result.job)
|
||||
if err != nil {
|
||||
return fmt.Errorf("投递结果写入失败，保留在途记录并禁止自动重投: %w", err)
|
||||
return fmt.Errorf("投递任务 %d 结果写入失败，已暂停领取任务并等待数据库恢复: %w", result.job.ID, err)
|
||||
}
|
||||
if terminal && message != nil {
|
||||
r.notify(ctx, message, job.EmailID)
|
||||
r.pending = nil
|
||||
if terminal && result.message != nil {
|
||||
r.notify(result.ctx, result.message, result.job.EmailID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -151,6 +151,10 @@ func claim(engine *xorm.Engine, job *models.OutboundDelivery) (bool, error) {
|
||||
}
|
||||
|
||||
func finish(engine *xorm.Engine, job models.OutboundDelivery, deliveryErr error) (bool, error) {
|
||||
return persistResult(engine, deliveryResult(job, deliveryErr))
|
||||
}
|
||||
|
||||
func deliveryResult(job models.OutboundDelivery, deliveryErr error) models.OutboundDelivery {
|
||||
job.State, job.LastError, job.NextAttempt = accepted, "", 0
|
||||
if deliveryErr != nil {
|
||||
// 不保存远端响应原文，避免凭据或恶意超长内容进入数据库和页面。
|
||||
@ -163,6 +167,10 @@ func finish(engine *xorm.Engine, job models.OutboundDelivery, deliveryErr error)
|
||||
job.NextAttempt = time.Now().Add(time.Minute * time.Duration(1<<(job.Attempts-1))).Unix()
|
||||
}
|
||||
}
|
||||
return job
|
||||
}
|
||||
|
||||
func persistResult(engine *xorm.Engine, job models.OutboundDelivery) (bool, error) {
|
||||
terminal := false
|
||||
err := db.Transaction(engine, func(session *xorm.Session) error {
|
||||
count, err := session.ID(job.ID).Where("state = ? AND attempts = ?", running, job.Attempts).Cols("state", "last_error", "next_attempt").Update(&job)
|
||||
@ -170,7 +178,17 @@ func finish(engine *xorm.Engine, job models.OutboundDelivery, deliveryErr error)
|
||||
return err
|
||||
}
|
||||
if count != 1 {
|
||||
return errors.New("投递任务状态已发生变化")
|
||||
// 提交响应丢失时，确认同一次尝试的相同结果已落库，不覆盖后续尝试。
|
||||
var stored models.OutboundDelivery
|
||||
found, readErr := session.ID(job.ID).Get(&stored)
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
if !found || stored.EmailID != job.EmailID || stored.Recipient != job.Recipient ||
|
||||
stored.Attempts != job.Attempts || stored.State != job.State ||
|
||||
stored.LastError != job.LastError || stored.NextAttempt != job.NextAttempt {
|
||||
return errors.New("投递任务状态已发生变化")
|
||||
}
|
||||
}
|
||||
terminal, err = syncStatus(session, job.EmailID)
|
||||
return err
|
||||
|
||||
@ -173,8 +173,8 @@ func TestResultFailureRollsBackAndNeverRedelivers(t *testing.T) {
|
||||
if err := r.process(jobs[0]); err == nil {
|
||||
t.Fatal("result failure was ignored")
|
||||
}
|
||||
if err := r.process(jobs[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
if err := r.tick(); err == nil {
|
||||
t.Fatal("result retry failure was ignored")
|
||||
}
|
||||
if calls.Load() != 1 {
|
||||
t.Fatalf("SMTP called %d times", calls.Load())
|
||||
@ -187,6 +187,77 @@ func TestResultFailureRollsBackAndNeverRedelivers(t *testing.T) {
|
||||
if job.State != running {
|
||||
t.Fatalf("state=%s", job.State)
|
||||
}
|
||||
if _, err := engine.Exec("DROP TRIGGER reject_result"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.tick(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.pending != nil || calls.Load() != 1 {
|
||||
t.Fatal("result was not recovered without another SMTP attempt")
|
||||
}
|
||||
var persisted models.OutboundDelivery
|
||||
if found, err := engine.ID(job.ID).Get(&persisted); err != nil || !found || persisted.State != accepted {
|
||||
t.Fatalf("result not persisted: %+v %v", persisted, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistResultAfterLostCommitResponse(t *testing.T) {
|
||||
for _, deliveryErr := range []error{nil, &textproto.Error{Code: 451, Msg: "later"}, &smtp.DeliveryUncertainError{Cause: io.EOF}} {
|
||||
engine, _, jobs := queueFixture(t)
|
||||
job := jobs[0]
|
||||
if ok, err := claim(engine, &job); err != nil || !ok {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := deliveryResult(job, deliveryErr)
|
||||
if _, err := persistResult(engine, result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := persistResult(engine, result); err != nil {
|
||||
t.Fatalf("same result must be idempotent: %v", err)
|
||||
}
|
||||
result.Attempts++
|
||||
if _, err := persistResult(engine, result); err == nil {
|
||||
t.Fatal("different attempt must not overwrite persisted result")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPendingResultStopsNewClaimsAndSurvivesRestartSafely(t *testing.T) {
|
||||
engine, _, jobs := queueFixture(t)
|
||||
if _, err := engine.Exec("CREATE TRIGGER reject_result BEFORE UPDATE ON user_email BEGIN SELECT RAISE(ABORT, 'injected failure'); END"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var calls int
|
||||
r := &Runtime{engine: engine, deliver: func(_ *context.Context, _ *parsemail.Email, _ string) error {
|
||||
calls++
|
||||
return nil
|
||||
}}
|
||||
for i := 0; i < 4; i++ {
|
||||
if err := r.tick(); err == nil {
|
||||
t.Fatal("persistence failure ignored")
|
||||
}
|
||||
}
|
||||
var second models.OutboundDelivery
|
||||
if _, err := engine.ID(jobs[1].ID).Get(&second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 1 || second.State != queued || second.Attempts != 0 {
|
||||
t.Fatalf("new delivery started during outage: calls=%d job=%+v", calls, second)
|
||||
}
|
||||
if _, err := engine.Exec("DROP TRIGGER reject_result"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recoverInterrupted(engine); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var first models.OutboundDelivery
|
||||
if _, err := engine.ID(jobs[0].ID).Get(&first); err != nil || first.State != uncertain {
|
||||
t.Fatalf("lost in-memory result must remain uncertain: %+v %v", first, err)
|
||||
}
|
||||
if _, err := persistResult(engine, r.pending.job); err == nil {
|
||||
t.Fatal("stale runtime overwrote restart recovery")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueuedJobsSurviveRestartAndDrain(t *testing.T) {
|
||||
|
||||
@ -203,6 +203,10 @@ func isAbsoluteUserCertificatePath(value string) bool {
|
||||
if filepath.IsAbs(value) {
|
||||
return true
|
||||
}
|
||||
// 非 Windows 主机也要识别盘符，避免把异平台路径静默拼到项目目录。
|
||||
if len(value) >= 2 && value[1] == ':' && ((value[0] >= 'A' && value[0] <= 'Z') || (value[0] >= 'a' && value[0] <= 'z')) {
|
||||
return true
|
||||
}
|
||||
volume := filepath.VolumeName(value)
|
||||
return volume != "" || strings.HasPrefix(value, `\\`) || strings.HasPrefix(value, "//")
|
||||
}
|
||||
@ -218,8 +222,8 @@ func validateUserCertificatePaths(privatePath, publicPath string) (string, strin
|
||||
}
|
||||
path := strings.TrimSpace(value)
|
||||
if isAbsoluteUserCertificatePath(path) {
|
||||
if filepath.VolumeName(path) != "" && !filepath.IsAbs(path) {
|
||||
return "", nil, newUserCertificateError("SSL 路径必须是完整的绝对路径")
|
||||
if !filepath.IsAbs(path) {
|
||||
return "", nil, newUserCertificateError("SSL 路径必须是当前服务器支持的完整绝对路径")
|
||||
}
|
||||
} else {
|
||||
path = filepath.Join(rootPath, path)
|
||||
|
||||
@ -54,12 +54,13 @@ func TestIsAbsoluteUserCertificatePathRecognizesWindowsPaths(t *testing.T) {
|
||||
cases := []string{
|
||||
`C:\outside\private.key`,
|
||||
`D:/outside/public.crt`,
|
||||
`C:private.key`,
|
||||
`\\server\share\private.key`,
|
||||
`//server/share/public.crt`,
|
||||
}
|
||||
for _, value := range cases {
|
||||
if !isAbsoluteUserCertificatePath(value) {
|
||||
t.Errorf("path %q was accepted", value)
|
||||
t.Errorf("path %q was not recognized", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -73,6 +74,17 @@ func TestIsAbsoluteUserCertificatePathRejectsEmptyAndAcceptsRelative(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUserCertificatePathsRejectsNonNativeAbsolutePath(t *testing.T) {
|
||||
for _, path := range []string{`C:private.key`, `C:\outside\private.key`, `\\server\share\private.key`} {
|
||||
if filepath.IsAbs(path) {
|
||||
continue
|
||||
}
|
||||
if _, _, err := validateUserCertificatePaths(path, path); err == nil || err.Error() != "SSL 路径必须是当前服务器支持的完整绝对路径" {
|
||||
t.Fatalf("non-native path %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyManagedSSLReferenceDoesNotPersistLegacyPaths(t *testing.T) {
|
||||
cfg := &config.Config{
|
||||
SSLPrivateKeyPath: "config/ssl/private.key",
|
||||
|
||||
@ -18,10 +18,13 @@
|
||||
| PROD-006 | P0 | 可信 TLS 和正式域名验证 | [!] | 仍使用自签名测试证书，不关闭证书校验 |
|
||||
| PROD-007 | P0 | 完整网页登录、发信和状态核对 | [!] | 需要有效测试账号和可信证书；不得绕过登录 |
|
||||
| PROD-008 | P0 | 超级管理员恢复完整交互 | [!] | IPC 权限已复核，正式密码代理交互及旧会话撤销未验收 |
|
||||
| PROD-009 | P1 | 备份与恢复演练 | [x] | 完整备份、归档及双库校验通过；异机恢复未演练 |
|
||||
| PROD-009 | P1 | 备份与恢复演练 | [x] | 同机独立网络和挂载空间恢复、HTTPS、内部 SMTP 入库通过；异机恢复未演练 |
|
||||
| PROD-010 | P1 | 内部 SMTP 与部署后安全回归 | [✓] | 五项真实收信、双库、产物、未认证 API 均通过 |
|
||||
| PROD-011 | P1 | 全量测试、竞态、数据库兼容性和压测 | [ ] | 本轮仅七包 132 个顶层用例；未运行真实 MySQL/PostgreSQL |
|
||||
| PROD-011 | P1 | 全量测试、竞态、数据库兼容性和压测 | [~] | 扩展至 60 包远程复测；竞态、真实 MySQL/PostgreSQL 和容量验收未完成 |
|
||||
| PROD-012 | P1 | 最新版本清库首次安装验收 | [ ] | 当前为保留数据升级，不能沿用旧版本清库结果 |
|
||||
| PROD-013 | P0 | 共享主机资源隔离与容量 | [!] | 首轮测试期间发生全机 OOM，MySQL 被终止后自动重启；必须补齐资源规划 |
|
||||
| PROD-014 | P1 | 投递结果写库恢复 | [x] | 数据库故障后仅重试结果落库、停止新领取，12 项远程队列测试通过 |
|
||||
| PROD-015 | P1 | 跨平台证书路径与插件测试 | [✓] | Linux 拒绝异平台路径误解析；保持证书校验、托管导入和插件目录权限要求 |
|
||||
|
||||
状态含义：`[ ]` 未开始，`[~]` 进行中，`[x]` 已完成待复审，`[✓]` 指定范围复审通过，`[!]` 暂缓或存在阻塞。
|
||||
|
||||
@ -47,7 +50,7 @@ SSH 脚本复用 `test/tools/remote_ssh.py`，强制核对已知主机密钥。
|
||||
- 每收件人独立记录 `queued`、`running`、`accepted`、`failed` 或 `uncertain`。成功收件人不随其他收件人的失败重投。
|
||||
- 临时失败最多五次，间隔 1、2、4、8 分钟；明确拒绝、证书问题、危险目标和结果不确定停止自动重试。
|
||||
- DATA 结果不确定、重启时仍在途、旧版本没有可靠尝试记录的邮件必须先核对实际收件结果，再决定人工处理。当前没有已验收的人工重投界面，不要直接改库重排。
|
||||
- 数据库读取失败不领取任务；结果写库失败只重试写入三次，仍失败则保留在途状态并记录错误。恢复数据库后需要核查日志及受控重启，不能把它理解成无限自动恢复。
|
||||
- 新版数据库读取失败不领取任务；结果写库失败时单工作器最多保留一份结果，暂停新领取，每秒仅重试写库，不再次调用 SMTP。数据库恢复后自动保存；进程中途退出仍将遗留在途记录转为待核对。相同尝试的相同结果允许重复保存，不覆盖后续尝试；线上部署状态以最新测试记录为准。
|
||||
- 当前仅允许单实例、单工作器使用同一队列库。没有验收多实例调度、吞吐容量、队列告警和故障期间的服务等级。
|
||||
- 现有邮件客户端 SMTP 提交、系统通知和其他调用方不自动等同于网页持久队列；分别验收其投递及失败处理。
|
||||
- 备用 MX 保留既有 25 STARTTLS、587 STARTTLS、465 TLS 顺序；禁止明文降级。各运营商的端口策略及可达性需要独立验证。
|
||||
@ -61,3 +64,9 @@ SSH 脚本复用 `test/tools/remote_ssh.py`，强制核对已知主机密钥。
|
||||
5. 在隔离远程目标完成备份恢复、全量与竞态测试、目标数据库兼容性和容量验证。最新版本重新清库安装前，先备份所有项目数据库并核实范围，不清理其他项目数据。
|
||||
|
||||
上述阻塞项解决后再做最终生产评审，不以构建成功、首页 200、内部 SMTP 250 或单次提交代替整体验收。
|
||||
|
||||
## 资源与测试隔离
|
||||
|
||||
47 服务器约 1.6 GiB 内存，无交换空间，另有 MySQL、Scert 等服务。首轮测试仅设置 Go 软内存上限，不能防止全机内存耗尽。新工具通过 systemd 设置 CPU、内存、进程数和最长运行时间上限，使用独立网络、私有临时目录，并禁止访问线上 `/opt/smail`、`/run/smail`。可用内存不足时拒绝开始测试，不自动放宽限制、不停止其他项目。
|
||||
|
||||
`GOMEMLIMIT` 是 Go 回收目标，不是进程或主机硬限制。本轮协议测试以 128 MiB cgroup 上限、25% CPU 配额和 64 MiB Go 回收目标完成，不能据此推导生产容量。生产仍需确定同机服务预算、并发认证开销、邮件大小与吞吐上限、告警和故障恢复指标。
|
||||
|
||||
@ -233,3 +233,47 @@ TLS 前未公布 AUTH，主动尝试明文 AUTH 返回 523。未认证外域邮
|
||||
- [!] 受信任 TLS、凭据轮换、正式管理员恢复交互、新密码登录与旧会话撤销、灾备恢复及全量兼容性验证仍未完成。
|
||||
|
||||
当前结论仍为：适合继续受控远程验收，不建议直接用于正式生产。后续门槛见《生产整改验收清单》。
|
||||
|
||||
## 2026 年 9 月 17 日投递结果恢复与扩展验收
|
||||
|
||||
本轮从 10:30 开始。继续只在 47 服务器执行业务和自动测试，本机仅源码检查、交叉编译与 SSH 操作。没有清理线上双库，没有删除历史日志和备份，没有修改其他项目配置。修改前检查点 `d3d4f5d`、`7f719ab` 已推送。
|
||||
|
||||
### 本轮整改
|
||||
|
||||
- 单工作器保留一个未落库投递结果；数据库故障期间停止新领取，恢复后只保存结果，不重复发送 SMTP。首次结果与退避时间固定；相同尝试的相同结果支持幂等保存，旧尝试不得覆盖新状态。
|
||||
- Linux 识别 Windows 盘符路径并明确拒绝非本机绝对路径，避免错误地拼接至项目目录。有效本机任意路径、证书及密钥校验和托管导入继续保留。
|
||||
- 插件测试分别验证普通用户创建目录被拒绝，以及 root 所有目录的原子写入，不放宽生产目录所有者校验。
|
||||
- IMAP 原始协议测试增加 15 秒读写期限，避免无限等待。
|
||||
- 新增逐包交叉编译、远程测试执行、systemd 资源隔离及备份恢复工具。逐轮日志单独保存，不覆盖上一轮结果。
|
||||
|
||||
### 首轮异常与处置
|
||||
|
||||
60 个测试包完成首轮执行，其中插件权限、异平台证书路径断言失败，IMAP 和 SMTP 包超时。现场负载曾达 74，内核记录 11:00:20 发生全机 OOM，MySQL 进程被系统终止后自动重启。首轮没有硬资源或网络隔离，不将此轮描述为无影响验收，也不能仅凭现有证据将全部压力归因于单个测试。
|
||||
|
||||
收敛后确认原测试执行器和测试二进制均已结束，Smail、MySQL、Scert 均为运行状态；Smail 主 PID `548293` 未变。没有停止其他项目进程。首轮日志保留在 `/var/tmp/smail-review-full-20260917/results.json` 及各包 `result.log`。
|
||||
|
||||
受限启动器第一次因私有临时目录遮住包目录而在测试前失败，补充项目目录映射后正常。IMAP 在 128 MiB cgroup 下首次被局部 OOM 终止，调整测试专属 Go 回收参数后通过，未削弱生产密码算法。192 MiB 尝试因可用内存不足被预检拒绝，没有执行。
|
||||
|
||||
### 定向复测
|
||||
|
||||
| 范围 | 顶层通过数 | 证据 |
|
||||
| --- | --- | --- |
|
||||
| 插件普通用户权限分支 | 17 | `results-20260917-110801/backend-hooks.log`；跳过 Windows 专属策略 |
|
||||
| 投递队列 | 12 | `results-20260917-110801/backend-services-outbox.log` |
|
||||
| 证书路径和导入 | 19 | `results-20260917-110801/backend-services-setup-ssl.log` |
|
||||
| 插件 root 原子写入 | 1 | `smail-plugin-root-review-20260917-1110.service` journal |
|
||||
| IMAP 全包 | 31 | `results-20260917-111234/backend-listen-imap_server.log` |
|
||||
| SMTP 全包 | 35 | `results-20260917-111234/backend-listen-smtp_server.log` |
|
||||
|
||||
上述相对日志均位于 `/var/tmp/smail-review-full-20260917/`。协议复测使用 128 MiB 硬上限、25% CPU、独立网络及禁止访问生产目录，峰值约 121.1 MiB，用时约 19 秒。SMTP 定向样本两项也在 64 MiB 上限下通过。协议超时在隔离环境中未重现，不据此宣称已定位原超时的唯一原因。
|
||||
|
||||
### 备份恢复演练
|
||||
|
||||
使用 `/var/backups/smail-before-outbox-20260917-100608/smail.tar.gz`，核对归档 SHA-256 `a2ed56666e9e691a63dfc6734ebe2a1e9c7bf2fe4ceb81cb55028e66ea52d016` 后，在 `/var/tmp/smail-review-restore-20260917-111022/` 恢复。
|
||||
|
||||
- 恢复前后双库完整性通过，逐表数量记录在受保护的 `result.json`。
|
||||
- 副本在独立挂载和网络命名空间启动，128 MiB 硬上限、25% CPU，HTTPS 就绪检查通过。
|
||||
- 仅连接副本网络空间内部 `127.0.0.1:25`，STARTTLS 后投递一封邮件，确认副本库新增一条，线上库没有该消息。
|
||||
- 线上服务主 PID 保持不变；演练结束后副本服务已停止，恢复数据和证据保留。
|
||||
|
||||
这是同机隔离恢复，不代表异机灾备、恢复时限承诺或最新版本清库安装验收。
|
||||
|
||||
45
test/tools/build_review_tests.ps1
Normal file
45
test/tools/build_review_tests.ps1
Normal file
@ -0,0 +1,45 @@
|
||||
param([string]$OutputName = 'full-review-20260917')
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$root = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..\..'))
|
||||
if ($OutputName -notmatch '^[a-z0-9-]+$') { throw '测试产物目录名称无效' }
|
||||
$output = Join-Path $root "out\$OutputName"
|
||||
New-Item -ItemType Directory -Path $output -Force | Out-Null
|
||||
$previous = @{}
|
||||
foreach ($name in @('GOOS', 'GOARCH', 'CGO_ENABLED')) {
|
||||
$previous[$name] = [Environment]::GetEnvironmentVariable($name, 'Process')
|
||||
}
|
||||
try {
|
||||
$env:GOOS = 'linux'
|
||||
$env:GOARCH = 'amd64'
|
||||
$env:CGO_ENABLED = '0'
|
||||
Push-Location (Join-Path $root 'backend')
|
||||
try {
|
||||
$packages = & go list -f '{{if or .TestGoFiles .XTestGoFiles}}{{.ImportPath}}{{end}}' ./...
|
||||
if ($LASTEXITCODE -ne 0) { throw '无法枚举测试包' }
|
||||
$manifest = @()
|
||||
foreach ($package in $packages) {
|
||||
if (-not $package.Trim()) { continue }
|
||||
$relative = $package.Substring('git.pcux.cn/xuans/Smail'.Length).TrimStart('/')
|
||||
$directory = if ($relative) { "backend/$relative" } else { 'backend' }
|
||||
$target = Join-Path $output $directory
|
||||
New-Item -ItemType Directory -Path $target -Force | Out-Null
|
||||
& go test -c -o (Join-Path $target 'review.test') $package
|
||||
if ($LASTEXITCODE -ne 0) { throw "测试编译失败：$package" }
|
||||
$manifest += @{ package = $package; directory = $directory }
|
||||
Write-Output "已编译：$package"
|
||||
}
|
||||
$json = ConvertTo-Json -InputObject $manifest -Depth 4
|
||||
[IO.File]::WriteAllText((Join-Path $output 'manifest.json'), $json, [Text.UTF8Encoding]::new($false))
|
||||
} finally { Pop-Location }
|
||||
Copy-Item -LiteralPath (Join-Path $root 'deploy') -Destination (Join-Path $output 'deploy') -Recurse -Force
|
||||
Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'run_review_tests.py') -Destination $output -Force
|
||||
Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'start_review_tests.py') -Destination $output -Force
|
||||
& tar -czf "$output.tar.gz" -C $output .
|
||||
if ($LASTEXITCODE -ne 0) { throw '测试产物归档失败' }
|
||||
Write-Output "仅完成交叉编译，未在本机运行测试：$output.tar.gz"
|
||||
} finally {
|
||||
foreach ($name in $previous.Keys) {
|
||||
[Environment]::SetEnvironmentVariable($name, $previous[$name], 'Process')
|
||||
}
|
||||
}
|
||||
129
test/tools/restore_review.py
Normal file
129
test/tools/restore_review.py
Normal file
@ -0,0 +1,129 @@
|
||||
"""在 47 服务器的独立命名空间恢复备份，不替换当前服务或删除文件。"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import tarfile
|
||||
import time
|
||||
|
||||
|
||||
def run(*args):
|
||||
return subprocess.check_output(args, text=True).strip()
|
||||
|
||||
|
||||
def database_counts(root):
|
||||
counts = {}
|
||||
for name in ('smail.db', 'smail-state.db'):
|
||||
with sqlite3.connect(f'file:{root / "data" / name}?mode=ro', uri=True) as database:
|
||||
assert database.execute('PRAGMA integrity_check').fetchone()[0] == 'ok'
|
||||
tables = database.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").fetchall()
|
||||
counts[name] = {
|
||||
table: database.execute('SELECT COUNT(*) FROM "' + table.replace('"', '""') + '"').fetchone()[0]
|
||||
for (table,) in tables
|
||||
}
|
||||
return counts
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('backup')
|
||||
parser.add_argument('--archive-sha', required=True)
|
||||
args = parser.parse_args()
|
||||
assert os.geteuid() == 0 and run('hostname') == 'ikun2'
|
||||
backup = Path(args.backup).resolve(strict=True)
|
||||
assert backup.parent == Path('/var/backups') and backup.name.startswith('smail-before-')
|
||||
archive = backup / 'smail.tar.gz'
|
||||
with archive.open('rb') as stream:
|
||||
assert hashlib.file_digest(stream, 'sha256').hexdigest() == args.archive_sha, '备份摘要不匹配'
|
||||
memory = dict(line.split(':', 1) for line in Path('/proc/meminfo').read_text().splitlines())
|
||||
assert int(memory['MemAvailable'].split()[0]) >= 256 * 1024, '可用内存不足，暂停恢复演练'
|
||||
stamp = time.strftime('%Y%m%d-%H%M%S')
|
||||
target = Path('/var/tmp') / ('smail-review-restore-' + stamp)
|
||||
target.mkdir(mode=0o700)
|
||||
with tarfile.open(archive) as stream:
|
||||
for member in stream.getmembers():
|
||||
path = Path(member.name)
|
||||
assert not path.is_absolute() and '..' not in path.parts and path.parts[0] == 'smail'
|
||||
assert member.isdir() or member.isfile(), '备份含特殊文件，停止自动恢复'
|
||||
subprocess.run(['tar', '-xzf', str(archive), '-C', str(target)], check=True)
|
||||
restored = target / 'smail'
|
||||
control = target / 'control'
|
||||
control.mkdir(mode=0o750)
|
||||
shutil.chown(control, user='smail', group='smail')
|
||||
before = database_counts(restored)
|
||||
live_pid = run('systemctl', 'show', 'smail.service', '--property=MainPID', '--value')
|
||||
unit = 'smail-restore-review-' + stamp
|
||||
result = {'backup': str(backup), 'restore': str(target), 'unit': unit, 'before': before}
|
||||
try:
|
||||
subprocess.run([
|
||||
'systemd-run', '--unit=' + unit, '--property=User=smail', '--property=Group=smail',
|
||||
'--property=PrivateNetwork=yes', '--property=PrivateTmp=yes',
|
||||
'--property=ProtectSystem=strict', '--property=ProtectHome=yes',
|
||||
'--property=NoNewPrivileges=yes', '--property=CapabilityBoundingSet=CAP_NET_BIND_SERVICE',
|
||||
'--property=AmbientCapabilities=CAP_NET_BIND_SERVICE',
|
||||
'--property=MemoryMax=128M', '--property=MemorySwapMax=0', '--property=CPUQuota=25%',
|
||||
'--property=TasksMax=128', '--property=RuntimeMaxSec=180s', '--property=OOMPolicy=stop',
|
||||
'--property=KillMode=control-group', '--property=TimeoutStopSec=10s',
|
||||
'--property=BindPaths=' + str(restored) + ':/opt/smail ' + str(control) + ':/run/smail',
|
||||
'--property=ReadWritePaths=/opt/smail /run/smail',
|
||||
'--property=WorkingDirectory=/opt/smail', '--setenv=SMAIL_ROOT=/opt/smail',
|
||||
'--setenv=GOMEMLIMIT=96MiB', '--setenv=GOMAXPROCS=1',
|
||||
'/opt/smail/smail',
|
||||
], check=True)
|
||||
ready = False
|
||||
for _ in range(60):
|
||||
pid = run('systemctl', 'show', unit, '--property=MainPID', '--value')
|
||||
if pid != '0':
|
||||
probe = subprocess.run([
|
||||
'nsenter', '--target', pid, '--net', 'curl', '--silent', '--fail', '--max-time', '2',
|
||||
'--noproxy', '*', '--cacert', str(restored / 'ssl/custom.crt'),
|
||||
'--resolve', 'mail.solit.cn:443:127.0.0.1', 'https://mail.solit.cn/api/ping',
|
||||
], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
if probe.returncode == 0:
|
||||
ready = True
|
||||
break
|
||||
time.sleep(1)
|
||||
assert ready, '恢复服务未就绪'
|
||||
assert os.readlink('/proc/' + pid + '/ns/net') != os.readlink('/proc/' + live_pid + '/ns/net')
|
||||
assert os.stat('/proc/' + pid + '/root/opt/smail/data/smail.db').st_ino == (restored / 'data/smail.db').stat().st_ino
|
||||
assert (restored / 'data/smail.db').stat().st_ino != Path('/opt/smail/data/smail.db').stat().st_ino
|
||||
probe = r'''
|
||||
import smtplib,ssl,sys
|
||||
from email.message import EmailMessage
|
||||
message=EmailMessage()
|
||||
message['From']='restore@review.invalid'
|
||||
message['To']='remotecheck@solit.cn'
|
||||
message['Subject']='restore-review'
|
||||
message['Message-ID']='<restore-'+sys.argv[2]+'@review.invalid>'
|
||||
message.set_content('restore database write check')
|
||||
context=ssl.create_default_context(cafile=sys.argv[1])
|
||||
with smtplib.SMTP('127.0.0.1',25,timeout=20) as client:
|
||||
client.ehlo('restore.invalid')
|
||||
client._host='mail.solit.cn'
|
||||
client.starttls(context=context)
|
||||
client.ehlo('restore.invalid')
|
||||
assert not client.send_message(message)
|
||||
'''
|
||||
subprocess.run(['nsenter', '--target', pid, '--net', 'python3', '-c', probe,
|
||||
str(restored / 'ssl/custom.crt'), stamp], check=True)
|
||||
with sqlite3.connect(f'file:{restored / "data/smail.db"}?mode=ro', uri=True) as database:
|
||||
assert database.execute('SELECT COUNT(*) FROM email WHERE msg_id=?', ('restore-' + stamp + '@review.invalid',)).fetchone()[0] == 1
|
||||
with sqlite3.connect('file:/opt/smail/data/smail.db?mode=ro', uri=True) as database:
|
||||
assert database.execute('SELECT COUNT(*) FROM email WHERE msg_id=?', ('restore-' + stamp + '@review.invalid',)).fetchone()[0] == 0
|
||||
result.update(ready=True, isolated=True, restored_smtp=True, live_pid_unchanged=live_pid == run(
|
||||
'systemctl', 'show', 'smail.service', '--property=MainPID', '--value'))
|
||||
finally:
|
||||
subprocess.run(['systemctl', 'stop', unit], check=True)
|
||||
result['after'] = database_counts(restored)
|
||||
result['service_log'] = run('journalctl', '-u', unit, '--no-pager', '-n', '80')
|
||||
(target / 'result.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print(json.dumps({key: value for key, value in result.items() if key not in ('before', 'after', 'service_log')}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
65
test/tools/run_review_tests.py
Normal file
65
test/tools/run_review_tests.py
Normal file
@ -0,0 +1,65 @@
|
||||
"""在远程独立目录逐包执行测试，保留全部日志和跳过记录。"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--only', action='append', default=[])
|
||||
parser.add_argument('--run', default='')
|
||||
args = parser.parse_args()
|
||||
root = Path(__file__).resolve().parent
|
||||
assert root.parent == Path('/var/tmp') and root.name.startswith('smail-review-')
|
||||
assert os.geteuid() != 0, '禁止使用 root 执行测试'
|
||||
environment = os.environ.copy()
|
||||
for name in list(environment):
|
||||
if name.startswith('SMAIL_'):
|
||||
environment.pop(name)
|
||||
environment['GOMAXPROCS'] = '1'
|
||||
environment['GOMEMLIMIT'] = '64MiB'
|
||||
environment['GOGC'] = '20'
|
||||
output = root / ('results-' + time.strftime('%Y%m%d-%H%M%S'))
|
||||
output.mkdir(mode=0o700)
|
||||
results = []
|
||||
for item in json.loads((root / 'manifest.json').read_text(encoding='utf-8')):
|
||||
if args.only and item['directory'] not in args.only:
|
||||
continue
|
||||
directory = (root / item['directory']).resolve()
|
||||
assert directory.is_relative_to(root)
|
||||
binary = directory / 'review.test'
|
||||
with binary.open('rb') as stream:
|
||||
assert stream.read(5) == b'\x7fELF\x02'
|
||||
started = time.monotonic()
|
||||
log = output / (item['directory'].replace('/', '-') + '.log')
|
||||
with log.open('w', encoding='utf-8') as stream:
|
||||
command = [str(binary), '-test.v', '-test.timeout=180s']
|
||||
if args.run:
|
||||
command.append('-test.run=' + args.run)
|
||||
process = subprocess.Popen(command, cwd=directory, env=environment,
|
||||
stdout=stream, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
try:
|
||||
code = process.wait(timeout=195)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait()
|
||||
code = 124
|
||||
lines = log.read_text(encoding='utf-8', errors='replace').splitlines()
|
||||
result = dict(item, code=code, log=str(log), seconds=round(time.monotonic() - started, 2),
|
||||
passed=sum(line.startswith('--- PASS:') for line in lines),
|
||||
failed=[line for line in lines if line.lstrip().startswith('--- FAIL:')],
|
||||
skipped=[line for line in lines if line.lstrip().startswith('--- SKIP:')])
|
||||
results.append(result)
|
||||
print(json.dumps(result, ensure_ascii=False), flush=True)
|
||||
(output / 'results.json').write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print('测试记录：' + str(output), flush=True)
|
||||
return int(any(item['code'] != 0 for item in results))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
47
test/tools/start_review_tests.py
Normal file
47
test/tools/start_review_tests.py
Normal file
@ -0,0 +1,47 @@
|
||||
"""通过受限 systemd 临时服务执行远程测试，不复用生产服务进程。"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('directory')
|
||||
parser.add_argument('--only', action='append', default=[])
|
||||
parser.add_argument('--run', default='')
|
||||
parser.add_argument('--memory-mib', type=int, choices=(64, 96, 128, 192), default=192)
|
||||
args = parser.parse_args()
|
||||
root = Path(args.directory).resolve(strict=True)
|
||||
assert os.geteuid() == 0
|
||||
assert root.parent == Path('/var/tmp') and root.name.startswith('smail-review-')
|
||||
memory = dict(line.split(':', 1) for line in Path('/proc/meminfo').read_text().splitlines())
|
||||
assert int(memory['MemAvailable'].split()[0]) >= (args.memory_mib + 128) * 1024, '可用内存不足，停止增加测试负载'
|
||||
unit = 'smail-tests-' + time.strftime('%Y%m%d-%H%M%S')
|
||||
command = [
|
||||
'systemd-run', '--wait', '--pipe', '--unit=' + unit,
|
||||
'--property=User=smail', '--property=Group=smail',
|
||||
'--property=MemoryMax=' + str(args.memory_mib) + 'M', '--property=MemorySwapMax=0',
|
||||
'--property=TasksMax=128', '--property=CPUQuota=25%',
|
||||
'--property=RuntimeMaxSec=1800s', '--property=TimeoutStopSec=10s',
|
||||
'--property=KillMode=control-group', '--property=OOMPolicy=stop',
|
||||
'--property=PrivateTmp=yes', '--property=PrivateNetwork=yes',
|
||||
'--property=BindPaths=' + str(root),
|
||||
'--property=ProtectSystem=strict', '--property=ProtectHome=yes',
|
||||
'--property=InaccessiblePaths=/opt/smail /run/smail',
|
||||
'--property=NoNewPrivileges=yes', '--property=ReadWritePaths=' + str(root),
|
||||
'--property=WorkingDirectory=' + str(root),
|
||||
'python3', str(root / 'run_review_tests.py'),
|
||||
]
|
||||
for directory in args.only:
|
||||
command.extend(['--only', directory])
|
||||
if args.run:
|
||||
command.extend(['--run', args.run])
|
||||
print('测试服务：' + unit, flush=True)
|
||||
return subprocess.run(command).returncode
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Loading…
Reference in New Issue
Block a user