From 421aa7303d3b361e295818a19ea4250ef63597e0 Mon Sep 17 00:00:00 2001 From: smail Date: Thu, 17 Sep 2026 11:28:52 +0800 Subject: [PATCH] =?UTF-8?q?=E8=A1=A5=E9=BD=90=E8=BF=9C=E7=A8=8B=E5=85=AD?= =?UTF-8?q?=E5=8D=81=E5=8C=85=E5=88=86=E6=89=B9=E9=AA=8C=E6=94=B6=E5=B9=B6?= =?UTF-8?q?=E9=98=B2=E6=AD=A2=E7=A9=BA=E6=B5=8B=E8=AF=95=E8=AF=AF=E6=8A=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/生产整改验收清单.md | 6 +- docs/远程服务器测试记录.md | 22 +++++++ test/tools/run_review_tests.py | 69 +++++++++++++++------ test/tools/start_review_tests.py | 5 +- test/tools/verify_review.py | 95 +++++++++++++++++++++++++++++ 5 files changed, 175 insertions(+), 22 deletions(-) create mode 100644 test/tools/verify_review.py diff --git a/docs/生产整改验收清单.md b/docs/生产整改验收清单.md index 6be41b5..e17ad0c 100644 --- a/docs/生产整改验收清单.md +++ b/docs/生产整改验收清单.md @@ -20,10 +20,10 @@ | PROD-008 | P0 | 超级管理员恢复完整交互 | [!] | IPC 权限已复核，正式密码代理交互及旧会话撤销未验收 | | PROD-009 | P1 | 备份与恢复演练 | [x] | 同机独立网络和挂载空间恢复、HTTPS、内部 SMTP 入库通过；异机恢复未演练 | | PROD-010 | P1 | 内部 SMTP 与部署后安全回归 | [✓] | 五项真实收信、双库、产物、未认证 API 均通过 | -| PROD-011 | P1 | 全量测试、竞态、数据库兼容性和压测 | [~] | 扩展至 60 包远程复测；竞态、真实 MySQL/PostgreSQL 和容量验收未完成 | +| PROD-011 | P1 | 全量测试、竞态、数据库兼容性和压测 | [~] | 60 包分批执行、958 个顶层用例通过；8 处跳过单列，竞态、真实跨库和容量验收未完成 | | PROD-012 | P1 | 最新版本清库首次安装验收 | [ ] | 当前为保留数据升级，不能沿用旧版本清库结果 | | PROD-013 | P0 | 共享主机资源隔离与容量 | [!] | 首轮测试期间发生全机 OOM，MySQL 被终止后自动重启；必须补齐资源规划 | -| PROD-014 | P1 | 投递结果写库恢复 | [x] | 数据库故障后仅重试结果落库、停止新领取，12 项远程队列测试通过 | +| PROD-014 | P1 | 投递结果写库恢复 | [✓] | 12 项远程测试及源码复审通过；持久故障后的进程退出仍需人工核对，不代表生产故障演练通过 | | PROD-015 | P1 | 跨平台证书路径与插件测试 | [✓] | Linux 拒绝异平台路径误解析；保持证书校验、托管导入和插件目录权限要求 | 状态含义：`[ ]` 未开始，`[~]` 进行中，`[x]` 已完成待复审，`[✓]` 指定范围复审通过，`[!]` 暂缓或存在阻塞。 @@ -70,3 +70,5 @@ SSH 脚本复用 `test/tools/remote_ssh.py`，强制核对已知主机密钥。 47 服务器约 1.6 GiB 内存，无交换空间，另有 MySQL、Scert 等服务。首轮测试仅设置 Go 软内存上限，不能防止全机内存耗尽。新工具通过 systemd 设置 CPU、内存、进程数和最长运行时间上限，使用独立网络、私有临时目录，并禁止访问线上 `/opt/smail`、`/run/smail`。可用内存不足时拒绝开始测试，不自动放宽限制、不停止其他项目。 `GOMEMLIMIT` 是 Go 回收目标，不是进程或主机硬限制。本轮协议测试以 128 MiB cgroup 上限、25% CPU 配额和 64 MiB Go 回收目标完成，不能据此推导生产容量。生产仍需确定同机服务预算、并发认证开销、邮件大小与吞吐上限、告警和故障恢复指标。 + +恢复服务整包执行曾在 128 MiB 上限触发测试 cgroup OOM，改为逐顶层用例独立进程后 14 项通过，峰值约 126.4 MiB。没有降低 Argon2id 的 64 MiB 内存参数；不能把拆分测试视为生产内存问题已解决。测试启动器默认 128 MiB，并对不存在的包、空清单和零匹配筛选返回失败；中断包保留运行中记录。 diff --git a/docs/远程服务器测试记录.md b/docs/远程服务器测试记录.md index c5b334e..64db4c8 100644 --- a/docs/远程服务器测试记录.md +++ b/docs/远程服务器测试记录.md @@ -277,3 +277,25 @@ TLS 前未公布 AUTH，主动尝试明文 AUTH 返回 523。未认证外域邮 - 线上服务主 PID 保持不变；演练结束后副本服务已停止，恢复数据和证据保留。 这是同机隔离恢复，不代表异机灾备、恢复时限承诺或最新版本清库安装验收。 + +### 分批全包复测结果 + +11:18 恢复服务整包测试达到 128 MiB 硬上限，systemd 返回 `oom-kill`。内核明确记录 `CONSTRAINT_MEMCG`，终止范围为 `smail-tests-20260917-111329.service`；该次没有重启线上 Smail、MySQL 或 Scert。不能称整轮一次性通过。 + +保留原失败证据后，将其余 22 包和恢复服务分别补跑，仍使用同一份 Linux 测试二进制。恢复服务采用逐顶层用例独立进程，14 项通过，用时约 75 秒，峰值约 126.4 MiB，未降低生产密码哈希强度。三批已完成记录按包去重并与 60 包清单核对，总计 958 个顶层用例通过： + +| 批次 | 完成包数 | 日志目录 | +| --- | --- | --- | +| 前段已完成包 | 37 | `results-20260917-111329` | +| 后段补跑 | 22 | `results-20260917-112034` | +| 恢复服务逐用例补跑 | 1 | `results-20260917-112305` | + +目录均位于 `/var/tmp/smail-review-full-20260917/`。以下 8 处跳过没有计为通过： + +- `TestMaster`：完整主服务集成测试未启用。 +- `TestMySQLIntegration`、`TestPostgresIntegration`：未提供真实专用测试数据库。 +- `TestPortableSQLIntegration/mysql`、`postgresql`、`mysql_postgresql`：真实跨数据库分支未启用。 +- `TestProductionPluginIsolationPolicy`：Windows 专属策略不能由 Linux 执行证明。 +- `TestSecuritySetupBrowserFixture`：浏览器夹具未启用，不能代替实际网页验收。 + +11:26 只读复核：Smail、MySQL、Scert 均运行，主 PID 分别为 `548293`、`549584`、`530794`；MySQL 重启计数仍为首轮事故后的 1。双库完整性通过、迁移版本 23、队列表为空、恢复 socket 为 `0600`，未认证用户信息及邮件列表均返回业务码 403。11:18:08 后未发现新增 OOM。此时线上版本尚为 `remote-13bcad90e832`，新代码未部署。 diff --git a/test/tools/run_review_tests.py b/test/tools/run_review_tests.py index 0157248..bfe3f10 100644 --- a/test/tools/run_review_tests.py +++ b/test/tools/run_review_tests.py @@ -4,6 +4,7 @@ import argparse import json import os from pathlib import Path +import re import signal import subprocess import time @@ -13,7 +14,10 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('--only', action='append', default=[]) parser.add_argument('--run', default='') + parser.add_argument('--per-test', action='store_true') args = parser.parse_args() + if args.per_test and args.run: + parser.error('逐用例模式不能同时使用正则筛选') root = Path(__file__).resolve().parent assert root.parent == Path('/var/tmp') and root.name.startswith('smail-review-') assert os.geteuid() != 0, '禁止使用 root 执行测试' @@ -24,10 +28,16 @@ def main(): environment['GOMAXPROCS'] = '1' environment['GOMEMLIMIT'] = '64MiB' environment['GOGC'] = '20' + manifest = json.loads((root / 'manifest.json').read_text(encoding='utf-8')) + if not manifest: + parser.error('测试清单为空') + unknown = set(args.only) - {item['directory'] for item in manifest} + if unknown: + parser.error('测试包不存在：' + ', '.join(sorted(unknown))) output = root / ('results-' + time.strftime('%Y%m%d-%H%M%S')) output.mkdir(mode=0o700) results = [] - for item in json.loads((root / 'manifest.json').read_text(encoding='utf-8')): + for item in manifest: if args.only and item['directory'] not in args.only: continue directory = (root / item['directory']).resolve() @@ -37,26 +47,47 @@ def main(): assert stream.read(5) == b'\x7fELF\x02' started = time.monotonic() log = output / (item['directory'].replace('/', '-') + '.log') - with log.open('w', encoding='utf-8') as stream: - command = [str(binary), '-test.v', '-test.timeout=180s'] - if args.run: - command.append('-test.run=' + args.run) - process = subprocess.Popen(command, cwd=directory, env=environment, - stdout=stream, stderr=subprocess.STDOUT, start_new_session=True) - try: - code = process.wait(timeout=195) - except subprocess.TimeoutExpired: - os.killpg(process.pid, signal.SIGKILL) - process.wait() - code = 124 - lines = log.read_text(encoding='utf-8', errors='replace').splitlines() - result = dict(item, code=code, log=str(log), seconds=round(time.monotonic() - started, 2), - passed=sum(line.startswith('--- PASS:') for line in lines), - failed=[line for line in lines if line.lstrip().startswith('--- FAIL:')], - skipped=[line for line in lines if line.lstrip().startswith('--- SKIP:')]) + patterns = [args.run] + if args.per_test: + listing = subprocess.check_output([str(binary), '-test.list=.'], cwd=directory, + env=environment, text=True, timeout=30) + names = [line for line in listing.splitlines() + if re.fullmatch(r'(?:Test|Example|Fuzz)\w+', line)] + if not names: + raise RuntimeError('未发现可运行用例：' + item['directory']) + patterns = ['^' + re.escape(name) + '$' for name in names] + result = dict(item, code=None, log=str(log), status='running', per_test=args.per_test) results.append(result) + summary = output / 'results.json' + summary.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding='utf-8') + code = 0 + with log.open('w', encoding='utf-8') as stream: + for pattern in patterns: + command = [str(binary), '-test.v', '-test.timeout=180s'] + if pattern: + command.append('-test.run=' + pattern) + process = subprocess.Popen(command, cwd=directory, env=environment, + stdout=stream, stderr=subprocess.STDOUT, start_new_session=True) + try: + current = process.wait(timeout=195) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + current = 124 + if current != 0: + code = current + lines = log.read_text(encoding='utf-8', errors='replace').splitlines() + passed = sum(line.startswith('--- PASS:') for line in lines) + skipped = [line for line in lines if line.lstrip().startswith('--- SKIP:')] + if not passed and not skipped and code == 0: + code = 2 + if args.per_test and passed + sum(line.startswith('--- SKIP:') for line in lines) != len(patterns): + code = code or 2 + result.update(code=code, status='finished', seconds=round(time.monotonic() - started, 2), + passed=passed, failed=[line for line in lines if line.lstrip().startswith('--- FAIL:')], + skipped=skipped) print(json.dumps(result, ensure_ascii=False), flush=True) - (output / 'results.json').write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding='utf-8') + summary.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding='utf-8') print('测试记录：' + str(output), flush=True) return int(any(item['code'] != 0 for item in results)) diff --git a/test/tools/start_review_tests.py b/test/tools/start_review_tests.py index 55f0756..61bc1c3 100644 --- a/test/tools/start_review_tests.py +++ b/test/tools/start_review_tests.py @@ -12,7 +12,8 @@ def main(): parser.add_argument('directory') parser.add_argument('--only', action='append', default=[]) parser.add_argument('--run', default='') - parser.add_argument('--memory-mib', type=int, choices=(64, 96, 128, 192), default=192) + parser.add_argument('--per-test', action='store_true') + parser.add_argument('--memory-mib', type=int, choices=(64, 96, 128, 192), default=128) args = parser.parse_args() root = Path(args.directory).resolve(strict=True) assert os.geteuid() == 0 @@ -39,6 +40,8 @@ def main(): command.extend(['--only', directory]) if args.run: command.extend(['--run', args.run]) + if args.per_test: + command.append('--per-test') print('测试服务：' + unit, flush=True) return subprocess.run(command).returncode diff --git a/test/tools/verify_review.py b/test/tools/verify_review.py new file mode 100644 index 0000000..c2f6ae1 --- /dev/null +++ b/test/tools/verify_review.py @@ -0,0 +1,95 @@ +"""只读核对远程分批测试覆盖及线上服务，不输出邮件、密码或会话内容。""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import sqlite3 +import stat +import subprocess + + +def run(*args): + return subprocess.check_output(args, text=True).strip() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--results', nargs='+', default=[]) + parser.add_argument('--main-sha') + parser.add_argument('--admin-sha') + parser.add_argument('--since', required=True) + args = parser.parse_args() + assert os.geteuid() == 0 and run('hostname') == 'ikun2' + root = Path('/opt/smail') + assert root.resolve() == root + summary = {} + if args.results: + test_root = Path('/var/tmp/smail-review-full-20260917') + expected = {item['directory'] for item in json.loads((test_root / 'manifest.json').read_text())} + completed = {} + for name in args.results: + directory = (test_root / name).resolve(strict=True) + assert directory.parent == test_root and directory.name.startswith('results-') + for item in json.loads((directory / 'results.json').read_text()): + assert item.get('code') == 0 and not item.get('failed'), '测试未通过或被中断' + assert item['directory'] not in completed, '测试包结果重复' + completed[item['directory']] = item + assert set(completed) == expected, '测试包覆盖不完整' + summary['tests'] = { + 'packages': len(completed), + 'passed': sum(item['passed'] for item in completed.values()), + 'skipped': {name: item['skipped'] for name, item in completed.items() if item['skipped']}, + 'per_test': [name for name, item in completed.items() if item.get('per_test')], + 'evidence': args.results, + } + services = {} + for name in ('smail.service', 'mysql.service', 'Scert.service'): + properties = run('systemctl', 'show', name, '--property=ActiveState,SubState,MainPID,NRestarts,ExecMainStatus') + services[name] = dict(line.split('=', 1) for line in properties.splitlines()) + assert services[name]['ActiveState'] == 'active' + summary['services'] = services + for name, checksum in (('smail', args.main_sha), ('smail-admin', args.admin_sha)): + if checksum: + with (root / name).open('rb') as stream: + actual = hashlib.file_digest(stream, 'sha256').hexdigest() + assert actual == checksum, '部署产物摘要不匹配' + summary[name + '_sha256'] = actual + summary['admin_version'] = run(str(root / 'smail-admin'), 'version') + summary['database_integrity'] = {} + for name in ('smail.db', 'smail-state.db'): + with sqlite3.connect(f'file:{root / "data" / name}?mode=ro', uri=True) as database: + integrity = database.execute('PRAGMA integrity_check').fetchall() + assert integrity == [('ok',)] + summary['database_integrity'][name] = 'ok' + with sqlite3.connect(f'file:{root / "data/smail.db"}?mode=ro', uri=True) as database: + summary['migration'] = database.execute('SELECT MAX(version) FROM schema_migration').fetchone()[0] + summary['queue_states'] = dict(database.execute('SELECT state,COUNT(*) FROM outbound_delivery GROUP BY state')) + control = Path('/run/smail/admin-control.sock').lstat() + assert stat.S_ISSOCK(control.st_mode) and stat.S_IMODE(control.st_mode) == 0o600 + assert control.st_uid == (root / 'data/smail.db').stat().st_uid + summary['control_socket_mode'] = oct(stat.S_IMODE(control.st_mode)) + curl = ['curl', '--silent', '--show-error', '--fail', '--max-time', '10', '--noproxy', '*', + '--cacert', str(root / 'ssl/custom.crt'), '--resolve', 'mail.solit.cn:443:127.0.0.1'] + run(*curl, 'https://mail.solit.cn/api/ping') + summary['unauthenticated'] = {} + for endpoint in ('/api/v1/user/info', '/api/v1/email/list'): + response = json.loads(run(*curl, '-X', 'POST', 'https://mail.solit.cn' + endpoint)) + assert response['errorNo'] == 403 and not response.get('data') + summary['unauthenticated'][endpoint] = response['errorNo'] + entries = run('journalctl', '-u', 'smail.service', '--since', args.since, '--output=json', '--no-pager') + levels = {} + for line in entries.splitlines(): + entry = json.loads(line) + priority = entry.get('PRIORITY', 'unknown') + levels[priority] = levels.get(priority, 0) + 1 + summary['journal_priorities'] = levels + kernel = run('journalctl', '-k', '--since', args.since, '--no-pager') + summary['oom_events'] = sum('oom-kill:constraint=' in line for line in kernel.splitlines()) + summary['global_oom_events'] = sum('oom-kill:constraint=CONSTRAINT_NONE' in line for line in kernel.splitlines()) + print(json.dumps(summary, ensure_ascii=False, indent=2)) + + +if __name__ == '__main__': + main()