This commit is contained in:
parent
1b2ba5a6a1
commit
421aa7303d
@ -20,10 +20,10 @@
|
||||
| PROD-008 | P0 | 超级管理员恢复完整交互 | [!] | IPC 权限已复核，正式密码代理交互及旧会话撤销未验收 |
|
||||
| PROD-009 | P1 | 备份与恢复演练 | [x] | 同机独立网络和挂载空间恢复、HTTPS、内部 SMTP 入库通过；异机恢复未演练 |
|
||||
| PROD-010 | P1 | 内部 SMTP 与部署后安全回归 | [✓] | 五项真实收信、双库、产物、未认证 API 均通过 |
|
||||
| PROD-011 | P1 | 全量测试、竞态、数据库兼容性和压测 | [~] | 扩展至 60 包远程复测；竞态、真实 MySQL/PostgreSQL 和容量验收未完成 |
|
||||
| PROD-011 | P1 | 全量测试、竞态、数据库兼容性和压测 | [~] | 60 包分批执行、958 个顶层用例通过；8 处跳过单列，竞态、真实跨库和容量验收未完成 |
|
||||
| PROD-012 | P1 | 最新版本清库首次安装验收 | [ ] | 当前为保留数据升级，不能沿用旧版本清库结果 |
|
||||
| PROD-013 | P0 | 共享主机资源隔离与容量 | [!] | 首轮测试期间发生全机 OOM，MySQL 被终止后自动重启；必须补齐资源规划 |
|
||||
| PROD-014 | P1 | 投递结果写库恢复 | [x] | 数据库故障后仅重试结果落库、停止新领取，12 项远程队列测试通过 |
|
||||
| PROD-014 | P1 | 投递结果写库恢复 | [✓] | 12 项远程测试及源码复审通过；持久故障后的进程退出仍需人工核对，不代表生产故障演练通过 |
|
||||
| PROD-015 | P1 | 跨平台证书路径与插件测试 | [✓] | Linux 拒绝异平台路径误解析；保持证书校验、托管导入和插件目录权限要求 |
|
||||
|
||||
状态含义：`[ ]` 未开始，`[~]` 进行中，`[x]` 已完成待复审，`[✓]` 指定范围复审通过，`[!]` 暂缓或存在阻塞。
|
||||
@ -70,3 +70,5 @@ SSH 脚本复用 `test/tools/remote_ssh.py`，强制核对已知主机密钥。
|
||||
47 服务器约 1.6 GiB 内存，无交换空间，另有 MySQL、Scert 等服务。首轮测试仅设置 Go 软内存上限，不能防止全机内存耗尽。新工具通过 systemd 设置 CPU、内存、进程数和最长运行时间上限，使用独立网络、私有临时目录，并禁止访问线上 `/opt/smail`、`/run/smail`。可用内存不足时拒绝开始测试，不自动放宽限制、不停止其他项目。
|
||||
|
||||
`GOMEMLIMIT` 是 Go 回收目标，不是进程或主机硬限制。本轮协议测试以 128 MiB cgroup 上限、25% CPU 配额和 64 MiB Go 回收目标完成，不能据此推导生产容量。生产仍需确定同机服务预算、并发认证开销、邮件大小与吞吐上限、告警和故障恢复指标。
|
||||
|
||||
恢复服务整包执行曾在 128 MiB 上限触发测试 cgroup OOM，改为逐顶层用例独立进程后 14 项通过，峰值约 126.4 MiB。没有降低 Argon2id 的 64 MiB 内存参数；不能把拆分测试视为生产内存问题已解决。测试启动器默认 128 MiB，并对不存在的包、空清单和零匹配筛选返回失败；中断包保留运行中记录。
|
||||
|
||||
@ -277,3 +277,25 @@ TLS 前未公布 AUTH，主动尝试明文 AUTH 返回 523。未认证外域邮
|
||||
- 线上服务主 PID 保持不变；演练结束后副本服务已停止，恢复数据和证据保留。
|
||||
|
||||
这是同机隔离恢复，不代表异机灾备、恢复时限承诺或最新版本清库安装验收。
|
||||
|
||||
### 分批全包复测结果
|
||||
|
||||
11:18 恢复服务整包测试达到 128 MiB 硬上限，systemd 返回 `oom-kill`。内核明确记录 `CONSTRAINT_MEMCG`，终止范围为 `smail-tests-20260917-111329.service`；该次没有重启线上 Smail、MySQL 或 Scert。不能称整轮一次性通过。
|
||||
|
||||
保留原失败证据后，将其余 22 包和恢复服务分别补跑，仍使用同一份 Linux 测试二进制。恢复服务采用逐顶层用例独立进程，14 项通过，用时约 75 秒，峰值约 126.4 MiB，未降低生产密码哈希强度。三批已完成记录按包去重并与 60 包清单核对，总计 958 个顶层用例通过：
|
||||
|
||||
| 批次 | 完成包数 | 日志目录 |
|
||||
| --- | --- | --- |
|
||||
| 前段已完成包 | 37 | `results-20260917-111329` |
|
||||
| 后段补跑 | 22 | `results-20260917-112034` |
|
||||
| 恢复服务逐用例补跑 | 1 | `results-20260917-112305` |
|
||||
|
||||
目录均位于 `/var/tmp/smail-review-full-20260917/`。以下 8 处跳过没有计为通过：
|
||||
|
||||
- `TestMaster`：完整主服务集成测试未启用。
|
||||
- `TestMySQLIntegration`、`TestPostgresIntegration`：未提供真实专用测试数据库。
|
||||
- `TestPortableSQLIntegration/mysql`、`postgresql`、`mysql_postgresql`：真实跨数据库分支未启用。
|
||||
- `TestProductionPluginIsolationPolicy`：Windows 专属策略不能由 Linux 执行证明。
|
||||
- `TestSecuritySetupBrowserFixture`：浏览器夹具未启用，不能代替实际网页验收。
|
||||
|
||||
11:26 只读复核：Smail、MySQL、Scert 均运行，主 PID 分别为 `548293`、`549584`、`530794`；MySQL 重启计数仍为首轮事故后的 1。双库完整性通过、迁移版本 23、队列表为空、恢复 socket 为 `0600`，未认证用户信息及邮件列表均返回业务码 403。11:18:08 后未发现新增 OOM。此时线上版本尚为 `remote-13bcad90e832`，新代码未部署。
|
||||
|
||||
@ -4,6 +4,7 @@ import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
@ -13,7 +14,10 @@ def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--only', action='append', default=[])
|
||||
parser.add_argument('--run', default='')
|
||||
parser.add_argument('--per-test', action='store_true')
|
||||
args = parser.parse_args()
|
||||
if args.per_test and args.run:
|
||||
parser.error('逐用例模式不能同时使用正则筛选')
|
||||
root = Path(__file__).resolve().parent
|
||||
assert root.parent == Path('/var/tmp') and root.name.startswith('smail-review-')
|
||||
assert os.geteuid() != 0, '禁止使用 root 执行测试'
|
||||
@ -24,10 +28,16 @@ def main():
|
||||
environment['GOMAXPROCS'] = '1'
|
||||
environment['GOMEMLIMIT'] = '64MiB'
|
||||
environment['GOGC'] = '20'
|
||||
manifest = json.loads((root / 'manifest.json').read_text(encoding='utf-8'))
|
||||
if not manifest:
|
||||
parser.error('测试清单为空')
|
||||
unknown = set(args.only) - {item['directory'] for item in manifest}
|
||||
if unknown:
|
||||
parser.error('测试包不存在：' + ', '.join(sorted(unknown)))
|
||||
output = root / ('results-' + time.strftime('%Y%m%d-%H%M%S'))
|
||||
output.mkdir(mode=0o700)
|
||||
results = []
|
||||
for item in json.loads((root / 'manifest.json').read_text(encoding='utf-8')):
|
||||
for item in manifest:
|
||||
if args.only and item['directory'] not in args.only:
|
||||
continue
|
||||
directory = (root / item['directory']).resolve()
|
||||
@ -37,26 +47,47 @@ def main():
|
||||
assert stream.read(5) == b'\x7fELF\x02'
|
||||
started = time.monotonic()
|
||||
log = output / (item['directory'].replace('/', '-') + '.log')
|
||||
with log.open('w', encoding='utf-8') as stream:
|
||||
command = [str(binary), '-test.v', '-test.timeout=180s']
|
||||
if args.run:
|
||||
command.append('-test.run=' + args.run)
|
||||
process = subprocess.Popen(command, cwd=directory, env=environment,
|
||||
stdout=stream, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
try:
|
||||
code = process.wait(timeout=195)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait()
|
||||
code = 124
|
||||
lines = log.read_text(encoding='utf-8', errors='replace').splitlines()
|
||||
result = dict(item, code=code, log=str(log), seconds=round(time.monotonic() - started, 2),
|
||||
passed=sum(line.startswith('--- PASS:') for line in lines),
|
||||
failed=[line for line in lines if line.lstrip().startswith('--- FAIL:')],
|
||||
skipped=[line for line in lines if line.lstrip().startswith('--- SKIP:')])
|
||||
patterns = [args.run]
|
||||
if args.per_test:
|
||||
listing = subprocess.check_output([str(binary), '-test.list=.'], cwd=directory,
|
||||
env=environment, text=True, timeout=30)
|
||||
names = [line for line in listing.splitlines()
|
||||
if re.fullmatch(r'(?:Test|Example|Fuzz)\w+', line)]
|
||||
if not names:
|
||||
raise RuntimeError('未发现可运行用例：' + item['directory'])
|
||||
patterns = ['^' + re.escape(name) + '$' for name in names]
|
||||
result = dict(item, code=None, log=str(log), status='running', per_test=args.per_test)
|
||||
results.append(result)
|
||||
summary = output / 'results.json'
|
||||
summary.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
code = 0
|
||||
with log.open('w', encoding='utf-8') as stream:
|
||||
for pattern in patterns:
|
||||
command = [str(binary), '-test.v', '-test.timeout=180s']
|
||||
if pattern:
|
||||
command.append('-test.run=' + pattern)
|
||||
process = subprocess.Popen(command, cwd=directory, env=environment,
|
||||
stdout=stream, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
try:
|
||||
current = process.wait(timeout=195)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait()
|
||||
current = 124
|
||||
if current != 0:
|
||||
code = current
|
||||
lines = log.read_text(encoding='utf-8', errors='replace').splitlines()
|
||||
passed = sum(line.startswith('--- PASS:') for line in lines)
|
||||
skipped = [line for line in lines if line.lstrip().startswith('--- SKIP:')]
|
||||
if not passed and not skipped and code == 0:
|
||||
code = 2
|
||||
if args.per_test and passed + sum(line.startswith('--- SKIP:') for line in lines) != len(patterns):
|
||||
code = code or 2
|
||||
result.update(code=code, status='finished', seconds=round(time.monotonic() - started, 2),
|
||||
passed=passed, failed=[line for line in lines if line.lstrip().startswith('--- FAIL:')],
|
||||
skipped=skipped)
|
||||
print(json.dumps(result, ensure_ascii=False), flush=True)
|
||||
(output / 'results.json').write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
summary.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print('测试记录：' + str(output), flush=True)
|
||||
return int(any(item['code'] != 0 for item in results))
|
||||
|
||||
|
||||
@ -12,7 +12,8 @@ def main():
|
||||
parser.add_argument('directory')
|
||||
parser.add_argument('--only', action='append', default=[])
|
||||
parser.add_argument('--run', default='')
|
||||
parser.add_argument('--memory-mib', type=int, choices=(64, 96, 128, 192), default=192)
|
||||
parser.add_argument('--per-test', action='store_true')
|
||||
parser.add_argument('--memory-mib', type=int, choices=(64, 96, 128, 192), default=128)
|
||||
args = parser.parse_args()
|
||||
root = Path(args.directory).resolve(strict=True)
|
||||
assert os.geteuid() == 0
|
||||
@ -39,6 +40,8 @@ def main():
|
||||
command.extend(['--only', directory])
|
||||
if args.run:
|
||||
command.extend(['--run', args.run])
|
||||
if args.per_test:
|
||||
command.append('--per-test')
|
||||
print('测试服务：' + unit, flush=True)
|
||||
return subprocess.run(command).returncode
|
||||
|
||||
|
||||
95
test/tools/verify_review.py
Normal file
95
test/tools/verify_review.py
Normal file
@ -0,0 +1,95 @@
|
||||
"""只读核对远程分批测试覆盖及线上服务，不输出邮件、密码或会话内容。"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import stat
|
||||
import subprocess
|
||||
|
||||
|
||||
def run(*args):
|
||||
return subprocess.check_output(args, text=True).strip()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--results', nargs='+', default=[])
|
||||
parser.add_argument('--main-sha')
|
||||
parser.add_argument('--admin-sha')
|
||||
parser.add_argument('--since', required=True)
|
||||
args = parser.parse_args()
|
||||
assert os.geteuid() == 0 and run('hostname') == 'ikun2'
|
||||
root = Path('/opt/smail')
|
||||
assert root.resolve() == root
|
||||
summary = {}
|
||||
if args.results:
|
||||
test_root = Path('/var/tmp/smail-review-full-20260917')
|
||||
expected = {item['directory'] for item in json.loads((test_root / 'manifest.json').read_text())}
|
||||
completed = {}
|
||||
for name in args.results:
|
||||
directory = (test_root / name).resolve(strict=True)
|
||||
assert directory.parent == test_root and directory.name.startswith('results-')
|
||||
for item in json.loads((directory / 'results.json').read_text()):
|
||||
assert item.get('code') == 0 and not item.get('failed'), '测试未通过或被中断'
|
||||
assert item['directory'] not in completed, '测试包结果重复'
|
||||
completed[item['directory']] = item
|
||||
assert set(completed) == expected, '测试包覆盖不完整'
|
||||
summary['tests'] = {
|
||||
'packages': len(completed),
|
||||
'passed': sum(item['passed'] for item in completed.values()),
|
||||
'skipped': {name: item['skipped'] for name, item in completed.items() if item['skipped']},
|
||||
'per_test': [name for name, item in completed.items() if item.get('per_test')],
|
||||
'evidence': args.results,
|
||||
}
|
||||
services = {}
|
||||
for name in ('smail.service', 'mysql.service', 'Scert.service'):
|
||||
properties = run('systemctl', 'show', name, '--property=ActiveState,SubState,MainPID,NRestarts,ExecMainStatus')
|
||||
services[name] = dict(line.split('=', 1) for line in properties.splitlines())
|
||||
assert services[name]['ActiveState'] == 'active'
|
||||
summary['services'] = services
|
||||
for name, checksum in (('smail', args.main_sha), ('smail-admin', args.admin_sha)):
|
||||
if checksum:
|
||||
with (root / name).open('rb') as stream:
|
||||
actual = hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
assert actual == checksum, '部署产物摘要不匹配'
|
||||
summary[name + '_sha256'] = actual
|
||||
summary['admin_version'] = run(str(root / 'smail-admin'), 'version')
|
||||
summary['database_integrity'] = {}
|
||||
for name in ('smail.db', 'smail-state.db'):
|
||||
with sqlite3.connect(f'file:{root / "data" / name}?mode=ro', uri=True) as database:
|
||||
integrity = database.execute('PRAGMA integrity_check').fetchall()
|
||||
assert integrity == [('ok',)]
|
||||
summary['database_integrity'][name] = 'ok'
|
||||
with sqlite3.connect(f'file:{root / "data/smail.db"}?mode=ro', uri=True) as database:
|
||||
summary['migration'] = database.execute('SELECT MAX(version) FROM schema_migration').fetchone()[0]
|
||||
summary['queue_states'] = dict(database.execute('SELECT state,COUNT(*) FROM outbound_delivery GROUP BY state'))
|
||||
control = Path('/run/smail/admin-control.sock').lstat()
|
||||
assert stat.S_ISSOCK(control.st_mode) and stat.S_IMODE(control.st_mode) == 0o600
|
||||
assert control.st_uid == (root / 'data/smail.db').stat().st_uid
|
||||
summary['control_socket_mode'] = oct(stat.S_IMODE(control.st_mode))
|
||||
curl = ['curl', '--silent', '--show-error', '--fail', '--max-time', '10', '--noproxy', '*',
|
||||
'--cacert', str(root / 'ssl/custom.crt'), '--resolve', 'mail.solit.cn:443:127.0.0.1']
|
||||
run(*curl, 'https://mail.solit.cn/api/ping')
|
||||
summary['unauthenticated'] = {}
|
||||
for endpoint in ('/api/v1/user/info', '/api/v1/email/list'):
|
||||
response = json.loads(run(*curl, '-X', 'POST', 'https://mail.solit.cn' + endpoint))
|
||||
assert response['errorNo'] == 403 and not response.get('data')
|
||||
summary['unauthenticated'][endpoint] = response['errorNo']
|
||||
entries = run('journalctl', '-u', 'smail.service', '--since', args.since, '--output=json', '--no-pager')
|
||||
levels = {}
|
||||
for line in entries.splitlines():
|
||||
entry = json.loads(line)
|
||||
priority = entry.get('PRIORITY', 'unknown')
|
||||
levels[priority] = levels.get(priority, 0) + 1
|
||||
summary['journal_priorities'] = levels
|
||||
kernel = run('journalctl', '-k', '--since', args.since, '--no-pager')
|
||||
summary['oom_events'] = sum('oom-kill:constraint=' in line for line in kernel.splitlines())
|
||||
summary['global_oom_events'] = sum('oom-kill:constraint=CONSTRAINT_NONE' in line for line in kernel.splitlines())
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in New Issue
Block a user