This commit is contained in:
parent
e14c538322
commit
3532f76c66
202
deploy/check_resources.py
Normal file
202
deploy/check_resources.py
Normal file
@ -0,0 +1,202 @@
|
||||
"""只读核对 Linux 主机上的 Smail 资源隔离,不执行部署、重启或压测。"""
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
from decimal import Decimal
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
|
||||
MIB = 1024 * 1024
|
||||
PROPERTIES = (
|
||||
'ActiveState', 'SubState', 'MainPID', 'InvocationID', 'ControlGroup', 'NRestarts',
|
||||
'MemoryHigh', 'MemoryMax', 'MemorySwapMax', 'TasksMax', 'OOMPolicy', 'KillMode',
|
||||
'Restart', 'RestartUSec', 'StartLimitIntervalUSec', 'StartLimitBurst', 'TimeoutStopUSec',
|
||||
)
|
||||
|
||||
|
||||
def systemd_state():
|
||||
result = subprocess.run(
|
||||
['systemctl', 'show', 'smail.service', '--property=' + ','.join(PROPERTIES)],
|
||||
check=True, capture_output=True, text=True, timeout=10,
|
||||
env={**os.environ, 'LC_ALL': 'C'},
|
||||
)
|
||||
state = dict(line.split('=', 1) for line in result.stdout.splitlines() if '=' in line)
|
||||
if any(name not in state for name in PROPERTIES):
|
||||
raise ValueError('服务管理器未提供完整资源属性')
|
||||
return {name: state[name] for name in PROPERTIES}
|
||||
|
||||
|
||||
def limit(value):
|
||||
if value in ('max', 'infinity'):
|
||||
return None
|
||||
number = int(value)
|
||||
if number < 0:
|
||||
raise ValueError('资源限额不能是负数')
|
||||
return number
|
||||
|
||||
|
||||
def go_memory_limit(environment):
|
||||
values = [item[len(b'GOMEMLIMIT='):] for item in environment.split(b'\0')
|
||||
if item.startswith(b'GOMEMLIMIT=')]
|
||||
if len(values) != 1:
|
||||
return None
|
||||
# 只允许合法的 Go 字节单位,不把其他环境变量或无效值写入报告。
|
||||
match = re.fullmatch(rb'([0-9]+)(B|KiB|MiB|GiB|TiB)?', values[0])
|
||||
if not match:
|
||||
return None
|
||||
units = {None: 1, b'B': 1, b'KiB': 1024, b'MiB': MIB,
|
||||
b'GiB': 1024 ** 3, b'TiB': 1024 ** 4}
|
||||
return int(match[1]) * units[match[2]]
|
||||
|
||||
|
||||
def duration_usec(value):
|
||||
if not re.fullmatch(r'(?:[0-9]+(?:\.[0-9]+)?(?:us|ms|s|min|h|d)\s*)+', value):
|
||||
return None
|
||||
units = {'us': 1, 'ms': 1000, 's': 1000000, 'min': 60000000,
|
||||
'h': 3600000000, 'd': 86400000000}
|
||||
return int(sum(Decimal(number) * units[unit] for number, unit in
|
||||
re.findall(r'([0-9]+(?:\.[0-9]+)?)(us|ms|s|min|h|d)', value)))
|
||||
|
||||
|
||||
def cpu_quota(value):
|
||||
quota, period = value.split()
|
||||
amount, interval = limit(quota), int(period)
|
||||
if interval <= 0 or (amount is not None and amount <= 0):
|
||||
raise ValueError('CPU 限额无效')
|
||||
return None if amount is None else amount / interval
|
||||
|
||||
|
||||
def collect():
|
||||
before = systemd_state()
|
||||
pid = int(before['MainPID'])
|
||||
if before['ActiveState'] != 'active' or before['SubState'] != 'running' or pid <= 0:
|
||||
raise ValueError('Smail 主服务未处于运行状态')
|
||||
proc = Path('/proc') / str(pid)
|
||||
base = Path('/sys/fs/cgroup').resolve(strict=True)
|
||||
group_name = before['ControlGroup']
|
||||
if not group_name.startswith('/') or '..' in Path(group_name).parts:
|
||||
raise ValueError('服务控制组路径无效')
|
||||
group = (base / group_name.lstrip('/')).resolve(strict=True)
|
||||
if group == base or base not in group.parents:
|
||||
raise ValueError('Smail 必须使用独立控制组')
|
||||
if ('0::' + group_name) not in (proc / 'cgroup').read_text().splitlines():
|
||||
raise ValueError('主进程没有加入预期的统一控制组')
|
||||
files = ('memory.high', 'memory.max', 'memory.current', 'memory.swap.max',
|
||||
'pids.max', 'pids.current', 'cpu.max')
|
||||
kernel = {name: (group / name).read_text().strip() for name in files}
|
||||
events = dict(line.split() for line in (group / 'memory.events').read_text().splitlines())
|
||||
parents = []
|
||||
parent = group.parent
|
||||
while True:
|
||||
limits = {name: (parent / name).read_text().strip() for name in files
|
||||
if (parent / name).exists()}
|
||||
parents.append({'path': str(parent.relative_to(base)), 'limits': limits})
|
||||
if parent == base:
|
||||
break
|
||||
parent = parent.parent
|
||||
memory = dict(line.split(':', 1) for line in Path('/proc/meminfo').read_text().splitlines())
|
||||
snapshot = {
|
||||
'time_utc': datetime.now(timezone.utc).isoformat(),
|
||||
'service': before, 'kernel': kernel, 'parents': parents,
|
||||
'oom': {name: int(events[name]) for name in ('oom', 'oom_kill')},
|
||||
'go_memory_bytes': go_memory_limit((proc / 'environ').read_bytes()),
|
||||
'host_available_bytes': int(memory['MemAvailable'].split()[0]) * 1024,
|
||||
'host_total_bytes': int(memory['MemTotal'].split()[0]) * 1024,
|
||||
}
|
||||
# 读取期间若发生重启或资源属性变更,旧、新进程的数据不能拼成通过证据。
|
||||
snapshot['stable'] = before == systemd_state()
|
||||
return snapshot
|
||||
|
||||
|
||||
def evaluate(snapshot, reserve_bytes):
|
||||
failures = []
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
failures.append(message)
|
||||
|
||||
service, kernel = snapshot['service'], snapshot['kernel']
|
||||
high, maximum = limit(kernel['memory.high']), limit(kernel['memory.max'])
|
||||
current, tasks = int(kernel['memory.current']), int(kernel['pids.current'])
|
||||
task_maximum = limit(kernel['pids.max'])
|
||||
go_limit = snapshot['go_memory_bytes']
|
||||
require(reserve_bytes >= 128 * MIB, '必须明确预留至少 128 MiB 的主机及同机服务增长余量')
|
||||
require(snapshot['stable'], '取证期间服务重启或资源配置发生变化,需要重新检查')
|
||||
require(high is not None and maximum is not None and 0 < high < maximum,
|
||||
'必须设置有限且递增的内存高水位与硬上限')
|
||||
require(limit(kernel['memory.swap.max']) == 0, '交换空间未限制为零,不符合当前部署预算')
|
||||
require(go_limit is not None and high is not None and 64 * MIB <= go_limit < high,
|
||||
'主进程缺少有效 GOMEMLIMIT,或没有为非 Go 内存预留高水位余量')
|
||||
require(high is not None and current < high, '当前内存已达到高水位或高水位未配置')
|
||||
require(task_maximum is not None and 0 < tasks < task_maximum, '任务数上限无效或已经耗尽')
|
||||
require(cpu_quota(kernel['cpu.max']) is not None, 'CPU 没有有限配额')
|
||||
for property_name, file_name in (('MemoryHigh', 'memory.high'), ('MemoryMax', 'memory.max'),
|
||||
('MemorySwapMax', 'memory.swap.max'), ('TasksMax', 'pids.max')):
|
||||
require(limit(service[property_name]) == limit(kernel[file_name]),
|
||||
property_name + ' 在服务管理器与实际控制组中不一致')
|
||||
require(service['OOMPolicy'] in ('stop', 'kill'), 'OOM 后未设置整组停止策略')
|
||||
require(service['KillMode'] == 'control-group', '停止服务时未覆盖整个进程组')
|
||||
require((duration_usec(service['TimeoutStopUSec']) or 0) >= 90 * 1000000,
|
||||
'停止等待不足 90 秒或未设置有效期限,无法覆盖队列及数据库收尾预算')
|
||||
require(service['Restart'] == 'on-failure', '服务重启策略不符合故障恢复要求')
|
||||
require((duration_usec(service['RestartUSec']) or 0) >= 5 * 1000000, '重启退避时间不足')
|
||||
require((duration_usec(service['StartLimitIntervalUSec']) or 0) >= 60 * 1000000
|
||||
and 1 <= int(service['StartLimitBurst']) <= 5, '缺少有效的启动频率限制')
|
||||
require(all(snapshot['oom'][name] == 0 for name in ('oom', 'oom_kill')),
|
||||
'当前服务控制组记录过 OOM,需要核查原因并重新验收')
|
||||
growth = None if maximum is None else max(0, maximum - current)
|
||||
if growth is not None:
|
||||
require(maximum < snapshot['host_total_bytes'], '单服务硬上限已达到主机总内存')
|
||||
require(snapshot['host_available_bytes'] - growth >= reserve_bytes,
|
||||
'服务增长到硬上限后,主机剩余预算不足')
|
||||
for parent in snapshot['parents']:
|
||||
values = parent['limits']
|
||||
for name in ('memory.high', 'memory.max'):
|
||||
bound = limit(values.get(name, 'max'))
|
||||
if bound is not None:
|
||||
require(bound - int(values['memory.current']) - growth >= reserve_bytes,
|
||||
'上级控制组 ' + parent['path'] + ' 的 ' + name + ' 余量不足')
|
||||
bound = limit(values.get('pids.max', 'max'))
|
||||
if bound is not None and task_maximum is not None:
|
||||
require(bound - int(values['pids.current']) >= task_maximum - tasks,
|
||||
'上级控制组 ' + parent['path'] + ' 的任务数余量不足')
|
||||
parent_cpu = cpu_quota(values.get('cpu.max', 'max 100000'))
|
||||
own_cpu = cpu_quota(kernel['cpu.max'])
|
||||
if parent_cpu is not None and own_cpu is not None:
|
||||
require(own_cpu <= parent_cpu, '上级控制组 ' + parent['path'] + ' 的 CPU 配额更小')
|
||||
return {
|
||||
'resource_checks_passed': not failures,
|
||||
'production_approved': False,
|
||||
'reserve_bytes': reserve_bytes,
|
||||
'remaining_growth_bytes': growth,
|
||||
'failures': failures,
|
||||
'scope': '仅检查单次资源快照,不替代容量、持续监控及完整生产验收',
|
||||
'snapshot': snapshot,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--reserve-mib', required=True, type=int,
|
||||
help='按同机服务增长与系统开销核算的保留内存,不少于 128 MiB')
|
||||
args = parser.parse_args()
|
||||
if args.reserve_mib < 128:
|
||||
parser.error('保留内存必须不少于 128 MiB,不能用零预算规避检查')
|
||||
try:
|
||||
if os.name != 'posix' or os.geteuid() != 0:
|
||||
raise ValueError('请在 Linux 部署主机使用 root 执行只读检查')
|
||||
report = evaluate(collect(), args.reserve_mib * MIB)
|
||||
except (OSError, ValueError, KeyError, subprocess.SubprocessError):
|
||||
# 系统命令的错误输出可能携带部署信息,不原样打印环境或异常内容。
|
||||
report = {'resource_checks_passed': False, 'production_approved': False,
|
||||
'failures': ['无法获得完整资源证据,请检查服务、root 权限及 cgroup v2 支持']}
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0 if report['resource_checks_passed'] else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@ -81,9 +81,9 @@ if ! getent passwd smail >/dev/null; then useradd --system --gid smail --home-di
|
||||
install -d -o root -g root -m 0755 "$runtime_dir"
|
||||
install -d -o root -g root -m 0750 /etc/smail
|
||||
if [ ! -e /etc/smail/smail.env ]; then
|
||||
# 只创建默认根目录;已存在的文件始终保留部署人员的自定义环境配置。
|
||||
# 只为新部署设置回收预算;已存在的文件始终保留部署人员的自定义环境配置。
|
||||
umask 0077
|
||||
printf '%s\n' 'SMAIL_ROOT=/opt/smail' > /etc/smail/smail.env
|
||||
printf '%s\n' 'SMAIL_ROOT=/opt/smail' 'GOMEMLIMIT=256MiB' > /etc/smail/smail.env
|
||||
fi
|
||||
chown root:root /etc/smail/smail.env
|
||||
chmod 0600 /etc/smail/smail.env
|
||||
@ -112,3 +112,5 @@ systemctl disable smail-admin >/dev/null 2>&1 || true
|
||||
echo "Smail 已安装到 $runtime_dir,旧程序和服务文件备份在 $backup_dir。"
|
||||
echo "配置、数据库和证书未覆盖;请使用 systemctl start smail 启动并检查运行状态。"
|
||||
echo "smail-admin 仅允许按需执行 start/stop,不支持开机自启动。"
|
||||
echo "主服务默认内存高水位 384 MiB、硬上限 512 MiB;不是容量承诺,请先核对同机资源预算。"
|
||||
echo "旧环境文件不会自动补写 GOMEMLIMIT;启动后请按《生产资源部署说明》执行只读资源预检。"
|
||||
|
||||
@ -2,6 +2,8 @@
|
||||
Description=Smail 邮件服务
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
StartLimitIntervalSec=300s
|
||||
StartLimitBurst=3
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
@ -12,7 +14,20 @@ EnvironmentFile=/etc/smail/smail.env
|
||||
WorkingDirectory=/opt/smail
|
||||
ExecStart=/opt/smail/smail
|
||||
Restart=on-failure
|
||||
RestartSec=5s
|
||||
RestartSec=10s
|
||||
# 队列等待 25 秒、数据库等待 35 秒,另留服务与日志收尾时间。
|
||||
TimeoutStopSec=90s
|
||||
KillMode=control-group
|
||||
OOMPolicy=stop
|
||||
# 默认预算只是部署起点;上线前须核对同机服务余量并完成容量验收。
|
||||
MemoryAccounting=true
|
||||
MemoryHigh=384M
|
||||
MemoryMax=512M
|
||||
MemorySwapMax=0
|
||||
CPUAccounting=true
|
||||
CPUQuota=100%
|
||||
TasksAccounting=true
|
||||
TasksMax=256
|
||||
RuntimeDirectory=smail
|
||||
RuntimeDirectoryMode=0750
|
||||
# 目录由 smail 进程拥有,允许主服务创建受保护的本机控制套接字。
|
||||
|
||||
@ -12,12 +12,14 @@
|
||||
|
||||
13:05,POP3 与 SMTP 错误分类修复的三包远程回归共 48 项通过,无跳过。13:10 同版候选程序通过空数据副本初始化、真实协议及三轮混合认证过载恢复;随后完成备份部署,线上为 `review-b47f36643547`。13:12 部署后内部 SMTP 五项真实收信、产物摘要、双库和安全边界复核通过。生产容量、资源规划和正式浏览器验收仍未通过。
|
||||
|
||||
15:21 起继续资源隔离整改。已补充 systemd 候选资源预算、OOM 整组收尾、启动限频、90 秒停止期限、只读资源预检及反例测试;本轮仅完成源码复审、语法检查和 Linux 测试程序交叉编译,未在本机执行应用或测试用例。当前会话无法通过 47 的 SSH 认证,没有上传或部署,也没有重新核实线上版本与资源。详见《生产资源部署说明》,不得把本轮源码变化计为 PROD-013 已通过。
|
||||
|
||||
## 任务状态
|
||||
|
||||
| 编号 | 优先级 | 任务 | 状态 | 验收边界 |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| PROD-001 | P0 | 移除脚本中的已知明文凭据 | [x] | 工作区已清理,历史泄露处置未结束 |
|
||||
| PROD-002 | P0 | 轮换泄露凭据、核查访问记录 | [!] | 旧 SSH 密码仍有效,需要协调替代登录及轮换 |
|
||||
| PROD-002 | P0 | 轮换泄露凭据、核查访问记录 | [!] | 前轮记录旧 SSH 密码仍有效;本轮不复用泄露密码,已有密钥无法登录,需要恢复受信任访问并确认轮换状态 |
|
||||
| PROD-003 | P0 | 处置 Git 历史中的凭据 | [!] | 未授权重写历史或强推;轮换应先于历史清理 |
|
||||
| PROD-004 | P0 | 持久化逐收件人投递、防不确定重投 | [x] | 相关自动测试通过,真实网页业务闭环待验收 |
|
||||
| PROD-005 | P1 | MX、备用路由、DNS 错误分类 | [✓] | 针对性测试通过,不代表各外部邮件供应商兼容性 |
|
||||
@ -28,7 +30,7 @@
|
||||
| PROD-010 | P1 | 内部 SMTP 与部署后安全回归 | [✓] | 五项真实收信、双库、产物、未认证 API 均通过 |
|
||||
| PROD-011 | P1 | 全量测试、竞态、数据库兼容性和压测 | [~] | 60 包分批执行、958 个顶层用例通过;8 处跳过单列,竞态、真实跨库和容量验收未完成 |
|
||||
| PROD-012 | P1 | 最新版本清库首次安装验收 | [~] | 新部署产物的空数据副本正式初始化 API、协议收信和认证过载恢复通过;真实浏览器首次安装待验收 |
|
||||
| PROD-013 | P0 | 共享主机资源隔离与容量 | [!] | 首轮测试期间发生全机 OOM,MySQL 被终止后自动重启;必须补齐资源规划 |
|
||||
| PROD-013 | P0 | 共享主机资源隔离与容量 | [!] | 首轮全机 OOM 后补充候选服务限额和只读预检;SSH 认证阻塞,尚未部署或测试,实际预算与容量未验收 |
|
||||
| PROD-014 | P1 | 投递结果写库恢复 | [✓] | 12 项远程测试及源码复审通过;持久故障后的进程退出仍需人工核对,不代表生产故障演练通过 |
|
||||
| PROD-015 | P1 | 跨平台证书路径与插件测试 | [✓] | Linux 拒绝异平台路径误解析;保持证书校验、托管导入和插件目录权限要求 |
|
||||
| PROD-016 | P0 | 密码计算全局并发和过载处理 | [✓] | 48 项远程回归及新部署产物三轮真实混合认证过载恢复通过;区分 SMTP 535/454、IMAP UNAVAILABLE、POP3 SYS/TEMP,不代表生产容量或浏览器验收通过 |
|
||||
|
||||
76
docs/生产资源部署说明.md
Normal file
76
docs/生产资源部署说明.md
Normal file
@ -0,0 +1,76 @@
|
||||
# 生产资源部署说明
|
||||
|
||||
## 状态与边界
|
||||
|
||||
本说明对应 2026 年 9 月 17 日的共享主机资源整改。以下配置和检查工具是待远程验收的候选变更,不表示已经部署到 47 服务器,也不是生产容量承诺。
|
||||
|
||||
- [x] 增加主服务资源上限、OOM 收尾与重启频率限制,等待远程复审。
|
||||
- [x] 增加只读资源预检和失败反例测试,等待远程执行。
|
||||
- [!] 当前会话没有可用的 47 服务器 SSH 登录凭据,无法取得实时资源快照或部署。
|
||||
- [ ] 明确同机 MySQL、Scert、操作系统的增长预算,选择 Smail 的实际限额。
|
||||
- [ ] 清空隔离验收实例的全部项目数据库,部署最新产物,执行安装、恢复、队列和内部 25 端口验收。
|
||||
- [ ] 记录峰值、持续运行、过载恢复和监控告警,完成资源项复审。
|
||||
|
||||
不要停止其他项目以通过测试,不自动扩充内存限额,不创建交换文件,不把旧记录中的可用内存当作当前资源。数据库清理只涉及获准的 Smail 测试实例,执行前必须备份并确认全部数据库的归属。
|
||||
|
||||
## 候选默认预算
|
||||
|
||||
正式安装模板新增如下配置,部署人员需要先核算宿主机预算。默认值不是根据完整生产负载测得的安全容量。
|
||||
|
||||
| 配置 | 候选默认值 | 用途 |
|
||||
| --- | --- | --- |
|
||||
| `GOMEMLIMIT` | `256MiB` | 新建环境文件中的 Go 回收目标,不是进程硬上限 |
|
||||
| `MemoryHigh` | `384M` | 控制组内存高水位 |
|
||||
| `MemoryMax` | `512M` | 主服务及同组子进程的内存硬上限 |
|
||||
| `MemorySwapMax` | `0` | 不以交换空间掩盖内存不足 |
|
||||
| `CPUQuota` | `100%` | 默认最多使用一个 CPU 的时间配额,不是整机所有 CPU |
|
||||
| `TasksMax` | `256` | 限制同组进程及线程数量,不等同于邮件用户数 |
|
||||
| `OOMPolicy` / `KillMode` | `stop` / `control-group` | OOM 或停止时收尾整个服务进程组 |
|
||||
| `Restart` / `RestartSec` | `on-failure` / `10s` | 故障后退避重启 |
|
||||
| `StartLimitIntervalSec` / `StartLimitBurst` | `300s` / `3` | 五分钟内最多三次启动,防止持续重启占用主机资源 |
|
||||
| `TimeoutStopSec` | `90s` | 覆盖队列 25 秒、数据库 35 秒及其他服务收尾时间 |
|
||||
|
||||
已有 `/etc/smail/smail.env` 始终保留,不自动添加或改写已有 `GOMEMLIMIT`。维护人员应在停服备份后人工检查该文件,保持 `root:root`、`0600` 权限。不要将整个环境文件输出到日志、聊天或验收文档。
|
||||
|
||||
若已有本机 systemd drop-in,它可能覆盖仓库模板。不能只看文件内容或 `daemon-reload` 成功;必须核对运行进程、服务管理器和内核控制组的实际值。
|
||||
|
||||
## 只读预检
|
||||
|
||||
工具只在 Linux 部署主机检查正在运行的 `smail.service`,要求 root、Python 3 和 cgroup v2。不会启动程序、改配置、重启服务、读写数据库或进行压测。
|
||||
|
||||
在服务器的本次审查源码目录运行:
|
||||
|
||||
```sh
|
||||
python3 deploy/check_resources.py --reserve-mib 256
|
||||
```
|
||||
|
||||
`256` 只是命令示例,必须按同机服务的增长及系统开销核算。工具要求显式指定且不得小于 `128`,不能为了通过检查降低已确定的预算。
|
||||
|
||||
预检逐项核对:
|
||||
|
||||
1. 服务处于运行状态,主 PID 位于本服务独立的统一控制组。
|
||||
2. 内存高水位、硬上限、交换与任务数的 systemd 属性和实际内核值相符,CPU 配额有限。
|
||||
3. 进程设置了有效的 `GOMEMLIMIT`,低于内存高水位,当前内存未达到高水位。
|
||||
4. 从当前占用增长到 `MemoryMax` 后,主机仍有指定的保留内存;同时检查上级控制组的内存、任务数和 CPU 约束。
|
||||
5. OOM 收尾、整个进程组停止、至少 90 秒且有界的停止期限、重启退避和启动频率限制有效,当前控制组没有累计 OOM 或 OOM kill。
|
||||
6. 取证前后 PID、调用标识和所查服务属性一致,避免把重启前后的数据拼成通过证据。
|
||||
|
||||
工具仅从进程环境解析 `GOMEMLIMIT` 的数值,不输出其他环境项。报告中 `resource_checks_passed=true` 只表示这一次快照通过资源规则,`production_approved` 始终是 `false`;退出码 `0` 不构成完整生产放行。取证失败、权限不足、cgroup v1、属性缺失或检查不通过时返回非零。
|
||||
|
||||
主机资源随其他服务负载变化。瞬时检查没有为 Smail 预留物理内存,也不证明 MySQL、Scert 已分别受到合适的限制;必须另行确定这些服务的预算、监控并验证持续运行。
|
||||
|
||||
## 部署与远程验收
|
||||
|
||||
1. 恢复受信任 SSH 登录,核对主机身份、线上版本、服务 PID、重启计数、完整预算及 OOM 记录。不要复用已泄露的历史密码。
|
||||
2. 先执行只读检查。旧服务缺少新规则时预期失败,保留结果作为整改前证据,不能通过重置计数掩盖 OOM。
|
||||
3. 用最新源码构建产物;在 47 的独立目录和隔离网络创建清洁数据库验收实例,保持硬资源上限。正式 unit 的检查使用 `systemd-analyze verify`,不得仅用字符串断言代替系统解析。
|
||||
4. 在受限的远程测试单元中执行 `python3 test/deployment/resource_policy_test.py`;也可使用统一部署测试入口 `node test/run.mjs deployment`。这些测试使用模拟快照和全新临时夹具,不访问业务数据库,不替代线上资源检查。
|
||||
5. 完成正式安装流程、真实网页发信队列、中文附件、内部 `127.0.0.1:25`、混合认证和受控故障恢复,记录内存峰值、延迟、繁忙响应、任务数、OOM 和同机服务状态。仅用小样本成功不能声明最大容量。
|
||||
6. 预算通过后安排停服窗口,备份双程序、unit、环境文件、所有数据库和密钥证书。安装程序不会自动停止正在运行的服务,部署时保留原数据与可回退产物。
|
||||
7. 部署后重新执行只读预检与内部 SMTP 回归,检查限额、线上摘要和其他服务状态;未通过时停止放行,按备份方案恢复,不扩大预算以掩盖失败。
|
||||
|
||||
触发启动频率限制后,先保存故障证据并处理原因,再由维护人员决定恢复启动。不能在定时任务里循环 `reset-failed`,不能把自动重启当作队列和数据库的故障验收。
|
||||
|
||||
## 本轮未证明的事项
|
||||
|
||||
尚未在 47 上执行本说明新增的测试或真实预检,尚未核实默认 512 MiB 硬上限适用于该主机。TLS、凭据轮换、完整浏览器流程、超管恢复撤销、竞态、真实跨库和异机恢复继续以《生产整改验收清单》为准,不因资源配置补齐而自动通过。
|
||||
@ -478,3 +478,21 @@ TLS 前未公布 AUTH,主动尝试明文 AUTH 返回 523。未认证外域邮
|
||||
部署后 Smail `active/running`,PID `563364`、重启计数 0;MySQL PID `549584`、Scert PID `530794` 均未改变,重启计数仍为 1 和 0。双库完整性正常,迁移 23,队列为空,恢复 socket `0600`,未认证用户信息与邮件列表业务码均为 403。双程序摘要与上表一致,13:11 以后没有新增 OOM。
|
||||
|
||||
本轮完成协议错误分类、完整响应回归、空数据 API 安装与混合认证过载恢复、备份升级及内部 SMTP 回归。可信 TLS、真实浏览器安装和发信闭环、正式超管恢复交互及撤销、泄露凭据处置、生产资源预算、竞态及跨库、异机恢复等门槛仍未验收,不能签署生产放行。
|
||||
|
||||
## 2026 年 9 月 17 日下午:资源防护候选,远程认证阻塞
|
||||
|
||||
15:21 起恢复工作。本轮先提交上一轮遗留的两份验收记录和 `runtime_review.py` 元组断言修正,检查点为 `e14c53832241aac4a9d052de82c1b450c5e30566`;推送首轮出现认证失败,重试成功,并以 `git ls-remote` 确认远程分支摘要一致。
|
||||
|
||||
当前会话、用户级和机器级环境均没有 `SMAIL_SSH_KEY` 或 `SMAIL_SSH_PASSWORD`。使用强制已知主机校验的连接工具、现有默认密钥、Smail 专用密钥及 OpenSSH 默认认证均未通过 47 的认证。没有关闭主机校验,没有从 Git 历史或旧日志提取泄露密码。本轮未登录服务器,未上传、重启、部署或清库;不能据此断言旧密码已撤销,也不能把 13:12 的运行状态当作本轮实时状态。
|
||||
|
||||
资源整改候选包括:
|
||||
|
||||
- 主服务内存高水位 384 MiB、硬上限 512 MiB、禁用交换配额、单 CPU 时间配额与 256 个任务上限。这些是待核算、待验收的模板值,不是 47 的已确认容量。
|
||||
- OOM 后整组停止、故障后 10 秒退避、五分钟最多三次启动;停止期限 90 秒,覆盖现有队列 25 秒与数据库 35 秒等待以及其他收尾。
|
||||
- 新安装环境文件增加 `GOMEMLIMIT=256MiB`,已有环境文件不改写;权限和备份逻辑保留。
|
||||
- 只读检查实际 cgroup v2、主进程环境中的 Go 内存数值、上级控制组、剩余增长预算、OOM 记录和服务取证稳定性;不记录其他环境变量。
|
||||
- 新增部署策略、缺失限额、预算不足、重启竞态、错误控制组和敏感信息隐藏的反例测试,并接入统一部署测试入口。测试尚未执行,不能登记通过数。
|
||||
|
||||
本机仅执行 Python 字节码语法编译、Node 语法检查、Bash `-n`、Git 差异检查和 `go test -c` 交叉编译;没有执行应用、测试用例或浏览器测试。交叉编译产物为 `out/resource-review-20260917/backend/cmd/smail-admin/review.test`,Go 构建信息确认 `go1.26.6`、`linux/amd64`、`CGO_ENABLED=0`。这些证据不替代远程 `systemd-analyze verify`、反例执行、真实资源预检或清库部署验收。
|
||||
|
||||
下一步需要通过受保护环境恢复受信任 SSH 访问,再在 47 的受限环境验收新增策略、核算同机预算、编译部署最新产物并按内部 `127.0.0.1:25` 继续业务验收。当前仍不具备生产放行证据。
|
||||
|
||||
20
test/deployment/resource-policy.test.mjs
Normal file
20
test/deployment/resource-policy.test.mjs
Normal file
@ -0,0 +1,20 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import path from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import test from 'node:test'
|
||||
|
||||
const root = fileURLToPath(new URL('../../', import.meta.url))
|
||||
|
||||
test('Linux 部署资源策略与只读预检反例', { skip: process.platform !== 'linux' }, () => {
|
||||
const result = spawnSync('python3', [path.join(root, 'test/deployment/resource_policy_test.py')], {
|
||||
cwd: root,
|
||||
encoding: 'utf8',
|
||||
timeout: 30000,
|
||||
env: { ...process.env, PYTHONDONTWRITEBYTECODE: '1' },
|
||||
})
|
||||
assert.ifError(result.error)
|
||||
assert.equal(result.status, 0, result.stdout + result.stderr)
|
||||
assert.match(result.stderr, /Ran [1-9][0-9]* tests/)
|
||||
process.stdout.write(result.stderr)
|
||||
})
|
||||
269
test/deployment/resource_policy_test.py
Normal file
269
test/deployment/resource_policy_test.py
Normal file
@ -0,0 +1,269 @@
|
||||
"""资源预检的反例和部署策略回归;仅在批准的远程测试环境运行。"""
|
||||
|
||||
import configparser
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SPEC = importlib.util.spec_from_file_location('check_resources', ROOT / 'deploy/check_resources.py')
|
||||
RESOURCES = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(RESOURCES)
|
||||
MIB = RESOURCES.MIB
|
||||
|
||||
|
||||
def snapshot():
|
||||
return {
|
||||
'stable': True, 'go_memory_bytes': 256 * MIB,
|
||||
'host_available_bytes': 1024 * MIB, 'host_total_bytes': 2048 * MIB,
|
||||
'oom': {'oom': 0, 'oom_kill': 0}, 'parents': [],
|
||||
'service': {
|
||||
'MemoryHigh': str(384 * MIB), 'MemoryMax': str(512 * MIB),
|
||||
'MemorySwapMax': '0', 'TasksMax': '256', 'OOMPolicy': 'stop',
|
||||
'KillMode': 'control-group', 'Restart': 'on-failure',
|
||||
'RestartUSec': '10s', 'StartLimitIntervalUSec': '5min', 'StartLimitBurst': '3',
|
||||
'TimeoutStopUSec': '1min 30s',
|
||||
},
|
||||
'kernel': {
|
||||
'memory.high': str(384 * MIB), 'memory.max': str(512 * MIB),
|
||||
'memory.current': str(128 * MIB), 'memory.swap.max': '0',
|
||||
'pids.max': '256', 'pids.current': '32', 'cpu.max': '100000 100000',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ResourcePolicyTests(unittest.TestCase):
|
||||
def test_valid_budget_is_not_production_approval(self):
|
||||
result = RESOURCES.evaluate(snapshot(), 256 * MIB)
|
||||
self.assertTrue(result['resource_checks_passed'])
|
||||
self.assertFalse(result['production_approved'])
|
||||
self.assertEqual(result['remaining_growth_bytes'], 384 * MIB)
|
||||
self.assertEqual(result['failures'], [])
|
||||
|
||||
def test_missing_and_unsafe_limits_fail_closed(self):
|
||||
cases = [
|
||||
('kernel', 'memory.high', 'max'),
|
||||
('kernel', 'memory.max', 'max'),
|
||||
('kernel', 'memory.high', str(512 * MIB)),
|
||||
('kernel', 'memory.high', '0'),
|
||||
('kernel', 'memory.current', str(384 * MIB)),
|
||||
('kernel', 'memory.swap.max', 'max'),
|
||||
('kernel', 'memory.swap.max', str(MIB)),
|
||||
('kernel', 'pids.max', 'max'),
|
||||
('kernel', 'pids.max', '0'),
|
||||
('kernel', 'pids.current', '256'),
|
||||
('kernel', 'cpu.max', 'max 100000'),
|
||||
('service', 'MemoryMax', 'infinity'),
|
||||
('service', 'MemoryHigh', str(256 * MIB)),
|
||||
('service', 'MemorySwapMax', 'infinity'),
|
||||
('service', 'TasksMax', '512'),
|
||||
('service', 'OOMPolicy', 'continue'),
|
||||
('service', 'KillMode', 'process'),
|
||||
('service', 'TimeoutStopUSec', '60s'),
|
||||
('service', 'TimeoutStopUSec', 'infinity'),
|
||||
('service', 'Restart', 'always'),
|
||||
('service', 'RestartUSec', '100ms'),
|
||||
('service', 'RestartUSec', 'infinity'),
|
||||
('service', 'StartLimitIntervalUSec', '0'),
|
||||
('service', 'StartLimitIntervalUSec', '5s'),
|
||||
('service', 'StartLimitBurst', '0'),
|
||||
('service', 'StartLimitBurst', '6'),
|
||||
('oom', 'oom', 1),
|
||||
('oom', 'oom_kill', 1),
|
||||
]
|
||||
for section, key, value in cases:
|
||||
with self.subTest(section=section, key=key, value=value):
|
||||
candidate = snapshot()
|
||||
candidate[section][key] = value
|
||||
report = RESOURCES.evaluate(candidate, 256 * MIB)
|
||||
self.assertFalse(report['resource_checks_passed'])
|
||||
self.assertTrue(report['failures'])
|
||||
|
||||
def test_process_budget_and_restart_race_fail_closed(self):
|
||||
for key, values in {
|
||||
'go_memory_bytes': [None, 0, 63 * MIB, 384 * MIB, 1024 * MIB],
|
||||
'stable': [False],
|
||||
'host_available_bytes': [128 * MIB, 512 * MIB],
|
||||
'host_total_bytes': [512 * MIB],
|
||||
}.items():
|
||||
for value in values:
|
||||
with self.subTest(key=key, value=value):
|
||||
candidate = snapshot()
|
||||
candidate[key] = value
|
||||
self.assertFalse(RESOURCES.evaluate(candidate, 256 * MIB)['resource_checks_passed'])
|
||||
self.assertFalse(RESOURCES.evaluate(snapshot(), 0)['resource_checks_passed'])
|
||||
|
||||
def test_ancestor_limits_cannot_be_ignored(self):
|
||||
baseline = {'memory.high': 'max', 'memory.max': 'max', 'memory.current': str(256 * MIB),
|
||||
'pids.max': 'max', 'pids.current': '64', 'cpu.max': 'max 100000'}
|
||||
cases = [('memory.high', str(768 * MIB)), ('memory.max', str(768 * MIB)),
|
||||
('pids.max', '256'), ('cpu.max', '50000 100000')]
|
||||
for key, value in cases:
|
||||
with self.subTest(key=key):
|
||||
candidate = snapshot()
|
||||
limits = {**baseline, key: value}
|
||||
candidate['parents'] = [{'path': 'system.slice', 'limits': limits}]
|
||||
result = RESOURCES.evaluate(candidate, 256 * MIB)
|
||||
self.assertFalse(result['resource_checks_passed'])
|
||||
self.assertTrue(any('system.slice' in failure for failure in result['failures']))
|
||||
candidate = snapshot()
|
||||
candidate['parents'] = [{'path': 'system.slice', 'limits': baseline}]
|
||||
self.assertTrue(RESOURCES.evaluate(candidate, 256 * MIB)['resource_checks_passed'])
|
||||
|
||||
def test_environment_parser_never_returns_credentials(self):
|
||||
environment = b'SMAIL_SECRET=must-not-appear\0GOMEMLIMIT=256MiB\0TOKEN=also-secret\0'
|
||||
self.assertEqual(RESOURCES.go_memory_limit(environment), 256 * MIB)
|
||||
for value in (b'', b'off', b'256MB', b'1.5GiB', b'secret', b'-1', b' 256MiB', b'\xff'):
|
||||
with self.subTest(value=value):
|
||||
self.assertIsNone(RESOURCES.go_memory_limit(b'GOMEMLIMIT=' + value + b'\0'))
|
||||
self.assertIsNone(RESOURCES.go_memory_limit(environment + b'GOMEMLIMIT=128MiB\0'))
|
||||
self.assertIsNone(RESOURCES.go_memory_limit(b'TOKEN=secret\0'))
|
||||
self.assertEqual(RESOURCES.go_memory_limit(b'GOMEMLIMIT=1024\0'), 1024)
|
||||
self.assertEqual(RESOURCES.go_memory_limit(b'GOMEMLIMIT=1GiB\0'), 1024 * MIB)
|
||||
|
||||
def test_duration_and_cpu_parsing(self):
|
||||
self.assertEqual(RESOURCES.duration_usec('5min'), 300000000)
|
||||
self.assertEqual(RESOURCES.duration_usec('1min 2.5s'), 62500000)
|
||||
self.assertEqual(RESOURCES.duration_usec('500ms'), 500000)
|
||||
for value in ('infinity', 'junk10s', '10s junk', '-1s', ''):
|
||||
self.assertIsNone(RESOURCES.duration_usec(value))
|
||||
self.assertEqual(RESOURCES.cpu_quota('25000 100000'), 0.25)
|
||||
self.assertIsNone(RESOURCES.cpu_quota('max 100000'))
|
||||
for value in ('0 100000', '100000 0', '-1 100000', 'unknown'):
|
||||
with self.assertRaises(ValueError):
|
||||
RESOURCES.cpu_quota(value)
|
||||
|
||||
def test_systemd_query_is_read_only_and_does_not_request_environment(self):
|
||||
output = '\n'.join(name + '=value' for name in RESOURCES.PROPERTIES)
|
||||
output += '\nEnvironment=TOKEN=must-not-appear'
|
||||
with patch.object(RESOURCES.subprocess, 'run', return_value=SimpleNamespace(stdout=output)) as run:
|
||||
result = RESOURCES.systemd_state()
|
||||
self.assertEqual(set(result), set(RESOURCES.PROPERTIES))
|
||||
self.assertNotIn('must-not-appear', json.dumps(result))
|
||||
command = run.call_args.args[0]
|
||||
self.assertEqual(command[:3], ['systemctl', 'show', 'smail.service'])
|
||||
self.assertNotIn('Environment', command[-1])
|
||||
self.assertNotIn('ExecStart', command[-1])
|
||||
self.assertNotIn('shell', run.call_args.kwargs)
|
||||
with patch.object(RESOURCES.subprocess, 'run', return_value=SimpleNamespace(stdout='MainPID=1')):
|
||||
with self.assertRaises(ValueError):
|
||||
RESOURCES.systemd_state()
|
||||
|
||||
def test_collect_refuses_inactive_service_before_reading_process(self):
|
||||
with patch.object(RESOURCES, 'systemd_state', return_value={
|
||||
'MainPID': '0', 'ActiveState': 'inactive', 'SubState': 'dead',
|
||||
}):
|
||||
with self.assertRaises(ValueError):
|
||||
RESOURCES.collect()
|
||||
|
||||
def collection_fixture(self):
|
||||
root = Path(tempfile.mkdtemp(prefix='smail-resource-'))
|
||||
base = root / 'sys/fs/cgroup'
|
||||
group = base / 'system.slice/smail.service'
|
||||
proc = root / 'proc/123'
|
||||
group.mkdir(parents=True)
|
||||
proc.mkdir(parents=True)
|
||||
for name, value in snapshot()['kernel'].items():
|
||||
(group / name).write_text(value, encoding='ascii')
|
||||
(group / 'memory.events').write_text('oom 0\noom_kill 0\n', encoding='ascii')
|
||||
(proc / 'cgroup').write_text('0::/system.slice/smail.service\n', encoding='ascii')
|
||||
(proc / 'environ').write_bytes(b'GOMEMLIMIT=256MiB\0TOKEN=must-not-appear\0')
|
||||
(root / 'proc/meminfo').write_text('MemAvailable: 1048576 kB\nMemTotal: 2097152 kB\n',
|
||||
encoding='ascii')
|
||||
state = {**snapshot()['service'], 'MainPID': '123', 'NRestarts': '0',
|
||||
'InvocationID': 'fixture', 'ActiveState': 'active', 'SubState': 'running',
|
||||
'ControlGroup': '/system.slice/smail.service'}
|
||||
|
||||
def locate(value):
|
||||
path = Path(value)
|
||||
if str(path).startswith('/proc') or str(path).startswith('/sys/fs/cgroup'):
|
||||
return root / str(path).lstrip('/')
|
||||
return path
|
||||
|
||||
return root, state, locate
|
||||
|
||||
def test_collect_reads_kernel_and_only_the_go_environment(self):
|
||||
root, state, locate = self.collection_fixture()
|
||||
with patch.object(RESOURCES, 'systemd_state', side_effect=[state, state.copy()]), \
|
||||
patch.object(RESOURCES, 'Path', side_effect=locate):
|
||||
result = RESOURCES.collect()
|
||||
self.assertTrue(result['stable'])
|
||||
self.assertEqual(result['go_memory_bytes'], 256 * MIB)
|
||||
self.assertEqual(result['host_available_bytes'], 1024 * MIB)
|
||||
self.assertNotIn('must-not-appear', json.dumps(result))
|
||||
self.assertEqual([parent['path'] for parent in result['parents']], ['system.slice', '.'])
|
||||
self.assertTrue(RESOURCES.evaluate(result, 256 * MIB)['resource_checks_passed'])
|
||||
|
||||
def test_collect_detects_service_restart(self):
|
||||
root, state, locate = self.collection_fixture()
|
||||
changed = {**state, 'InvocationID': 'new-process'}
|
||||
with patch.object(RESOURCES, 'systemd_state', side_effect=[state, changed]), \
|
||||
patch.object(RESOURCES, 'Path', side_effect=locate):
|
||||
result = RESOURCES.collect()
|
||||
self.assertFalse(result['stable'])
|
||||
self.assertFalse(RESOURCES.evaluate(result, 256 * MIB)['resource_checks_passed'])
|
||||
|
||||
def test_collect_refuses_pid_in_another_group(self):
|
||||
root, state, locate = self.collection_fixture()
|
||||
(root / 'proc/123/cgroup').write_text('0::/system.slice/other.service\n', encoding='ascii')
|
||||
with patch.object(RESOURCES, 'systemd_state', return_value=state), \
|
||||
patch.object(RESOURCES, 'Path', side_effect=locate):
|
||||
with self.assertRaises(ValueError):
|
||||
RESOURCES.collect()
|
||||
|
||||
def test_cli_redacts_collection_errors_and_fails(self):
|
||||
output = io.StringIO()
|
||||
with patch('sys.argv', ['check_resources.py', '--reserve-mib', '256']), \
|
||||
patch.object(RESOURCES.os, 'name', 'posix'), \
|
||||
patch.object(RESOURCES.os, 'geteuid', return_value=0, create=True), \
|
||||
patch.object(RESOURCES, 'collect', side_effect=OSError('TOKEN=must-not-appear')), \
|
||||
patch('sys.stdout', output):
|
||||
status = RESOURCES.main()
|
||||
self.assertEqual(status, 1)
|
||||
self.assertNotIn('TOKEN', output.getvalue())
|
||||
self.assertFalse(json.loads(output.getvalue())['production_approved'])
|
||||
|
||||
def test_cli_reports_success_without_production_approval(self):
|
||||
output = io.StringIO()
|
||||
with patch('sys.argv', ['check_resources.py', '--reserve-mib', '256']), \
|
||||
patch.object(RESOURCES.os, 'name', 'posix'), \
|
||||
patch.object(RESOURCES.os, 'geteuid', return_value=0, create=True), \
|
||||
patch.object(RESOURCES, 'collect', return_value=snapshot()), \
|
||||
patch('sys.stdout', output):
|
||||
self.assertEqual(RESOURCES.main(), 0)
|
||||
self.assertTrue(json.loads(output.getvalue())['resource_checks_passed'])
|
||||
self.assertFalse(json.loads(output.getvalue())['production_approved'])
|
||||
|
||||
def test_deployment_unit_has_bounded_resources_and_restart_policy(self):
|
||||
unit = configparser.ConfigParser(interpolation=None)
|
||||
unit.read(ROOT / 'deploy/systemd/smail.service', encoding='utf-8')
|
||||
values = unit['Service']
|
||||
for key, value in {'MemoryHigh': '384M', 'MemoryMax': '512M', 'MemorySwapMax': '0',
|
||||
'CPUQuota': '100%', 'TasksMax': '256', 'OOMPolicy': 'stop',
|
||||
'KillMode': 'control-group', 'Restart': 'on-failure',
|
||||
'RestartSec': '10s', 'TimeoutStopSec': '90s',
|
||||
'MemoryAccounting': 'true', 'CPUAccounting': 'true',
|
||||
'TasksAccounting': 'true'}.items():
|
||||
self.assertEqual(values[key], value)
|
||||
self.assertEqual(unit['Unit']['StartLimitIntervalSec'], '300s')
|
||||
self.assertEqual(unit['Unit']['StartLimitBurst'], '3')
|
||||
|
||||
def test_installer_preserves_existing_environment_budget(self):
|
||||
source = (ROOT / 'deploy/install-systemd.sh').read_text(encoding='utf-8')
|
||||
guard = source.index('if [ ! -e /etc/smail/smail.env ]; then')
|
||||
end = source.index('\nfi', guard)
|
||||
assignment = source.index("'GOMEMLIMIT=256MiB'")
|
||||
self.assertTrue(guard < assignment < end)
|
||||
self.assertEqual(source.count('GOMEMLIMIT='), 1)
|
||||
self.assertIn('chmod 0600 /etc/smail/smail.env', source)
|
||||
self.assertIn('cp -p -- /etc/smail/smail.env "$backup_dir/smail.env"', source)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
Loading…
Reference in New Issue
Block a user