diff --git a/deploy/nginx/acme-renew.env.example b/deploy/nginx/acme-renew.env.example new file mode 100644 index 0000000..ed741ba --- /dev/null +++ b/deploy/nginx/acme-renew.env.example @@ -0,0 +1,5 @@ +# 仅在五个证书域名的公网 80 验证路径全部通过真实挑战后启用。 +# 正式文件为 /etc/smail/acme-renew.env，root 所有，权限 0600。 +# HTTP 模式复用 Nginx 的 /var/lib/smail-acme，不监听新端口，不停止服务。 +# 使用 DNS 模式时设为 dns，并在 acme-dns.env 中配置专用服务商凭据。 +SMAIL_ACME_CHALLENGE=http diff --git a/deploy/nginx/renew_certificate.py b/deploy/nginx/renew_certificate.py index 4723551..8d05444 100644 --- a/deploy/nginx/renew_certificate.py +++ b/deploy/nginx/renew_certificate.py @@ -1,9 +1,10 @@ #!/usr/bin/python3 -"""使用 DNS 验证续期，不停止 Nginx，候选证书通过校验后才发布。""" +'''使用显式 DNS 或 Webroot 验证续期，不停止 Nginx，校验通过后才发布。''' import os from pathlib import Path import re +import stat import subprocess import deploy_certificate as deploy @@ -12,6 +13,7 @@ import deploy_certificate as deploy ROOT = Path('/var/lib/smail-acme-tls') DOMAINS = ('solit.cn', 'mail.solit.cn', 'smtp.solit.cn', 'imap.solit.cn', 'pop.solit.cn') RENEW_TIMEOUT = 600 +WEBROOT = Path('/var/lib/smail-acme') def dns_provider(): @@ -21,6 +23,25 @@ def dns_provider(): return provider +def challenge_arguments(): + method = os.environ.get('SMAIL_ACME_CHALLENGE', 'dns') + if method == 'dns': + return ['--dns', dns_provider()] + if method != 'http': + raise RuntimeError('验证方式必须为 dns 或 http，不允许自动回退到停机验证') + private = ROOT.lstat() + if (ROOT.resolve(strict=True) != ROOT or not stat.S_ISDIR(private.st_mode) + or private.st_uid != 0 or private.st_mode & 0o077): + raise RuntimeError('ACME 账号目录必须由 root 独占，禁止公开访问') + for directory in (WEBROOT, WEBROOT / '.well-known', WEBROOT / '.well-known/acme-challenge'): + info = directory.lstat() + if (directory.resolve(strict=True) != directory or not stat.S_ISDIR(info.st_mode) + or info.st_uid != 0 or info.st_mode & 0o022 + or info.st_mode & 0o005 != 0o005): + raise RuntimeError('HTTP 验证目录必须由 root 管理、可读取且不能被其他用户写入') + return ['--http', '--http.webroot', str(WEBROOT)] + + def main(): if os.geteuid() != 0: raise RuntimeError('必须使用 root 执行证书续期') @@ -33,19 +54,20 @@ def main(): raise RuntimeError('无法判断线上证书有效期，拒绝续期') due = result.returncode == 1 if due: - provider = dns_provider() + challenge = challenge_arguments() deploy.check_nginx_paths() subprocess.run(['nginx', '-t'], check=True) subprocess.run(['systemctl', 'is-active', '--quiet', 'nginx'], check=True) command = ['/usr/local/libexec/smail-acme/lego', '--path', str(ROOT), - '--email', '', '--accept-tos', '--key-type', 'rsa2048', '--dns', provider, + '--email', '', '--accept-tos', '--key-type', 'rsa2048', *challenge, '--server', 'https://acme-v02.api.letsencrypt.org/directory', '--http-timeout', '20', '--cert.timeout', '30'] for domain in DOMAINS: command.extend(['--domains', domain]) command.extend(['renew', '--days', '30', '--reuse-key', '--no-random-sleep']) subprocess.run(command, check=True, timeout=RENEW_TIMEOUT, - env={**os.environ, 'GOMEMLIMIT': '64MiB'}) + env={**os.environ, 'GOMEMLIMIT': '64MiB'}, + umask=0o022 if '--http' in challenge else 0o077) else: print('线上证书有效期大于 30 天，本次不申请新证书') deploy.publish() diff --git a/deploy/nginx/smail-certificate-renew.service b/deploy/nginx/smail-certificate-renew.service index 9c8101d..b095c4a 100644 --- a/deploy/nginx/smail-certificate-renew.service +++ b/deploy/nginx/smail-certificate-renew.service @@ -7,6 +7,7 @@ Wants=network-online.target Type=oneshot ExecStart=/usr/local/libexec/smail-acme/renew_certificate.py EnvironmentFile=-/etc/smail/acme-dns.env +EnvironmentFile=-/etc/smail/acme-renew.env TimeoutStartSec=720 TimeoutStopSec=15 MemoryHigh=96M diff --git a/docs/HTTP无停机续期验收记录.md b/docs/HTTP无停机续期验收记录.md new file mode 100644 index 0000000..b5ef70c --- /dev/null +++ b/docs/HTTP无停机续期验收记录.md @@ -0,0 +1,54 @@ +# HTTP 无停机续期验收记录 + +## 范围与结论 + +时间为 2026 年 9 月 18 日 02:15 至 02:27，Asia/Shanghai。测试只在 `47.108.78.12`、主机名 `ikun2` 执行。本机仅编辑、Git、传输和静态检查。修改前基线提交及推送为 `cfa41e1`。 + +- [✓] 五个域名均解析到 47 服务器；现有 Nginx 模板已经为验证路径提供独立 Webroot。 +- [✓] 候选增加显式 HTTP 模式，不停止 Nginx，不绑定验证端口，不从 DNS 自动降级。 +- [✓] 41 项远程隔离回归通过，含原有证书校验、真实隔离 Nginx 平滑重载及失败回退，以及 HTTP 目录、掩码、安全失败行为。 +- [✓] 全新测试账号的真实公网挑战失败期间，两站点 138 次 HTTPS 探测通过，正式文件与四服务进程不变，验证文件由 ACME 客户端清理，测试单元已停止。 +- [!] 五个域名的真实 HTTP-01 挑战均被 CA 判为 403；HTTP 模式不能启用，未取得或部署测试证书。 +- [!] 线上仍是旧续期脚本和定时任务；专用 DNS 授权、正式新证书发布、外部告警仍未验收，不满足生产放行。 + +## 候选实现 + +`deploy/nginx/renew_certificate.py` 默认仍为 DNS 模式。设置 `SMAIL_ACME_CHALLENGE=http` 时，使用 `--http --http.webroot /var/lib/smail-acme`，不启用 lego 内置 HTTP 服务器。未知或空验证方式拒绝执行；DNS 缺少服务商时不自动切换方式。 + +HTTP 模式要求 ACME 账号目录由 root 独占且没有符号链接；公开验证目录及两层子目录必须为 root 所有、其他用户不可写且可读取、遍历。lego 4.34.0 的 Webroot 实现以 0644 创建验证文件，服务级 `UMask=0077` 会使它变为 0600。因此仅 HTTP 子进程设置 `umask=0022`，父进程和 DNS 子进程保持 0077。账号与证书目录仍为 root 独占；公开目录不保存账号、证书或私钥。 + +候选 systemd 服务增加可选 `/etc/smail/acme-renew.env`。仓库 `acme-renew.env.example` 仅为示例，不代表已经配置或允许上线。实际候选未安装到 `/usr/local/libexec/smail-acme`，正式 service 和 timer 均未替换。 + +## 隔离回归 + +目录为 `/var/tmp/smail-review-http-renew-20260918`。 + +- `tests-022115.txt`：39 项通过，2.594 秒，峰值 23.6 MiB，交换空间为零。 +- `tests-022511.txt`：增加掩码及私有账号目录回归后，41 项通过，零跳过；单元 `smail-webroot-tests-20260918-022511` 运行 2.217 秒，峰值 23.9 MiB，交换空间为零。 + +单元使用独立网络、私有临时目录、严格只读系统，线上应用、正式 ACME 状态、管理 socket、Nginx 配置和公开 Webroot 均不可访问。唯一可写业务目录是本轮测试目录，内存上限 128 MiB。测试以 root 执行；真实 Nginx 仅运行在隔离网络，不影响主机服务。 + +## 全新公网挑战 + +工具为 `test/deployment/webroot_renewal_drill.py`。显式参数 `--confirm-staging-challenge`、目标主机、Nginx 模板摘要、五域名解析和可用内存检查通过后，创建空的 root 独占目录，不复制任何既有 ACME 账号或授权缓存。 + +- 证据目录：`/var/tmp/smail-review-http-challenge-20260918-022531`。 +- 独立账号目录：`/var/lib/smail-acme-http-review-20260918-022531`，权限 0700，仅保存在服务器。 +- ACME 单元：`smail-http-challenge-20260918-022531`，运行 34.751 秒，峰值 10.6 MiB，交换空间为零，退出码 1。 +- `acme.log` 记录新账号密钥生成、五份新授权、五次 HTTP-01 求解及 403 失败；不是缓存授权续期。 +- 单元无法访问线上应用、管理 socket、Nginx 配置及正式 ACME 状态；仅独立账号目录和公开验证子目录可写。未调用正式部署钩子。 + +`result.json` 为 `failed`，但 `servicesUnchanged`、`protectedUnchanged`、`challengeFilesRestored`、`probesPassed`、`testUnitStopped` 均为真。两个站点累计 138 次可信 HTTPS 探测全部通过。Nginx 主进程 578671、Smail 574584、Scert 589832、MySQL 589785 及 InvocationID 均保持不变。 + +## 403 的证据边界 + +02:27 只读核查显示，服务器自身通过回环或公网 IP 访问不存在的验证文件均返回 Nginx 的 404，没有错误跳转。Nginx 访问日志在 02:25:38、43、48、54、59 记录来自 CA 验证节点的五个实际挑战请求，状态均为 200，其中两次发送 87 字节，三次发送字节数为零；错误日志没有对应文件权限错误。 + +与此同时，CA 报告五个域名均收到 403。这证明本站成功响应不足以代表 CA 验证成功；尚未确定是某条公网路径、其他验证节点还是中间设备造成差异。没有据此宣称云厂商或本站某条规则是已确认根因，也未通过重复申请掩盖失败。 + +## 后续门槛 + +1. 优先取得 Smail 专用最小权限 DNS 授权，在远程受保护文件配置，不借用其他项目凭据，不在聊天或仓库保存秘密。 +2. 如继续 HTTP 方案，先定位公网 403 差异，再用全新测试账号确认五个域名全部通过；当前不能直接部署。 +3. 验证正式签发、同密钥发布、平滑重载、七个 TLS 入口及故障回退，再同步安装续期脚本、发布脚本和 service，避免新旧锁协议混用。 +4. 接入真实失败及到期告警；业务浏览器、管理员恢复、容量及灾难恢复等原发布门槛不因此缩减。 diff --git a/docs/无停机续期整改记录.md b/docs/无停机续期整改记录.md index 0bc6fa8..03692fa 100644 --- a/docs/无停机续期整改记录.md +++ b/docs/无停机续期整改记录.md @@ -1,5 +1,11 @@ # 无停机续期整改记录 +## 2026 年 9 月 18 日 HTTP 备用方案复核 + +02:25 使用全新测试 CA 账号完成五域名真实 HTTP-01 挑战尝试，全部被 CA 判为 403，未取得证书。Nginx 同时段访问日志中五个对应请求均记录为 200，不能将失败直接归因于本站文件权限，也不能仅凭本站日志批准公网挑战可用。两个站点 138 次连续 HTTPS 探测全部成功，线上四个服务进程、正式证书、私钥、应用和旧续期任务均未变化。 + +候选新增显式 `SMAIL_ACME_CHALLENGE=http`，仅使用现有 Nginx Webroot，不监听新端口、不停止服务、不自动从 DNS 降级。41 项远程隔离回归通过，包括 HTTP 子进程掩码与 root 独占账号目录检查。HTTP 模式尚未启用，DNS 专用授权仍缺失，正式换证和外部告警仍未验收。详情见《HTTP无停机续期验收记录》；下方保留此前 DNS 整改阶段的证据。 + ## 范围与当前结论 远程主机时间：2026-09-18 00:00 至 00:35，时区为 Asia/Shanghai。测试仅在 `47.108.78.12` 执行，本机仅编辑和传输。本文中的日期以远程命令及证据时间为准。 diff --git a/docs/生产整改验收清单.md b/docs/生产整改验收清单.md index e5121a0..21f449f 100644 --- a/docs/生产整改验收清单.md +++ b/docs/生产整改验收清单.md @@ -2,6 +2,14 @@ ## 范围与结论 +### 当前进展：无停机 HTTP 备用方案真实挑战受阻 + +- [✓] 候选支持显式 HTTP Webroot，不抢占端口、不停止 Nginx；41 项远程隔离回归通过，保留默认 DNS 及校验、回退机制。 +- [✓] 02:25 全新测试账号挑战失败期间，两站点 138 次 HTTPS 探测通过，四个线上服务进程、正式文件和旧续期任务均未变化。 +- [!] CA 对五域名 HTTP 挑战均报告 403，但对应 Nginx 日志为 200，公网差异仍待定位，不能启用 HTTP 模式。DNS 专用授权仍缺失，线上无停机自动续期尚未落地，整体生产放行仍不通过。 + +详见《HTTP无停机续期验收记录》。以下各阶段结论仅覆盖各自验证范围。 + ### 当前进展：完整候选反代隔离运行通过 - [✓] 2026 年 9 月 18 日，`review-c1dd533` 完整 Linux 候选六文件上传至 47 服务器，长度、摘要、ELF 格式核对通过；主前端和 Telegram 界面均使用新生产构建。 diff --git a/test/deployment/test_certificate_renewal.py b/test/deployment/test_certificate_renewal.py index c1d9344..76d9823 100644 --- a/test/deployment/test_certificate_renewal.py +++ b/test/deployment/test_certificate_renewal.py @@ -96,6 +96,7 @@ class RenewalTests(unittest.TestCase): return subprocess.CompletedProcess(command, 1) if command[0].endswith('/lego'): self.assertEqual(kwargs['timeout'], 600) + self.assertEqual(kwargs['umask'], 0o077) self.assertEqual(command[command.index('--dns') + 1], 'alidns') self.assertNotIn('--tls', command) self.assertIn('--server', command) diff --git a/test/deployment/test_certificate_webroot.py b/test/deployment/test_certificate_webroot.py new file mode 100644 index 0000000..5f09df9 --- /dev/null +++ b/test/deployment/test_certificate_webroot.py @@ -0,0 +1,152 @@ +'''无停机 HTTP 验证选项回归，仅使用隔离目录和模拟续期命令。''' + +from contextlib import nullcontext +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + + +BASE = Path(__file__).resolve().parent +SCRIPTS = BASE if (BASE / 'renew_certificate.py').exists() else BASE.parents[1] / 'deploy/nginx' +sys.path.insert(0, str(SCRIPTS)) +import renew_certificate as renew + + +class WebrootTests(unittest.TestCase): + def setUp(self): + self.directory = Path(tempfile.mkdtemp(prefix='webroot-tests-', dir=Path.cwd())) + self.webroot = self.directory / 'public' + self.challenge = self.webroot / '.well-known/acme-challenge' + self.challenge.mkdir(parents=True) + for path in (self.webroot, self.challenge.parent, self.challenge): + path.chmod(0o755) + for context in (patch.object(renew, 'WEBROOT', self.webroot), + patch.object(renew, 'ROOT', self.directory), + patch.dict(os.environ, {'SMAIL_ACME_CHALLENGE': 'http'}, clear=True)): + context.start() + self.addCleanup(context.stop) + + def execute(self, failure=None, due=True): + calls = [] + + def run(command, **kwargs): + calls.append(command) + if '-checkend' in command: + return subprocess.CompletedProcess(command, 1 if due else 0) + if command[0].endswith('/lego'): + self.assertEqual(kwargs['timeout'], 600) + self.assertEqual(kwargs['umask'], 0o022) + self.assertEqual(command[command.index('--http.webroot') + 1], str(self.webroot)) + for required in ('--http', '--reuse-key', '--no-random-sleep'): + self.assertIn(required, command) + for forbidden in ('--dns', '--tls', '--http.port'): + self.assertNotIn(forbidden, command) + if failure: + raise failure + return subprocess.CompletedProcess(command, 0) + + with patch.object(renew.os, 'geteuid', return_value=0), \ + patch.object(renew.deploy, 'certificate_lock', return_value=nullcontext()), \ + patch.object(renew.deploy, 'check_nginx_paths'), \ + patch.object(renew.deploy, 'publish') as publish, \ + patch.object(renew.subprocess, 'run', side_effect=run): + if failure: + with self.assertRaises(type(failure)): + renew.main() + publish.assert_not_called() + else: + renew.main() + publish.assert_called_once() + self.assertFalse(any(command[:2] in (['systemctl', 'stop'], ['systemctl', 'start']) + for command in calls)) + return calls + + def test_http_explicitly_uses_existing_webroot(self): + self.assertEqual(renew.challenge_arguments(), ['--http', '--http.webroot', str(self.webroot)]) + + def test_due_http_success_reuses_key_without_port_binding(self): + self.assertTrue(any(command[0].endswith('/lego') for command in self.execute())) + + def test_due_http_failure_does_not_publish_or_stop(self): + self.execute(subprocess.CalledProcessError(1, 'lego')) + + def test_due_http_timeout_does_not_publish_or_stop(self): + self.execute(subprocess.TimeoutExpired('lego', 600)) + + def test_not_due_does_not_request_new_certificate(self): + self.assertFalse(any(command[0].endswith('/lego') for command in self.execute(due=False))) + + def test_invalid_method_does_not_fall_back(self): + for method in ('', 'tls', 'HTTP', 'http dns'): + with self.subTest(method=method), patch.dict(os.environ, {'SMAIL_ACME_CHALLENGE': method}): + with self.assertRaises(RuntimeError): + renew.challenge_arguments() + + def test_dns_remains_explicit_and_requires_provider(self): + with patch.dict(os.environ, {'SMAIL_ACME_CHALLENGE': 'dns'}): + with self.assertRaises(RuntimeError): + renew.challenge_arguments() + with patch.dict(os.environ, {'SMAIL_ACME_DNS_PROVIDER': 'alidns'}): + self.assertEqual(renew.challenge_arguments(), ['--dns', 'alidns']) + + def test_missing_directory_is_rejected(self): + with patch.object(renew, 'WEBROOT', self.directory / 'missing'): + with self.assertRaises(FileNotFoundError): + renew.challenge_arguments() + + def test_public_account_directory_is_rejected(self): + self.directory.chmod(0o755) + try: + with self.assertRaises(RuntimeError): + renew.challenge_arguments() + finally: + self.directory.chmod(0o700) + + def test_child_umask_preserves_private_parent_and_readable_challenge(self): + token = self.challenge / 'token' + key = self.directory / 'key' + previous = os.umask(0o077) + try: + subprocess.run([ + sys.executable, '-c', + 'import os,sys; fd=os.open(sys.argv[1],os.O_CREAT|os.O_EXCL|os.O_WRONLY,0o644); os.close(fd)', + str(token), + ], check=True, umask=0o022, timeout=10) + key.write_bytes(b'private-test-data') + self.assertEqual(token.stat().st_mode & 0o777, 0o644) + self.assertEqual(key.stat().st_mode & 0o777, 0o600) + self.assertEqual(self.directory.stat().st_mode & 0o777, 0o700) + finally: + os.umask(previous) + + def test_unsafe_permissions_on_each_directory_are_rejected(self): + for directory in (self.webroot, self.challenge.parent, self.challenge): + for mode in (0o775, 0o777, 0o750, 0o700): + with self.subTest(directory=directory.name, mode=oct(mode)): + directory.chmod(mode) + try: + with self.assertRaises(RuntimeError): + renew.challenge_arguments() + finally: + directory.chmod(0o755) + + def test_symlink_directory_is_rejected(self): + link = self.directory / 'linked' + link.symlink_to(self.webroot, target_is_directory=True) + with patch.object(renew, 'WEBROOT', link), self.assertRaises(RuntimeError): + renew.challenge_arguments() + + def test_regular_file_is_not_webroot(self): + path = self.directory / 'file' + path.write_bytes(b'not-a-directory') + path.chmod(0o755) + with patch.object(renew, 'WEBROOT', path), self.assertRaises(RuntimeError): + renew.challenge_arguments() + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/test/deployment/webroot_renewal_drill.py b/test/deployment/webroot_renewal_drill.py new file mode 100644 index 0000000..8f63188 --- /dev/null +++ b/test/deployment/webroot_renewal_drill.py @@ -0,0 +1,177 @@ +'''使用全新 ACME 测试账号验证公网 HTTP 挑战，不停服，不发布测试证书。''' + +import argparse +from datetime import datetime, timezone +import hashlib +import http.client +import json +import os +from pathlib import Path +import socket +import subprocess +import threading +import time + +import nginx_certificate_cutover as monitor +import renew_certificate as renew + + +SERVER = 'https://acme-staging-v02.api.letsencrypt.org/directory' +CONFIG = Path('/etc/nginx/conf.d/smail.conf') +CONFIG_SHA = 'bf6816c38f9c639ab17fd7344fff4b06afc5a2388f74318e18c60b8b3311ebd8' + + +def run(*command): + return subprocess.check_output(command, text=True, timeout=15).strip() + + +def file_state(path): + info = path.stat() + return {'sha256': hashlib.sha256(path.read_bytes()).hexdigest(), + 'uid': info.st_uid, 'gid': info.st_gid, 'mode': oct(info.st_mode & 0o777)} + + +def service_state(): + return {name: run('systemctl', 'show', name, '--property=ActiveState,MainPID,InvocationID,NRestarts') + for name in ('smail.service', 'nginx.service', 'Scert.service', 'mysql.service')} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--confirm-staging-challenge', action='store_true') + args = parser.parse_args() + if not args.confirm_staging_challenge or os.geteuid() != 0 or run('hostname') != 'ikun2': + parser.error('只允许在 47 服务器明确执行测试 CA 的全新 HTTP 挑战') + if file_state(CONFIG)['sha256'] != CONFIG_SHA: + parser.error('Nginx 配置与已审查模板不符，停止演练') + memory = dict(line.split(':', 1) for line in Path('/proc/meminfo').read_text().splitlines()) + if int(memory['MemAvailable'].split()[0]) < 256 * 1024: + parser.error('可用内存不足，停止演练') + run('nginx', '-t') + run('systemctl', 'is-active', 'nginx', 'smail', 'Scert', 'mysql') + for domain in renew.DOMAINS: + addresses = {entry[4][0] for entry in socket.getaddrinfo(domain, 80, type=socket.SOCK_STREAM)} + if addresses != {'47.108.78.12'}: + parser.error('域名解析与本次授权目标不符：' + domain) + connection = http.client.HTTPConnection('127.0.0.1', 80, timeout=5) + try: + connection.request('GET', '/.well-known/acme-challenge/preflight-absent', headers={'Host': domain}) + response = connection.getresponse() + response.read() + if response.status != 404: + parser.error('验证路径未返回预期状态：' + domain) + finally: + connection.close() + os.umask(0o077) + stamp = time.strftime('%Y%m%d-%H%M%S') + evidence = Path('/var/tmp') / ('smail-review-http-challenge-' + stamp) + state = Path('/var/lib') / ('smail-acme-http-review-' + stamp) + evidence.mkdir(mode=0o700) + state.mkdir(mode=0o700) + assert not list(state.iterdir()), '必须使用全新账号目录，不能复用授权缓存' + renew.ROOT = state + os.environ['SMAIL_ACME_CHALLENGE'] = 'http' + challenge = renew.challenge_arguments() + protected = [Path('/opt/smail/smail'), Path('/opt/smail/smail-admin'), + Path('/opt/smail/ssl/custom.crt'), Path('/opt/smail/ssl/custom.key'), + Path('/usr/local/libexec/smail-acme/renew_certificate.py'), + Path('/usr/local/libexec/smail-acme/deploy_certificate.py'), + Path('/etc/systemd/system/smail-certificate-renew.service')] + protected.extend(sorted(Path('/etc/nginx/conf.d').glob('*.conf'))) + protected.extend(sorted(Path('/var/lib/smail-acme-tls/certificates').glob('solit.cn.*'))) + before = {str(path): file_state(path) for path in protected} + before_services = service_state() + challenge_dir = renew.WEBROOT / '.well-known/acme-challenge' + before_tokens = {path.name: file_state(path) for path in challenge_dir.iterdir()} + report = {'status': 'running', 'startedUTC': datetime.now(timezone.utc).isoformat(), + 'stagingOnly': True, 'freshAccount': True, 'productionDeploymentExercised': False, + 'evidence': str(evidence), 'state': str(state), 'beforeServices': before_services, + 'protectedBefore': before, 'samples': []} + stop = threading.Event() + + def sample(): + for host in ('mail.solit.cn', 'web.apmmk.cn'): + value = {'host': host, 'timeUTC': datetime.now(timezone.utc).isoformat()} + try: + value['bytes'] = len(monitor.request(host, 443)) + value['ok'] = True + except Exception as error: + value.update(ok=False, error=type(error).__name__) + report['samples'].append(value) + + def watch(): + while not stop.wait(0.5): + sample() + + unit = 'smail-http-challenge-' + stamp + command = [ + 'systemd-run', '--wait', '--pipe', '--unit=' + unit, + '--property=Type=exec', '--property=UMask=0022', + '--property=ProtectSystem=strict', '--property=ProtectHome=yes', + '--property=NoNewPrivileges=yes', '--property=PrivateTmp=yes', + '--property=InaccessiblePaths=/opt/smail /run/smail /etc/nginx /var/lib/smail-acme-tls', + '--property=ReadWritePaths=' + str(state) + ' ' + str(challenge_dir), + '--property=CapabilityBoundingSet=', '--property=MemoryHigh=96M', + '--property=MemoryMax=128M', '--property=MemorySwapMax=0', '--property=CPUQuota=100%', + '--property=TasksMax=64', '--property=RuntimeMaxSec=600s', + '--property=TimeoutStopSec=15s', '--property=KillMode=control-group', + '--property=OOMPolicy=stop', '--setenv=GOMEMLIMIT=64MiB', + '/usr/local/libexec/smail-acme/lego', '--path', str(state), '--server', SERVER, + '--email', '', '--accept-tos', '--key-type', 'rsa2048', + '--http-timeout', '20', '--cert.timeout', '30', *challenge, + ] + for domain in renew.DOMAINS: + command.extend(['--domains', domain]) + command.append('run') + thread = threading.Thread(target=watch, daemon=True) + with renew.deploy.certificate_lock(): + try: + sample() + assert all(item['ok'] for item in report['samples']), '演练前业务探测失败' + thread.start() + with (evidence / 'acme.log').open('xb') as log: + result = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, timeout=630) + report['exitCode'] = result.returncode + assert result.returncode == 0, '全新 HTTP 挑战失败，日志已保留' + certificate = state / 'certificates/solit.cn.crt' + key = state / 'certificates/solit.cn.key' + public_cert = run('openssl', 'x509', '-in', str(certificate), '-pubkey', '-noout') + public_key = run('openssl', 'pkey', '-in', str(key), '-pubout') + assert public_cert == public_key + for domain in renew.DOMAINS: + run('openssl', 'x509', '-in', str(certificate), '-checkhost', domain, '-noout') + report['certificate'] = file_state(certificate) + report['domains'] = list(renew.DOMAINS) + report['certificateMetadata'] = run('openssl', 'x509', '-in', str(certificate), + '-noout', '-issuer', '-dates', '-ext', 'subjectAltName') + report['status'] = 'passed' + except BaseException as error: + report.update(status='failed', error=type(error).__name__) + raise + finally: + try: + stopped = subprocess.run(['systemctl', 'stop', unit], capture_output=True, text=True, timeout=25) + report['testUnitStopped'] = stopped.returncode == 0 or 'not loaded' in stopped.stderr + sample() + finally: + stop.set() + if thread.ident: + thread.join(timeout=10) + report['afterServices'] = service_state() + report['servicesUnchanged'] = report['afterServices'] == before_services + report['protectedUnchanged'] = all(file_state(path) == before[str(path)] for path in protected) + report['challengeFilesRestored'] = before_tokens == { + path.name: file_state(path) for path in challenge_dir.iterdir()} + report['probesPassed'] = bool(report['samples']) and all(item['ok'] for item in report['samples']) + if not all(report.get(key) for key in ('testUnitStopped', 'servicesUnchanged', + 'protectedUnchanged', 'challengeFilesRestored', 'probesPassed')): + report['status'] = 'failed' + report['finishedUTC'] = datetime.now(timezone.utc).isoformat() + (evidence / 'result.json').write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding='utf-8') + print(json.dumps({key: value for key, value in report.items() + if key not in ('samples', 'protectedBefore')}, ensure_ascii=False, indent=2)) + assert report['status'] == 'passed' + + +if __name__ == '__main__': + main()