diff --git a/deploy/nginx/acme-dns.env.example b/deploy/nginx/acme-dns.env.example new file mode 100644 index 0000000..5e67db6 --- /dev/null +++ b/deploy/nginx/acme-dns.env.example @@ -0,0 +1,6 @@ +# 仅为阿里云 DNS 配置示例，实际服务商和授权范围必须先确认。 +# 正式文件位于 /etc/smail/acme-dns.env，归 root 所有，权限为 0600。 +# 禁止将填写后的文件、密钥或环境变量输出提交到仓库。 +SMAIL_ACME_DNS_PROVIDER=alidns +ALICLOUD_ACCESS_KEY= +ALICLOUD_SECRET_KEY= diff --git a/deploy/nginx/deploy_certificate.py b/deploy/nginx/deploy_certificate.py index 8672efd..83a7b55 100644 --- a/deploy/nginx/deploy_certificate.py +++ b/deploy/nginx/deploy_certificate.py @@ -2,24 +2,83 @@ """校验证书并同步至 Smail，私钥与回退副本仅在服务器保存。""" from datetime import datetime, timezone +from contextlib import contextmanager +import fcntl import hashlib import os from pathlib import Path import pwd -import shutil +import shlex +import socket +import ssl +import stat import subprocess +import time import uuid DOMAINS = ('solit.cn', 'mail.solit.cn', 'smtp.solit.cn', 'imap.solit.cn', 'pop.solit.cn') LINEAGE = Path('/var/lib/smail-acme-tls/certificates') TARGET = Path('/opt/smail/ssl') +BACKUPS = Path('/var/backups') +LOCK = Path('/run/smail-certificate-renew.lock') +CA_FILE = '/etc/ssl/certs/ca-certificates.crt' +LIVE_PORTS = (443, 8002) def run(*args, **kwargs): + kwargs.setdefault('timeout', 15) return subprocess.run(args, check=True, capture_output=True, **kwargs).stdout +@contextmanager +def certificate_lock(): + fd = os.open(LOCK, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + with os.fdopen(fd, 'a') as stream: + info = os.fstat(stream.fileno()) + if (not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_nlink != 1 + or info.st_mode & 0o077): + raise RuntimeError('证书锁文件不安全') + fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + yield + + +def read_file(path, uid): + if path.parent.resolve(strict=True) != path.parent: + raise RuntimeError('证书目录不能包含符号链接') + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + with os.fdopen(fd, 'rb') as stream: + info = os.fstat(stream.fileno()) + if (not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_uid != uid + or info.st_mode & 0o077 or info.st_size > 1024 * 1024): + raise RuntimeError('证书文件类型、属主、权限或大小不符合要求') + data = stream.read(1024 * 1024 + 1) + if len(data) != info.st_size: + raise RuntimeError('读取期间证书发生变化') + return data + + +def check_nginx_paths(): + configuration = run('nginx', '-T').decode('utf-8') + found = set() + expected = {'ssl_certificate': TARGET / 'custom.crt', + 'ssl_certificate_key': TARGET / 'custom.key'} + for line in configuration.splitlines(): + if not line.lstrip().startswith(('ssl_certificate ', 'ssl_certificate\t', + 'ssl_certificate_key ', 'ssl_certificate_key\t')): + continue + tokens = shlex.split(line, comments=True) + if len(tokens) < 2 or tokens[0] not in expected: + continue + value = tokens[1].rstrip(';') + if value.startswith('/var/lib/smail-acme') or value.startswith(str(LINEAGE) + '/'): + raise RuntimeError('Nginx 仍引用签发目录，必须先迁移到已校验的线上证书路径') + if value == str(expected[tokens[0]]): + found.add(tokens[0]) + if found != set(expected): + raise RuntimeError('未找到 Nginx 的 Smail 线上证书路径，拒绝发布') + + def replace_file(path, data, uid, gid): staged = path.with_name(path.name + '.new-' + uuid.uuid4().hex) fd = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) @@ -36,51 +95,98 @@ def replace_file(path, data, uid, gid): os.close(fd) +def validate(directory): + for domain in DOMAINS: + run('openssl', 'verify', '-CAfile', CA_FILE, '-purpose', 'sslserver', + '-untrusted', str(directory / 'candidate.issuer.crt'), '-verify_hostname', domain, + str(directory / 'candidate.crt')) + run('openssl', 'x509', '-in', str(directory / 'candidate.crt'), '-checkend', '604800', '-noout') + public_cert = run('openssl', 'x509', '-in', str(directory / 'candidate.crt'), '-pubkey', '-noout') + public_key = run('openssl', 'pkey', '-in', str(directory / 'candidate.key'), '-pubout') + if public_cert != public_key: + raise RuntimeError('证书与私钥不匹配') + # 普通续期只允许复用密钥，避免两个文件替换之间出现不匹配；密钥轮换另行维护。 + previous_key = run('openssl', 'pkey', '-in', str(directory / 'previous.key'), '-pubout') + previous_cert = run('openssl', 'x509', '-in', str(directory / 'previous.crt'), '-pubkey', '-noout') + if public_key != previous_key or previous_key != previous_cert: + raise RuntimeError('无停机续期必须复用现有密钥，密钥轮换需要单独流程') + + +def verify_live_certificate(fingerprint): + context = ssl.create_default_context(cafile=CA_FILE) + for port in LIVE_PORTS: + deadline = time.monotonic() + 12 + while True: + try: + with socket.create_connection(('127.0.0.1', port), timeout=2) as connection: + with context.wrap_socket(connection, server_hostname='mail.solit.cn') as secure: + actual = hashlib.sha256(secure.getpeercert(binary_form=True)).hexdigest() + if actual == fingerprint: + break + raise RuntimeError('线上入口未返回预期证书') + except (OSError, RuntimeError): + if time.monotonic() >= deadline: + raise RuntimeError('线上证书验证失败，端口=' + str(port)) from None + time.sleep(0.25) + + +def fingerprint(path): + return hashlib.sha256(run('openssl', 'x509', '-in', str(path), '-outform', 'DER')).hexdigest() + + +def publish(): + check_nginx_paths() + run('systemctl', 'is-active', '--quiet', 'nginx') + identity = pwd.getpwnam('smail') + sources = {'candidate.crt': read_file(LINEAGE / 'solit.cn.crt', 0), + 'candidate.key': read_file(LINEAGE / 'solit.cn.key', 0), + 'candidate.issuer.crt': read_file(LINEAGE / 'solit.cn.issuer.crt', 0), + 'previous.crt': read_file(TARGET / 'custom.crt', identity.pw_uid), + 'previous.key': read_file(TARGET / 'custom.key', identity.pw_uid)} + stamp = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S') + '-' + uuid.uuid4().hex[:8] + backup = BACKUPS / ('smail-certificate-' + stamp) + backup.mkdir(mode=0o700) + print('证书校验及回退快照=' + str(backup), flush=True) + for name, data in sources.items(): + replace_file(backup / name, data, 0, 0) + # 校验与发布使用同一份受保护快照，不重新读取可能变化的签发目录。 + validate(backup) + expected = fingerprint(backup / 'candidate.crt') + changed = [suffix for suffix in ('key', 'crt') + if sources['candidate.' + suffix] != sources['previous.' + suffix]] + if not changed: + try: + verify_live_certificate(expected) + except RuntimeError: + # 恢复上次在文件发布后、重载前中断的任务；不能仅因文件相同就宣告成功。 + run('nginx', '-t') + run('systemctl', 'reload', 'nginx') + verify_live_certificate(expected) + print('证书校验通过，线上入口与运行文件已同步') + return + try: + for suffix in changed: + replace_file(TARGET / ('custom.' + suffix), sources['candidate.' + suffix], + identity.pw_uid, identity.pw_gid) + run('nginx', '-t') + run('systemctl', 'reload', 'nginx') + verify_live_certificate(expected) + except Exception: + for suffix in changed: + replace_file(TARGET / ('custom.' + suffix), sources['previous.' + suffix], + identity.pw_uid, identity.pw_gid) + run('nginx', '-t') + run('systemctl', 'reload', 'nginx') + verify_live_certificate(fingerprint(backup / 'previous.crt')) + raise + print('证书已同步，SHA256=' + expected) + + def main(): if os.geteuid() != 0: raise RuntimeError('必须使用 root 执行证书同步') - # 先验证信任链、服务域名、有效期和密钥匹配，再修改运行文件。 - for domain in DOMAINS: - run('openssl', 'verify', '-CAfile', '/etc/ssl/certs/ca-certificates.crt', - '-untrusted', str(LINEAGE / 'solit.cn.issuer.crt'), '-verify_hostname', domain, - str(LINEAGE / 'solit.cn.crt')) - run('openssl', 'x509', '-in', str(LINEAGE / 'solit.cn.crt'), '-checkend', '604800', '-noout') - public_cert = run('openssl', 'x509', '-in', str(LINEAGE / 'solit.cn.crt'), '-pubkey', '-noout') - public_key = run('openssl', 'pkey', '-in', str(LINEAGE / 'solit.cn.key'), '-pubout') - if public_cert != public_key: - raise RuntimeError('证书与私钥不匹配') - if TARGET.is_symlink() or TARGET.resolve() != TARGET: - raise RuntimeError('证书目标目录不能包含符号链接') - pairs = [('solit.cn.key', 'custom.key'), ('solit.cn.crt', 'custom.crt')] - for _, name in pairs: - path = TARGET / name - if path.is_symlink() or not path.is_file() or path.stat().st_nlink != 1: - raise RuntimeError('现有证书文件类型不符合要求') - if all((LINEAGE / source).read_bytes() == (TARGET / name).read_bytes() for source, name in pairs): - print('证书校验通过，运行文件已同步') - return - identity = pwd.getpwnam('smail') - stamp = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S') + '-' + uuid.uuid4().hex[:8] - backup = Path('/var/backups') / ('smail-certificate-' + stamp) - backup.mkdir(mode=0o700) - for _, name in pairs: - path = TARGET / name - if path.is_symlink() or not path.is_file(): - raise RuntimeError('现有证书文件类型不符合要求') - shutil.copy2(path, backup / name) - os.chmod(backup / name, 0o600) - try: - for source, name in pairs: - replace_file(TARGET / name, (LINEAGE / source).read_bytes(), identity.pw_uid, identity.pw_gid) - run('nginx', '-t') - run('systemctl', 'reload', 'nginx') - except Exception: - for _, name in pairs: - replace_file(TARGET / name, (backup / name).read_bytes(), identity.pw_uid, identity.pw_gid) - raise - fingerprint = hashlib.sha256(run('openssl', 'x509', '-in', str(TARGET / 'custom.crt'), '-outform', 'DER')).hexdigest() - print('证书已同步，SHA256=' + fingerprint) - print('回退副本=' + str(backup)) + with certificate_lock(): + publish() if __name__ == '__main__': diff --git a/deploy/nginx/renew_certificate.py b/deploy/nginx/renew_certificate.py index 63863f7..4723551 100644 --- a/deploy/nginx/renew_certificate.py +++ b/deploy/nginx/renew_certificate.py @@ -1,44 +1,54 @@ #!/usr/bin/python3 -"""仅在临近到期时占用 443 续期，任何正常异常路径均恢复 Nginx。""" +"""使用 DNS 验证续期，不停止 Nginx，候选证书通过校验后才发布。""" -import fcntl import os from pathlib import Path +import re import subprocess +import deploy_certificate as deploy + ROOT = Path('/var/lib/smail-acme-tls') DOMAINS = ('solit.cn', 'mail.solit.cn', 'smtp.solit.cn', 'imap.solit.cn', 'pop.solit.cn') +RENEW_TIMEOUT = 600 + + +def dns_provider(): + provider = os.environ.get('SMAIL_ACME_DNS_PROVIDER', '') + if not re.fullmatch(r'[a-z][a-z0-9_]{0,63}', provider) or provider == 'exec': + raise RuntimeError('未配置有效的 DNS 服务商，拒绝续期；不会回退到停机验证') + return provider def main(): if os.geteuid() != 0: raise RuntimeError('必须使用 root 执行证书续期') - with open('/run/smail-certificate-renew.lock', 'a') as lock: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - certificate = str(ROOT / 'certificates/solit.cn.crt') + with deploy.certificate_lock(): + certificate = str(deploy.TARGET / 'custom.crt') subprocess.run(['openssl', 'x509', '-in', certificate, '-noout'], check=True) - due = subprocess.run(['openssl', 'x509', '-in', certificate, '-noout', '-checkend', '2592000'], - capture_output=True).returncode != 0 + result = subprocess.run(['openssl', 'x509', '-in', certificate, '-noout', '-checkend', '2592000'], + capture_output=True) + if result.returncode not in (0, 1): + raise RuntimeError('无法判断线上证书有效期，拒绝续期') + due = result.returncode == 1 if due: + provider = dns_provider() + deploy.check_nginx_paths() subprocess.run(['nginx', '-t'], check=True) subprocess.run(['systemctl', 'is-active', '--quiet', 'nginx'], check=True) command = ['/usr/local/libexec/smail-acme/lego', '--path', str(ROOT), - '--email', '', '--accept-tos', '--key-type', 'rsa2048', '--tls', + '--email', '', '--accept-tos', '--key-type', 'rsa2048', '--dns', provider, + '--server', 'https://acme-v02.api.letsencrypt.org/directory', '--http-timeout', '20', '--cert.timeout', '30'] for domain in DOMAINS: command.extend(['--domains', domain]) command.extend(['renew', '--days', '30', '--reuse-key', '--no-random-sleep']) - # systemd 的 ExecStopPost 另行覆盖进程被终止时的服务恢复。 - try: - subprocess.run(['systemctl', 'stop', 'nginx'], check=True) - subprocess.run(command, check=True, timeout=150, - env={**os.environ, 'GOMEMLIMIT': '64MiB'}) - finally: - subprocess.run(['systemctl', 'start', 'nginx'], check=True) + subprocess.run(command, check=True, timeout=RENEW_TIMEOUT, + env={**os.environ, 'GOMEMLIMIT': '64MiB'}) else: - print('证书有效期大于 30 天，本次不暂停 Web 服务') - subprocess.run(['/usr/local/libexec/smail-acme/deploy_certificate.py'], check=True) + print('线上证书有效期大于 30 天，本次不申请新证书') + deploy.publish() if __name__ == '__main__': diff --git a/deploy/nginx/smail-certificate-renew.service b/deploy/nginx/smail-certificate-renew.service index dcc1cbc..9c8101d 100644 --- a/deploy/nginx/smail-certificate-renew.service +++ b/deploy/nginx/smail-certificate-renew.service @@ -6,8 +6,9 @@ Wants=network-online.target [Service] Type=oneshot ExecStart=/usr/local/libexec/smail-acme/renew_certificate.py -ExecStopPost=/usr/bin/systemctl start nginx.service -TimeoutStartSec=240 +EnvironmentFile=-/etc/smail/acme-dns.env +TimeoutStartSec=720 +TimeoutStopSec=15 MemoryHigh=96M MemoryMax=128M MemorySwapMax=0 diff --git a/deploy/nginx/smail.conf b/deploy/nginx/smail.conf index c637dd5..ec1deb9 100644 --- a/deploy/nginx/smail.conf +++ b/deploy/nginx/smail.conf @@ -32,8 +32,8 @@ server { listen 443 ssl; listen [::]:443 ssl; server_name solit.cn smtp.solit.cn imap.solit.cn pop.solit.cn; - ssl_certificate /var/lib/smail-acme-tls/certificates/solit.cn.crt; - ssl_certificate_key /var/lib/smail-acme-tls/certificates/solit.cn.key; + ssl_certificate /opt/smail/ssl/custom.crt; + ssl_certificate_key /opt/smail/ssl/custom.key; return 308 https://mail.solit.cn$request_uri; } @@ -41,8 +41,8 @@ server { listen 443 ssl; listen [::]:443 ssl; server_name mail.solit.cn; - ssl_certificate /var/lib/smail-acme-tls/certificates/solit.cn.crt; - ssl_certificate_key /var/lib/smail-acme-tls/certificates/solit.cn.key; + ssl_certificate /opt/smail/ssl/custom.crt; + ssl_certificate_key /opt/smail/ssl/custom.key; location / { proxy_pass https://127.0.0.1:8002; diff --git a/docs/Nginx反代与证书部署说明.md b/docs/Nginx反代与证书部署说明.md index 76dd61e..fc5bc06 100644 --- a/docs/Nginx反代与证书部署说明.md +++ b/docs/Nginx反代与证书部署说明.md @@ -1,5 +1,13 @@ # Nginx 反代与证书部署说明 +## 最新整改状态 + +远程主机 2026-09-18 00:22 已将 Nginx 证书路径迁移为 `/opt/smail/ssl/custom.crt`、`custom.key`，不再直接使用签发目录。只对 Smail 站点执行路径迁移和平滑重载，正式证书、应用程序、其他站点配置未改动。58 次双站 HTTPS 探测和预先建立的下载连接通过，七个 TLS/STARTTLS 入口再次通过。 + +仓库中的续期和部署脚本已改为 DNS 验证、校验快照、平滑重载、实际指纹检查及失败回退，27 项远程隔离回归通过。由于缺少专用 DNS 授权和真实挑战证据，线上续期脚本及 systemd 单元尚未替换，旧自动任务仍然存在到期停机风险。不得把仓库候选代码、未到期检查或路径迁移当作无停机续期已经启用。 + +实时配置确认 Scert 已使用 `web.apmmk.cn:443`，不再监听旧记录中的 8003/8004。本轮没有修改其配置。完整证据、失败过程和启用门槛见《无停机续期整改记录》；以下签发和维护窗口章节保留原部署过程。 + ## 已部署拓扑 2026-09-17 在 47.108.78.12 部署，本机不运行应用测试。 @@ -27,7 +35,7 @@ - 有效期：2026-09-17 13:54:40 UTC 至 2026-12-16 13:54:39 UTC。 - 叶证书 SHA256：`608fb55e3b8add3426a63f672debd86a58389a1a78d0961b9603b2e1abecae43`。 - ACME 状态目录：`/var/lib/smail-acme-tls`，0700，仅 root 使用。 -- Nginx 证书：上述目录下的 `certificates/solit.cn.crt` 和 `solit.cn.key`。 +- Nginx 证书：现为 `/opt/smail/ssl/custom.crt`、`custom.key`；初次部署曾直接使用上述签发目录，最新整改已分离。 - Smail 证书：`/opt/smail/ssl/custom.crt` 和 `custom.key`，属主 smail，0600。新握手热加载，不为换证重启邮件服务。 HTTP-01 测试失败，CA 报告部分节点收到 403 或连接重置，本机 Nginx 仅记录部分节点的 200。外部路径原因尚未定位，不能断言具体运营商策略。改用 TLS-ALPN-01 后，五个域名的测试和正式签发均成功，没有使用 DNS 密钥。 @@ -36,6 +44,8 @@ lego 固定为项目同版本 v4.34.0，官方发布包 SHA256 为 `c681c09d3f2a ## 续期运维 +本节描述当前仍在运行的旧续期任务，不是仓库中待启用的 DNS 候选。两个新脚本必须成套部署，不能将带互斥锁的新部署脚本单独混入持锁后再启动子进程的旧续期实现。 + 脚本安装在 `/usr/local/libexec/smail-acme/`。`smail-certificate-renew.timer` 每日 04:00 加 0 至 30 分钟错峰运行；未到期时校验证书并确认运行文件已同步，不暂停 Web。剩余不足 30 天时执行续期，复用密钥，避免证书与私钥切换间隙。 续期会暂停 Nginx 使用公网 443。执行上限 150 秒，服务总超时 240 秒，正常异常路径使用 `finally` 恢复，systemd `ExecStopPost` 额外覆盖任务终止后的恢复。邮件服务不随 Nginx 停止。此方案存在短暂 Web 维护窗口，不是无中断续期。 diff --git a/docs/无停机续期整改记录.md b/docs/无停机续期整改记录.md new file mode 100644 index 0000000..ad00508 --- /dev/null +++ b/docs/无停机续期整改记录.md @@ -0,0 +1,70 @@ +# 无停机续期整改记录 + +## 范围与当前结论 + +远程主机时间：2026-09-18 00:00 至 00:24，时区为 Asia/Shanghai。测试仅在 `47.108.78.12` 执行，本机仅编辑和传输。本文中的日期以远程命令及证据时间为准。 + +- [✓] 明确原停机原因：线上旧脚本使用 TLS-ALPN-01 独占公网 443，并显式停止整个 Nginx 服务。 +- [✓] Nginx 已从签发目录迁移到 `/opt/smail/ssl/custom.crt`、`custom.key`，与 Smail 共用已发布证书；8001/8002、上游 TLS 校验及其他站点配置未改动。 +- [✓] 27 项远程隔离回归通过，包括真实独立 Nginx 换证、平滑重载和失败回退后的实际 TLS 指纹验证。 +- [✓] 线上配置迁移期间两站点 58 次 HTTPS 请求全部成功，一条迁移前已建立的下载连接完整返回 312,377 字节；主服务进程未重启。 +- [✓] 候选脚本在真实未到期状态完成系统信任、域名、密钥、权限和线上证书一致性检查；七个 TLS/STARTTLS 入口再次通过。 +- [!] DNS-01 候选未替换线上续期脚本和 systemd 定时任务。尚无 Smail 专用 DNS 凭据配置，不得借用 Scert 凭据，也不能以模拟结果代替真实挑战。 +- [!] 全新 DNS 挑战、正式新证书签发发布、外部失败和到期告警未验收；整体生产放行仍不通过。 + +本轮没有清库、发送业务邮件、重置账号、重编译应用或停止共享服务。正式证书、程序及现有续期脚本均未替换，不构成最新应用首次安装或业务浏览器验收。 + +## 代码变更 + +`deploy/nginx/renew_certificate.py` 使用 `SMAIL_ACME_DNS_PROVIDER` 指定 DNS 服务商，不再调用 `stop/start nginx`，不允许配置缺失时退回停机验证。依据线上实际使用证书判断是否临近到期，而不是依据签发目录中的候选证书。ACME 正式服务地址显式指定，命令超时为 600 秒，保留密钥复用及互斥控制。 + +`deploy/nginx/deploy_certificate.py` 与续期入口共用同一进程锁。候选、旧证书和旧私钥首先保存到 root 专属快照，再对同一份快照进行系统信任链、五个域名、服务用途、剩余至少七天有效期以及私钥匹配验证。普通续期只允许复用已有密钥，密钥轮换另行验收。 + +发布后执行 `nginx -t`、平滑重载，并实际检查 443 和 8002 的系统信任及证书指纹。失败时恢复旧文件、重新重载并检查旧指纹。文件已经一致但线上仍使用旧证书时，会补做重载，避免上次任务在写文件后被中断导致永久不同步。校验快照和回退副本保留在 `/var/backups/smail-certificate-*`，不会自动删除。 + +候选 service 移除了 `ExecStopPost` 启动 Nginx 的旧钩子，支持 `/etc/smail/acme-dns.env`，总超时 720 秒。原有 96/128 MiB、禁用交换空间、CPU 和任务数限额保留。`acme-dns.env.example` 只包含空白凭据示例，不是可直接启用的正式授权。 + +历史维护窗口工具 `test/deployment/renewal_drill.py` 增加旧续期脚本摘要限定，拒绝将原先针对停机实现的演练用于新的 DNS 实现。 + +修正 `test/tools/remote_review.py`：项目审查上传目录使用 `0700`，不再把已保护的目录改回 `0755`。已通过真实上传后的远程权限检查确认。 + +## 隔离测试证据 + +证据目录：`/var/tmp/smail-review-cert-safe-20260918`。 + +最终日志：`tests-final-2.txt`。单元 `smail-cert-safe-tests-20260918-0025` 返回成功，27 项通过，零跳过；运行约 3.609 秒，内存峰值约 22.3 MiB，交换空间为零。这不是生产容量结论。 + +测试使用独立网络空间和只读系统目录，禁止访问线上应用、正式 ACME 状态、管理 socket 及 Nginx 配置。测试证书由隔离 CA 签发，只有隔离验证上下文信任，不导入系统信任库。真实 Nginx 实例监听隔离空间的 18443/18002，测试完成后仅终止该测试实例。 + +覆盖申请失败及超时不停止业务、不发布候选；未配置或非法服务商拒绝；不可信链、错误域名、临近到期、私钥不匹配及密钥轮换拒绝；并发、锁权限、证书权限、符号链接拒绝；重载失败和线上指纹失败回退；中断后的重载补偿；候选文件在快照后变化不能影响发布内容。 + +第一轮 23 项模拟及文件系统回归通过。新增真实 Nginx 测试首次失败，是该实例试图使用被只读保护的主机 `/var/lib/nginx/body`，并非生产服务故障；随后将其全部临时路径指向隔离测试目录。失败日志 `tests-final.txt` 与最终通过日志均保留，未放宽隔离边界。 + +`candidate-live-check.txt` 为候选脚本的真实未到期检查，成功返回；运行约 450 毫秒，峰值约 11.8 MiB。应用和 ACME 目录只读挂载；本次未申请证书、未改正式文件，也没有替换线上定时任务。其受保护快照为 `/var/backups/smail-certificate-20260917T162339-e3f3d284`，名称时间为 UTC。 + +## 线上迁移证据 + +最终目录：`/var/tmp/smail-review-cert-safe-20260918-cutover-final`。 + +- `cutover-result.json`：完整前后摘要、服务状态、连续探测和下载证据。 +- `smail.conf.before`：原 Smail 站点配置，迁移工具只接受已核对的原配置摘要和仅变更两条证书路径的候选配置。 +- `cutover-output.txt`：单元 `smail-cert-cutover-20260918-4` 执行输出。 +- `transport-result.json`：七个 TLS/STARTTLS 入口、Host/SNI 和 HTTP 跳转复测。 + +迁移时间为本地 00:22:31 至 00:22:38，仅执行一次平滑重载。Nginx 主进程 `578671`、Smail 主进程 `574584` 及各自 InvocationID 前后一致。两站点探测使用 `mail.solit.cn:443` 和 `web.apmmk.cn:443`，没有关闭系统信任或域名校验。 + +下载使用线上实际资源 `/assets/main-app-96760af3.js`，重载前建立连接并读取前 1024 字节，重载后读取余下内容，与独立请求返回的完整内容比较相同。此项证明本次已有下载连接完成，不扩大为任意长连接或长期可用性保证。 + +Smail 站点配置 SHA-256 从 `2c6e42ce1c996947a13b35bddd9840888a4bd9e7e2de7480954daa3fafbb18a4` 变为 `bf6816c38f9c639ab17fd7344fff4b06afc5a2388f74318e18c60b8b3311ebd8`。其他站点配置、正式证书、私钥、应用双程序和线上续期脚本摘要未变。 + +迁移前有三次拒绝或失败，均未修改线上配置：第一次权限检查发现上传目录变为 0755；第二次发现入口 JS 仅为小型引导脚本，不能充当持续下载样本；第三次按旧记录探测 Scert 8004 被拒绝。实时核查确认 Scert 已迁移至按域名共享 443，工具随后按当前实际入口执行，未恢复或修改 Scert 配置。对应日志及失败结果保留在主证据目录和 `-cutover` 目录。 + +## 启用 DNS 续期前的门槛 + +1. 确认 `solit.cn` 实际 DNS 服务商及专用最小权限授权。凭据只保存在远程受保护文件，不在聊天、仓库、普通文档或日志中传递；示例采用阿里云不代表已获得该账号授权。 +2. 使用全新、独立的 ACME 测试账号完成五个域名的真实 DNS-01 挑战，验证 TXT 创建、传播及清理，不以已有授权缓存代替挑战。 +3. 验证 DNS 拒绝、超时与清理失败期间两个站点及邮件服务仍可用，正式证书和私钥保持不变。 +4. 在受保护备份和持续探测下同步安装新的续期脚本、部署脚本和 service，再完成正式签发及七个入口的新指纹检查。不能只替换其中一个脚本：旧续期脚本在持锁期间通过子进程调用部署脚本，与新版本共用锁的执行方式不同，混用会导致锁冲突。 +5. 接入真实外部续期失败、证书临近到期告警，完成一次失败告警送达及恢复验证，再将无停机续期列为验收通过。 + +截至本轮结束，线上仍使用旧续期定时任务，证书有效期至 2026-12-16。保留旧任务不代表批准未来停机维护窗口；必须在进入到期续期窗口前完成 DNS 迁移。发布验收清单中的其他业务、安全、容量和恢复门槛继续有效。 diff --git a/docs/生产整改验收清单.md b/docs/生产整改验收清单.md index 5929e40..fd83159 100644 --- a/docs/生产整改验收清单.md +++ b/docs/生产整改验收清单.md @@ -2,9 +2,20 @@ ## 范围与结论 -### 当前状态：2026 年 9 月 17 日晚间复测 +### 当前状态：远程主机 2026 年 9 月 18 日凌晨无停机续期整改 -本节与《晚间验收复测记录》作为当前状态入口；下方带时间的旧记录保留追溯，不应把早期 SSH 失败、自签名或单次资源失败当作当前结论。已部署程序仍为 `review-2263779`，本轮没有编译或替换应用。 +- [✓] Nginx 已迁移到已发布证书目录，不再直接读取 ACME 签发产物；Smail 8001/8002 和其他站点配置未修改。 +- [✓] 27 项远程隔离回归通过，含真实独立 Nginx 换证、重载和回退后的 TLS 指纹检查。 +- [✓] 线上平滑重载期间两个站点 58 次 HTTPS 探测成功，已有连接完整下载 312,377 字节，Nginx 与 Smail 主进程未重启；七个 TLS/STARTTLS 入口复测通过。 +- [✓] DNS 候选脚本在真实未到期状态校验通过，但没有签发新证书或替换自动任务。 +- [!] Smail 专用 DNS 授权、全新 DNS 挑战、正式新证书发布、外部告警尚未验收；线上续期脚本和定时任务仍为旧版本，不能宣称无停机自动续期已启用。 +- [!] 实时配置显示 Scert 已按域名共享 443，旧记录中的 8003/8004 不再是当前入口。其他生产门槛继续有效，整体结论仍为“不满足生产放行”。 + +本轮无业务数据写入，不清库、不重编译应用、不替代首次安装和浏览器业务验收。详情及失败记录见《无停机续期整改记录》。 + +### 历史状态：2026 年 9 月 17 日晚间复测 + +本节与《晚间验收复测记录》保留当时状态；当前以顶部较新章节为准，不应把早期 SSH 失败、自签名或单次资源失败当作当前结论。已部署程序仍为 `review-2263779`，本轮没有编译或替换应用。 - [✓] SSH 可用；8001/8002 与 Nginx 反代、正式可信证书、七个 TLS/STARTTLS 入口复测通过。 - [✓] 六项远程隔离续期回归通过；备份摘要、证书有效期与关键文件权限复核通过。 diff --git a/test/deployment/nginx_certificate_cutover.py b/test/deployment/nginx_certificate_cutover.py new file mode 100644 index 0000000..55c8649 --- /dev/null +++ b/test/deployment/nginx_certificate_cutover.py @@ -0,0 +1,229 @@ +"""在远程主机迁移 Smail 的 Nginx 证书路径，保留回退和连续探测证据。""" + +import argparse +from datetime import datetime, timezone +import hashlib +from html.parser import HTMLParser +import http.client +import json +import os +from pathlib import Path +import re +import socket +import ssl +import subprocess +import threading +import time +import uuid + +import deploy_certificate as deploy + + +CONFIG = Path('/etc/nginx/conf.d/smail.conf') +ROOT = Path('/var/tmp') +LIMIT = 8 * 1024 * 1024 + + +def run(*command): + return subprocess.run(command, check=True, capture_output=True, timeout=15).stdout + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def state(): + return run('systemctl', 'show', 'nginx', 'smail', '-p', 'Id', '-p', 'MainPID', + '-p', 'ActiveState', '-p', 'InvocationID').decode() + + +def request(host, port, path='/', slow=False): + raw = socket.socket() + raw.settimeout(15 if slow else 3) + if slow: + raw.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 8192) + try: + raw.connect(('127.0.0.1', port)) + secure = ssl.create_default_context().wrap_socket(raw, server_hostname=host) + except BaseException: + raw.close() + raise + try: + authority = host if port == 443 else host + ':' + str(port) + secure.sendall(('GET ' + path + ' HTTP/1.1\r\nHost: ' + authority + + '\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n').encode()) + response = http.client.HTTPResponse(secure) + response.begin() + if response.status != 200: + raise RuntimeError('HTTPS 状态异常：' + str(response.status)) + if slow: + return secure, response + with response: + data = response.read(LIMIT + 1) + if len(data) > LIMIT: + raise RuntimeError('响应超出本次探测上限') + secure.close() + return data + except BaseException: + secure.close() + raise + + +class Scripts(HTMLParser): + def __init__(self): + super().__init__() + self.paths = [] + + def handle_starttag(self, tag, attrs): + source = dict(attrs).get('src', '') + if tag == 'script' and source.startswith('/assets/') and source.endswith('.js'): + self.paths.append(source) + + +def atomic_config(data, mode): + staged = CONFIG.with_name(CONFIG.name + '.candidate-' + uuid.uuid4().hex) + fd = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode) + with os.fdopen(fd, 'wb') as stream: + os.fchmod(stream.fileno(), mode) + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.replace(staged, CONFIG) + fd = os.open(CONFIG.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--evidence-dir', required=True, type=Path) + parser.add_argument('--expected-config-sha256', required=True) + parser.add_argument('--asset-path', help='用于下载连续性检查的已部署大型脚本路径') + args = parser.parse_args() + evidence = args.evidence_dir.resolve(strict=True) + if (os.geteuid() != 0 or evidence.parent != ROOT + or not evidence.name.startswith('smail-review-cert-safe-') + or evidence.stat().st_uid != 0 or evidence.stat().st_mode & 0o077): + parser.error('必须在远程项目专用且受保护的证据目录中执行') + report_path = evidence / 'cutover-result.json' + if report_path.exists(): + parser.error('禁止覆盖已有迁移证据') + if args.asset_path and not re.fullmatch(r'/assets/[a-zA-Z0-9_-]+\.js', args.asset_path): + parser.error('下载探测仅允许项目静态脚本路径') + previous = CONFIG.read_bytes() + if digest(previous) != args.expected_config_sha256 or CONFIG.is_symlink(): + parser.error('线上配置与已审查版本不符') + proposed = (evidence / 'smail.conf').read_bytes() + expected = previous.replace( + b'/var/lib/smail-acme-tls/certificates/solit.cn.crt', b'/opt/smail/ssl/custom.crt').replace( + b'/var/lib/smail-acme-tls/certificates/solit.cn.key', b'/opt/smail/ssl/custom.key') + if proposed != expected or proposed == previous: + parser.error('候选配置必须且只能改变 Smail 证书读取路径') + run('nginx', '-t') + run('systemctl', 'is-active', '--quiet', 'nginx', 'smail') + protected = [Path('/opt/smail/smail'), Path('/opt/smail/smail-admin'), + Path('/opt/smail/ssl/custom.crt'), Path('/opt/smail/ssl/custom.key'), + Path('/usr/local/libexec/smail-acme/renew_certificate.py'), + Path('/usr/local/libexec/smail-acme/deploy_certificate.py'), + Path('/etc/systemd/system/smail-certificate-renew.service')] + protected.extend(path for path in Path('/etc/nginx/conf.d').glob('*.conf') if path != CONFIG) + protected_before = {str(path): digest(path.read_bytes()) for path in protected} + before_state = state() + backup = evidence / 'smail.conf.before' + with backup.open('xb') as stream: + stream.write(previous) + backup.chmod(0o600) + report = {'startedUTC': datetime.now(timezone.utc).isoformat(), 'status': 'running', + 'beforeConfigSHA256': digest(previous), 'afterConfigSHA256': digest(proposed), + 'beforeState': before_state, 'productionCertificateChanged': False, + 'dnsRenewalActivated': False, 'protectedBefore': protected_before, 'samples': []} + stop = threading.Event() + threads = [] + pending = None + changed = False + mode = CONFIG.stat().st_mode & 0o777 + + def monitor(host, port): + while not stop.is_set(): + sample = {'utc': datetime.now(timezone.utc).isoformat(), 'host': host, 'port': port} + try: + request(host, port) + sample['ok'] = True + except Exception as error: + sample.update(ok=False, error=type(error).__name__) + report['samples'].append(sample) + stop.wait(0.2) + + try: + scripts = Scripts() + scripts.feed(request('mail.solit.cn', 443).decode('utf-8')) + if not scripts.paths: + raise RuntimeError('未找到用于既有连接测试的真实静态资源') + asset = args.asset_path or scripts.paths[0] + expected_body = request('mail.solit.cn', 443, asset) + if len(expected_body) < 65536: + raise RuntimeError('静态资源过小，无法进行本次下载连续性检查') + for host, port in (('mail.solit.cn', 443), ('web.apmmk.cn', 443)): + thread = threading.Thread(target=monitor, args=(host, port)) + thread.start() + threads.append(thread) + time.sleep(2) + if len(report['samples']) < 4 or any(not item['ok'] for item in report['samples']): + raise RuntimeError('迁移前连续探测未通过') + secure, response = request('mail.solit.cn', 443, asset, slow=True) + pending = (secure, response) + prefix = response.read(1024) + report['connectionOpenedBeforeReload'] = True + changed = True + atomic_config(proposed, mode) + run('nginx', '-t') + run('systemctl', 'reload', 'nginx') + report['reloadedUTC'] = datetime.now(timezone.utc).isoformat() + body = prefix + response.read(LIMIT + 1) + if body != expected_body: + raise RuntimeError('重载前已建立的 HTTPS 下载未完整完成') + report['download'] = {'path': asset, 'bytes': len(body), 'sha256': digest(body), 'completed': True} + time.sleep(4) + stop.set() + for thread in threads: + thread.join(timeout=5) + if any(thread.is_alive() for thread in threads): + raise RuntimeError('连续探测未在规定时间内结束') + after_state = state() + report['afterState'] = after_state + if after_state != before_state: + raise RuntimeError('迁移期间主服务进程或运行状态发生变化') + if any(digest(path.read_bytes()) != protected_before[str(path)] for path in protected): + raise RuntimeError('迁移期间保护范围内的文件发生变化') + if any(not item['ok'] for item in report['samples']): + raise RuntimeError('迁移期间连续 HTTPS 探测失败') + report['protectedFilesUnchanged'] = True + report['status'] = 'passed' + except BaseException as error: + report['status'] = 'failed' + report['errorType'] = type(error).__name__ + if changed: + atomic_config(previous, mode) + run('nginx', '-t') + run('systemctl', 'reload', 'nginx') + report['configRolledBack'] = True + raise + finally: + stop.set() + for thread in threads: + thread.join(timeout=5) + if pending is not None: + pending[1].close() + pending[0].close() + report['finishedUTC'] = datetime.now(timezone.utc).isoformat() + report_path.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + report_path.chmod(0o600) + print(json.dumps({key: value for key, value in report.items() if key != 'samples'}, indent=2)) + print('HTTPS 探测次数=' + str(len(report['samples']))) + + +if __name__ == '__main__': + with deploy.certificate_lock(): + main() diff --git a/test/deployment/renewal_drill.py b/test/deployment/renewal_drill.py index b62992a..a5a9ef4 100644 --- a/test/deployment/renewal_drill.py +++ b/test/deployment/renewal_drill.py @@ -14,6 +14,7 @@ from unittest.mock import patch SERVER = 'https://acme-staging-v02.api.letsencrypt.org/directory' SOURCE = Path('/usr/local/libexec/smail-acme/renew_certificate.py') +LEGACY_SOURCE_SHA256 = '6d0abbe9466f8becd61b1cb3c750bfffb938d16c5a9bcae0b96d1451227d4216' DOMAINS = {'solit.cn', 'mail.solit.cn', 'smtp.solit.cn', 'imap.solit.cn', 'pop.solit.cn'} RUN = subprocess.run @@ -53,6 +54,8 @@ def main(): parser.error('演练目录必须归 root 所有且禁止其他用户访问') if (evidence / 'result.json').exists(): parser.error('禁止覆盖已有演练结果，请使用新的证据目录') + if digest(SOURCE) != LEGACY_SOURCE_SHA256: + parser.error('本工具仅用于已封存的停机续期实现，不能用于 DNS 无停机续期验收') run(['nginx', '-t'], check=True) run(['systemctl', 'is-active', '--quiet', 'nginx'], check=True) diff --git a/test/deployment/test_certificate_renewal.py b/test/deployment/test_certificate_renewal.py index e1a78ba..c1d9344 100644 --- a/test/deployment/test_certificate_renewal.py +++ b/test/deployment/test_certificate_renewal.py @@ -1,36 +1,54 @@ """证书更新脚本的隔离回归，不操作生产文件或服务。""" -import importlib.util +from contextlib import nullcontext +import os from pathlib import Path +import signal import subprocess +import sys +import tempfile +from types import SimpleNamespace import unittest -from unittest.mock import mock_open, patch +from unittest.mock import patch BASE = Path(__file__).resolve().parent +SCRIPTS = BASE if (BASE / 'deploy_certificate.py').exists() else BASE.parents[1] / 'deploy/nginx' +sys.path.insert(0, str(SCRIPTS)) -def load(name): - path = BASE / (name + '.py') - if not path.exists(): - path = BASE.parents[1] / 'deploy/nginx' / (name + '.py') - spec = importlib.util.spec_from_file_location(name, path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module +import deploy_certificate as deploy +import renew_certificate as renew +import nginx_certificate_cutover as cutover -renew = load('renew_certificate') -deploy = load('deploy_certificate') +class CutoverTests(unittest.TestCase): + def test_atomic_config_preserves_mode_under_restrictive_umask(self): + directory = Path(tempfile.mkdtemp(prefix='cutover-tests-', dir=Path.cwd())) + path = directory / 'smail.conf' + previous = os.umask(0o077) + try: + with patch.object(cutover, 'CONFIG', path): + cutover.atomic_config(b'candidate\n', 0o644) + self.assertEqual(path.read_bytes(), b'candidate\n') + self.assertEqual(path.stat().st_mode & 0o777, 0o644) + cutover.atomic_config(b'previous\n', 0o644) + self.assertEqual(path.read_bytes(), b'previous\n') + self.assertEqual(path.stat().st_mode & 0o777, 0o644) + finally: + os.umask(previous) class RenewalTests(unittest.TestCase): - def execute(self, callback): + def execute(self, callback, provider='alidns'): with patch.object(renew.os, 'geteuid', return_value=0), \ - patch('builtins.open', mock_open()), \ - patch.object(renew.fcntl, 'flock'), \ + patch.object(deploy, 'certificate_lock', return_value=nullcontext()), \ + patch.object(deploy, 'check_nginx_paths'), \ + patch.object(deploy, 'publish') as publish, \ + patch.dict(os.environ, {'SMAIL_ACME_DNS_PROVIDER': provider}), \ patch.object(renew.subprocess, 'run', side_effect=callback): renew.main() + return publish def test_not_due_keeps_nginx_running(self): calls = [] @@ -39,13 +57,14 @@ class RenewalTests(unittest.TestCase): calls.append(command) return subprocess.CompletedProcess(command, 0) - self.execute(run) + publish = self.execute(run, provider='') self.assertNotIn(['systemctl', 'stop', 'nginx'], calls) - self.assertIn(['/usr/local/libexec/smail-acme/deploy_certificate.py'], calls) + self.assertFalse(any(command[0].endswith('/lego') for command in calls)) + publish.assert_called_once() - def test_due_failure_restores_nginx(self): + def test_due_failure_never_stops_nginx_or_publishes(self): for failure in (subprocess.CalledProcessError(1, 'lego'), - subprocess.TimeoutExpired('lego', 150)): + subprocess.TimeoutExpired('lego', renew.RENEW_TIMEOUT)): with self.subTest(failure=type(failure).__name__): calls = [] @@ -57,10 +76,16 @@ class RenewalTests(unittest.TestCase): raise failure return subprocess.CompletedProcess(command, 0) - with self.assertRaises(type(failure)): - self.execute(run) - self.assertEqual(calls[-1], ['systemctl', 'start', 'nginx']) - self.assertNotIn(['/usr/local/libexec/smail-acme/deploy_certificate.py'], calls) + with patch.object(deploy, 'certificate_lock', return_value=nullcontext()), \ + patch.object(deploy, 'check_nginx_paths'), \ + patch.object(deploy, 'publish') as publish, \ + patch.dict(os.environ, {'SMAIL_ACME_DNS_PROVIDER': 'alidns'}), \ + patch.object(renew.subprocess, 'run', side_effect=run), \ + self.assertRaises(type(failure)): + renew.main() + publish.assert_not_called() + self.assertFalse(any(command[:2] in (['systemctl', 'stop'], ['systemctl', 'start']) + for command in calls)) def test_due_success_is_bounded_and_reuses_key(self): calls = [] @@ -70,38 +95,336 @@ class RenewalTests(unittest.TestCase): if '-checkend' in command: return subprocess.CompletedProcess(command, 1) if command[0].endswith('/lego'): - self.assertEqual(kwargs['timeout'], 150) + self.assertEqual(kwargs['timeout'], 600) + self.assertEqual(command[command.index('--dns') + 1], 'alidns') + self.assertNotIn('--tls', command) + self.assertIn('--server', command) self.assertIn('--reuse-key', command) self.assertIn('--no-random-sleep', command) return subprocess.CompletedProcess(command, 0) + publish = self.execute(run) + publish.assert_called_once() + self.assertTrue(any(command[0].endswith('/lego') for command in calls)) + self.assertFalse(any(command[:2] in (['systemctl', 'stop'], ['systemctl', 'start']) + for command in calls)) + + def test_due_uses_live_certificate_not_candidate(self): + def run(command, **kwargs): + if command[0] == 'openssl': + self.assertEqual(command[command.index('-in') + 1], str(deploy.TARGET / 'custom.crt')) + return subprocess.CompletedProcess(command, 0) + self.execute(run) - self.assertLess(calls.index(['systemctl', 'start', 'nginx']), - calls.index(['/usr/local/libexec/smail-acme/deploy_certificate.py'])) + + def test_missing_or_unsafe_provider_fails_before_lego(self): + for provider in ('', '--tls', 'exec', 'alidns --tls', '../alidns'): + calls = [] + + def run(command, **kwargs): + calls.append(command) + return subprocess.CompletedProcess(command, 1 if '-checkend' in command else 0) + + with self.subTest(provider=provider), self.assertRaises(RuntimeError): + self.execute(run, provider) + self.assertFalse(any(command[0].endswith('/lego') for command in calls)) + + def test_invalid_expiry_check_fails_closed(self): + def run(command, **kwargs): + return subprocess.CompletedProcess(command, 2 if '-checkend' in command else 0) + + with self.assertRaisesRegex(RuntimeError, '无法判断'): + self.execute(run) def test_non_root_is_rejected(self): with patch.object(renew.os, 'geteuid', return_value=1000), \ self.assertRaises(RuntimeError): renew.main() - def test_untrusted_certificate_never_replaces_files(self): - with patch.object(deploy.os, 'geteuid', return_value=0), \ - patch.object(deploy, 'run', side_effect=subprocess.CalledProcessError(1, 'openssl')), \ - patch.object(deploy, 'replace_file') as replace, \ + +class DeploymentTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + if os.geteuid() != 0: + raise RuntimeError('本回归必须在远程隔离 root 单元中执行') + cls.artifacts = Path(tempfile.mkdtemp(prefix='certificate-tests-', dir=Path.cwd())) + cls.ca = cls.artifacts / 'ca.crt' + cls.ca_key = cls.artifacts / 'ca.key' + cls.key = cls.artifacts / 'server.key' + cls.csr = cls.artifacts / 'server.csr' + cls.old = cls.artifacts / 'old.crt' + cls.new = cls.artifacts / 'new.crt' + cls.real_run = staticmethod(deploy.run) + cls.real_run('openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', + '-keyout', str(cls.ca_key), '-out', str(cls.ca), '-days', '120', + '-subj', '/CN=Smail isolated test CA', + '-addext', 'keyUsage=critical,keyCertSign,cRLSign') + cls.real_run('openssl', 'req', '-new', '-newkey', 'rsa:2048', '-nodes', + '-keyout', str(cls.key), '-out', str(cls.csr), '-subj', '/CN=solit.cn', + '-addext', 'basicConstraints=critical,CA:FALSE', + '-addext', 'keyUsage=critical,digitalSignature,keyEncipherment', + '-addext', 'extendedKeyUsage=serverAuth', + '-addext', 'subjectAltName=' + ','.join('DNS:' + d for d in deploy.DOMAINS)) + for target, serial, days in ((cls.old, '1', '20'), (cls.new, '2', '90')): + cls.real_run('openssl', 'x509', '-req', '-in', str(cls.csr), '-CA', str(cls.ca), + '-CAkey', str(cls.ca_key), '-set_serial', serial, '-days', days, + '-copy_extensions', 'copy', '-out', str(target)) + print('保留隔离证据目录=' + str(cls.artifacts), flush=True) + + def setUp(self): + self.directory = self.artifacts / self._testMethodName + self.directory.mkdir(mode=0o700) + self.source = self.directory / 'candidate' + self.target = self.directory / 'live' + self.backups = self.directory / 'backups' + for directory in (self.source, self.target, self.backups): + directory.mkdir(mode=0o700) + for path, value in ((self.source / 'solit.cn.crt', self.new.read_bytes()), + (self.source / 'solit.cn.key', self.key.read_bytes()), + (self.source / 'solit.cn.issuer.crt', self.ca.read_bytes()), + (self.target / 'custom.crt', self.old.read_bytes()), + (self.target / 'custom.key', self.key.read_bytes())): + path.write_bytes(value) + path.chmod(0o600) + self.calls = [] + self.failure = None + self.failed_once = False + self.config = ('ssl_certificate ' + str(self.target / 'custom.crt') + ';\n' + + 'ssl_certificate_key ' + str(self.target / 'custom.key') + ';\n') + for name, value in (('LINEAGE', self.source), ('TARGET', self.target), + ('BACKUPS', self.backups), ('CA_FILE', str(self.ca)), + ('LOCK', self.directory / 'renew.lock')): + patcher = patch.object(deploy, name, value) + patcher.start() + self.addCleanup(patcher.stop) + identity = patch.object(deploy.pwd, 'getpwnam', return_value=SimpleNamespace(pw_uid=0, pw_gid=0)) + identity.start() + self.addCleanup(identity.stop) + commands = patch.object(deploy, 'run', side_effect=self.command) + commands.start() + self.addCleanup(commands.stop) + live = patch.object(deploy, 'verify_live_certificate') + self.real_verify = deploy.verify_live_certificate + self.live = live.start() + self.addCleanup(live.stop) + + def command(self, *command, **kwargs): + self.calls.append(command) + if command == self.failure and not self.failed_once: + self.failed_once = True + raise subprocess.CalledProcessError(1, command) + if command == ('nginx', '-T'): + return self.config.encode() + if command[0] in ('nginx', 'systemctl'): + return b'' + return self.real_run(*command, **kwargs) + + def assert_old_untouched(self): + self.assertEqual((self.target / 'custom.crt').read_bytes(), self.old.read_bytes()) + self.assertEqual((self.target / 'custom.key').read_bytes(), self.key.read_bytes()) + + def test_publish_valid_certificate_reloads_without_restart(self): + deploy.main() + self.assertEqual((self.target / 'custom.crt').read_bytes(), self.new.read_bytes()) + self.assertIn(('systemctl', 'reload', 'nginx'), self.calls) + self.assertNotIn(('systemctl', 'stop', 'nginx'), self.calls) + self.assertNotIn(('systemctl', 'start', 'nginx'), self.calls) + self.live.assert_called_once_with(deploy.fingerprint(self.new)) + for path in self.backups.rglob('*'): + self.assertEqual(path.stat().st_mode & 0o077, 0) + + def test_unchanged_certificate_checks_live_without_reload(self): + (self.source / 'solit.cn.crt').write_bytes(self.old.read_bytes()) + deploy.main() + self.assertNotIn(('systemctl', 'reload', 'nginx'), self.calls) + self.live.assert_called_once() + + def test_interrupted_publish_is_reconciled_when_files_already_match(self): + (self.source / 'solit.cn.crt').write_bytes(self.old.read_bytes()) + self.live.side_effect = [RuntimeError('上次重载前中断'), None] + deploy.main() + self.assertEqual(self.calls.count(('systemctl', 'reload', 'nginx')), 1) + self.assertEqual(self.live.call_count, 2) + + def test_failed_reconciliation_does_not_report_success(self): + (self.source / 'solit.cn.crt').write_bytes(self.old.read_bytes()) + self.live.side_effect = RuntimeError('线上证书不匹配') + with self.assertRaisesRegex(RuntimeError, '线上证书不匹配'): + deploy.main() + self.assertEqual(self.live.call_count, 2) + self.assert_old_untouched() + + def test_untrusted_certificate_does_not_change_live_files(self): + with patch.object(deploy, 'CA_FILE', '/etc/ssl/certs/ca-certificates.crt'), \ self.assertRaises(subprocess.CalledProcessError): deploy.main() - replace.assert_not_called() + self.assert_old_untouched() + self.assertNotIn(('systemctl', 'reload', 'nginx'), self.calls) - def test_mismatched_private_key_never_replaces_files(self): - def run(*command, **kwargs): - return b'private-public-key' if command[1] == 'pkey' else b'certificate-public-key' - - with patch.object(deploy.os, 'geteuid', return_value=0), \ - patch.object(deploy, 'run', side_effect=run), \ - patch.object(deploy, 'replace_file') as replace, \ - self.assertRaisesRegex(RuntimeError, '不匹配'): + def test_wrong_domain_does_not_change_live_files(self): + with patch.object(deploy, 'DOMAINS', ('wrong.example',)), \ + self.assertRaises(subprocess.CalledProcessError): deploy.main() - replace.assert_not_called() + self.assert_old_untouched() + + def test_short_lived_certificate_is_rejected(self): + self.real_run('openssl', 'x509', '-req', '-in', str(self.csr), '-CA', str(self.ca), + '-CAkey', str(self.ca_key), '-set_serial', '3', '-days', '1', + '-copy_extensions', 'copy', '-out', str(self.source / 'solit.cn.crt')) + with self.assertRaises(subprocess.CalledProcessError): + deploy.main() + self.assert_old_untouched() + + def test_mismatched_key_is_rejected(self): + (self.source / 'solit.cn.key').write_bytes(self.ca_key.read_bytes()) + with self.assertRaisesRegex(RuntimeError, '不匹配'): + deploy.main() + self.assert_old_untouched() + + def test_key_rotation_is_rejected(self): + (self.target / 'custom.key').write_bytes(self.ca_key.read_bytes()) + with self.assertRaisesRegex(RuntimeError, '复用现有密钥'): + deploy.main() + self.assertEqual((self.target / 'custom.crt').read_bytes(), self.old.read_bytes()) + + def test_nginx_still_using_candidate_is_rejected(self): + self.config += 'ssl_certificate ' + str(self.source / 'solit.cn.crt') + ';\n' + with self.assertRaisesRegex(RuntimeError, '签发目录'): + deploy.main() + self.assert_old_untouched() + + def test_missing_live_nginx_paths_is_rejected(self): + self.config = 'ssl_certificate /some/other/site.crt;\n' + with self.assertRaisesRegex(RuntimeError, '未找到'): + deploy.main() + self.assert_old_untouched() + + def test_nginx_test_failure_rolls_back_and_reloads_old(self): + self.failure = ('nginx', '-t') + with self.assertRaises(subprocess.CalledProcessError): + deploy.main() + self.assert_old_untouched() + self.assertEqual(self.calls.count(('nginx', '-t')), 2) + self.assertIn(('systemctl', 'reload', 'nginx'), self.calls) + self.live.assert_called_once_with(deploy.fingerprint(self.old)) + + def test_reload_failure_rolls_back_and_reloads_old(self): + self.failure = ('systemctl', 'reload', 'nginx') + with self.assertRaises(subprocess.CalledProcessError): + deploy.main() + self.assert_old_untouched() + self.assertEqual(self.calls.count(('systemctl', 'reload', 'nginx')), 2) + + def test_live_probe_failure_rolls_back_and_verifies_old(self): + self.live.side_effect = [RuntimeError('模拟线上证书不匹配'), None] + with self.assertRaisesRegex(RuntimeError, '模拟'): + deploy.main() + self.assert_old_untouched() + self.assertEqual(self.live.call_count, 2) + self.live.assert_called_with(deploy.fingerprint(self.old)) + + def test_unreadable_private_permissions_are_rejected(self): + (self.source / 'solit.cn.key').chmod(0o644) + with self.assertRaisesRegex(RuntimeError, '权限'): + deploy.main() + self.assert_old_untouched() + + def test_symlink_source_is_rejected(self): + link = self.directory / 'linked-source' + link.symlink_to(self.source, target_is_directory=True) + with patch.object(deploy, 'LINEAGE', link), self.assertRaisesRegex(RuntimeError, '符号链接'): + deploy.main() + self.assert_old_untouched() + + def test_concurrent_deployment_is_rejected(self): + with deploy.certificate_lock(), self.assertRaises(BlockingIOError): + deploy.main() + self.assert_old_untouched() + + def test_unsafe_lock_permissions_are_rejected(self): + deploy.LOCK.touch(mode=0o644) + deploy.LOCK.chmod(0o644) + with self.assertRaisesRegex(RuntimeError, '锁文件'): + deploy.main() + self.assert_old_untouched() + + def test_candidate_mutation_after_snapshot_cannot_change_published_bytes(self): + validate = deploy.validate + + def mutate(snapshot): + (self.source / 'solit.cn.crt').write_bytes(b'changed-after-snapshot') + validate(snapshot) + + with patch.object(deploy, 'validate', side_effect=mutate): + deploy.main() + self.assertEqual((self.target / 'custom.crt').read_bytes(), self.new.read_bytes()) + + def test_real_isolated_nginx_reload_and_rollback(self): + configuration = self.directory / 'nginx.conf' + configuration.write_text( + 'worker_processes 1;\n' + + 'pid ' + str(self.directory / 'nginx.pid') + ';\n' + + 'error_log ' + str(self.directory / 'nginx-error.log') + ';\n' + + 'events { worker_connections 32; }\n' + + 'http {\naccess_log off;\n' + + ''.join(name + '_temp_path ' + str(self.directory / name) + ';\n' + for name in ('client_body', 'proxy', 'fastcgi', 'uwsgi', 'scgi')) + + 'server {\n' + + 'listen 127.0.0.1:18443 ssl;\nlisten 127.0.0.1:18002 ssl;\n' + + 'server_name mail.solit.cn;\n' + self.config + + 'location / { return 200 "ready"; }\n}\n}\n', encoding='utf-8') + arguments = ('nginx', '-c', str(configuration), '-p', str(self.directory)) + + def isolated(*command, **kwargs): + self.calls.append(command) + if command == ('nginx', '-T'): + return self.real_run(*arguments, '-T') + if command == ('nginx', '-t'): + return self.real_run(*arguments, '-t') + if command == ('systemctl', 'is-active', '--quiet', 'nginx'): + if process.poll() is not None: + raise RuntimeError('隔离 Nginx 未运行') + return b'' + if command == ('systemctl', 'reload', 'nginx'): + return self.real_run(*arguments, '-s', 'reload') + if command[0] == 'systemctl': + raise AssertionError('隔离测试禁止操作主机服务') + return self.real_run(*command, **kwargs) + + with (self.directory / 'nginx-start.log').open('wb') as log: + process = subprocess.Popen((*arguments, '-g', 'daemon off;'), stdout=log, stderr=log) + try: + with patch.object(deploy, 'run', side_effect=isolated), \ + patch.object(deploy, 'LIVE_PORTS', (18443, 18002)), \ + patch.object(deploy, 'verify_live_certificate', side_effect=self.real_verify): + self.real_verify(deploy.fingerprint(self.old)) + deploy.main() + self.real_verify(deploy.fingerprint(self.new)) + self.assertIsNone(process.poll()) + (self.source / 'solit.cn.crt').write_bytes(self.old.read_bytes()) + attempted = False + + def fail_after_live_update(expected): + nonlocal attempted + self.real_verify(expected) + if not attempted: + attempted = True + raise RuntimeError('模拟新证书上线后健康检查失败') + + with patch.object(deploy, 'verify_live_certificate', side_effect=fail_after_live_update), \ + self.assertRaisesRegex(RuntimeError, '健康检查失败'): + deploy.main() + self.real_verify(deploy.fingerprint(self.new)) + self.assertEqual((self.target / 'custom.crt').read_bytes(), self.new.read_bytes()) + self.assertIsNone(process.poll()) + finally: + if process.poll() is None: + process.send_signal(signal.SIGQUIT) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) if __name__ == '__main__': diff --git a/test/tools/remote_review.py b/test/tools/remote_review.py index 1c03934..1996e32 100644 --- a/test/tools/remote_review.py +++ b/test/tools/remote_review.py @@ -22,7 +22,7 @@ def main(): or len(target.parts) < 5 or target.parts[1:3] != ("var", "tmp") or not target.parts[3].startswith("smail-review-")): raise ValueError("上传路径必须位于项目审查临时目录") - command = "install -d -m 0755 " + shlex.quote(str(target.parent)) + command = "install -d -m 0700 " + shlex.quote(str(target.parent)) _, output, error = client.exec_command(command) if output.channel.recv_exit_status() != 0: raise RuntimeError(error.read().decode("utf-8", "replace"))