整改基线:封存无停机续期候选及远程回归证据
Security Checks / security (push) Waiting to run

This commit is contained in:
xuans committed 2026-09-18 00:33:42 +08:00
1 parent 06c49e88b7
commit 9aff6f9dfb
12 files changed
+879 -110

No files matched your search

@@ -0,0 +1,229 @@
"""在远程主机迁移 Smail 的 Nginx 证书路径,保留回退和连续探测证据。"""
import argparse
from datetime import datetime, timezone
import hashlib
from html.parser import HTMLParser
import http.client
import json
import os
from pathlib import Path
import re
import socket
import ssl
import subprocess
import threading
import time
import uuid
import deploy_certificate as deploy
CONFIG = Path('/etc/nginx/conf.d/smail.conf')
ROOT = Path('/var/tmp')
LIMIT = 8 * 1024 * 1024
def run(*command):
return subprocess.run(command, check=True, capture_output=True, timeout=15).stdout
def digest(data):
return hashlib.sha256(data).hexdigest()
def state():
return run('systemctl', 'show', 'nginx', 'smail', '-p', 'Id', '-p', 'MainPID',
'-p', 'ActiveState', '-p', 'InvocationID').decode()
def request(host, port, path='/', slow=False):
raw = socket.socket()
raw.settimeout(15 if slow else 3)
if slow:
raw.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 8192)
try:
raw.connect(('127.0.0.1', port))
secure = ssl.create_default_context().wrap_socket(raw, server_hostname=host)
except BaseException:
raw.close()
raise
try:
authority = host if port == 443 else host + ':' + str(port)
secure.sendall(('GET ' + path + ' HTTP/1.1\r\nHost: ' + authority
+ '\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n').encode())
response = http.client.HTTPResponse(secure)
response.begin()
if response.status != 200:
raise RuntimeError('HTTPS 状态异常:' + str(response.status))
if slow:
return secure, response
with response:
data = response.read(LIMIT + 1)
if len(data) > LIMIT:
raise RuntimeError('响应超出本次探测上限')
secure.close()
return data
except BaseException:
secure.close()
raise
class Scripts(HTMLParser):
def __init__(self):
super().__init__()
self.paths = []
def handle_starttag(self, tag, attrs):
source = dict(attrs).get('src', '')
if tag == 'script' and source.startswith('/assets/') and source.endswith('.js'):
self.paths.append(source)
def atomic_config(data, mode):
staged = CONFIG.with_name(CONFIG.name + '.candidate-' + uuid.uuid4().hex)
fd = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode)
with os.fdopen(fd, 'wb') as stream:
os.fchmod(stream.fileno(), mode)
stream.write(data)
stream.flush()
os.fsync(stream.fileno())
os.replace(staged, CONFIG)
fd = os.open(CONFIG.parent, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(fd)
finally:
os.close(fd)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--evidence-dir', required=True, type=Path)
parser.add_argument('--expected-config-sha256', required=True)
parser.add_argument('--asset-path', help='用于下载连续性检查的已部署大型脚本路径')
args = parser.parse_args()
evidence = args.evidence_dir.resolve(strict=True)
if (os.geteuid() != 0 or evidence.parent != ROOT
or not evidence.name.startswith('smail-review-cert-safe-')
or evidence.stat().st_uid != 0 or evidence.stat().st_mode & 0o077):
parser.error('必须在远程项目专用且受保护的证据目录中执行')
report_path = evidence / 'cutover-result.json'
if report_path.exists():
parser.error('禁止覆盖已有迁移证据')
if args.asset_path and not re.fullmatch(r'/assets/[a-zA-Z0-9_-]+\.js', args.asset_path):
parser.error('下载探测仅允许项目静态脚本路径')
previous = CONFIG.read_bytes()
if digest(previous) != args.expected_config_sha256 or CONFIG.is_symlink():
parser.error('线上配置与已审查版本不符')
proposed = (evidence / 'smail.conf').read_bytes()
expected = previous.replace(
b'/var/lib/smail-acme-tls/certificates/solit.cn.crt', b'/opt/smail/ssl/custom.crt').replace(
b'/var/lib/smail-acme-tls/certificates/solit.cn.key', b'/opt/smail/ssl/custom.key')
if proposed != expected or proposed == previous:
parser.error('候选配置必须且只能改变 Smail 证书读取路径')
run('nginx', '-t')
run('systemctl', 'is-active', '--quiet', 'nginx', 'smail')
protected = [Path('/opt/smail/smail'), Path('/opt/smail/smail-admin'),
Path('/opt/smail/ssl/custom.crt'), Path('/opt/smail/ssl/custom.key'),
Path('/usr/local/libexec/smail-acme/renew_certificate.py'),
Path('/usr/local/libexec/smail-acme/deploy_certificate.py'),
Path('/etc/systemd/system/smail-certificate-renew.service')]
protected.extend(path for path in Path('/etc/nginx/conf.d').glob('*.conf') if path != CONFIG)
protected_before = {str(path): digest(path.read_bytes()) for path in protected}
before_state = state()
backup = evidence / 'smail.conf.before'
with backup.open('xb') as stream:
stream.write(previous)
backup.chmod(0o600)
report = {'startedUTC': datetime.now(timezone.utc).isoformat(), 'status': 'running',
'beforeConfigSHA256': digest(previous), 'afterConfigSHA256': digest(proposed),
'beforeState': before_state, 'productionCertificateChanged': False,
'dnsRenewalActivated': False, 'protectedBefore': protected_before, 'samples': []}
stop = threading.Event()
threads = []
pending = None
changed = False
mode = CONFIG.stat().st_mode & 0o777
def monitor(host, port):
while not stop.is_set():
sample = {'utc': datetime.now(timezone.utc).isoformat(), 'host': host, 'port': port}
try:
request(host, port)
sample['ok'] = True
except Exception as error:
sample.update(ok=False, error=type(error).__name__)
report['samples'].append(sample)
stop.wait(0.2)
try:
scripts = Scripts()
scripts.feed(request('mail.solit.cn', 443).decode('utf-8'))
if not scripts.paths:
raise RuntimeError('未找到用于既有连接测试的真实静态资源')
asset = args.asset_path or scripts.paths[0]
expected_body = request('mail.solit.cn', 443, asset)
if len(expected_body) < 65536:
raise RuntimeError('静态资源过小,无法进行本次下载连续性检查')
for host, port in (('mail.solit.cn', 443), ('web.apmmk.cn', 443)):
thread = threading.Thread(target=monitor, args=(host, port))
thread.start()
threads.append(thread)
time.sleep(2)
if len(report['samples']) < 4 or any(not item['ok'] for item in report['samples']):
raise RuntimeError('迁移前连续探测未通过')
secure, response = request('mail.solit.cn', 443, asset, slow=True)
pending = (secure, response)
prefix = response.read(1024)
report['connectionOpenedBeforeReload'] = True
changed = True
atomic_config(proposed, mode)
run('nginx', '-t')
run('systemctl', 'reload', 'nginx')
report['reloadedUTC'] = datetime.now(timezone.utc).isoformat()
body = prefix + response.read(LIMIT + 1)
if body != expected_body:
raise RuntimeError('重载前已建立的 HTTPS 下载未完整完成')
report['download'] = {'path': asset, 'bytes': len(body), 'sha256': digest(body), 'completed': True}
time.sleep(4)
stop.set()
for thread in threads:
thread.join(timeout=5)
if any(thread.is_alive() for thread in threads):
raise RuntimeError('连续探测未在规定时间内结束')
after_state = state()
report['afterState'] = after_state
if after_state != before_state:
raise RuntimeError('迁移期间主服务进程或运行状态发生变化')
if any(digest(path.read_bytes()) != protected_before[str(path)] for path in protected):
raise RuntimeError('迁移期间保护范围内的文件发生变化')
if any(not item['ok'] for item in report['samples']):
raise RuntimeError('迁移期间连续 HTTPS 探测失败')
report['protectedFilesUnchanged'] = True
report['status'] = 'passed'
except BaseException as error:
report['status'] = 'failed'
report['errorType'] = type(error).__name__
if changed:
atomic_config(previous, mode)
run('nginx', '-t')
run('systemctl', 'reload', 'nginx')
report['configRolledBack'] = True
raise
finally:
stop.set()
for thread in threads:
thread.join(timeout=5)
if pending is not None:
pending[1].close()
pending[0].close()
report['finishedUTC'] = datetime.now(timezone.utc).isoformat()
report_path.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
report_path.chmod(0o600)
print(json.dumps({key: value for key, value in report.items() if key != 'samples'}, indent=2))
print('HTTPS 探测次数=' + str(len(report['samples'])))
if __name__ == '__main__':
with deploy.certificate_lock():
main()
+3
View File
@@ -14,6 +14,7 @@ from unittest.mock import patch
SERVER = 'https://acme-staging-v02.api.letsencrypt.org/directory'
SOURCE = Path('/usr/local/libexec/smail-acme/renew_certificate.py')
LEGACY_SOURCE_SHA256 = '6d0abbe9466f8becd61b1cb3c750bfffb938d16c5a9bcae0b96d1451227d4216'
DOMAINS = {'solit.cn', 'mail.solit.cn', 'smtp.solit.cn', 'imap.solit.cn', 'pop.solit.cn'}
RUN = subprocess.run
@@ -53,6 +54,8 @@ def main():
parser.error('演练目录必须归 root 所有且禁止其他用户访问')
if (evidence / 'result.json').exists():
parser.error('禁止覆盖已有演练结果,请使用新的证据目录')
if digest(SOURCE) != LEGACY_SOURCE_SHA256:
parser.error('本工具仅用于已封存的停机续期实现,不能用于 DNS 无停机续期验收')
run(['nginx', '-t'], check=True)
run(['systemctl', 'is-active', '--quiet', 'nginx'], check=True)
+363 -40
View File
@@ -1,36 +1,54 @@
"""证书更新脚本的隔离回归,不操作生产文件或服务。"""
import importlib.util
from contextlib import nullcontext
import os
from pathlib import Path
import signal
import subprocess
import sys
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import mock_open, patch
from unittest.mock import patch
BASE = Path(__file__).resolve().parent
SCRIPTS = BASE if (BASE / 'deploy_certificate.py').exists() else BASE.parents[1] / 'deploy/nginx'
sys.path.insert(0, str(SCRIPTS))
def load(name):
path = BASE / (name + '.py')
if not path.exists():
path = BASE.parents[1] / 'deploy/nginx' / (name + '.py')
spec = importlib.util.spec_from_file_location(name, path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
import deploy_certificate as deploy
import renew_certificate as renew
import nginx_certificate_cutover as cutover
renew = load('renew_certificate')
deploy = load('deploy_certificate')
class CutoverTests(unittest.TestCase):
def test_atomic_config_preserves_mode_under_restrictive_umask(self):
directory = Path(tempfile.mkdtemp(prefix='cutover-tests-', dir=Path.cwd()))
path = directory / 'smail.conf'
previous = os.umask(0o077)
try:
with patch.object(cutover, 'CONFIG', path):
cutover.atomic_config(b'candidate\n', 0o644)
self.assertEqual(path.read_bytes(), b'candidate\n')
self.assertEqual(path.stat().st_mode & 0o777, 0o644)
cutover.atomic_config(b'previous\n', 0o644)
self.assertEqual(path.read_bytes(), b'previous\n')
self.assertEqual(path.stat().st_mode & 0o777, 0o644)
finally:
os.umask(previous)
class RenewalTests(unittest.TestCase):
def execute(self, callback):
def execute(self, callback, provider='alidns'):
with patch.object(renew.os, 'geteuid', return_value=0), \
patch('builtins.open', mock_open()), \
patch.object(renew.fcntl, 'flock'), \
patch.object(deploy, 'certificate_lock', return_value=nullcontext()), \
patch.object(deploy, 'check_nginx_paths'), \
patch.object(deploy, 'publish') as publish, \
patch.dict(os.environ, {'SMAIL_ACME_DNS_PROVIDER': provider}), \
patch.object(renew.subprocess, 'run', side_effect=callback):
renew.main()
return publish
def test_not_due_keeps_nginx_running(self):
calls = []
@@ -39,13 +57,14 @@ class RenewalTests(unittest.TestCase):
calls.append(command)
return subprocess.CompletedProcess(command, 0)
self.execute(run)
publish = self.execute(run, provider='')
self.assertNotIn(['systemctl', 'stop', 'nginx'], calls)
self.assertIn(['/usr/local/libexec/smail-acme/deploy_certificate.py'], calls)
self.assertFalse(any(command[0].endswith('/lego') for command in calls))
publish.assert_called_once()
def test_due_failure_restores_nginx(self):
def test_due_failure_never_stops_nginx_or_publishes(self):
for failure in (subprocess.CalledProcessError(1, 'lego'),
subprocess.TimeoutExpired('lego', 150)):
subprocess.TimeoutExpired('lego', renew.RENEW_TIMEOUT)):
with self.subTest(failure=type(failure).__name__):
calls = []
@@ -57,10 +76,16 @@ class RenewalTests(unittest.TestCase):
raise failure
return subprocess.CompletedProcess(command, 0)
with self.assertRaises(type(failure)):
self.execute(run)
self.assertEqual(calls[-1], ['systemctl', 'start', 'nginx'])
self.assertNotIn(['/usr/local/libexec/smail-acme/deploy_certificate.py'], calls)
with patch.object(deploy, 'certificate_lock', return_value=nullcontext()), \
patch.object(deploy, 'check_nginx_paths'), \
patch.object(deploy, 'publish') as publish, \
patch.dict(os.environ, {'SMAIL_ACME_DNS_PROVIDER': 'alidns'}), \
patch.object(renew.subprocess, 'run', side_effect=run), \
self.assertRaises(type(failure)):
renew.main()
publish.assert_not_called()
self.assertFalse(any(command[:2] in (['systemctl', 'stop'], ['systemctl', 'start'])
for command in calls))
def test_due_success_is_bounded_and_reuses_key(self):
calls = []
@@ -70,38 +95,336 @@ class RenewalTests(unittest.TestCase):
if '-checkend' in command:
return subprocess.CompletedProcess(command, 1)
if command[0].endswith('/lego'):
self.assertEqual(kwargs['timeout'], 150)
self.assertEqual(kwargs['timeout'], 600)
self.assertEqual(command[command.index('--dns') + 1], 'alidns')
self.assertNotIn('--tls', command)
self.assertIn('--server', command)
self.assertIn('--reuse-key', command)
self.assertIn('--no-random-sleep', command)
return subprocess.CompletedProcess(command, 0)
publish = self.execute(run)
publish.assert_called_once()
self.assertTrue(any(command[0].endswith('/lego') for command in calls))
self.assertFalse(any(command[:2] in (['systemctl', 'stop'], ['systemctl', 'start'])
for command in calls))
def test_due_uses_live_certificate_not_candidate(self):
def run(command, **kwargs):
if command[0] == 'openssl':
self.assertEqual(command[command.index('-in') + 1], str(deploy.TARGET / 'custom.crt'))
return subprocess.CompletedProcess(command, 0)
self.execute(run)
self.assertLess(calls.index(['systemctl', 'start', 'nginx']),
calls.index(['/usr/local/libexec/smail-acme/deploy_certificate.py']))
def test_missing_or_unsafe_provider_fails_before_lego(self):
for provider in ('', '--tls', 'exec', 'alidns --tls', '../alidns'):
calls = []
def run(command, **kwargs):
calls.append(command)
return subprocess.CompletedProcess(command, 1 if '-checkend' in command else 0)
with self.subTest(provider=provider), self.assertRaises(RuntimeError):
self.execute(run, provider)
self.assertFalse(any(command[0].endswith('/lego') for command in calls))
def test_invalid_expiry_check_fails_closed(self):
def run(command, **kwargs):
return subprocess.CompletedProcess(command, 2 if '-checkend' in command else 0)
with self.assertRaisesRegex(RuntimeError, '无法判断'):
self.execute(run)
def test_non_root_is_rejected(self):
with patch.object(renew.os, 'geteuid', return_value=1000), \
self.assertRaises(RuntimeError):
renew.main()
def test_untrusted_certificate_never_replaces_files(self):
with patch.object(deploy.os, 'geteuid', return_value=0), \
patch.object(deploy, 'run', side_effect=subprocess.CalledProcessError(1, 'openssl')), \
patch.object(deploy, 'replace_file') as replace, \
class DeploymentTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
if os.geteuid() != 0:
raise RuntimeError('本回归必须在远程隔离 root 单元中执行')
cls.artifacts = Path(tempfile.mkdtemp(prefix='certificate-tests-', dir=Path.cwd()))
cls.ca = cls.artifacts / 'ca.crt'
cls.ca_key = cls.artifacts / 'ca.key'
cls.key = cls.artifacts / 'server.key'
cls.csr = cls.artifacts / 'server.csr'
cls.old = cls.artifacts / 'old.crt'
cls.new = cls.artifacts / 'new.crt'
cls.real_run = staticmethod(deploy.run)
cls.real_run('openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes',
'-keyout', str(cls.ca_key), '-out', str(cls.ca), '-days', '120',
'-subj', '/CN=Smail isolated test CA',
'-addext', 'keyUsage=critical,keyCertSign,cRLSign')
cls.real_run('openssl', 'req', '-new', '-newkey', 'rsa:2048', '-nodes',
'-keyout', str(cls.key), '-out', str(cls.csr), '-subj', '/CN=solit.cn',
'-addext', 'basicConstraints=critical,CA:FALSE',
'-addext', 'keyUsage=critical,digitalSignature,keyEncipherment',
'-addext', 'extendedKeyUsage=serverAuth',
'-addext', 'subjectAltName=' + ','.join('DNS:' + d for d in deploy.DOMAINS))
for target, serial, days in ((cls.old, '1', '20'), (cls.new, '2', '90')):
cls.real_run('openssl', 'x509', '-req', '-in', str(cls.csr), '-CA', str(cls.ca),
'-CAkey', str(cls.ca_key), '-set_serial', serial, '-days', days,
'-copy_extensions', 'copy', '-out', str(target))
print('保留隔离证据目录=' + str(cls.artifacts), flush=True)
def setUp(self):
self.directory = self.artifacts / self._testMethodName
self.directory.mkdir(mode=0o700)
self.source = self.directory / 'candidate'
self.target = self.directory / 'live'
self.backups = self.directory / 'backups'
for directory in (self.source, self.target, self.backups):
directory.mkdir(mode=0o700)
for path, value in ((self.source / 'solit.cn.crt', self.new.read_bytes()),
(self.source / 'solit.cn.key', self.key.read_bytes()),
(self.source / 'solit.cn.issuer.crt', self.ca.read_bytes()),
(self.target / 'custom.crt', self.old.read_bytes()),
(self.target / 'custom.key', self.key.read_bytes())):
path.write_bytes(value)
path.chmod(0o600)
self.calls = []
self.failure = None
self.failed_once = False
self.config = ('ssl_certificate ' + str(self.target / 'custom.crt') + ';\n'
+ 'ssl_certificate_key ' + str(self.target / 'custom.key') + ';\n')
for name, value in (('LINEAGE', self.source), ('TARGET', self.target),
('BACKUPS', self.backups), ('CA_FILE', str(self.ca)),
('LOCK', self.directory / 'renew.lock')):
patcher = patch.object(deploy, name, value)
patcher.start()
self.addCleanup(patcher.stop)
identity = patch.object(deploy.pwd, 'getpwnam', return_value=SimpleNamespace(pw_uid=0, pw_gid=0))
identity.start()
self.addCleanup(identity.stop)
commands = patch.object(deploy, 'run', side_effect=self.command)
commands.start()
self.addCleanup(commands.stop)
live = patch.object(deploy, 'verify_live_certificate')
self.real_verify = deploy.verify_live_certificate
self.live = live.start()
self.addCleanup(live.stop)
def command(self, *command, **kwargs):
self.calls.append(command)
if command == self.failure and not self.failed_once:
self.failed_once = True
raise subprocess.CalledProcessError(1, command)
if command == ('nginx', '-T'):
return self.config.encode()
if command[0] in ('nginx', 'systemctl'):
return b''
return self.real_run(*command, **kwargs)
def assert_old_untouched(self):
self.assertEqual((self.target / 'custom.crt').read_bytes(), self.old.read_bytes())
self.assertEqual((self.target / 'custom.key').read_bytes(), self.key.read_bytes())
def test_publish_valid_certificate_reloads_without_restart(self):
deploy.main()
self.assertEqual((self.target / 'custom.crt').read_bytes(), self.new.read_bytes())
self.assertIn(('systemctl', 'reload', 'nginx'), self.calls)
self.assertNotIn(('systemctl', 'stop', 'nginx'), self.calls)
self.assertNotIn(('systemctl', 'start', 'nginx'), self.calls)
self.live.assert_called_once_with(deploy.fingerprint(self.new))
for path in self.backups.rglob('*'):
self.assertEqual(path.stat().st_mode & 0o077, 0)
def test_unchanged_certificate_checks_live_without_reload(self):
(self.source / 'solit.cn.crt').write_bytes(self.old.read_bytes())
deploy.main()
self.assertNotIn(('systemctl', 'reload', 'nginx'), self.calls)
self.live.assert_called_once()
def test_interrupted_publish_is_reconciled_when_files_already_match(self):
(self.source / 'solit.cn.crt').write_bytes(self.old.read_bytes())
self.live.side_effect = [RuntimeError('上次重载前中断'), None]
deploy.main()
self.assertEqual(self.calls.count(('systemctl', 'reload', 'nginx')), 1)
self.assertEqual(self.live.call_count, 2)
def test_failed_reconciliation_does_not_report_success(self):
(self.source / 'solit.cn.crt').write_bytes(self.old.read_bytes())
self.live.side_effect = RuntimeError('线上证书不匹配')
with self.assertRaisesRegex(RuntimeError, '线上证书不匹配'):
deploy.main()
self.assertEqual(self.live.call_count, 2)
self.assert_old_untouched()
def test_untrusted_certificate_does_not_change_live_files(self):
with patch.object(deploy, 'CA_FILE', '/etc/ssl/certs/ca-certificates.crt'), \
self.assertRaises(subprocess.CalledProcessError):
deploy.main()
replace.assert_not_called()
self.assert_old_untouched()
self.assertNotIn(('systemctl', 'reload', 'nginx'), self.calls)
def test_mismatched_private_key_never_replaces_files(self):
def run(*command, **kwargs):
return b'private-public-key' if command[1] == 'pkey' else b'certificate-public-key'
with patch.object(deploy.os, 'geteuid', return_value=0), \
patch.object(deploy, 'run', side_effect=run), \
patch.object(deploy, 'replace_file') as replace, \
self.assertRaisesRegex(RuntimeError, '不匹配'):
def test_wrong_domain_does_not_change_live_files(self):
with patch.object(deploy, 'DOMAINS', ('wrong.example',)), \
self.assertRaises(subprocess.CalledProcessError):
deploy.main()
replace.assert_not_called()
self.assert_old_untouched()
def test_short_lived_certificate_is_rejected(self):
self.real_run('openssl', 'x509', '-req', '-in', str(self.csr), '-CA', str(self.ca),
'-CAkey', str(self.ca_key), '-set_serial', '3', '-days', '1',
'-copy_extensions', 'copy', '-out', str(self.source / 'solit.cn.crt'))
with self.assertRaises(subprocess.CalledProcessError):
deploy.main()
self.assert_old_untouched()
def test_mismatched_key_is_rejected(self):
(self.source / 'solit.cn.key').write_bytes(self.ca_key.read_bytes())
with self.assertRaisesRegex(RuntimeError, '不匹配'):
deploy.main()
self.assert_old_untouched()
def test_key_rotation_is_rejected(self):
(self.target / 'custom.key').write_bytes(self.ca_key.read_bytes())
with self.assertRaisesRegex(RuntimeError, '复用现有密钥'):
deploy.main()
self.assertEqual((self.target / 'custom.crt').read_bytes(), self.old.read_bytes())
def test_nginx_still_using_candidate_is_rejected(self):
self.config += 'ssl_certificate ' + str(self.source / 'solit.cn.crt') + ';\n'
with self.assertRaisesRegex(RuntimeError, '签发目录'):
deploy.main()
self.assert_old_untouched()
def test_missing_live_nginx_paths_is_rejected(self):
self.config = 'ssl_certificate /some/other/site.crt;\n'
with self.assertRaisesRegex(RuntimeError, '未找到'):
deploy.main()
self.assert_old_untouched()
def test_nginx_test_failure_rolls_back_and_reloads_old(self):
self.failure = ('nginx', '-t')
with self.assertRaises(subprocess.CalledProcessError):
deploy.main()
self.assert_old_untouched()
self.assertEqual(self.calls.count(('nginx', '-t')), 2)
self.assertIn(('systemctl', 'reload', 'nginx'), self.calls)
self.live.assert_called_once_with(deploy.fingerprint(self.old))
def test_reload_failure_rolls_back_and_reloads_old(self):
self.failure = ('systemctl', 'reload', 'nginx')
with self.assertRaises(subprocess.CalledProcessError):
deploy.main()
self.assert_old_untouched()
self.assertEqual(self.calls.count(('systemctl', 'reload', 'nginx')), 2)
def test_live_probe_failure_rolls_back_and_verifies_old(self):
self.live.side_effect = [RuntimeError('模拟线上证书不匹配'), None]
with self.assertRaisesRegex(RuntimeError, '模拟'):
deploy.main()
self.assert_old_untouched()
self.assertEqual(self.live.call_count, 2)
self.live.assert_called_with(deploy.fingerprint(self.old))
def test_unreadable_private_permissions_are_rejected(self):
(self.source / 'solit.cn.key').chmod(0o644)
with self.assertRaisesRegex(RuntimeError, '权限'):
deploy.main()
self.assert_old_untouched()
def test_symlink_source_is_rejected(self):
link = self.directory / 'linked-source'
link.symlink_to(self.source, target_is_directory=True)
with patch.object(deploy, 'LINEAGE', link), self.assertRaisesRegex(RuntimeError, '符号链接'):
deploy.main()
self.assert_old_untouched()
def test_concurrent_deployment_is_rejected(self):
with deploy.certificate_lock(), self.assertRaises(BlockingIOError):
deploy.main()
self.assert_old_untouched()
def test_unsafe_lock_permissions_are_rejected(self):
deploy.LOCK.touch(mode=0o644)
deploy.LOCK.chmod(0o644)
with self.assertRaisesRegex(RuntimeError, '锁文件'):
deploy.main()
self.assert_old_untouched()
def test_candidate_mutation_after_snapshot_cannot_change_published_bytes(self):
validate = deploy.validate
def mutate(snapshot):
(self.source / 'solit.cn.crt').write_bytes(b'changed-after-snapshot')
validate(snapshot)
with patch.object(deploy, 'validate', side_effect=mutate):
deploy.main()
self.assertEqual((self.target / 'custom.crt').read_bytes(), self.new.read_bytes())
def test_real_isolated_nginx_reload_and_rollback(self):
configuration = self.directory / 'nginx.conf'
configuration.write_text(
'worker_processes 1;\n'
+ 'pid ' + str(self.directory / 'nginx.pid') + ';\n'
+ 'error_log ' + str(self.directory / 'nginx-error.log') + ';\n'
+ 'events { worker_connections 32; }\n'
+ 'http {\naccess_log off;\n'
+ ''.join(name + '_temp_path ' + str(self.directory / name) + ';\n'
for name in ('client_body', 'proxy', 'fastcgi', 'uwsgi', 'scgi'))
+ 'server {\n'
+ 'listen 127.0.0.1:18443 ssl;\nlisten 127.0.0.1:18002 ssl;\n'
+ 'server_name mail.solit.cn;\n' + self.config
+ 'location / { return 200 "ready"; }\n}\n}\n', encoding='utf-8')
arguments = ('nginx', '-c', str(configuration), '-p', str(self.directory))
def isolated(*command, **kwargs):
self.calls.append(command)
if command == ('nginx', '-T'):
return self.real_run(*arguments, '-T')
if command == ('nginx', '-t'):
return self.real_run(*arguments, '-t')
if command == ('systemctl', 'is-active', '--quiet', 'nginx'):
if process.poll() is not None:
raise RuntimeError('隔离 Nginx 未运行')
return b''
if command == ('systemctl', 'reload', 'nginx'):
return self.real_run(*arguments, '-s', 'reload')
if command[0] == 'systemctl':
raise AssertionError('隔离测试禁止操作主机服务')
return self.real_run(*command, **kwargs)
with (self.directory / 'nginx-start.log').open('wb') as log:
process = subprocess.Popen((*arguments, '-g', 'daemon off;'), stdout=log, stderr=log)
try:
with patch.object(deploy, 'run', side_effect=isolated), \
patch.object(deploy, 'LIVE_PORTS', (18443, 18002)), \
patch.object(deploy, 'verify_live_certificate', side_effect=self.real_verify):
self.real_verify(deploy.fingerprint(self.old))
deploy.main()
self.real_verify(deploy.fingerprint(self.new))
self.assertIsNone(process.poll())
(self.source / 'solit.cn.crt').write_bytes(self.old.read_bytes())
attempted = False
def fail_after_live_update(expected):
nonlocal attempted
self.real_verify(expected)
if not attempted:
attempted = True
raise RuntimeError('模拟新证书上线后健康检查失败')
with patch.object(deploy, 'verify_live_certificate', side_effect=fail_after_live_update), \
self.assertRaisesRegex(RuntimeError, '健康检查失败'):
deploy.main()
self.real_verify(deploy.fingerprint(self.new))
self.assertEqual((self.target / 'custom.crt').read_bytes(), self.new.read_bytes())
self.assertIsNone(process.poll())
finally:
if process.poll() is None:
process.send_signal(signal.SIGQUIT)
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=10)
if __name__ == '__main__':
+1 -1
View File
@@ -22,7 +22,7 @@ def main():
or len(target.parts) < 5 or target.parts[1:3] != ("var", "tmp")
or not target.parts[3].startswith("smail-review-")):
raise ValueError("上传路径必须位于项目审查临时目录")
command = "install -d -m 0755 " + shlex.quote(str(target.parent))
command = "install -d -m 0700 " + shlex.quote(str(target.parent))
_, output, error = client.exec_command(command)
if output.channel.recv_exit_status() != 0:
raise RuntimeError(error.read().decode("utf-8", "replace"))