1 parent
06c49e88b7
commit
9aff6f9dfb
12 files changed
+879
-110
No files matched your search
@@ -0,0 +1,229 @@
|
||||
"""在远程主机迁移 Smail 的 Nginx 证书路径,保留回退和连续探测证据。"""
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
from html.parser import HTMLParser
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import deploy_certificate as deploy
|
||||
|
||||
|
||||
CONFIG = Path('/etc/nginx/conf.d/smail.conf')
|
||||
ROOT = Path('/var/tmp')
|
||||
LIMIT = 8 * 1024 * 1024
|
||||
|
||||
|
||||
def run(*command):
|
||||
return subprocess.run(command, check=True, capture_output=True, timeout=15).stdout
|
||||
|
||||
|
||||
def digest(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def state():
|
||||
return run('systemctl', 'show', 'nginx', 'smail', '-p', 'Id', '-p', 'MainPID',
|
||||
'-p', 'ActiveState', '-p', 'InvocationID').decode()
|
||||
|
||||
|
||||
def request(host, port, path='/', slow=False):
|
||||
raw = socket.socket()
|
||||
raw.settimeout(15 if slow else 3)
|
||||
if slow:
|
||||
raw.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 8192)
|
||||
try:
|
||||
raw.connect(('127.0.0.1', port))
|
||||
secure = ssl.create_default_context().wrap_socket(raw, server_hostname=host)
|
||||
except BaseException:
|
||||
raw.close()
|
||||
raise
|
||||
try:
|
||||
authority = host if port == 443 else host + ':' + str(port)
|
||||
secure.sendall(('GET ' + path + ' HTTP/1.1\r\nHost: ' + authority
|
||||
+ '\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n').encode())
|
||||
response = http.client.HTTPResponse(secure)
|
||||
response.begin()
|
||||
if response.status != 200:
|
||||
raise RuntimeError('HTTPS 状态异常:' + str(response.status))
|
||||
if slow:
|
||||
return secure, response
|
||||
with response:
|
||||
data = response.read(LIMIT + 1)
|
||||
if len(data) > LIMIT:
|
||||
raise RuntimeError('响应超出本次探测上限')
|
||||
secure.close()
|
||||
return data
|
||||
except BaseException:
|
||||
secure.close()
|
||||
raise
|
||||
|
||||
|
||||
class Scripts(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.paths = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
source = dict(attrs).get('src', '')
|
||||
if tag == 'script' and source.startswith('/assets/') and source.endswith('.js'):
|
||||
self.paths.append(source)
|
||||
|
||||
|
||||
def atomic_config(data, mode):
|
||||
staged = CONFIG.with_name(CONFIG.name + '.candidate-' + uuid.uuid4().hex)
|
||||
fd = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode)
|
||||
with os.fdopen(fd, 'wb') as stream:
|
||||
os.fchmod(stream.fileno(), mode)
|
||||
stream.write(data)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(staged, CONFIG)
|
||||
fd = os.open(CONFIG.parent, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--evidence-dir', required=True, type=Path)
|
||||
parser.add_argument('--expected-config-sha256', required=True)
|
||||
parser.add_argument('--asset-path', help='用于下载连续性检查的已部署大型脚本路径')
|
||||
args = parser.parse_args()
|
||||
evidence = args.evidence_dir.resolve(strict=True)
|
||||
if (os.geteuid() != 0 or evidence.parent != ROOT
|
||||
or not evidence.name.startswith('smail-review-cert-safe-')
|
||||
or evidence.stat().st_uid != 0 or evidence.stat().st_mode & 0o077):
|
||||
parser.error('必须在远程项目专用且受保护的证据目录中执行')
|
||||
report_path = evidence / 'cutover-result.json'
|
||||
if report_path.exists():
|
||||
parser.error('禁止覆盖已有迁移证据')
|
||||
if args.asset_path and not re.fullmatch(r'/assets/[a-zA-Z0-9_-]+\.js', args.asset_path):
|
||||
parser.error('下载探测仅允许项目静态脚本路径')
|
||||
previous = CONFIG.read_bytes()
|
||||
if digest(previous) != args.expected_config_sha256 or CONFIG.is_symlink():
|
||||
parser.error('线上配置与已审查版本不符')
|
||||
proposed = (evidence / 'smail.conf').read_bytes()
|
||||
expected = previous.replace(
|
||||
b'/var/lib/smail-acme-tls/certificates/solit.cn.crt', b'/opt/smail/ssl/custom.crt').replace(
|
||||
b'/var/lib/smail-acme-tls/certificates/solit.cn.key', b'/opt/smail/ssl/custom.key')
|
||||
if proposed != expected or proposed == previous:
|
||||
parser.error('候选配置必须且只能改变 Smail 证书读取路径')
|
||||
run('nginx', '-t')
|
||||
run('systemctl', 'is-active', '--quiet', 'nginx', 'smail')
|
||||
protected = [Path('/opt/smail/smail'), Path('/opt/smail/smail-admin'),
|
||||
Path('/opt/smail/ssl/custom.crt'), Path('/opt/smail/ssl/custom.key'),
|
||||
Path('/usr/local/libexec/smail-acme/renew_certificate.py'),
|
||||
Path('/usr/local/libexec/smail-acme/deploy_certificate.py'),
|
||||
Path('/etc/systemd/system/smail-certificate-renew.service')]
|
||||
protected.extend(path for path in Path('/etc/nginx/conf.d').glob('*.conf') if path != CONFIG)
|
||||
protected_before = {str(path): digest(path.read_bytes()) for path in protected}
|
||||
before_state = state()
|
||||
backup = evidence / 'smail.conf.before'
|
||||
with backup.open('xb') as stream:
|
||||
stream.write(previous)
|
||||
backup.chmod(0o600)
|
||||
report = {'startedUTC': datetime.now(timezone.utc).isoformat(), 'status': 'running',
|
||||
'beforeConfigSHA256': digest(previous), 'afterConfigSHA256': digest(proposed),
|
||||
'beforeState': before_state, 'productionCertificateChanged': False,
|
||||
'dnsRenewalActivated': False, 'protectedBefore': protected_before, 'samples': []}
|
||||
stop = threading.Event()
|
||||
threads = []
|
||||
pending = None
|
||||
changed = False
|
||||
mode = CONFIG.stat().st_mode & 0o777
|
||||
|
||||
def monitor(host, port):
|
||||
while not stop.is_set():
|
||||
sample = {'utc': datetime.now(timezone.utc).isoformat(), 'host': host, 'port': port}
|
||||
try:
|
||||
request(host, port)
|
||||
sample['ok'] = True
|
||||
except Exception as error:
|
||||
sample.update(ok=False, error=type(error).__name__)
|
||||
report['samples'].append(sample)
|
||||
stop.wait(0.2)
|
||||
|
||||
try:
|
||||
scripts = Scripts()
|
||||
scripts.feed(request('mail.solit.cn', 443).decode('utf-8'))
|
||||
if not scripts.paths:
|
||||
raise RuntimeError('未找到用于既有连接测试的真实静态资源')
|
||||
asset = args.asset_path or scripts.paths[0]
|
||||
expected_body = request('mail.solit.cn', 443, asset)
|
||||
if len(expected_body) < 65536:
|
||||
raise RuntimeError('静态资源过小,无法进行本次下载连续性检查')
|
||||
for host, port in (('mail.solit.cn', 443), ('web.apmmk.cn', 443)):
|
||||
thread = threading.Thread(target=monitor, args=(host, port))
|
||||
thread.start()
|
||||
threads.append(thread)
|
||||
time.sleep(2)
|
||||
if len(report['samples']) < 4 or any(not item['ok'] for item in report['samples']):
|
||||
raise RuntimeError('迁移前连续探测未通过')
|
||||
secure, response = request('mail.solit.cn', 443, asset, slow=True)
|
||||
pending = (secure, response)
|
||||
prefix = response.read(1024)
|
||||
report['connectionOpenedBeforeReload'] = True
|
||||
changed = True
|
||||
atomic_config(proposed, mode)
|
||||
run('nginx', '-t')
|
||||
run('systemctl', 'reload', 'nginx')
|
||||
report['reloadedUTC'] = datetime.now(timezone.utc).isoformat()
|
||||
body = prefix + response.read(LIMIT + 1)
|
||||
if body != expected_body:
|
||||
raise RuntimeError('重载前已建立的 HTTPS 下载未完整完成')
|
||||
report['download'] = {'path': asset, 'bytes': len(body), 'sha256': digest(body), 'completed': True}
|
||||
time.sleep(4)
|
||||
stop.set()
|
||||
for thread in threads:
|
||||
thread.join(timeout=5)
|
||||
if any(thread.is_alive() for thread in threads):
|
||||
raise RuntimeError('连续探测未在规定时间内结束')
|
||||
after_state = state()
|
||||
report['afterState'] = after_state
|
||||
if after_state != before_state:
|
||||
raise RuntimeError('迁移期间主服务进程或运行状态发生变化')
|
||||
if any(digest(path.read_bytes()) != protected_before[str(path)] for path in protected):
|
||||
raise RuntimeError('迁移期间保护范围内的文件发生变化')
|
||||
if any(not item['ok'] for item in report['samples']):
|
||||
raise RuntimeError('迁移期间连续 HTTPS 探测失败')
|
||||
report['protectedFilesUnchanged'] = True
|
||||
report['status'] = 'passed'
|
||||
except BaseException as error:
|
||||
report['status'] = 'failed'
|
||||
report['errorType'] = type(error).__name__
|
||||
if changed:
|
||||
atomic_config(previous, mode)
|
||||
run('nginx', '-t')
|
||||
run('systemctl', 'reload', 'nginx')
|
||||
report['configRolledBack'] = True
|
||||
raise
|
||||
finally:
|
||||
stop.set()
|
||||
for thread in threads:
|
||||
thread.join(timeout=5)
|
||||
if pending is not None:
|
||||
pending[1].close()
|
||||
pending[0].close()
|
||||
report['finishedUTC'] = datetime.now(timezone.utc).isoformat()
|
||||
report_path.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
||||
report_path.chmod(0o600)
|
||||
print(json.dumps({key: value for key, value in report.items() if key != 'samples'}, indent=2))
|
||||
print('HTTPS 探测次数=' + str(len(report['samples'])))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
with deploy.certificate_lock():
|
||||
main()
|
||||
@@ -14,6 +14,7 @@ from unittest.mock import patch
|
||||
|
||||
SERVER = 'https://acme-staging-v02.api.letsencrypt.org/directory'
|
||||
SOURCE = Path('/usr/local/libexec/smail-acme/renew_certificate.py')
|
||||
LEGACY_SOURCE_SHA256 = '6d0abbe9466f8becd61b1cb3c750bfffb938d16c5a9bcae0b96d1451227d4216'
|
||||
DOMAINS = {'solit.cn', 'mail.solit.cn', 'smtp.solit.cn', 'imap.solit.cn', 'pop.solit.cn'}
|
||||
RUN = subprocess.run
|
||||
|
||||
@@ -53,6 +54,8 @@ def main():
|
||||
parser.error('演练目录必须归 root 所有且禁止其他用户访问')
|
||||
if (evidence / 'result.json').exists():
|
||||
parser.error('禁止覆盖已有演练结果,请使用新的证据目录')
|
||||
if digest(SOURCE) != LEGACY_SOURCE_SHA256:
|
||||
parser.error('本工具仅用于已封存的停机续期实现,不能用于 DNS 无停机续期验收')
|
||||
run(['nginx', '-t'], check=True)
|
||||
run(['systemctl', 'is-active', '--quiet', 'nginx'], check=True)
|
||||
|
||||
|
||||
@@ -1,36 +1,54 @@
|
||||
"""证书更新脚本的隔离回归,不操作生产文件或服务。"""
|
||||
|
||||
import importlib.util
|
||||
from contextlib import nullcontext
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import mock_open, patch
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
BASE = Path(__file__).resolve().parent
|
||||
SCRIPTS = BASE if (BASE / 'deploy_certificate.py').exists() else BASE.parents[1] / 'deploy/nginx'
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
|
||||
def load(name):
|
||||
path = BASE / (name + '.py')
|
||||
if not path.exists():
|
||||
path = BASE.parents[1] / 'deploy/nginx' / (name + '.py')
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
import deploy_certificate as deploy
|
||||
import renew_certificate as renew
|
||||
import nginx_certificate_cutover as cutover
|
||||
|
||||
|
||||
renew = load('renew_certificate')
|
||||
deploy = load('deploy_certificate')
|
||||
class CutoverTests(unittest.TestCase):
|
||||
def test_atomic_config_preserves_mode_under_restrictive_umask(self):
|
||||
directory = Path(tempfile.mkdtemp(prefix='cutover-tests-', dir=Path.cwd()))
|
||||
path = directory / 'smail.conf'
|
||||
previous = os.umask(0o077)
|
||||
try:
|
||||
with patch.object(cutover, 'CONFIG', path):
|
||||
cutover.atomic_config(b'candidate\n', 0o644)
|
||||
self.assertEqual(path.read_bytes(), b'candidate\n')
|
||||
self.assertEqual(path.stat().st_mode & 0o777, 0o644)
|
||||
cutover.atomic_config(b'previous\n', 0o644)
|
||||
self.assertEqual(path.read_bytes(), b'previous\n')
|
||||
self.assertEqual(path.stat().st_mode & 0o777, 0o644)
|
||||
finally:
|
||||
os.umask(previous)
|
||||
|
||||
|
||||
class RenewalTests(unittest.TestCase):
|
||||
def execute(self, callback):
|
||||
def execute(self, callback, provider='alidns'):
|
||||
with patch.object(renew.os, 'geteuid', return_value=0), \
|
||||
patch('builtins.open', mock_open()), \
|
||||
patch.object(renew.fcntl, 'flock'), \
|
||||
patch.object(deploy, 'certificate_lock', return_value=nullcontext()), \
|
||||
patch.object(deploy, 'check_nginx_paths'), \
|
||||
patch.object(deploy, 'publish') as publish, \
|
||||
patch.dict(os.environ, {'SMAIL_ACME_DNS_PROVIDER': provider}), \
|
||||
patch.object(renew.subprocess, 'run', side_effect=callback):
|
||||
renew.main()
|
||||
return publish
|
||||
|
||||
def test_not_due_keeps_nginx_running(self):
|
||||
calls = []
|
||||
@@ -39,13 +57,14 @@ class RenewalTests(unittest.TestCase):
|
||||
calls.append(command)
|
||||
return subprocess.CompletedProcess(command, 0)
|
||||
|
||||
self.execute(run)
|
||||
publish = self.execute(run, provider='')
|
||||
self.assertNotIn(['systemctl', 'stop', 'nginx'], calls)
|
||||
self.assertIn(['/usr/local/libexec/smail-acme/deploy_certificate.py'], calls)
|
||||
self.assertFalse(any(command[0].endswith('/lego') for command in calls))
|
||||
publish.assert_called_once()
|
||||
|
||||
def test_due_failure_restores_nginx(self):
|
||||
def test_due_failure_never_stops_nginx_or_publishes(self):
|
||||
for failure in (subprocess.CalledProcessError(1, 'lego'),
|
||||
subprocess.TimeoutExpired('lego', 150)):
|
||||
subprocess.TimeoutExpired('lego', renew.RENEW_TIMEOUT)):
|
||||
with self.subTest(failure=type(failure).__name__):
|
||||
calls = []
|
||||
|
||||
@@ -57,10 +76,16 @@ class RenewalTests(unittest.TestCase):
|
||||
raise failure
|
||||
return subprocess.CompletedProcess(command, 0)
|
||||
|
||||
with self.assertRaises(type(failure)):
|
||||
self.execute(run)
|
||||
self.assertEqual(calls[-1], ['systemctl', 'start', 'nginx'])
|
||||
self.assertNotIn(['/usr/local/libexec/smail-acme/deploy_certificate.py'], calls)
|
||||
with patch.object(deploy, 'certificate_lock', return_value=nullcontext()), \
|
||||
patch.object(deploy, 'check_nginx_paths'), \
|
||||
patch.object(deploy, 'publish') as publish, \
|
||||
patch.dict(os.environ, {'SMAIL_ACME_DNS_PROVIDER': 'alidns'}), \
|
||||
patch.object(renew.subprocess, 'run', side_effect=run), \
|
||||
self.assertRaises(type(failure)):
|
||||
renew.main()
|
||||
publish.assert_not_called()
|
||||
self.assertFalse(any(command[:2] in (['systemctl', 'stop'], ['systemctl', 'start'])
|
||||
for command in calls))
|
||||
|
||||
def test_due_success_is_bounded_and_reuses_key(self):
|
||||
calls = []
|
||||
@@ -70,38 +95,336 @@ class RenewalTests(unittest.TestCase):
|
||||
if '-checkend' in command:
|
||||
return subprocess.CompletedProcess(command, 1)
|
||||
if command[0].endswith('/lego'):
|
||||
self.assertEqual(kwargs['timeout'], 150)
|
||||
self.assertEqual(kwargs['timeout'], 600)
|
||||
self.assertEqual(command[command.index('--dns') + 1], 'alidns')
|
||||
self.assertNotIn('--tls', command)
|
||||
self.assertIn('--server', command)
|
||||
self.assertIn('--reuse-key', command)
|
||||
self.assertIn('--no-random-sleep', command)
|
||||
return subprocess.CompletedProcess(command, 0)
|
||||
|
||||
publish = self.execute(run)
|
||||
publish.assert_called_once()
|
||||
self.assertTrue(any(command[0].endswith('/lego') for command in calls))
|
||||
self.assertFalse(any(command[:2] in (['systemctl', 'stop'], ['systemctl', 'start'])
|
||||
for command in calls))
|
||||
|
||||
def test_due_uses_live_certificate_not_candidate(self):
|
||||
def run(command, **kwargs):
|
||||
if command[0] == 'openssl':
|
||||
self.assertEqual(command[command.index('-in') + 1], str(deploy.TARGET / 'custom.crt'))
|
||||
return subprocess.CompletedProcess(command, 0)
|
||||
|
||||
self.execute(run)
|
||||
self.assertLess(calls.index(['systemctl', 'start', 'nginx']),
|
||||
calls.index(['/usr/local/libexec/smail-acme/deploy_certificate.py']))
|
||||
|
||||
def test_missing_or_unsafe_provider_fails_before_lego(self):
|
||||
for provider in ('', '--tls', 'exec', 'alidns --tls', '../alidns'):
|
||||
calls = []
|
||||
|
||||
def run(command, **kwargs):
|
||||
calls.append(command)
|
||||
return subprocess.CompletedProcess(command, 1 if '-checkend' in command else 0)
|
||||
|
||||
with self.subTest(provider=provider), self.assertRaises(RuntimeError):
|
||||
self.execute(run, provider)
|
||||
self.assertFalse(any(command[0].endswith('/lego') for command in calls))
|
||||
|
||||
def test_invalid_expiry_check_fails_closed(self):
|
||||
def run(command, **kwargs):
|
||||
return subprocess.CompletedProcess(command, 2 if '-checkend' in command else 0)
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, '无法判断'):
|
||||
self.execute(run)
|
||||
|
||||
def test_non_root_is_rejected(self):
|
||||
with patch.object(renew.os, 'geteuid', return_value=1000), \
|
||||
self.assertRaises(RuntimeError):
|
||||
renew.main()
|
||||
|
||||
def test_untrusted_certificate_never_replaces_files(self):
|
||||
with patch.object(deploy.os, 'geteuid', return_value=0), \
|
||||
patch.object(deploy, 'run', side_effect=subprocess.CalledProcessError(1, 'openssl')), \
|
||||
patch.object(deploy, 'replace_file') as replace, \
|
||||
|
||||
class DeploymentTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if os.geteuid() != 0:
|
||||
raise RuntimeError('本回归必须在远程隔离 root 单元中执行')
|
||||
cls.artifacts = Path(tempfile.mkdtemp(prefix='certificate-tests-', dir=Path.cwd()))
|
||||
cls.ca = cls.artifacts / 'ca.crt'
|
||||
cls.ca_key = cls.artifacts / 'ca.key'
|
||||
cls.key = cls.artifacts / 'server.key'
|
||||
cls.csr = cls.artifacts / 'server.csr'
|
||||
cls.old = cls.artifacts / 'old.crt'
|
||||
cls.new = cls.artifacts / 'new.crt'
|
||||
cls.real_run = staticmethod(deploy.run)
|
||||
cls.real_run('openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes',
|
||||
'-keyout', str(cls.ca_key), '-out', str(cls.ca), '-days', '120',
|
||||
'-subj', '/CN=Smail isolated test CA',
|
||||
'-addext', 'keyUsage=critical,keyCertSign,cRLSign')
|
||||
cls.real_run('openssl', 'req', '-new', '-newkey', 'rsa:2048', '-nodes',
|
||||
'-keyout', str(cls.key), '-out', str(cls.csr), '-subj', '/CN=solit.cn',
|
||||
'-addext', 'basicConstraints=critical,CA:FALSE',
|
||||
'-addext', 'keyUsage=critical,digitalSignature,keyEncipherment',
|
||||
'-addext', 'extendedKeyUsage=serverAuth',
|
||||
'-addext', 'subjectAltName=' + ','.join('DNS:' + d for d in deploy.DOMAINS))
|
||||
for target, serial, days in ((cls.old, '1', '20'), (cls.new, '2', '90')):
|
||||
cls.real_run('openssl', 'x509', '-req', '-in', str(cls.csr), '-CA', str(cls.ca),
|
||||
'-CAkey', str(cls.ca_key), '-set_serial', serial, '-days', days,
|
||||
'-copy_extensions', 'copy', '-out', str(target))
|
||||
print('保留隔离证据目录=' + str(cls.artifacts), flush=True)
|
||||
|
||||
def setUp(self):
|
||||
self.directory = self.artifacts / self._testMethodName
|
||||
self.directory.mkdir(mode=0o700)
|
||||
self.source = self.directory / 'candidate'
|
||||
self.target = self.directory / 'live'
|
||||
self.backups = self.directory / 'backups'
|
||||
for directory in (self.source, self.target, self.backups):
|
||||
directory.mkdir(mode=0o700)
|
||||
for path, value in ((self.source / 'solit.cn.crt', self.new.read_bytes()),
|
||||
(self.source / 'solit.cn.key', self.key.read_bytes()),
|
||||
(self.source / 'solit.cn.issuer.crt', self.ca.read_bytes()),
|
||||
(self.target / 'custom.crt', self.old.read_bytes()),
|
||||
(self.target / 'custom.key', self.key.read_bytes())):
|
||||
path.write_bytes(value)
|
||||
path.chmod(0o600)
|
||||
self.calls = []
|
||||
self.failure = None
|
||||
self.failed_once = False
|
||||
self.config = ('ssl_certificate ' + str(self.target / 'custom.crt') + ';\n'
|
||||
+ 'ssl_certificate_key ' + str(self.target / 'custom.key') + ';\n')
|
||||
for name, value in (('LINEAGE', self.source), ('TARGET', self.target),
|
||||
('BACKUPS', self.backups), ('CA_FILE', str(self.ca)),
|
||||
('LOCK', self.directory / 'renew.lock')):
|
||||
patcher = patch.object(deploy, name, value)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
identity = patch.object(deploy.pwd, 'getpwnam', return_value=SimpleNamespace(pw_uid=0, pw_gid=0))
|
||||
identity.start()
|
||||
self.addCleanup(identity.stop)
|
||||
commands = patch.object(deploy, 'run', side_effect=self.command)
|
||||
commands.start()
|
||||
self.addCleanup(commands.stop)
|
||||
live = patch.object(deploy, 'verify_live_certificate')
|
||||
self.real_verify = deploy.verify_live_certificate
|
||||
self.live = live.start()
|
||||
self.addCleanup(live.stop)
|
||||
|
||||
def command(self, *command, **kwargs):
|
||||
self.calls.append(command)
|
||||
if command == self.failure and not self.failed_once:
|
||||
self.failed_once = True
|
||||
raise subprocess.CalledProcessError(1, command)
|
||||
if command == ('nginx', '-T'):
|
||||
return self.config.encode()
|
||||
if command[0] in ('nginx', 'systemctl'):
|
||||
return b''
|
||||
return self.real_run(*command, **kwargs)
|
||||
|
||||
def assert_old_untouched(self):
|
||||
self.assertEqual((self.target / 'custom.crt').read_bytes(), self.old.read_bytes())
|
||||
self.assertEqual((self.target / 'custom.key').read_bytes(), self.key.read_bytes())
|
||||
|
||||
def test_publish_valid_certificate_reloads_without_restart(self):
|
||||
deploy.main()
|
||||
self.assertEqual((self.target / 'custom.crt').read_bytes(), self.new.read_bytes())
|
||||
self.assertIn(('systemctl', 'reload', 'nginx'), self.calls)
|
||||
self.assertNotIn(('systemctl', 'stop', 'nginx'), self.calls)
|
||||
self.assertNotIn(('systemctl', 'start', 'nginx'), self.calls)
|
||||
self.live.assert_called_once_with(deploy.fingerprint(self.new))
|
||||
for path in self.backups.rglob('*'):
|
||||
self.assertEqual(path.stat().st_mode & 0o077, 0)
|
||||
|
||||
def test_unchanged_certificate_checks_live_without_reload(self):
|
||||
(self.source / 'solit.cn.crt').write_bytes(self.old.read_bytes())
|
||||
deploy.main()
|
||||
self.assertNotIn(('systemctl', 'reload', 'nginx'), self.calls)
|
||||
self.live.assert_called_once()
|
||||
|
||||
def test_interrupted_publish_is_reconciled_when_files_already_match(self):
|
||||
(self.source / 'solit.cn.crt').write_bytes(self.old.read_bytes())
|
||||
self.live.side_effect = [RuntimeError('上次重载前中断'), None]
|
||||
deploy.main()
|
||||
self.assertEqual(self.calls.count(('systemctl', 'reload', 'nginx')), 1)
|
||||
self.assertEqual(self.live.call_count, 2)
|
||||
|
||||
def test_failed_reconciliation_does_not_report_success(self):
|
||||
(self.source / 'solit.cn.crt').write_bytes(self.old.read_bytes())
|
||||
self.live.side_effect = RuntimeError('线上证书不匹配')
|
||||
with self.assertRaisesRegex(RuntimeError, '线上证书不匹配'):
|
||||
deploy.main()
|
||||
self.assertEqual(self.live.call_count, 2)
|
||||
self.assert_old_untouched()
|
||||
|
||||
def test_untrusted_certificate_does_not_change_live_files(self):
|
||||
with patch.object(deploy, 'CA_FILE', '/etc/ssl/certs/ca-certificates.crt'), \
|
||||
self.assertRaises(subprocess.CalledProcessError):
|
||||
deploy.main()
|
||||
replace.assert_not_called()
|
||||
self.assert_old_untouched()
|
||||
self.assertNotIn(('systemctl', 'reload', 'nginx'), self.calls)
|
||||
|
||||
def test_mismatched_private_key_never_replaces_files(self):
|
||||
def run(*command, **kwargs):
|
||||
return b'private-public-key' if command[1] == 'pkey' else b'certificate-public-key'
|
||||
|
||||
with patch.object(deploy.os, 'geteuid', return_value=0), \
|
||||
patch.object(deploy, 'run', side_effect=run), \
|
||||
patch.object(deploy, 'replace_file') as replace, \
|
||||
self.assertRaisesRegex(RuntimeError, '不匹配'):
|
||||
def test_wrong_domain_does_not_change_live_files(self):
|
||||
with patch.object(deploy, 'DOMAINS', ('wrong.example',)), \
|
||||
self.assertRaises(subprocess.CalledProcessError):
|
||||
deploy.main()
|
||||
replace.assert_not_called()
|
||||
self.assert_old_untouched()
|
||||
|
||||
def test_short_lived_certificate_is_rejected(self):
|
||||
self.real_run('openssl', 'x509', '-req', '-in', str(self.csr), '-CA', str(self.ca),
|
||||
'-CAkey', str(self.ca_key), '-set_serial', '3', '-days', '1',
|
||||
'-copy_extensions', 'copy', '-out', str(self.source / 'solit.cn.crt'))
|
||||
with self.assertRaises(subprocess.CalledProcessError):
|
||||
deploy.main()
|
||||
self.assert_old_untouched()
|
||||
|
||||
def test_mismatched_key_is_rejected(self):
|
||||
(self.source / 'solit.cn.key').write_bytes(self.ca_key.read_bytes())
|
||||
with self.assertRaisesRegex(RuntimeError, '不匹配'):
|
||||
deploy.main()
|
||||
self.assert_old_untouched()
|
||||
|
||||
def test_key_rotation_is_rejected(self):
|
||||
(self.target / 'custom.key').write_bytes(self.ca_key.read_bytes())
|
||||
with self.assertRaisesRegex(RuntimeError, '复用现有密钥'):
|
||||
deploy.main()
|
||||
self.assertEqual((self.target / 'custom.crt').read_bytes(), self.old.read_bytes())
|
||||
|
||||
def test_nginx_still_using_candidate_is_rejected(self):
|
||||
self.config += 'ssl_certificate ' + str(self.source / 'solit.cn.crt') + ';\n'
|
||||
with self.assertRaisesRegex(RuntimeError, '签发目录'):
|
||||
deploy.main()
|
||||
self.assert_old_untouched()
|
||||
|
||||
def test_missing_live_nginx_paths_is_rejected(self):
|
||||
self.config = 'ssl_certificate /some/other/site.crt;\n'
|
||||
with self.assertRaisesRegex(RuntimeError, '未找到'):
|
||||
deploy.main()
|
||||
self.assert_old_untouched()
|
||||
|
||||
def test_nginx_test_failure_rolls_back_and_reloads_old(self):
|
||||
self.failure = ('nginx', '-t')
|
||||
with self.assertRaises(subprocess.CalledProcessError):
|
||||
deploy.main()
|
||||
self.assert_old_untouched()
|
||||
self.assertEqual(self.calls.count(('nginx', '-t')), 2)
|
||||
self.assertIn(('systemctl', 'reload', 'nginx'), self.calls)
|
||||
self.live.assert_called_once_with(deploy.fingerprint(self.old))
|
||||
|
||||
def test_reload_failure_rolls_back_and_reloads_old(self):
|
||||
self.failure = ('systemctl', 'reload', 'nginx')
|
||||
with self.assertRaises(subprocess.CalledProcessError):
|
||||
deploy.main()
|
||||
self.assert_old_untouched()
|
||||
self.assertEqual(self.calls.count(('systemctl', 'reload', 'nginx')), 2)
|
||||
|
||||
def test_live_probe_failure_rolls_back_and_verifies_old(self):
|
||||
self.live.side_effect = [RuntimeError('模拟线上证书不匹配'), None]
|
||||
with self.assertRaisesRegex(RuntimeError, '模拟'):
|
||||
deploy.main()
|
||||
self.assert_old_untouched()
|
||||
self.assertEqual(self.live.call_count, 2)
|
||||
self.live.assert_called_with(deploy.fingerprint(self.old))
|
||||
|
||||
def test_unreadable_private_permissions_are_rejected(self):
|
||||
(self.source / 'solit.cn.key').chmod(0o644)
|
||||
with self.assertRaisesRegex(RuntimeError, '权限'):
|
||||
deploy.main()
|
||||
self.assert_old_untouched()
|
||||
|
||||
def test_symlink_source_is_rejected(self):
|
||||
link = self.directory / 'linked-source'
|
||||
link.symlink_to(self.source, target_is_directory=True)
|
||||
with patch.object(deploy, 'LINEAGE', link), self.assertRaisesRegex(RuntimeError, '符号链接'):
|
||||
deploy.main()
|
||||
self.assert_old_untouched()
|
||||
|
||||
def test_concurrent_deployment_is_rejected(self):
|
||||
with deploy.certificate_lock(), self.assertRaises(BlockingIOError):
|
||||
deploy.main()
|
||||
self.assert_old_untouched()
|
||||
|
||||
def test_unsafe_lock_permissions_are_rejected(self):
|
||||
deploy.LOCK.touch(mode=0o644)
|
||||
deploy.LOCK.chmod(0o644)
|
||||
with self.assertRaisesRegex(RuntimeError, '锁文件'):
|
||||
deploy.main()
|
||||
self.assert_old_untouched()
|
||||
|
||||
def test_candidate_mutation_after_snapshot_cannot_change_published_bytes(self):
|
||||
validate = deploy.validate
|
||||
|
||||
def mutate(snapshot):
|
||||
(self.source / 'solit.cn.crt').write_bytes(b'changed-after-snapshot')
|
||||
validate(snapshot)
|
||||
|
||||
with patch.object(deploy, 'validate', side_effect=mutate):
|
||||
deploy.main()
|
||||
self.assertEqual((self.target / 'custom.crt').read_bytes(), self.new.read_bytes())
|
||||
|
||||
def test_real_isolated_nginx_reload_and_rollback(self):
|
||||
configuration = self.directory / 'nginx.conf'
|
||||
configuration.write_text(
|
||||
'worker_processes 1;\n'
|
||||
+ 'pid ' + str(self.directory / 'nginx.pid') + ';\n'
|
||||
+ 'error_log ' + str(self.directory / 'nginx-error.log') + ';\n'
|
||||
+ 'events { worker_connections 32; }\n'
|
||||
+ 'http {\naccess_log off;\n'
|
||||
+ ''.join(name + '_temp_path ' + str(self.directory / name) + ';\n'
|
||||
for name in ('client_body', 'proxy', 'fastcgi', 'uwsgi', 'scgi'))
|
||||
+ 'server {\n'
|
||||
+ 'listen 127.0.0.1:18443 ssl;\nlisten 127.0.0.1:18002 ssl;\n'
|
||||
+ 'server_name mail.solit.cn;\n' + self.config
|
||||
+ 'location / { return 200 "ready"; }\n}\n}\n', encoding='utf-8')
|
||||
arguments = ('nginx', '-c', str(configuration), '-p', str(self.directory))
|
||||
|
||||
def isolated(*command, **kwargs):
|
||||
self.calls.append(command)
|
||||
if command == ('nginx', '-T'):
|
||||
return self.real_run(*arguments, '-T')
|
||||
if command == ('nginx', '-t'):
|
||||
return self.real_run(*arguments, '-t')
|
||||
if command == ('systemctl', 'is-active', '--quiet', 'nginx'):
|
||||
if process.poll() is not None:
|
||||
raise RuntimeError('隔离 Nginx 未运行')
|
||||
return b''
|
||||
if command == ('systemctl', 'reload', 'nginx'):
|
||||
return self.real_run(*arguments, '-s', 'reload')
|
||||
if command[0] == 'systemctl':
|
||||
raise AssertionError('隔离测试禁止操作主机服务')
|
||||
return self.real_run(*command, **kwargs)
|
||||
|
||||
with (self.directory / 'nginx-start.log').open('wb') as log:
|
||||
process = subprocess.Popen((*arguments, '-g', 'daemon off;'), stdout=log, stderr=log)
|
||||
try:
|
||||
with patch.object(deploy, 'run', side_effect=isolated), \
|
||||
patch.object(deploy, 'LIVE_PORTS', (18443, 18002)), \
|
||||
patch.object(deploy, 'verify_live_certificate', side_effect=self.real_verify):
|
||||
self.real_verify(deploy.fingerprint(self.old))
|
||||
deploy.main()
|
||||
self.real_verify(deploy.fingerprint(self.new))
|
||||
self.assertIsNone(process.poll())
|
||||
(self.source / 'solit.cn.crt').write_bytes(self.old.read_bytes())
|
||||
attempted = False
|
||||
|
||||
def fail_after_live_update(expected):
|
||||
nonlocal attempted
|
||||
self.real_verify(expected)
|
||||
if not attempted:
|
||||
attempted = True
|
||||
raise RuntimeError('模拟新证书上线后健康检查失败')
|
||||
|
||||
with patch.object(deploy, 'verify_live_certificate', side_effect=fail_after_live_update), \
|
||||
self.assertRaisesRegex(RuntimeError, '健康检查失败'):
|
||||
deploy.main()
|
||||
self.real_verify(deploy.fingerprint(self.new))
|
||||
self.assertEqual((self.target / 'custom.crt').read_bytes(), self.new.read_bytes())
|
||||
self.assertIsNone(process.poll())
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.send_signal(signal.SIGQUIT)
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait(timeout=10)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -22,7 +22,7 @@ def main():
|
||||
or len(target.parts) < 5 or target.parts[1:3] != ("var", "tmp")
|
||||
or not target.parts[3].startswith("smail-review-")):
|
||||
raise ValueError("上传路径必须位于项目审查临时目录")
|
||||
command = "install -d -m 0755 " + shlex.quote(str(target.parent))
|
||||
command = "install -d -m 0700 " + shlex.quote(str(target.parent))
|
||||
_, output, error = client.exec_command(command)
|
||||
if output.channel.recv_exit_status() != 0:
|
||||
raise RuntimeError(error.read().decode("utf-8", "replace"))
|
||||
|
||||
Reference in new issue
Block a user