完成完整候选反代隔离验收并补齐收尾回归证据

This commit is contained in:
xuans committed 2026-09-18 02:17:40 +08:00
1 parent 74fb8ad65e
commit 45078da64a
6 files changed
+399 -17

No files matched your search

+70
View File
@@ -0,0 +1,70 @@
'''隔离验收收尾回归,模拟异常,不停止真实服务。'''
from pathlib import Path
import subprocess
import sys
import unittest
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
import runtime_review as review
class RuntimeCleanupTest(unittest.TestCase):
app = 'smail-runtime-review-20260918-020000'
probe = 'smail-runtime-probe-20260918-020000'
proxy = 'smail-runtime-nginx-20260918-020000'
def test_resources_then_stop_all(self):
result = {}
with patch.object(review.subprocess, 'check_output', return_value='Result=success') as stats, \
patch.object(review, 'stop_test_units', return_value=[]) as stop:
review.collect_and_stop(result, self.app, self.probe, self.proxy)
self.assertEqual(stats.call_count, 2)
stop.assert_called_once_with([self.proxy, self.probe, self.app])
self.assertEqual(result['cleanup_errors'], [])
self.assertEqual(result['resource_errors'], [])
def test_resource_timeout_still_stops_all(self):
result = {}
with patch.object(review.subprocess, 'check_output', side_effect=subprocess.TimeoutExpired('show', 15)), \
patch.object(review, 'stop_test_units', return_value=[]) as stop:
review.collect_and_stop(result, self.app, self.probe, self.proxy)
self.assertEqual(len(result['resource_errors']), 2)
stop.assert_called_once_with([self.proxy, self.probe, self.app])
def test_stop_timeout_and_kill_error_do_not_skip_remaining(self):
success = subprocess.CompletedProcess([], 0, '', '')
with patch.object(review.subprocess, 'run', side_effect=[
subprocess.TimeoutExpired('stop', 25), OSError(), success, success,
]) as run:
errors = review.stop_test_units([self.proxy, self.probe, self.app])
self.assertEqual([item['action'] for item in errors], ['stop', 'kill'])
self.assertEqual(run.call_args_list[-1].args[0], ['systemctl', 'stop', self.app])
def test_missing_unit_is_not_failure(self):
missing = subprocess.CompletedProcess([], 5, '', 'Unit not loaded.')
with patch.object(review.subprocess, 'run', return_value=missing):
self.assertEqual(review.stop_test_units([self.probe]), [])
def test_stop_failure_attempts_kill_and_reports_failure(self):
failed = subprocess.CompletedProcess([], 1, '', 'Failed to stop unit')
success = subprocess.CompletedProcess([], 0, '', '')
with patch.object(review.subprocess, 'run', side_effect=[failed, success]) as run:
errors = review.stop_test_units([self.app])
self.assertEqual(errors[0]['error'], 'CalledProcessError')
self.assertEqual(run.call_args_list[1].args[0], [
'systemctl', 'kill', '--kill-whom=all', '--signal=SIGKILL', self.app,
])
def test_reject_business_units_before_any_stop(self):
for name in ('nginx', 'smail.service', self.app + ' other', '../mysql.service'):
with self.subTest(name=name), patch.object(review.subprocess, 'run') as run:
with self.assertRaises(AssertionError):
review.stop_test_units([self.app, name])
run.assert_not_called()
if __name__ == '__main__':
unittest.main()
+21 -7
View File
@@ -26,20 +26,26 @@ ADDRESS = ACCOUNT + '@runtime.invalid'
class LocalHTTPS(http.client.HTTPSConnection):
def connect(self):
raw = socket.create_connection(('127.0.0.1', self.port), self.timeout)
raw = socket.create_connection(('127.0.0.1', self.port), self.timeout, self.source_address)
self.sock = self._context.wrap_socket(raw, server_hostname=self.host)
class Client:
def __init__(self, context=None):
def __init__(self, context=None, port=None, source=None):
self.context = context
self.port = port or (443 if context else 80)
self.source = (source, 0) if source else None
self.cookies = SimpleCookie()
def request(self, path, payload=None, method='POST'):
connection = (LocalHTTPS(HOST, 443, timeout=30, context=self.context)
if self.context else http.client.HTTPConnection('127.0.0.1', 80, timeout=30))
def request(self, path, payload=None, method='POST', extra_headers=None):
connection = (LocalHTTPS(HOST, self.port, timeout=30, context=self.context,
source_address=self.source)
if self.context else http.client.HTTPConnection('127.0.0.1', self.port, timeout=30))
origin = 'https://' + HOST if self.context else 'http://127.0.0.1'
if self.port != (443 if self.context else 80):
origin += ':' + str(self.port)
headers = {'Origin': origin, 'Content-Type': 'application/json'}
headers.update(extra_headers or {})
if self.cookies:
headers['Cookie'] = '; '.join(item.key + '=' + item.coded_value for item in self.cookies.values())
if 'smail_csrf' in self.cookies:
@@ -138,6 +144,7 @@ def main():
parser = argparse.ArgumentParser()
parser.add_argument('root', type=Path)
parser.add_argument('--live-net', required=True)
parser.add_argument('--proxy', action='store_true')
args = parser.parse_args()
assert str(Path('/proc/self/ns/net').readlink()) != args.live_net, '禁止在宿主网络执行测试'
root = args.root.resolve(strict=True)
@@ -145,7 +152,8 @@ def main():
result = {'stage': 'setup', 'passed': False}
started = time.monotonic()
try:
client = Client()
setup_port = 8001 if args.proxy else 80
client = Client(port=setup_port)
for _ in range(60):
try:
assert client.request('/', method='GET')[0] == 200
@@ -156,7 +164,7 @@ def main():
raise AssertionError('初始化服务未就绪')
assert 'smail_setup_token' in client.cookies, '初始化令牌未下发'
assert client.cookies['smail_setup_token']['httponly']
assert Client().request('/api/v1/setup', {'step': 'database', 'action': 'get'})[0] == 403
assert Client(port=setup_port).request('/api/v1/setup', {'step': 'database', 'action': 'get'})[0] == 403
assert not (root / 'data/smail.db').exists(), '业务库必须从空数据开始'
setup = '/api/v1/setup'
client.success(setup, {'step': 'database', 'action': 'set', 'db_type': 'sqlite', 'db_dsn': ''})
@@ -185,6 +193,9 @@ def main():
assert 'smail_csrf' in client.cookies
assert client.success('/api/v1/user/info').get('account') == ACCOUNT
result['setup'] = {'empty_database': True, 'token_required': True, 'login': True, 'session': True}
if args.proxy:
from runtime_proxy import configure_proxy
result['proxy'] = configure_proxy(client, context, login, Client, LocalHTTPS)
result['stage'] = 'protocols'
codes = {}
for protocol in ('smtp', 'imap', 'pop3'):
@@ -256,6 +267,9 @@ def main():
assert client.success('/api/v1/login', login).get('account') == ACCOUNT
round_result['recovered'] = True
(root.parent / 'probe-result.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
if args.proxy:
from runtime_proxy import verify_proxy_sources
result['proxy']['sources'] = verify_proxy_sources(context, login, Client)
result['stage'] = 'complete'
result['passed'] = True
finally:
+146
View File
@@ -0,0 +1,146 @@
'''复用部署模板,在独立网络中验收 Nginx 与 8001/8002,不访问线上实例。'''
import http.client
import shutil
import time
HOST = 'mail.runtime.invalid'
def prepare_nginx(target, root, template):
directory = target / 'nginx'
directory.mkdir(mode=0o750)
shutil.chown(directory, user='smail', group='smail')
source = template.read_text(encoding='utf-8')
assert 'proxy_pass https://127.0.0.1:8002;' in source
assert 'proxy_ssl_verify on;' in source
assert 'proxy_set_header X-Forwarded-For $remote_addr;' in source
site = source.replace('solit.cn', 'runtime.invalid')
site = site.replace('/opt/smail/ssl/custom.crt', str(root / 'ssl/runtime.crt'))
site = site.replace('/opt/smail/ssl/custom.key', str(root / 'ssl/runtime.key'))
site = site.replace('/etc/ssl/certs/ca-certificates.crt', str(root / 'ssl/runtime.crt'))
site = site.replace('/var/lib/smail-acme', str(directory / 'acme'))
(directory / 'site.conf').write_text(site, encoding='utf-8')
config = '\n'.join([
'worker_processes 1;',
'pid ' + str(directory / 'nginx.pid') + ';',
'error_log ' + str(directory / 'error.log') + ' warn;',
'events { worker_connections 128; }',
'http {',
' access_log off;',
*(' ' + name + '_temp_path ' + str(directory / name) + ';'
for name in ('client_body', 'proxy', 'fastcgi', 'uwsgi', 'scgi')),
' include ' + str(directory / 'site.conf') + ';',
'}',
'',
])
(directory / 'nginx.conf').write_text(config, encoding='utf-8')
for name in ('site.conf', 'nginx.conf'):
path = directory / name
path.chmod(0o640)
shutil.chown(path, user='root', group='smail')
def configure_proxy(client, context, login, client_type, connection_type):
status = client.success('/api/v1/config/status', method='GET')
assert status['available']
payload = client.success('/api/v1/config/base', method='GET')
assert payload['http']['port'] == 8001 and payload['http']['httpsPort'] == 8002
payload.pop('databasePasswordStatus', None)
payload['http']['publicHttpsPort'] = 443
payload['http']['trustedProxyCidrs'] = ['127.0.0.1/32']
request = {'expectedGeneration': status['activeGeneration'], 'payload': payload,
'confirmation': {'confirmed': True, 'method': 'recent_auth'}}
client.success('/api/v1/config/base/validate', request)
candidate = client.success('/api/v1/config/base', request, 'PUT')
client.success('/api/v1/config/base/apply', {
'generation': candidate['generation'], 'candidateId': candidate['candidateId'],
})
for _ in range(30):
try:
active = client.success('/api/v1/config/base', method='GET')
assert active['http']['publicHttpsPort'] == 443
assert active['http']['trustedProxyCidrs'] == ['127.0.0.1/32']
break
except (OSError, http.client.HTTPException, AssertionError):
time.sleep(1)
else:
raise AssertionError('反代配置未通过正式接口生效')
assert active['http']['port'] == 8001 and active['http']['httpsPort'] == 8002
assert client_type(context, port=8002).success('/api/v1/login', login)['account'] == login['account']
redirects = {}
for port in (80, 8001):
connection = http.client.HTTPConnection('127.0.0.1', port, timeout=10)
try:
connection.request('GET', '/login?review=port', headers={'Host': HOST})
response = connection.getresponse()
response.read()
assert response.status in (301, 302, 307, 308), (port, response.status)
location = response.getheader('Location')
assert location == 'https://' + HOST + '/login?review=port', (port, location)
redirects[str(port)] = location
finally:
connection.close()
recovery = verify_recovery_page(context, connection_type)
return {'configuration_api': True, 'internal_http': 8001, 'internal_https': 8002,
'public_https': 443, 'upstream_tls_verified': True, 'redirects': redirects,
'recovery': recovery}
def verify_recovery_page(context, connection_type):
from html.parser import HTMLParser
class PageParser(HTMLParser):
def __init__(self):
super().__init__()
self.login_url = None
self.scripts = []
def handle_starttag(self, tag, attrs):
attributes = dict(attrs)
if tag == 'div' and attributes.get('id') == 'app':
self.login_url = attributes.get('data-login-url')
if tag == 'script' and attributes.get('src'):
self.scripts.append(attributes['src'])
def fetch(path):
connection = connection_type(HOST, 443, timeout=10, context=context)
try:
connection.request('GET', path)
response = connection.getresponse()
content = response.read()
return response.status, dict(response.getheaders()), content
finally:
connection.close()
status, headers, _ = fetch('/public-password-recovery')
assert status == 302 and headers.get('Location') == 'https://' + HOST + '/rest/'
status, headers, content = fetch('/rest/')
assert status == 200 and headers.get('Cache-Control') == 'no-store'
page = PageParser()
page.feed(content.decode('utf-8'))
assert page.login_url == 'https://' + HOST + '/#/login'
assert page.scripts, '恢复页面缺少生产构建入口'
for script in page.scripts:
assert script.startswith('/assets/') and '..' not in script
status, headers, content = fetch(script)
assert status == 200 and content and 'javascript' in headers.get('Content-Type', '')
return {'entry_redirect': True, 'login_url': page.login_url,
'no_store': True, 'scripts_loaded': len(page.scripts)}
def verify_proxy_sources(context, login, client_type):
first = client_type(context, source='127.0.0.2')
second = client_type(context, source='127.0.0.3')
wrong = {**login, 'password': 'invalid-proxy-review-password'}
for _ in range(10):
status, data, _ = first.request('/api/v1/login', wrong)
assert status == 200 and data is not None and data['errorNo'] != 0
status, data, _ = first.request('/api/v1/login', login, extra_headers={
'X-Forwarded-For': '127.0.0.3', 'X-Real-IP': '127.0.0.3',
})
assert status == 200 and data is not None and data['errorNo'] != 0, '伪造来源绕过登录限流'
assert second.success('/api/v1/login', login)['account'] == login['account'], '不同来源被错误合并限流'
return {'blocked_source': '127.0.0.2', 'unaffected_source': '127.0.0.3',
'forged_headers_rejected': True, 'independent_login': True}
+79 -10
View File
@@ -5,6 +5,7 @@ import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import sqlite3
import subprocess
@@ -16,14 +17,51 @@ def run(*args):
def service_state():
return {name: run('systemctl', 'show', name, '--property=MainPID,NRestarts,ActiveState')
for name in ('smail.service', 'mysql.service', 'Scert.service')}
return {name: run('systemctl', 'show', name, '--property=MainPID,NRestarts,ActiveState,InvocationID')
for name in ('smail.service', 'nginx.service', 'mysql.service', 'Scert.service')}
def stop_test_units(names):
assert all(re.fullmatch(r'smail-runtime-(review|probe|nginx)-[0-9]{8}-[0-9]{6}', name)
for name in names), '只能停止本轮隔离测试单元'
errors = []
for name in names:
try:
stopped = subprocess.run(['systemctl', 'stop', name], capture_output=True,
text=True, timeout=25)
if stopped.returncode and 'not loaded' not in stopped.stderr:
raise subprocess.CalledProcessError(stopped.returncode, 'systemctl')
except (OSError, subprocess.SubprocessError) as error:
errors.append({'unit': name, 'action': 'stop', 'error': type(error).__name__})
try:
subprocess.run(['systemctl', 'kill', '--kill-whom=all', '--signal=SIGKILL', name],
check=True, capture_output=True, timeout=15)
except (OSError, subprocess.SubprocessError) as kill_error:
errors.append({'unit': name, 'action': 'kill', 'error': type(kill_error).__name__})
return errors
def collect_and_stop(result, unit, probe_unit, proxy_unit=None):
result['resource_errors'] = []
try:
for key, name in [('resources', unit)] + ([('nginx_resources', proxy_unit)] if proxy_unit else []):
try:
result[key] = subprocess.check_output([
'systemctl', 'show', name,
'--property=MemoryPeak,MemoryCurrent,MemorySwapPeak,Result,ExecMainStatus,NRestarts',
], text=True, timeout=15).strip()
except (OSError, subprocess.SubprocessError) as error:
result['resource_errors'].append({'unit': name, 'error': type(error).__name__})
finally:
names = ([proxy_unit] if proxy_unit else []) + [probe_unit, unit]
result['cleanup_errors'] = stop_test_units(names)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--main-sha', required=True)
parser.add_argument('--binary', type=Path, default=Path('/opt/smail/smail'))
parser.add_argument('--nginx-template', type=Path)
args = parser.parse_args()
assert os.geteuid() == 0 and run('hostname') == 'ikun2'
os.umask(0o027)
@@ -42,7 +80,8 @@ def main():
with (live / 'smail').open('rb') as stream:
live_sha = hashlib.file_digest(stream, 'sha256').hexdigest()
memory = dict(line.split(':', 1) for line in Path('/proc/meminfo').read_text().splitlines())
assert int(memory['MemAvailable'].split()[0]) >= 272 * 1024, '可用内存不足,暂停隔离验收'
required_memory = 320 if args.nginx_template else 272
assert int(memory['MemAvailable'].split()[0]) >= required_memory * 1024, '可用内存不足,暂停隔离验收'
stamp = time.strftime('%Y%m%d-%H%M%S')
target = Path('/var/tmp') / ('smail-review-runtime-' + stamp)
target.mkdir(mode=0o750)
@@ -55,6 +94,8 @@ def main():
shutil.copy2(binary, root / 'smail')
(root / 'smail').chmod(0o755)
shutil.copy2(Path(__file__).with_name('runtime_probe.py'), target / 'runtime_probe.py')
if args.nginx_template:
shutil.copy2(Path(__file__).with_name('runtime_proxy.py'), target / 'runtime_proxy.py')
subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '2',
'-keyout', str(root / 'ssl/runtime.key'), '-out', str(root / 'ssl/runtime.crt'),
'-subj', '/CN=mail.runtime.invalid', '-addext',
@@ -68,8 +109,19 @@ def main():
before = service_state()
unit = 'smail-runtime-review-' + stamp
probe_unit = 'smail-runtime-probe-' + stamp
proxy_unit = 'smail-runtime-nginx-' + stamp
result = {'target': str(target), 'unit': unit, 'main_sha256': args.main_sha,
'source': str(binary), 'live_sha256': live_sha, 'before': before}
'source': str(binary), 'live_sha256': live_sha, 'before': before,
'topology': 'nginx-8001-8002' if args.nginx_template else 'direct-80-443'}
service_env = []
probe_args = []
if args.nginx_template:
from runtime_proxy import prepare_nginx
prepare_nginx(target, root, args.nginx_template)
result['nginx_template_sha256'] = hashlib.sha256(args.nginx_template.read_bytes()).hexdigest()
service_env = ['--setenv=setup_port=8001', '--setenv=SMAIL_HTTP_PORT=8001',
'--setenv=SMAIL_HTTPS_PORT=8002']
probe_args = ['--proxy']
started = time.strftime('%Y-%m-%dT%H:%M:%S')
try:
subprocess.run([
@@ -85,7 +137,7 @@ def main():
'--property=BindPaths=' + str(root) + ':/opt/smail ' + str(control) + ':/run/smail',
'--property=ReadWritePaths=/opt/smail /run/smail',
'--property=WorkingDirectory=/opt/smail', '--setenv=SMAIL_ROOT=/opt/smail',
'--setenv=GOMEMLIMIT=64MiB', '--setenv=GOMAXPROCS=1', '/opt/smail/smail',
'--setenv=GOMEMLIMIT=64MiB', '--setenv=GOMAXPROCS=1', *service_env, '/opt/smail/smail',
], check=True)
pid = run('systemctl', 'show', unit, '--property=MainPID', '--value')
assert pid != '0'
@@ -93,6 +145,24 @@ def main():
assert str(Path('/proc/' + pid + '/ns/net').readlink()) != live_net
assert Path('/proc/' + pid + '/root/opt/smail/smail').stat().st_ino == (root / 'smail').stat().st_ino
assert (root / 'smail').stat().st_ino != (live / 'smail').stat().st_ino
if args.nginx_template:
subprocess.run([
'systemd-run', '--unit=' + proxy_unit, '--property=Type=exec',
'--property=User=smail', '--property=Group=smail',
'--property=NetworkNamespacePath=/proc/' + pid + '/ns/net',
'--property=PrivateTmp=no', '--property=ProtectSystem=strict',
'--property=ProtectHome=yes', '--property=NoNewPrivileges=yes',
'--property=InaccessiblePaths=/opt/smail /run/smail /etc/nginx',
'--property=ReadWritePaths=' + str(target / 'nginx'),
'--property=CapabilityBoundingSet=CAP_NET_BIND_SERVICE',
'--property=AmbientCapabilities=CAP_NET_BIND_SERVICE',
'--property=MemoryMax=32M', '--property=MemorySwapMax=0',
'--property=CPUQuota=25%', '--property=TasksMax=16',
'--property=RuntimeMaxSec=300s', '--property=OOMPolicy=stop',
'--property=KillMode=control-group', '--property=TimeoutStopSec=10s',
'/usr/sbin/nginx', '-p', str(target / 'nginx'),
'-c', str(target / 'nginx/nginx.conf'), '-g', 'daemon off;',
], check=True)
probe = subprocess.run([
'systemd-run', '--wait', '--pipe', '--unit=' + probe_unit,
'--property=User=smail', '--property=Group=smail',
@@ -103,7 +173,7 @@ def main():
'--property=MemorySwapMax=0', '--property=CPUQuota=25%', '--property=TasksMax=32',
'--property=RuntimeMaxSec=240s', '--property=OOMPolicy=stop',
'--property=KillMode=control-group', '--property=TimeoutStopSec=10s',
'python3', str(target / 'runtime_probe.py'), str(root), '--live-net', live_net,
'python3', str(target / 'runtime_probe.py'), str(root), '--live-net', live_net, *probe_args,
], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=270)
(target / 'probe.log').write_text(probe.stdout, encoding='utf-8')
result['probe_code'] = probe.returncode
@@ -116,10 +186,7 @@ def main():
assert database.execute('SELECT COUNT(*) FROM email WHERE msg_id=?', (message_id,)).fetchone()[0] == 0
result['live_message_absent'] = True
finally:
result['resources'] = run('systemctl', 'show', unit,
'--property=MemoryPeak,MemoryCurrent,Result,ExecMainStatus,NRestarts')
subprocess.run(['systemctl', 'stop', unit], check=True)
subprocess.run(['systemctl', 'stop', probe_unit], check=False, stderr=subprocess.DEVNULL)
collect_and_stop(result, unit, probe_unit, proxy_unit if args.nginx_template else None)
result['after'] = service_state()
with (live / 'smail').open('rb') as stream:
result['live_unchanged'] = (result['after'] == before
@@ -136,6 +203,8 @@ def main():
(target / 'result.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
print(json.dumps(result, ensure_ascii=False, indent=2))
assert result['live_unchanged'] and result['oom_events'] == 0
assert not result['cleanup_errors'], '测试单元未全部停止,需要核查保留的结果'
assert not result['resource_errors'], '资源证据不完整,需要核查保留的结果'
assert result['integrity'] == {'smail.db': [('ok',)], 'smail-state.db': [('ok',)]}