补充云端验收证据并加固测试凭据检查

This commit is contained in:
xuans committed 2026-09-21 00:21:50 +08:00
1 parent 316bc1400a
commit 8122a5ff6b
17 files changed
+833 -47

No files matched your search

+31
View File
@@ -1,8 +1,11 @@
'''隔离验收收尾回归,模拟异常,不停止真实服务。'''
from pathlib import Path
from contextlib import closing
import subprocess
import sqlite3
import sys
import tempfile
import unittest
from unittest.mock import patch
@@ -16,6 +19,34 @@ class RuntimeCleanupTest(unittest.TestCase):
probe = 'smail-runtime-probe-20260918-020000'
proxy = 'smail-runtime-nginx-20260918-020000'
def test_single_database_integrity_and_legacy_rejection(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / 'data').mkdir()
with closing(sqlite3.connect(root / 'data/smail.db')) as database:
database.execute('CREATE TABLE config_records (id TEXT)')
self.assertEqual(review.database_integrity(root), {'smail.db': [('ok',)]})
(root / 'data/smail-state.db').touch()
with self.assertRaises(AssertionError):
review.database_integrity(root)
def test_missing_database_is_not_created_by_verifier(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / 'data').mkdir()
with self.assertRaises(AssertionError):
review.database_integrity(root)
self.assertEqual(list((root / 'data').iterdir()), [])
def test_missing_history_table_is_rejected(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / 'data').mkdir()
with closing(sqlite3.connect(root / 'data/smail.db')) as database:
database.execute('CREATE TABLE other (id TEXT)')
with self.assertRaises(AssertionError):
review.database_integrity(root)
def test_resources_then_stop_all(self):
result = {}
with patch.object(review.subprocess, 'check_output', return_value='Result=success') as stats, \
+63
View File
@@ -0,0 +1,63 @@
"""冷备归档与协议正文校验反例,不启动任何真实服务。"""
from email.message import EmailMessage
from io import BytesIO
from pathlib import Path
import sys
import tarfile
import tempfile
import unittest
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
from runtime_probe import verify_message_content
from runtime_restore import archive_members, tree_digest
class RuntimeRestoreTest(unittest.TestCase):
def test_archive_rejects_traversal_links_and_devices(self):
for name, kind in [('instance/../outside', tarfile.REGTYPE),
('/instance/file', tarfile.REGTYPE),
('other/file', tarfile.REGTYPE),
('instance/link', tarfile.SYMTYPE),
('instance/hard', tarfile.LNKTYPE),
('instance/device', tarfile.CHRTYPE)]:
with self.subTest(name=name), tarfile.open(fileobj=BytesIO(), mode='w') as archive:
member = tarfile.TarInfo(name)
member.type = kind
archive.addfile(member)
with self.assertRaises(AssertionError):
archive_members(archive)
def test_archive_accepts_only_regular_instance_content(self):
with tarfile.open(fileobj=BytesIO(), mode='w') as archive:
member = tarfile.TarInfo('instance')
member.type = tarfile.DIRTYPE
archive.addfile(member)
archive.addfile(tarfile.TarInfo('instance/smail.db'))
self.assertEqual(len(archive_members(archive)), 2)
def test_digest_detects_modified_backup_bytes(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
path = root / 'smail.db'
path.write_bytes(b'original')
before = tree_digest(root)
path.write_bytes(b'changed')
self.assertNotEqual(tree_digest(root), before)
def test_mime_checks_body_filename_message_and_attachment_bytes(self):
message = EmailMessage()
message['Message-ID'] = '<acceptance@runtime.invalid>'
message.set_content('中文验收正文')
message.add_attachment(b'payload', maintype='text', subtype='plain', filename='验收附件.txt')
raw = message.as_bytes()
verify_message_content(raw, 'acceptance@runtime.invalid', '中文验收正文', b'payload')
for identifier, body, attachment in [('wrong', '中文验收正文', b'payload'),
('acceptance@runtime.invalid', '缺失正文', b'payload'),
('acceptance@runtime.invalid', '中文验收正文', b'corrupted')]:
with self.subTest(identifier=identifier, body=body), self.assertRaises(AssertionError):
verify_message_content(raw, identifier, body, attachment)
if __name__ == '__main__':
unittest.main()
+55
View File
@@ -0,0 +1,55 @@
"""凭据扫描回归使用合成值,不复制真实凭据。"""
from pathlib import Path
import os
import runpy
import sys
import types
import unittest
from unittest.mock import Mock, patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
from check_tracked_secrets import scan_source
class TrackedSecretsTest(unittest.TestCase):
def test_cloud_key_literal_is_rejected_without_value(self):
secret = 'LTAI' + 'a' * 20
result = scan_source('config.txt', 'key=' + secret)
self.assertEqual(result, [('config.txt', 1, 'credential_literal')])
self.assertNotIn(secret, repr(result))
def test_python_cloud_secret_assignment_is_rejected(self):
result = scan_source('tool.py', 'ak="sample"\nsk="sample-secret"\n')
self.assertEqual(len(result), 2)
self.assertNotIn('sample-secret', repr(result))
def test_environment_lookup_is_allowed(self):
self.assertEqual(scan_source('tool.py', 'import os\nak=os.environ["KEY"]\nsk=os.environ["SECRET"]'), [])
def test_private_key_is_rejected(self):
header = '-----BEGIN RSA PRIVATE' + ' KEY-----'
self.assertEqual(scan_source('key.pem', header), [('key.pem', 1, 'credential_literal')])
def test_python_parse_failure_is_not_silently_skipped(self):
self.assertEqual(scan_source('tool.py', 'if:'), [('tool.py', 1, 'python_parse_failed')])
def test_missing_cloud_environment_rejects_before_network(self):
tools = Path(__file__).resolve().parents[1] / 'tools'
ssh_module = types.ModuleType('paramiko')
ssh_module.SSHClient = Mock(side_effect=AssertionError('unexpected SSH operation'))
for name in ('add_dns.py', 'add_missing_dns.py', 'add_missing_dns_v2.py', 'check_alidns.py'):
for environment, missing in (({}, 'SMAIL_ALIDNS_ACCESS_KEY_ID'),
({'SMAIL_ALIDNS_ACCESS_KEY_ID': 'sample'}, 'SMAIL_ALIDNS_ACCESS_KEY_SECRET')):
with self.subTest(name=name, missing=missing), \
patch.dict(os.environ, environment, clear=True), \
patch.dict(sys.modules, {'paramiko': ssh_module}), \
patch('urllib.request.urlopen', side_effect=AssertionError('unexpected HTTP operation')), \
self.assertRaises(KeyError) as error:
runpy.run_path(str(tools / name), run_name='__main__')
self.assertEqual(error.exception.args, (missing,))
ssh_module.SSHClient.assert_not_called()
if __name__ == '__main__':
unittest.main()
+4 -3
View File
@@ -1,5 +1,9 @@
import os
import urllib.request, urllib.parse, hmac, hashlib, base64, time, json, paramiko, subprocess
ak = os.environ['SMAIL_ALIDNS_ACCESS_KEY_ID']
sk = os.environ['SMAIL_ALIDNS_ACCESS_KEY_SECRET']
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.RejectPolicy())
from remote_ssh import connect_remote
@@ -18,9 +22,6 @@ print('DKIM Value:', dkim_val)
dmarc_val = 'v=DMARC1; p=quarantine; rua=mailto:admin@solit.cn'
print('DMARC Value:', dmarc_val)
ak = 'ENV_ALIDNS_ACCESS_KEY_ID'
sk = 'ENV_ALIDNS_ACCESS_KEY_SECRET'
def aliyun_request(params):
public_params = {
'Format': 'JSON',
+3 -2
View File
@@ -1,7 +1,8 @@
import os
import urllib.request, urllib.parse, hmac, hashlib, base64, time, json
ak = 'ENV_ALIDNS_ACCESS_KEY_ID'
sk = 'ENV_ALIDNS_ACCESS_KEY_SECRET'
ak = os.environ['SMAIL_ALIDNS_ACCESS_KEY_ID']
sk = os.environ['SMAIL_ALIDNS_ACCESS_KEY_SECRET']
def aliyun_request(params):
public_params = {
+3 -2
View File
@@ -1,7 +1,8 @@
import os
import urllib.request, urllib.parse, hmac, hashlib, base64, time, json
ak = 'ENV_ALIDNS_ACCESS_KEY_ID'
sk = 'ENV_ALIDNS_ACCESS_KEY_SECRET'
ak = os.environ['SMAIL_ALIDNS_ACCESS_KEY_ID']
sk = os.environ['SMAIL_ALIDNS_ACCESS_KEY_SECRET']
def percent_encode(val):
res = urllib.parse.quote(str(val), safe='')
+3 -2
View File
@@ -1,7 +1,8 @@
import os
import urllib.request, urllib.parse, hmac, hashlib, base64, time, json
ak = 'ENV_ALIDNS_ACCESS_KEY_ID'
sk = 'ENV_ALIDNS_ACCESS_KEY_SECRET'
ak = os.environ['SMAIL_ALIDNS_ACCESS_KEY_ID']
sk = os.environ['SMAIL_ALIDNS_ACCESS_KEY_SECRET']
def aliyun_request(params):
public_params = {
+12 -10
View File
@@ -1,18 +1,20 @@
from playwright.sync_api import sync_playwright
import os
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page()
page.on(console, lambda msg: print(f[CONSOLE {msg.type}] {msg.text}))
page.on(pageerror, lambda exc: print(f[PAGE ERROR] {exc}))
response = page.goto(http://47.108.78.12/)
print(Status:, response.status)
page.on('console', lambda msg: print(f'控制台类型:{msg.type}'))
page.on('pageerror', lambda exc: print(f'页面错误类型:{type(exc).__name__}'))
response = page.goto(os.environ['SMAIL_TEST_URL'])
print('状态码:', response.status)
content = page.content()
print(HTML length:, len(content))
next_btn = page.locator(#next)
print(next locator count:, next_btn.count())
buttons = page.locator(button)
print(Total buttons:, buttons.count())
print('页面长度:', len(content))
next_btn = page.locator('#next')
print('下一步按钮数量:', next_btn.count())
buttons = page.locator('button')
print('按钮总数:', buttons.count())
for i in range(buttons.count()):
print(fButton {i}: text={buttons.nth(i).inner_text()} id={buttons.nth(i).get_attribute('id')})
print(f'按钮 {i}:{buttons.nth(i).inner_text()}')
browser.close()
+58
View File
@@ -0,0 +1,58 @@
"""检查已跟踪文件中的云凭据,报告只包含位置,不回显敏感值。"""
import ast
from pathlib import Path
import re
import subprocess
import sys
ACCESS_KEY = re.compile(r'LTAI[A-Za-z0-9]{16,}')
PRIVATE_KEY = re.compile(r'-----BEGIN (?:[A-Z0-9 ]+ )?PRIVATE' + r' KEY-----')
def scan_source(name, source):
findings = set()
for number, line in enumerate(source.splitlines(), 1):
if ACCESS_KEY.search(line) or PRIVATE_KEY.search(line):
findings.add((name, number, 'credential_literal'))
if name.endswith('.py'):
try:
tree = ast.parse(source)
except SyntaxError:
findings.add((name, 1, 'python_parse_failed'))
return sorted(findings)
for node in ast.walk(tree):
if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.Constant):
continue
if not isinstance(node.value.value, str) or not node.value.value:
continue
for target in node.targets:
if isinstance(target, ast.Name) and target.id.lower() in {'ak', 'sk'}:
findings.add((name, node.lineno, 'cloud_key_assignment'))
return sorted(findings)
def main():
root = Path(__file__).resolve().parents[2]
result = subprocess.run(['git', 'ls-files', '-z'], cwd=root, check=True, capture_output=True)
findings = []
for name in result.stdout.decode('utf-8').split('\0'):
if not name:
continue
path = root / name
if not path.is_file():
continue
try:
source = path.read_text(encoding='utf-8-sig')
except UnicodeDecodeError:
continue
findings.extend(scan_source(name, source))
for name, line, category in findings:
print(f'{name}:{line}: {category}')
print('凭据检查:' + ('不通过' if findings else '通过'))
return 1 if findings else 0
if __name__ == '__main__':
sys.exit(main())
+101 -3
View File
@@ -4,6 +4,8 @@ import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
from email.message import EmailMessage
from email.parser import BytesParser
from email.policy import default as email_policy
import http.client
from http.cookies import SimpleCookie
import imaplib
@@ -15,6 +17,7 @@ import smtplib
import socket
import sqlite3
import ssl
import sys
import threading
import time
@@ -93,6 +96,48 @@ class LocalPOP(poplib.POP3_SSL):
return self.context.wrap_socket(raw, server_hostname=self.host)
def verify_message_content(raw, message_id, body, attachment):
message = BytesParser(policy=email_policy).parsebytes(raw)
assert str(message['Message-ID']).strip('<>') == message_id
text = message.get_body(preferencelist=('plain',))
assert text is not None and body in text.get_content()
files = list(message.iter_attachments())
assert len(files) == 1 and files[0].get_filename() == '验收附件.txt'
assert files[0].get_payload(decode=True) == attachment
def verify_protocol_read(context, codes, message_id, body, attachment):
# 通过协议取回真实正文,不以数据库存在或欢迎响应替代取信验收。
client = LocalIMAP(HOST, 993, ssl_context=context, timeout=30)
try:
assert client.login(ADDRESS, codes['imap'])[0] == 'OK'
assert client.select('INBOX', readonly=True)[0] == 'OK'
status, matches = client.uid('SEARCH', None, 'HEADER', 'Message-ID', '"<' + message_id + '>"')
assert status == 'OK' and matches[0].split(), 'IMAP 收件箱没有测试邮件'
status, contents = client.uid('FETCH', matches[0].split()[-1], '(BODY.PEEK[])')
assert status == 'OK'
raw = next(item[1] for item in contents if isinstance(item, tuple))
verify_message_content(raw, message_id, body, attachment)
finally:
client.logout()
client = LocalPOP(HOST, 995, context=context, timeout=30)
try:
client.user(ADDRESS)
client.pass_(codes['pop3'])
found = False
for item in client.list()[1]:
raw = b'\r\n'.join(client.retr(int(item.split()[0]))[1])
message = BytesParser(policy=email_policy).parsebytes(raw)
if str(message['Message-ID']).strip('<>') == message_id:
verify_message_content(raw, message_id, body, attachment)
found = True
break
assert found, 'POP3 没有取到测试邮件'
finally:
client.quit()
return {'imap_inbox': True, 'imap_body_attachment': True, 'pop3_body_attachment': True}
def protocol_operation(protocol, context, code):
if protocol == 'smtp':
client = smtp_connect(context)
@@ -140,15 +185,61 @@ def protocol_operation(protocol, context, code):
return invoke
def verify_restored(root, context, password, message_id, port):
client = Client(context, port=port)
for _ in range(60):
try:
assert client.request('/api/ping', method='GET')[0] == 200
break
except (OSError, AssertionError):
time.sleep(1)
else:
raise AssertionError('恢复实例未就绪')
assert client.success('/api/v1/login', {'account': ACCOUNT, 'password': password})['account'] == ACCOUNT
assert client.success('/api/v1/user/info')['account'] == ACCOUNT
client.success('/api/v1/security/reauth', {'password': password})
codes = {}
for protocol in ('smtp', 'imap', 'pop3'):
codes[protocol] = client.success('/api/v1/security/credentials/protocols/generate', {'protocol': protocol})['code']
assert protocol_operation(protocol, context, codes[protocol])() == 'success'
body = '内部25端口真实收信,检查中文正文和附件。'
attachment = '隔离验收附件内容'.encode('utf-8')
result = {'login': True, 'session': True, 'protocol_login': list(codes),
'historical_mail': verify_protocol_read(context, codes, message_id, body, attachment)}
message = EmailMessage()
message['From'] = 'restore@sender.invalid'
message['To'] = ADDRESS
message['Subject'] = '备份恢复后中文收信验收'
restored_id = message_id + '.restored'
message['Message-ID'] = '<' + restored_id + '>'
message.set_content(body)
message.add_attachment(attachment, maintype='text', subtype='plain', filename='验收附件.txt')
with smtp_connect(context) as smtp:
assert not smtp.send_message(message)
result['new_mail'] = verify_protocol_read(context, codes, restored_id, body, attachment)
result['passed'] = True
(root.parent / 'restore-probe-result.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
print(json.dumps(result, ensure_ascii=False, indent=2))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('root', type=Path)
parser.add_argument('--live-net', required=True)
parser.add_argument('--proxy', action='store_true')
parser.add_argument('--password-stdin', action='store_true')
parser.add_argument('--restore-message-id')
args = parser.parse_args()
assert str(Path('/proc/self/ns/net').readlink()) != args.live_net, '禁止在宿主网络执行测试'
root = args.root.resolve(strict=True)
assert root.parent.name.startswith('smail-review-runtime-') and root.name == 'instance'
assert root.parent.name.startswith(('smail-review-runtime-', 'smail-review-restore-')) and root.name == 'instance'
password = sys.stdin.readline().strip() if args.password_stdin else secrets.token_urlsafe(9)
assert len(password) == 12
if args.restore_message_id:
assert args.password_stdin
context = ssl.create_default_context(cafile=str(root / 'ssl/runtime.crt'))
verify_restored(root, context, password, args.restore_message_id, 8002 if args.proxy else 443)
return
result = {'stage': 'setup', 'passed': False}
started = time.monotonic()
try:
@@ -170,7 +261,6 @@ def main():
client.success(setup, {'step': 'database', 'action': 'set', 'db_type': 'sqlite', 'db_dsn': ''})
with sqlite3.connect(f'file:{root / "data/smail.db"}?mode=ro', uri=True) as database:
assert database.execute('SELECT COUNT(*) FROM user').fetchone()[0] == 0
password = secrets.token_urlsafe(9)
client.success(setup, {'step': 'password', 'action': 'set', 'account': ACCOUNT, 'password': password})
client.success(setup, {'step': 'domain', 'action': 'set', 'smtp_domain': 'runtime.invalid',
'web_domain': HOST, 'multi_domain': ''})
@@ -193,6 +283,12 @@ def main():
assert 'smail_csrf' in client.cookies
assert client.success('/api/v1/user/info').get('account') == ACCOUNT
result['setup'] = {'empty_database': True, 'token_required': True, 'login': True, 'session': True}
# 首次登录先完成正式安全设置,不绕过恢复方式要求直接启用协议。
client.success('/api/v1/security/reauth', {'password': password})
recovery = client.success('/api/v1/security/credentials/recovery-codes/generate', {'count': 10})
assert len(recovery['codes']) == 10
recovery.clear()
result['security_setup'] = {'recovery_codes_generated': 10, 'plaintext_logged': False}
if args.proxy:
from runtime_proxy import configure_proxy
result['proxy'] = configure_proxy(client, context, login, Client, LocalHTTPS)
@@ -213,7 +309,7 @@ def main():
result['smtp_invalid_credentials'] = 535
result['stage'] = 'smtp'
message = EmailMessage()
message['From'] = 'probe@runtime.invalid'
message['From'] = 'probe@sender.invalid'
message['To'] = ADDRESS
message['Subject'] = '隔离首次安装中文收信验收'
message_id = 'runtime-' + root.parent.name + '@runtime.invalid'
@@ -234,6 +330,8 @@ def main():
(rows[0][0],)).fetchall()
assert owners == [(ACCOUNT,)]
result['smtp'] = {'message_id': message_id, 'stored': True, 'owners': 1, 'attachment': True}
result['stage'] = 'protocol_read'
result['protocol_read'] = verify_protocol_read(context, codes, message_id, body, attachment)
result['stage'] = 'overload'
rounds = []
result['overload'] = rounds
+123
View File
@@ -0,0 +1,123 @@
"""对已停止的隔离实例执行完整冷备、空目录还原与业务读取,不操作正式实例。"""
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import tarfile
import time
from runtime_review import collect_and_stop, database_integrity, run, service_state
def archive_members(archive):
members = archive.getmembers()
for member in members:
path = Path(member.name)
assert not path.is_absolute() and '..' not in path.parts
assert path.parts and path.parts[0] == 'instance'
assert member.isdir() or member.isfile(), '备份包含链接或特殊文件'
return members
def tree_digest(root):
result = {}
for path in sorted(root.rglob('*')):
assert not path.is_symlink(), '隔离备份拒绝链接'
if path.is_file():
with path.open('rb') as stream:
result[str(path.relative_to(root))] = hashlib.file_digest(stream, 'sha256').hexdigest()
return result
def verify_backup_restore(source, evidence, password, proxy):
assert os.geteuid() == 0 and run('hostname') == 'ikun2'
source = source.resolve(strict=True)
assert source.parent.parent == Path('/var/tmp')
assert source.parent.name.startswith('smail-review-runtime-') and source.name == 'instance'
assert run('systemctl', 'show', evidence['unit'], '--property=MainPID', '--value') == '0'
database_integrity(source)
before = service_state()
source_digest = tree_digest(source)
stamp = time.strftime('%Y%m%d-%H%M%S')
target = Path('/var/tmp') / ('smail-review-restore-' + stamp)
target.mkdir(mode=0o750)
shutil.chown(target, user='smail', group='smail')
archive_path = target / 'cold-backup.tar.gz'
with tarfile.open(archive_path, 'x:gz') as archive:
archive.add(source, arcname='instance')
archive_path.chmod(0o600)
with tarfile.open(archive_path, 'r:gz') as archive:
members = archive_members(archive)
assert not (target / 'instance').exists()
archive.extractall(target, members=members, filter='data')
root = target / 'instance'
assert tree_digest(root) == source_digest
assert (root / 'data/smail.db').stat().st_ino != (source / 'data/smail.db').stat().st_ino
for path in [root, *root.rglob('*')]:
shutil.chown(path, user='smail', group='smail')
control = target / 'control'
control.mkdir(mode=0o750)
shutil.chown(control, user='smail', group='smail')
probe_path = target / 'runtime_probe.py'
shutil.copy2(Path(__file__).with_name('runtime_probe.py'), probe_path)
probe_path.chmod(0o640)
shutil.chown(probe_path, user='root', group='smail')
unit = 'smail-runtime-review-' + stamp
probe_unit = 'smail-runtime-probe-' + stamp
result = {'source': str(source), 'target': str(target), 'file_count': len(source_digest),
'archive_sha256': hashlib.sha256(archive_path.read_bytes()).hexdigest(),
'empty_target': True, 'byte_identical': True, 'cold_backup': True}
started = time.monotonic()
try:
subprocess.run([
'systemd-run', '--unit=' + unit, '--property=Type=exec',
'--property=User=smail', '--property=Group=smail',
'--property=PrivateNetwork=yes', '--property=PrivateTmp=yes',
'--property=ProtectSystem=strict', '--property=ProtectHome=yes',
'--property=NoNewPrivileges=yes', '--property=CapabilityBoundingSet=CAP_NET_BIND_SERVICE',
'--property=AmbientCapabilities=CAP_NET_BIND_SERVICE',
'--property=MemoryMax=192M', '--property=MemorySwapMax=0', '--property=CPUQuota=25%',
'--property=TasksMax=128', '--property=RuntimeMaxSec=180s', '--property=OOMPolicy=stop',
'--property=KillMode=control-group', '--property=TimeoutStopSec=10s',
'--property=BindPaths=' + str(root) + ':/opt/smail ' + str(control) + ':/run/smail',
'--property=ReadWritePaths=/opt/smail /run/smail',
'--property=WorkingDirectory=/opt/smail', '--setenv=SMAIL_ROOT=/opt/smail',
'--setenv=GOMEMLIMIT=64MiB', '--setenv=GOMAXPROCS=1', '/opt/smail/smail',
], check=True)
pid = run('systemctl', 'show', unit, '--property=MainPID', '--value')
live_net = str(Path('/proc/self/ns/net').readlink())
assert pid != '0' and str(Path('/proc/' + pid + '/ns/net').readlink()) != live_net
assert Path('/proc/' + pid + '/root/opt/smail/smail').stat().st_ino == (root / 'smail').stat().st_ino
probe = subprocess.run([
'systemd-run', '--wait', '--pipe', '--unit=' + probe_unit,
'--property=User=smail', '--property=Group=smail',
'--property=NetworkNamespacePath=/proc/' + pid + '/ns/net',
'--property=ProtectSystem=strict', '--property=ProtectHome=yes',
'--property=NoNewPrivileges=yes', '--property=InaccessiblePaths=/opt/smail /run/smail',
'--property=ReadWritePaths=' + str(target), '--property=MemoryMax=48M',
'--property=MemorySwapMax=0', '--property=CPUQuota=25%', '--property=TasksMax=32',
'--property=RuntimeMaxSec=150s', '--property=OOMPolicy=stop',
'--property=KillMode=control-group', '--property=TimeoutStopSec=10s',
'python3', str(probe_path), str(root), '--live-net', live_net, '--password-stdin',
'--restore-message-id', evidence['probe']['smtp']['message_id'], *(['--proxy'] if proxy else []),
], input=password + '\n', text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=165)
(target / 'probe.log').write_text(probe.stdout, encoding='utf-8')
result['probe_code'] = probe.returncode
assert probe.returncode == 0, '还原业务验收失败,请检查保留的日志'
result['probe'] = json.loads((target / 'restore-probe-result.json').read_text())
assert result['probe']['passed']
finally:
collect_and_stop(result, unit, probe_unit)
result['live_unchanged'] = service_state() == before
result['source_unchanged'] = tree_digest(source) == source_digest
result['seconds'] = round(time.monotonic() - started, 2)
result['integrity'] = database_integrity(root)
(target / 'service.log').write_text(run('journalctl', '-u', unit, '--no-pager'), encoding='utf-8')
(target / 'result.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
assert result['live_unchanged'] and result['source_unchanged']
assert not result['cleanup_errors'] and not result['resource_errors']
assert 'Result=success' in result['resources'] and 'MemorySwapPeak=0' in result['resources']
return result
+38 -11
View File
@@ -1,11 +1,13 @@
"""在 47 服务器创建空数据副本并进行受限验收,不停止线上服务或删除文件。"""
import argparse
from contextlib import closing
import hashlib
import json
import os
from pathlib import Path
import re
import secrets
import shutil
import sqlite3
import subprocess
@@ -21,6 +23,18 @@ def service_state():
for name in ('smail.service', 'nginx.service', 'mysql.service', 'Scert.service')}
def database_integrity(root):
# 验收器不能创建数据库,也不能允许第二个状态库。
data = root / 'data'
assert not (data / 'smail-state.db').exists(), '单数据库实例生成了旧状态库'
assert sorted(path.name for path in data.glob('*.db')) == ['smail.db']
with closing(sqlite3.connect((data / 'smail.db').as_uri() + '?mode=ro', uri=True)) as database:
integrity = database.execute('PRAGMA integrity_check').fetchall()
assert database.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='config_records'").fetchone()[0] == 1
assert integrity == [('ok',)]
return {'smail.db': integrity}
def stop_test_units(names):
assert all(re.fullmatch(r'smail-runtime-(review|probe|nginx)-[0-9]{8}-[0-9]{6}', name)
for name in names), '只能停止本轮隔离测试单元'
@@ -62,6 +76,8 @@ def main():
parser.add_argument('--main-sha', required=True)
parser.add_argument('--binary', type=Path, default=Path('/opt/smail/smail'))
parser.add_argument('--nginx-template', type=Path)
parser.add_argument('--memory-max-mib', type=int, choices=(128, 192, 256), default=128)
parser.add_argument('--restore', action='store_true')
args = parser.parse_args()
assert os.geteuid() == 0 and run('hostname') == 'ikun2'
os.umask(0o027)
@@ -80,7 +96,7 @@ def main():
with (live / 'smail').open('rb') as stream:
live_sha = hashlib.file_digest(stream, 'sha256').hexdigest()
memory = dict(line.split(':', 1) for line in Path('/proc/meminfo').read_text().splitlines())
required_memory = 320 if args.nginx_template else 272
required_memory = args.memory_max_mib + (192 if args.nginx_template else 144)
assert int(memory['MemAvailable'].split()[0]) >= required_memory * 1024, '可用内存不足,暂停隔离验收'
stamp = time.strftime('%Y%m%d-%H%M%S')
target = Path('/var/tmp') / ('smail-review-runtime-' + stamp)
@@ -94,8 +110,12 @@ def main():
shutil.copy2(binary, root / 'smail')
(root / 'smail').chmod(0o755)
shutil.copy2(Path(__file__).with_name('runtime_probe.py'), target / 'runtime_probe.py')
(target / 'runtime_probe.py').chmod(0o640)
shutil.chown(target / 'runtime_probe.py', user='root', group='smail')
if args.nginx_template:
shutil.copy2(Path(__file__).with_name('runtime_proxy.py'), target / 'runtime_proxy.py')
(target / 'runtime_proxy.py').chmod(0o640)
shutil.chown(target / 'runtime_proxy.py', user='root', group='smail')
subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '2',
'-keyout', str(root / 'ssl/runtime.key'), '-out', str(root / 'ssl/runtime.crt'),
'-subj', '/CN=mail.runtime.invalid', '-addext',
@@ -111,6 +131,7 @@ def main():
probe_unit = 'smail-runtime-probe-' + stamp
proxy_unit = 'smail-runtime-nginx-' + stamp
result = {'target': str(target), 'unit': unit, 'main_sha256': args.main_sha,
'memory_max_mib': args.memory_max_mib,
'source': str(binary), 'live_sha256': live_sha, 'before': before,
'topology': 'nginx-8001-8002' if args.nginx_template else 'direct-80-443'}
service_env = []
@@ -123,6 +144,7 @@ def main():
'--setenv=SMAIL_HTTPS_PORT=8002']
probe_args = ['--proxy']
started = time.strftime('%Y-%m-%dT%H:%M:%S')
password = secrets.token_urlsafe(9)
try:
subprocess.run([
'systemd-run', '--unit=' + unit, '--property=Type=exec',
@@ -131,7 +153,8 @@ def main():
'--property=ProtectSystem=strict', '--property=ProtectHome=yes',
'--property=NoNewPrivileges=yes', '--property=CapabilityBoundingSet=CAP_NET_BIND_SERVICE',
'--property=AmbientCapabilities=CAP_NET_BIND_SERVICE',
'--property=MemoryMax=128M', '--property=MemorySwapMax=0', '--property=CPUQuota=25%',
'--property=MemoryMax=' + str(args.memory_max_mib) + 'M',
'--property=MemorySwapMax=0', '--property=CPUQuota=25%',
'--property=TasksMax=128', '--property=RuntimeMaxSec=300s', '--property=OOMPolicy=stop',
'--property=KillMode=control-group', '--property=TimeoutStopSec=10s',
'--property=BindPaths=' + str(root) + ':/opt/smail ' + str(control) + ':/run/smail',
@@ -173,8 +196,9 @@ def main():
'--property=MemorySwapMax=0', '--property=CPUQuota=25%', '--property=TasksMax=32',
'--property=RuntimeMaxSec=240s', '--property=OOMPolicy=stop',
'--property=KillMode=control-group', '--property=TimeoutStopSec=10s',
'python3', str(target / 'runtime_probe.py'), str(root), '--live-net', live_net, *probe_args,
], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=270)
'python3', str(target / 'runtime_probe.py'), str(root), '--live-net', live_net,
'--password-stdin', *probe_args,
], input=password + '\n', text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=270)
(target / 'probe.log').write_text(probe.stdout, encoding='utf-8')
result['probe_code'] = probe.returncode
if (target / 'probe-result.json').exists():
@@ -191,12 +215,10 @@ def main():
with (live / 'smail').open('rb') as stream:
result['live_unchanged'] = (result['after'] == before
and hashlib.file_digest(stream, 'sha256').hexdigest() == live_sha)
result['integrity'] = {}
for name in ('smail.db', 'smail-state.db'):
path = root / 'data' / name
if path.exists():
with sqlite3.connect(f'file:{path}?mode=ro', uri=True) as database:
result['integrity'][name] = database.execute('PRAGMA integrity_check').fetchall()
try:
result['integrity'] = database_integrity(root)
except (AssertionError, sqlite3.Error) as error:
result['integrity'] = {'error': type(error).__name__}
(target / 'service.log').write_text(run('journalctl', '-u', unit, '--no-pager'), encoding='utf-8')
kernel = run('journalctl', '-k', '--since', started, '--no-pager')
result['oom_events'] = sum('oom-kill:constraint=' in line for line in kernel.splitlines())
@@ -205,7 +227,12 @@ def main():
assert result['live_unchanged'] and result['oom_events'] == 0
assert not result['cleanup_errors'], '测试单元未全部停止,需要核查保留的结果'
assert not result['resource_errors'], '资源证据不完整,需要核查保留的结果'
assert result['integrity'] == {'smail.db': [('ok',)], 'smail-state.db': [('ok',)]}
assert result['integrity'] == {'smail.db': [('ok',)]}
if args.restore:
from runtime_restore import verify_backup_restore
restored = verify_backup_restore(root, result, password, args.nginx_template is not None)
(target / 'restore-result.json').write_text(json.dumps(restored, ensure_ascii=False, indent=2), encoding='utf-8')
print(json.dumps({'restore': restored}, ensure_ascii=False, indent=2))
if __name__ == '__main__':