1 parent
05f7490c07
commit
f265cc62f5
17 files changed
+833
-47
No files matched your search
@@ -1,8 +1,11 @@
|
||||
'''隔离验收收尾回归,模拟异常,不停止真实服务。'''
|
||||
|
||||
from pathlib import Path
|
||||
from contextlib import closing
|
||||
import subprocess
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -16,6 +19,34 @@ class RuntimeCleanupTest(unittest.TestCase):
|
||||
probe = 'smail-runtime-probe-20260918-020000'
|
||||
proxy = 'smail-runtime-nginx-20260918-020000'
|
||||
|
||||
def test_single_database_integrity_and_legacy_rejection(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / 'data').mkdir()
|
||||
with closing(sqlite3.connect(root / 'data/smail.db')) as database:
|
||||
database.execute('CREATE TABLE config_records (id TEXT)')
|
||||
self.assertEqual(review.database_integrity(root), {'smail.db': [('ok',)]})
|
||||
(root / 'data/smail-state.db').touch()
|
||||
with self.assertRaises(AssertionError):
|
||||
review.database_integrity(root)
|
||||
|
||||
def test_missing_database_is_not_created_by_verifier(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / 'data').mkdir()
|
||||
with self.assertRaises(AssertionError):
|
||||
review.database_integrity(root)
|
||||
self.assertEqual(list((root / 'data').iterdir()), [])
|
||||
|
||||
def test_missing_history_table_is_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / 'data').mkdir()
|
||||
with closing(sqlite3.connect(root / 'data/smail.db')) as database:
|
||||
database.execute('CREATE TABLE other (id TEXT)')
|
||||
with self.assertRaises(AssertionError):
|
||||
review.database_integrity(root)
|
||||
|
||||
def test_resources_then_stop_all(self):
|
||||
result = {}
|
||||
with patch.object(review.subprocess, 'check_output', return_value='Result=success') as stats, \
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
"""冷备归档与协议正文校验反例,不启动任何真实服务。"""
|
||||
|
||||
from email.message import EmailMessage
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
|
||||
from runtime_probe import verify_message_content
|
||||
from runtime_restore import archive_members, tree_digest
|
||||
|
||||
|
||||
class RuntimeRestoreTest(unittest.TestCase):
|
||||
def test_archive_rejects_traversal_links_and_devices(self):
|
||||
for name, kind in [('instance/../outside', tarfile.REGTYPE),
|
||||
('/instance/file', tarfile.REGTYPE),
|
||||
('other/file', tarfile.REGTYPE),
|
||||
('instance/link', tarfile.SYMTYPE),
|
||||
('instance/hard', tarfile.LNKTYPE),
|
||||
('instance/device', tarfile.CHRTYPE)]:
|
||||
with self.subTest(name=name), tarfile.open(fileobj=BytesIO(), mode='w') as archive:
|
||||
member = tarfile.TarInfo(name)
|
||||
member.type = kind
|
||||
archive.addfile(member)
|
||||
with self.assertRaises(AssertionError):
|
||||
archive_members(archive)
|
||||
|
||||
def test_archive_accepts_only_regular_instance_content(self):
|
||||
with tarfile.open(fileobj=BytesIO(), mode='w') as archive:
|
||||
member = tarfile.TarInfo('instance')
|
||||
member.type = tarfile.DIRTYPE
|
||||
archive.addfile(member)
|
||||
archive.addfile(tarfile.TarInfo('instance/smail.db'))
|
||||
self.assertEqual(len(archive_members(archive)), 2)
|
||||
|
||||
def test_digest_detects_modified_backup_bytes(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
path = root / 'smail.db'
|
||||
path.write_bytes(b'original')
|
||||
before = tree_digest(root)
|
||||
path.write_bytes(b'changed')
|
||||
self.assertNotEqual(tree_digest(root), before)
|
||||
|
||||
def test_mime_checks_body_filename_message_and_attachment_bytes(self):
|
||||
message = EmailMessage()
|
||||
message['Message-ID'] = '<acceptance@runtime.invalid>'
|
||||
message.set_content('中文验收正文')
|
||||
message.add_attachment(b'payload', maintype='text', subtype='plain', filename='验收附件.txt')
|
||||
raw = message.as_bytes()
|
||||
verify_message_content(raw, 'acceptance@runtime.invalid', '中文验收正文', b'payload')
|
||||
for identifier, body, attachment in [('wrong', '中文验收正文', b'payload'),
|
||||
('acceptance@runtime.invalid', '缺失正文', b'payload'),
|
||||
('acceptance@runtime.invalid', '中文验收正文', b'corrupted')]:
|
||||
with self.subTest(identifier=identifier, body=body), self.assertRaises(AssertionError):
|
||||
verify_message_content(raw, identifier, body, attachment)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,55 @@
|
||||
"""凭据扫描回归使用合成值,不复制真实凭据。"""
|
||||
|
||||
from pathlib import Path
|
||||
import os
|
||||
import runpy
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
|
||||
from check_tracked_secrets import scan_source
|
||||
|
||||
|
||||
class TrackedSecretsTest(unittest.TestCase):
|
||||
def test_cloud_key_literal_is_rejected_without_value(self):
|
||||
secret = 'LTAI' + 'a' * 20
|
||||
result = scan_source('config.txt', 'key=' + secret)
|
||||
self.assertEqual(result, [('config.txt', 1, 'credential_literal')])
|
||||
self.assertNotIn(secret, repr(result))
|
||||
|
||||
def test_python_cloud_secret_assignment_is_rejected(self):
|
||||
result = scan_source('tool.py', 'ak="sample"\nsk="sample-secret"\n')
|
||||
self.assertEqual(len(result), 2)
|
||||
self.assertNotIn('sample-secret', repr(result))
|
||||
|
||||
def test_environment_lookup_is_allowed(self):
|
||||
self.assertEqual(scan_source('tool.py', 'import os\nak=os.environ["KEY"]\nsk=os.environ["SECRET"]'), [])
|
||||
|
||||
def test_private_key_is_rejected(self):
|
||||
header = '-----BEGIN RSA PRIVATE' + ' KEY-----'
|
||||
self.assertEqual(scan_source('key.pem', header), [('key.pem', 1, 'credential_literal')])
|
||||
|
||||
def test_python_parse_failure_is_not_silently_skipped(self):
|
||||
self.assertEqual(scan_source('tool.py', 'if:'), [('tool.py', 1, 'python_parse_failed')])
|
||||
|
||||
def test_missing_cloud_environment_rejects_before_network(self):
|
||||
tools = Path(__file__).resolve().parents[1] / 'tools'
|
||||
ssh_module = types.ModuleType('paramiko')
|
||||
ssh_module.SSHClient = Mock(side_effect=AssertionError('unexpected SSH operation'))
|
||||
for name in ('add_dns.py', 'add_missing_dns.py', 'add_missing_dns_v2.py', 'check_alidns.py'):
|
||||
for environment, missing in (({}, 'SMAIL_ALIDNS_ACCESS_KEY_ID'),
|
||||
({'SMAIL_ALIDNS_ACCESS_KEY_ID': 'sample'}, 'SMAIL_ALIDNS_ACCESS_KEY_SECRET')):
|
||||
with self.subTest(name=name, missing=missing), \
|
||||
patch.dict(os.environ, environment, clear=True), \
|
||||
patch.dict(sys.modules, {'paramiko': ssh_module}), \
|
||||
patch('urllib.request.urlopen', side_effect=AssertionError('unexpected HTTP operation')), \
|
||||
self.assertRaises(KeyError) as error:
|
||||
runpy.run_path(str(tools / name), run_name='__main__')
|
||||
self.assertEqual(error.exception.args, (missing,))
|
||||
ssh_module.SSHClient.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -1,5 +1,9 @@
|
||||
import os
|
||||
import urllib.request, urllib.parse, hmac, hashlib, base64, time, json, paramiko, subprocess
|
||||
|
||||
ak = os.environ['SMAIL_ALIDNS_ACCESS_KEY_ID']
|
||||
sk = os.environ['SMAIL_ALIDNS_ACCESS_KEY_SECRET']
|
||||
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(paramiko.RejectPolicy())
|
||||
from remote_ssh import connect_remote
|
||||
@@ -18,9 +22,6 @@ print('DKIM Value:', dkim_val)
|
||||
dmarc_val = 'v=DMARC1; p=quarantine; rua=mailto:admin@solit.cn'
|
||||
print('DMARC Value:', dmarc_val)
|
||||
|
||||
ak = 'LTAI5t5ihQv8vf7jS4CNKThu'
|
||||
sk = 'zzJODyPWOInEnI8ueQVPLNw2HzbfKx'
|
||||
|
||||
def aliyun_request(params):
|
||||
public_params = {
|
||||
'Format': 'JSON',
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import os
|
||||
import urllib.request, urllib.parse, hmac, hashlib, base64, time, json
|
||||
|
||||
ak = 'LTAI5t5ihQv8vf7jS4CNKThu'
|
||||
sk = 'zzJODyPWOInEnI8ueQVPLNw2HzbfKx'
|
||||
ak = os.environ['SMAIL_ALIDNS_ACCESS_KEY_ID']
|
||||
sk = os.environ['SMAIL_ALIDNS_ACCESS_KEY_SECRET']
|
||||
|
||||
def aliyun_request(params):
|
||||
public_params = {
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import os
|
||||
import urllib.request, urllib.parse, hmac, hashlib, base64, time, json
|
||||
|
||||
ak = 'LTAI5t5ihQv8vf7jS4CNKThu'
|
||||
sk = 'zzJODyPWOInEnI8ueQVPLNw2HzbfKx'
|
||||
ak = os.environ['SMAIL_ALIDNS_ACCESS_KEY_ID']
|
||||
sk = os.environ['SMAIL_ALIDNS_ACCESS_KEY_SECRET']
|
||||
|
||||
def percent_encode(val):
|
||||
res = urllib.parse.quote(str(val), safe='')
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import os
|
||||
import urllib.request, urllib.parse, hmac, hashlib, base64, time, json
|
||||
|
||||
ak = 'LTAI5t5ihQv8vf7jS4CNKThu'
|
||||
sk = 'zzJODyPWOInEnI8ueQVPLNw2HzbfKx'
|
||||
ak = os.environ['SMAIL_ALIDNS_ACCESS_KEY_ID']
|
||||
sk = os.environ['SMAIL_ALIDNS_ACCESS_KEY_SECRET']
|
||||
|
||||
def aliyun_request(params):
|
||||
public_params = {
|
||||
|
||||
+12
-10
@@ -1,18 +1,20 @@
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
import os
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True)
|
||||
page = browser.new_page()
|
||||
page.on(console, lambda msg: print(f[CONSOLE {msg.type}] {msg.text}))
|
||||
page.on(pageerror, lambda exc: print(f[PAGE ERROR] {exc}))
|
||||
response = page.goto(http://47.108.78.12/)
|
||||
print(Status:, response.status)
|
||||
page.on('console', lambda msg: print(f'控制台类型:{msg.type}'))
|
||||
page.on('pageerror', lambda exc: print(f'页面错误类型:{type(exc).__name__}'))
|
||||
response = page.goto(os.environ['SMAIL_TEST_URL'])
|
||||
print('状态码:', response.status)
|
||||
content = page.content()
|
||||
print(HTML length:, len(content))
|
||||
next_btn = page.locator(#next)
|
||||
print(next locator count:, next_btn.count())
|
||||
buttons = page.locator(button)
|
||||
print(Total buttons:, buttons.count())
|
||||
print('页面长度:', len(content))
|
||||
next_btn = page.locator('#next')
|
||||
print('下一步按钮数量:', next_btn.count())
|
||||
buttons = page.locator('button')
|
||||
print('按钮总数:', buttons.count())
|
||||
for i in range(buttons.count()):
|
||||
print(fButton {i}: text={buttons.nth(i).inner_text()} id={buttons.nth(i).get_attribute('id')})
|
||||
print(f'按钮 {i}:{buttons.nth(i).inner_text()}')
|
||||
browser.close()
|
||||
@@ -0,0 +1,58 @@
|
||||
"""检查已跟踪文件中的云凭据,报告只包含位置,不回显敏感值。"""
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
ACCESS_KEY = re.compile(r'LTAI[A-Za-z0-9]{16,}')
|
||||
PRIVATE_KEY = re.compile(r'-----BEGIN (?:[A-Z0-9 ]+ )?PRIVATE' + r' KEY-----')
|
||||
|
||||
|
||||
def scan_source(name, source):
|
||||
findings = set()
|
||||
for number, line in enumerate(source.splitlines(), 1):
|
||||
if ACCESS_KEY.search(line) or PRIVATE_KEY.search(line):
|
||||
findings.add((name, number, 'credential_literal'))
|
||||
if name.endswith('.py'):
|
||||
try:
|
||||
tree = ast.parse(source)
|
||||
except SyntaxError:
|
||||
findings.add((name, 1, 'python_parse_failed'))
|
||||
return sorted(findings)
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.Constant):
|
||||
continue
|
||||
if not isinstance(node.value.value, str) or not node.value.value:
|
||||
continue
|
||||
for target in node.targets:
|
||||
if isinstance(target, ast.Name) and target.id.lower() in {'ak', 'sk'}:
|
||||
findings.add((name, node.lineno, 'cloud_key_assignment'))
|
||||
return sorted(findings)
|
||||
|
||||
|
||||
def main():
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
result = subprocess.run(['git', 'ls-files', '-z'], cwd=root, check=True, capture_output=True)
|
||||
findings = []
|
||||
for name in result.stdout.decode('utf-8').split('\0'):
|
||||
if not name:
|
||||
continue
|
||||
path = root / name
|
||||
if not path.is_file():
|
||||
continue
|
||||
try:
|
||||
source = path.read_text(encoding='utf-8-sig')
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
findings.extend(scan_source(name, source))
|
||||
for name, line, category in findings:
|
||||
print(f'{name}:{line}: {category}')
|
||||
print('凭据检查:' + ('不通过' if findings else '通过'))
|
||||
return 1 if findings else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
+101
-3
@@ -4,6 +4,8 @@ import argparse
|
||||
import base64
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from email.message import EmailMessage
|
||||
from email.parser import BytesParser
|
||||
from email.policy import default as email_policy
|
||||
import http.client
|
||||
from http.cookies import SimpleCookie
|
||||
import imaplib
|
||||
@@ -15,6 +17,7 @@ import smtplib
|
||||
import socket
|
||||
import sqlite3
|
||||
import ssl
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
@@ -93,6 +96,48 @@ class LocalPOP(poplib.POP3_SSL):
|
||||
return self.context.wrap_socket(raw, server_hostname=self.host)
|
||||
|
||||
|
||||
def verify_message_content(raw, message_id, body, attachment):
|
||||
message = BytesParser(policy=email_policy).parsebytes(raw)
|
||||
assert str(message['Message-ID']).strip('<>') == message_id
|
||||
text = message.get_body(preferencelist=('plain',))
|
||||
assert text is not None and body in text.get_content()
|
||||
files = list(message.iter_attachments())
|
||||
assert len(files) == 1 and files[0].get_filename() == '验收附件.txt'
|
||||
assert files[0].get_payload(decode=True) == attachment
|
||||
|
||||
|
||||
def verify_protocol_read(context, codes, message_id, body, attachment):
|
||||
# 通过协议取回真实正文,不以数据库存在或欢迎响应替代取信验收。
|
||||
client = LocalIMAP(HOST, 993, ssl_context=context, timeout=30)
|
||||
try:
|
||||
assert client.login(ADDRESS, codes['imap'])[0] == 'OK'
|
||||
assert client.select('INBOX', readonly=True)[0] == 'OK'
|
||||
status, matches = client.uid('SEARCH', None, 'HEADER', 'Message-ID', '"<' + message_id + '>"')
|
||||
assert status == 'OK' and matches[0].split(), 'IMAP 收件箱没有测试邮件'
|
||||
status, contents = client.uid('FETCH', matches[0].split()[-1], '(BODY.PEEK[])')
|
||||
assert status == 'OK'
|
||||
raw = next(item[1] for item in contents if isinstance(item, tuple))
|
||||
verify_message_content(raw, message_id, body, attachment)
|
||||
finally:
|
||||
client.logout()
|
||||
client = LocalPOP(HOST, 995, context=context, timeout=30)
|
||||
try:
|
||||
client.user(ADDRESS)
|
||||
client.pass_(codes['pop3'])
|
||||
found = False
|
||||
for item in client.list()[1]:
|
||||
raw = b'\r\n'.join(client.retr(int(item.split()[0]))[1])
|
||||
message = BytesParser(policy=email_policy).parsebytes(raw)
|
||||
if str(message['Message-ID']).strip('<>') == message_id:
|
||||
verify_message_content(raw, message_id, body, attachment)
|
||||
found = True
|
||||
break
|
||||
assert found, 'POP3 没有取到测试邮件'
|
||||
finally:
|
||||
client.quit()
|
||||
return {'imap_inbox': True, 'imap_body_attachment': True, 'pop3_body_attachment': True}
|
||||
|
||||
|
||||
def protocol_operation(protocol, context, code):
|
||||
if protocol == 'smtp':
|
||||
client = smtp_connect(context)
|
||||
@@ -140,15 +185,61 @@ def protocol_operation(protocol, context, code):
|
||||
return invoke
|
||||
|
||||
|
||||
def verify_restored(root, context, password, message_id, port):
|
||||
client = Client(context, port=port)
|
||||
for _ in range(60):
|
||||
try:
|
||||
assert client.request('/api/ping', method='GET')[0] == 200
|
||||
break
|
||||
except (OSError, AssertionError):
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise AssertionError('恢复实例未就绪')
|
||||
assert client.success('/api/v1/login', {'account': ACCOUNT, 'password': password})['account'] == ACCOUNT
|
||||
assert client.success('/api/v1/user/info')['account'] == ACCOUNT
|
||||
client.success('/api/v1/security/reauth', {'password': password})
|
||||
codes = {}
|
||||
for protocol in ('smtp', 'imap', 'pop3'):
|
||||
codes[protocol] = client.success('/api/v1/security/credentials/protocols/generate', {'protocol': protocol})['code']
|
||||
assert protocol_operation(protocol, context, codes[protocol])() == 'success'
|
||||
body = '内部25端口真实收信,检查中文正文和附件。'
|
||||
attachment = '隔离验收附件内容'.encode('utf-8')
|
||||
result = {'login': True, 'session': True, 'protocol_login': list(codes),
|
||||
'historical_mail': verify_protocol_read(context, codes, message_id, body, attachment)}
|
||||
message = EmailMessage()
|
||||
message['From'] = 'restore@sender.invalid'
|
||||
message['To'] = ADDRESS
|
||||
message['Subject'] = '备份恢复后中文收信验收'
|
||||
restored_id = message_id + '.restored'
|
||||
message['Message-ID'] = '<' + restored_id + '>'
|
||||
message.set_content(body)
|
||||
message.add_attachment(attachment, maintype='text', subtype='plain', filename='验收附件.txt')
|
||||
with smtp_connect(context) as smtp:
|
||||
assert not smtp.send_message(message)
|
||||
result['new_mail'] = verify_protocol_read(context, codes, restored_id, body, attachment)
|
||||
result['passed'] = True
|
||||
(root.parent / 'restore-probe-result.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('root', type=Path)
|
||||
parser.add_argument('--live-net', required=True)
|
||||
parser.add_argument('--proxy', action='store_true')
|
||||
parser.add_argument('--password-stdin', action='store_true')
|
||||
parser.add_argument('--restore-message-id')
|
||||
args = parser.parse_args()
|
||||
assert str(Path('/proc/self/ns/net').readlink()) != args.live_net, '禁止在宿主网络执行测试'
|
||||
root = args.root.resolve(strict=True)
|
||||
assert root.parent.name.startswith('smail-review-runtime-') and root.name == 'instance'
|
||||
assert root.parent.name.startswith(('smail-review-runtime-', 'smail-review-restore-')) and root.name == 'instance'
|
||||
password = sys.stdin.readline().strip() if args.password_stdin else secrets.token_urlsafe(9)
|
||||
assert len(password) == 12
|
||||
if args.restore_message_id:
|
||||
assert args.password_stdin
|
||||
context = ssl.create_default_context(cafile=str(root / 'ssl/runtime.crt'))
|
||||
verify_restored(root, context, password, args.restore_message_id, 8002 if args.proxy else 443)
|
||||
return
|
||||
result = {'stage': 'setup', 'passed': False}
|
||||
started = time.monotonic()
|
||||
try:
|
||||
@@ -170,7 +261,6 @@ def main():
|
||||
client.success(setup, {'step': 'database', 'action': 'set', 'db_type': 'sqlite', 'db_dsn': ''})
|
||||
with sqlite3.connect(f'file:{root / "data/smail.db"}?mode=ro', uri=True) as database:
|
||||
assert database.execute('SELECT COUNT(*) FROM user').fetchone()[0] == 0
|
||||
password = secrets.token_urlsafe(9)
|
||||
client.success(setup, {'step': 'password', 'action': 'set', 'account': ACCOUNT, 'password': password})
|
||||
client.success(setup, {'step': 'domain', 'action': 'set', 'smtp_domain': 'runtime.invalid',
|
||||
'web_domain': HOST, 'multi_domain': ''})
|
||||
@@ -193,6 +283,12 @@ def main():
|
||||
assert 'smail_csrf' in client.cookies
|
||||
assert client.success('/api/v1/user/info').get('account') == ACCOUNT
|
||||
result['setup'] = {'empty_database': True, 'token_required': True, 'login': True, 'session': True}
|
||||
# 首次登录先完成正式安全设置,不绕过恢复方式要求直接启用协议。
|
||||
client.success('/api/v1/security/reauth', {'password': password})
|
||||
recovery = client.success('/api/v1/security/credentials/recovery-codes/generate', {'count': 10})
|
||||
assert len(recovery['codes']) == 10
|
||||
recovery.clear()
|
||||
result['security_setup'] = {'recovery_codes_generated': 10, 'plaintext_logged': False}
|
||||
if args.proxy:
|
||||
from runtime_proxy import configure_proxy
|
||||
result['proxy'] = configure_proxy(client, context, login, Client, LocalHTTPS)
|
||||
@@ -213,7 +309,7 @@ def main():
|
||||
result['smtp_invalid_credentials'] = 535
|
||||
result['stage'] = 'smtp'
|
||||
message = EmailMessage()
|
||||
message['From'] = 'probe@runtime.invalid'
|
||||
message['From'] = 'probe@sender.invalid'
|
||||
message['To'] = ADDRESS
|
||||
message['Subject'] = '隔离首次安装中文收信验收'
|
||||
message_id = 'runtime-' + root.parent.name + '@runtime.invalid'
|
||||
@@ -234,6 +330,8 @@ def main():
|
||||
(rows[0][0],)).fetchall()
|
||||
assert owners == [(ACCOUNT,)]
|
||||
result['smtp'] = {'message_id': message_id, 'stored': True, 'owners': 1, 'attachment': True}
|
||||
result['stage'] = 'protocol_read'
|
||||
result['protocol_read'] = verify_protocol_read(context, codes, message_id, body, attachment)
|
||||
result['stage'] = 'overload'
|
||||
rounds = []
|
||||
result['overload'] = rounds
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
"""对已停止的隔离实例执行完整冷备、空目录还原与业务读取,不操作正式实例。"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
import time
|
||||
|
||||
from runtime_review import collect_and_stop, database_integrity, run, service_state
|
||||
|
||||
|
||||
def archive_members(archive):
|
||||
members = archive.getmembers()
|
||||
for member in members:
|
||||
path = Path(member.name)
|
||||
assert not path.is_absolute() and '..' not in path.parts
|
||||
assert path.parts and path.parts[0] == 'instance'
|
||||
assert member.isdir() or member.isfile(), '备份包含链接或特殊文件'
|
||||
return members
|
||||
|
||||
|
||||
def tree_digest(root):
|
||||
result = {}
|
||||
for path in sorted(root.rglob('*')):
|
||||
assert not path.is_symlink(), '隔离备份拒绝链接'
|
||||
if path.is_file():
|
||||
with path.open('rb') as stream:
|
||||
result[str(path.relative_to(root))] = hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
return result
|
||||
|
||||
|
||||
def verify_backup_restore(source, evidence, password, proxy):
|
||||
assert os.geteuid() == 0 and run('hostname') == 'ikun2'
|
||||
source = source.resolve(strict=True)
|
||||
assert source.parent.parent == Path('/var/tmp')
|
||||
assert source.parent.name.startswith('smail-review-runtime-') and source.name == 'instance'
|
||||
assert run('systemctl', 'show', evidence['unit'], '--property=MainPID', '--value') == '0'
|
||||
database_integrity(source)
|
||||
before = service_state()
|
||||
source_digest = tree_digest(source)
|
||||
stamp = time.strftime('%Y%m%d-%H%M%S')
|
||||
target = Path('/var/tmp') / ('smail-review-restore-' + stamp)
|
||||
target.mkdir(mode=0o750)
|
||||
shutil.chown(target, user='smail', group='smail')
|
||||
archive_path = target / 'cold-backup.tar.gz'
|
||||
with tarfile.open(archive_path, 'x:gz') as archive:
|
||||
archive.add(source, arcname='instance')
|
||||
archive_path.chmod(0o600)
|
||||
with tarfile.open(archive_path, 'r:gz') as archive:
|
||||
members = archive_members(archive)
|
||||
assert not (target / 'instance').exists()
|
||||
archive.extractall(target, members=members, filter='data')
|
||||
root = target / 'instance'
|
||||
assert tree_digest(root) == source_digest
|
||||
assert (root / 'data/smail.db').stat().st_ino != (source / 'data/smail.db').stat().st_ino
|
||||
for path in [root, *root.rglob('*')]:
|
||||
shutil.chown(path, user='smail', group='smail')
|
||||
control = target / 'control'
|
||||
control.mkdir(mode=0o750)
|
||||
shutil.chown(control, user='smail', group='smail')
|
||||
probe_path = target / 'runtime_probe.py'
|
||||
shutil.copy2(Path(__file__).with_name('runtime_probe.py'), probe_path)
|
||||
probe_path.chmod(0o640)
|
||||
shutil.chown(probe_path, user='root', group='smail')
|
||||
unit = 'smail-runtime-review-' + stamp
|
||||
probe_unit = 'smail-runtime-probe-' + stamp
|
||||
result = {'source': str(source), 'target': str(target), 'file_count': len(source_digest),
|
||||
'archive_sha256': hashlib.sha256(archive_path.read_bytes()).hexdigest(),
|
||||
'empty_target': True, 'byte_identical': True, 'cold_backup': True}
|
||||
started = time.monotonic()
|
||||
try:
|
||||
subprocess.run([
|
||||
'systemd-run', '--unit=' + unit, '--property=Type=exec',
|
||||
'--property=User=smail', '--property=Group=smail',
|
||||
'--property=PrivateNetwork=yes', '--property=PrivateTmp=yes',
|
||||
'--property=ProtectSystem=strict', '--property=ProtectHome=yes',
|
||||
'--property=NoNewPrivileges=yes', '--property=CapabilityBoundingSet=CAP_NET_BIND_SERVICE',
|
||||
'--property=AmbientCapabilities=CAP_NET_BIND_SERVICE',
|
||||
'--property=MemoryMax=192M', '--property=MemorySwapMax=0', '--property=CPUQuota=25%',
|
||||
'--property=TasksMax=128', '--property=RuntimeMaxSec=180s', '--property=OOMPolicy=stop',
|
||||
'--property=KillMode=control-group', '--property=TimeoutStopSec=10s',
|
||||
'--property=BindPaths=' + str(root) + ':/opt/smail ' + str(control) + ':/run/smail',
|
||||
'--property=ReadWritePaths=/opt/smail /run/smail',
|
||||
'--property=WorkingDirectory=/opt/smail', '--setenv=SMAIL_ROOT=/opt/smail',
|
||||
'--setenv=GOMEMLIMIT=64MiB', '--setenv=GOMAXPROCS=1', '/opt/smail/smail',
|
||||
], check=True)
|
||||
pid = run('systemctl', 'show', unit, '--property=MainPID', '--value')
|
||||
live_net = str(Path('/proc/self/ns/net').readlink())
|
||||
assert pid != '0' and str(Path('/proc/' + pid + '/ns/net').readlink()) != live_net
|
||||
assert Path('/proc/' + pid + '/root/opt/smail/smail').stat().st_ino == (root / 'smail').stat().st_ino
|
||||
probe = subprocess.run([
|
||||
'systemd-run', '--wait', '--pipe', '--unit=' + probe_unit,
|
||||
'--property=User=smail', '--property=Group=smail',
|
||||
'--property=NetworkNamespacePath=/proc/' + pid + '/ns/net',
|
||||
'--property=ProtectSystem=strict', '--property=ProtectHome=yes',
|
||||
'--property=NoNewPrivileges=yes', '--property=InaccessiblePaths=/opt/smail /run/smail',
|
||||
'--property=ReadWritePaths=' + str(target), '--property=MemoryMax=48M',
|
||||
'--property=MemorySwapMax=0', '--property=CPUQuota=25%', '--property=TasksMax=32',
|
||||
'--property=RuntimeMaxSec=150s', '--property=OOMPolicy=stop',
|
||||
'--property=KillMode=control-group', '--property=TimeoutStopSec=10s',
|
||||
'python3', str(probe_path), str(root), '--live-net', live_net, '--password-stdin',
|
||||
'--restore-message-id', evidence['probe']['smtp']['message_id'], *(['--proxy'] if proxy else []),
|
||||
], input=password + '\n', text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=165)
|
||||
(target / 'probe.log').write_text(probe.stdout, encoding='utf-8')
|
||||
result['probe_code'] = probe.returncode
|
||||
assert probe.returncode == 0, '还原业务验收失败,请检查保留的日志'
|
||||
result['probe'] = json.loads((target / 'restore-probe-result.json').read_text())
|
||||
assert result['probe']['passed']
|
||||
finally:
|
||||
collect_and_stop(result, unit, probe_unit)
|
||||
result['live_unchanged'] = service_state() == before
|
||||
result['source_unchanged'] = tree_digest(source) == source_digest
|
||||
result['seconds'] = round(time.monotonic() - started, 2)
|
||||
result['integrity'] = database_integrity(root)
|
||||
(target / 'service.log').write_text(run('journalctl', '-u', unit, '--no-pager'), encoding='utf-8')
|
||||
(target / 'result.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
assert result['live_unchanged'] and result['source_unchanged']
|
||||
assert not result['cleanup_errors'] and not result['resource_errors']
|
||||
assert 'Result=success' in result['resources'] and 'MemorySwapPeak=0' in result['resources']
|
||||
return result
|
||||
@@ -1,11 +1,13 @@
|
||||
"""在 47 服务器创建空数据副本并进行受限验收,不停止线上服务或删除文件。"""
|
||||
|
||||
import argparse
|
||||
from contextlib import closing
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
@@ -21,6 +23,18 @@ def service_state():
|
||||
for name in ('smail.service', 'nginx.service', 'mysql.service', 'Scert.service')}
|
||||
|
||||
|
||||
def database_integrity(root):
|
||||
# 验收器不能创建数据库,也不能允许第二个状态库。
|
||||
data = root / 'data'
|
||||
assert not (data / 'smail-state.db').exists(), '单数据库实例生成了旧状态库'
|
||||
assert sorted(path.name for path in data.glob('*.db')) == ['smail.db']
|
||||
with closing(sqlite3.connect((data / 'smail.db').as_uri() + '?mode=ro', uri=True)) as database:
|
||||
integrity = database.execute('PRAGMA integrity_check').fetchall()
|
||||
assert database.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='config_records'").fetchone()[0] == 1
|
||||
assert integrity == [('ok',)]
|
||||
return {'smail.db': integrity}
|
||||
|
||||
|
||||
def stop_test_units(names):
|
||||
assert all(re.fullmatch(r'smail-runtime-(review|probe|nginx)-[0-9]{8}-[0-9]{6}', name)
|
||||
for name in names), '只能停止本轮隔离测试单元'
|
||||
@@ -62,6 +76,8 @@ def main():
|
||||
parser.add_argument('--main-sha', required=True)
|
||||
parser.add_argument('--binary', type=Path, default=Path('/opt/smail/smail'))
|
||||
parser.add_argument('--nginx-template', type=Path)
|
||||
parser.add_argument('--memory-max-mib', type=int, choices=(128, 192, 256), default=128)
|
||||
parser.add_argument('--restore', action='store_true')
|
||||
args = parser.parse_args()
|
||||
assert os.geteuid() == 0 and run('hostname') == 'ikun2'
|
||||
os.umask(0o027)
|
||||
@@ -80,7 +96,7 @@ def main():
|
||||
with (live / 'smail').open('rb') as stream:
|
||||
live_sha = hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
memory = dict(line.split(':', 1) for line in Path('/proc/meminfo').read_text().splitlines())
|
||||
required_memory = 320 if args.nginx_template else 272
|
||||
required_memory = args.memory_max_mib + (192 if args.nginx_template else 144)
|
||||
assert int(memory['MemAvailable'].split()[0]) >= required_memory * 1024, '可用内存不足,暂停隔离验收'
|
||||
stamp = time.strftime('%Y%m%d-%H%M%S')
|
||||
target = Path('/var/tmp') / ('smail-review-runtime-' + stamp)
|
||||
@@ -94,8 +110,12 @@ def main():
|
||||
shutil.copy2(binary, root / 'smail')
|
||||
(root / 'smail').chmod(0o755)
|
||||
shutil.copy2(Path(__file__).with_name('runtime_probe.py'), target / 'runtime_probe.py')
|
||||
(target / 'runtime_probe.py').chmod(0o640)
|
||||
shutil.chown(target / 'runtime_probe.py', user='root', group='smail')
|
||||
if args.nginx_template:
|
||||
shutil.copy2(Path(__file__).with_name('runtime_proxy.py'), target / 'runtime_proxy.py')
|
||||
(target / 'runtime_proxy.py').chmod(0o640)
|
||||
shutil.chown(target / 'runtime_proxy.py', user='root', group='smail')
|
||||
subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '2',
|
||||
'-keyout', str(root / 'ssl/runtime.key'), '-out', str(root / 'ssl/runtime.crt'),
|
||||
'-subj', '/CN=mail.runtime.invalid', '-addext',
|
||||
@@ -111,6 +131,7 @@ def main():
|
||||
probe_unit = 'smail-runtime-probe-' + stamp
|
||||
proxy_unit = 'smail-runtime-nginx-' + stamp
|
||||
result = {'target': str(target), 'unit': unit, 'main_sha256': args.main_sha,
|
||||
'memory_max_mib': args.memory_max_mib,
|
||||
'source': str(binary), 'live_sha256': live_sha, 'before': before,
|
||||
'topology': 'nginx-8001-8002' if args.nginx_template else 'direct-80-443'}
|
||||
service_env = []
|
||||
@@ -123,6 +144,7 @@ def main():
|
||||
'--setenv=SMAIL_HTTPS_PORT=8002']
|
||||
probe_args = ['--proxy']
|
||||
started = time.strftime('%Y-%m-%dT%H:%M:%S')
|
||||
password = secrets.token_urlsafe(9)
|
||||
try:
|
||||
subprocess.run([
|
||||
'systemd-run', '--unit=' + unit, '--property=Type=exec',
|
||||
@@ -131,7 +153,8 @@ def main():
|
||||
'--property=ProtectSystem=strict', '--property=ProtectHome=yes',
|
||||
'--property=NoNewPrivileges=yes', '--property=CapabilityBoundingSet=CAP_NET_BIND_SERVICE',
|
||||
'--property=AmbientCapabilities=CAP_NET_BIND_SERVICE',
|
||||
'--property=MemoryMax=128M', '--property=MemorySwapMax=0', '--property=CPUQuota=25%',
|
||||
'--property=MemoryMax=' + str(args.memory_max_mib) + 'M',
|
||||
'--property=MemorySwapMax=0', '--property=CPUQuota=25%',
|
||||
'--property=TasksMax=128', '--property=RuntimeMaxSec=300s', '--property=OOMPolicy=stop',
|
||||
'--property=KillMode=control-group', '--property=TimeoutStopSec=10s',
|
||||
'--property=BindPaths=' + str(root) + ':/opt/smail ' + str(control) + ':/run/smail',
|
||||
@@ -173,8 +196,9 @@ def main():
|
||||
'--property=MemorySwapMax=0', '--property=CPUQuota=25%', '--property=TasksMax=32',
|
||||
'--property=RuntimeMaxSec=240s', '--property=OOMPolicy=stop',
|
||||
'--property=KillMode=control-group', '--property=TimeoutStopSec=10s',
|
||||
'python3', str(target / 'runtime_probe.py'), str(root), '--live-net', live_net, *probe_args,
|
||||
], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=270)
|
||||
'python3', str(target / 'runtime_probe.py'), str(root), '--live-net', live_net,
|
||||
'--password-stdin', *probe_args,
|
||||
], input=password + '\n', text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=270)
|
||||
(target / 'probe.log').write_text(probe.stdout, encoding='utf-8')
|
||||
result['probe_code'] = probe.returncode
|
||||
if (target / 'probe-result.json').exists():
|
||||
@@ -191,12 +215,10 @@ def main():
|
||||
with (live / 'smail').open('rb') as stream:
|
||||
result['live_unchanged'] = (result['after'] == before
|
||||
and hashlib.file_digest(stream, 'sha256').hexdigest() == live_sha)
|
||||
result['integrity'] = {}
|
||||
for name in ('smail.db', 'smail-state.db'):
|
||||
path = root / 'data' / name
|
||||
if path.exists():
|
||||
with sqlite3.connect(f'file:{path}?mode=ro', uri=True) as database:
|
||||
result['integrity'][name] = database.execute('PRAGMA integrity_check').fetchall()
|
||||
try:
|
||||
result['integrity'] = database_integrity(root)
|
||||
except (AssertionError, sqlite3.Error) as error:
|
||||
result['integrity'] = {'error': type(error).__name__}
|
||||
(target / 'service.log').write_text(run('journalctl', '-u', unit, '--no-pager'), encoding='utf-8')
|
||||
kernel = run('journalctl', '-k', '--since', started, '--no-pager')
|
||||
result['oom_events'] = sum('oom-kill:constraint=' in line for line in kernel.splitlines())
|
||||
@@ -205,7 +227,12 @@ def main():
|
||||
assert result['live_unchanged'] and result['oom_events'] == 0
|
||||
assert not result['cleanup_errors'], '测试单元未全部停止,需要核查保留的结果'
|
||||
assert not result['resource_errors'], '资源证据不完整,需要核查保留的结果'
|
||||
assert result['integrity'] == {'smail.db': [('ok',)], 'smail-state.db': [('ok',)]}
|
||||
assert result['integrity'] == {'smail.db': [('ok',)]}
|
||||
if args.restore:
|
||||
from runtime_restore import verify_backup_restore
|
||||
restored = verify_backup_restore(root, result, password, args.nginx_template is not None)
|
||||
(target / 'restore-result.json').write_text(json.dumps(restored, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print(json.dumps({'restore': restored}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in new issue
Block a user