补齐SQLite复制中断恢复及云端部署验收
Security Checks / security (push) Waiting to run

This commit is contained in:
xuans committed 2026-09-21 06:27:57 +08:00
1 parent 2de629fb79
commit d8b5a5fb23
7 files changed
+405 -5

No files matched your search

@@ -0,0 +1,78 @@
"""中断验收的范围、种子数据和误报防护测试,不终止任何真实服务。"""
from contextlib import closing
import http.client
from pathlib import Path
import sqlite3
import sys
import tempfile
import unittest
from unittest.mock import Mock, patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
import runtime_interruption as interruption
class RuntimeInterruptionTest(unittest.TestCase):
def test_occupied_target_requires_structured_error_contract(self):
payload = {'errorNo': 409, 'data': {'code': 'DATABASE_MIGRATION_TARGET_EXISTS'}}
self.assertEqual(interruption.assert_occupied_target(409, payload)['status'], 409)
for status, invalid in ((200, payload), (409, None),
(409, {'errorNo': 409, 'data': 'DATABASE_MIGRATION_TARGET_EXISTS'}),
(409, {'errorNo': 409, 'data': {'code': 'OTHER'}}),
(409, {'errorNo': 0, 'data': payload['data']})):
with self.assertRaises(AssertionError):
interruption.assert_occupied_target(status, invalid)
def test_formal_instance_rejected_before_service_command(self):
with patch.object(interruption, 'service_state') as state, patch.object(interruption.subprocess, 'run') as run:
with self.assertRaises(AssertionError):
interruption.verify_interruption(Path('/opt/smail'), 'smail.service', 'not-a-secret', {})
state.assert_not_called()
run.assert_not_called()
def test_seed_missing_database_is_not_created(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / 'data').mkdir()
with self.assertRaises(FileNotFoundError):
interruption.seed_source(root)
self.assertFalse((root / 'data/smail.db').exists())
def test_seed_preserves_existing_mail_and_schema(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / 'data').mkdir()
path = root / 'data/smail.db'
with closing(sqlite3.connect(path)) as database:
database.execute('CREATE TABLE email (id INTEGER PRIMARY KEY, msg_id TEXT UNIQUE, text TEXT)')
database.execute("INSERT INTO email VALUES (1, 'original', 'body')")
database.commit()
result = interruption.seed_source(root)
self.assertEqual(result['added_rows'], 1024)
with closing(sqlite3.connect(path)) as database:
self.assertEqual(database.execute('SELECT COUNT(*) FROM email').fetchone()[0], 1025)
self.assertEqual(database.execute('SELECT * FROM email WHERE id=1').fetchone(), (1, 'original', 'body'))
self.assertEqual(database.execute('SELECT COUNT(*) FROM sqlite_master WHERE type=\'table\'').fetchone()[0], 1)
def test_completed_request_cannot_count_as_interruption(self):
client = Mock()
client.success.return_value = {'migration': {'id': 'one'}, 'token': 'not-a-secret'}
for response in ((200, {}, {}), (500, {}, {})):
client.request.return_value = response
with patch.object(interruption, 'login', return_value=(client, None)), \
patch.object(interruption, 'clear_pending_restart', return_value=1):
with self.assertRaises(AssertionError):
interruption.prepare(Path('/unused'), 'not-a-secret')
def test_disconnected_request_requires_controller_process_evidence(self):
client = Mock()
client.success.return_value = {'migration': {'id': 'one'}, 'token': 'not-a-secret'}
client.request.side_effect = http.client.RemoteDisconnected()
with patch.object(interruption, 'login', return_value=(client, None)), \
patch.object(interruption, 'clear_pending_restart', return_value=1):
self.assertEqual(interruption.prepare(Path('/unused'), 'not-a-secret'), {'request_interrupted': True})
if __name__ == '__main__':
unittest.main()
+232
View File
@@ -0,0 +1,232 @@
"""在隔离 SQLite 实例复制事务未提交时终止进程,保留残留目标并验证恢复。"""
import argparse
from contextlib import closing
from email.message import EmailMessage
import hashlib
import http.client
import json
import os
from pathlib import Path
import re
import shutil
import sqlite3
import subprocess
import sys
import threading
import time
from runtime_migration import (
clear_pending_restart, database_fingerprint, login, read_database,
root_database, run_probe, service_state,
)
from runtime_probe import ACCOUNT, ADDRESS, smtp_connect, verify_protocol_read
def assert_scope(root, unit):
assert root.parent.parent == Path('/var/tmp')
assert re.fullmatch(r'smail-review-runtime-\d{8}-\d{6}', root.parent.name)
assert root.name == 'instance' and root.resolve(strict=True) == root
assert unit == root.parent.name.replace('smail-review-runtime-', 'smail-runtime-review-')
assert (root / 'data').resolve(strict=True) == root / 'data'
def seed_source(root):
path = root / 'data/smail.db'
assert path.resolve(strict=True) == path and path.is_file()
# 使用本轮服务身份和独占写事务扩充已有邮件表,不新增结构,不连接正式数据库。
with closing(sqlite3.connect(path.as_uri() + '?mode=rw', uri=True)) as database:
database.execute('BEGIN IMMEDIATE')
columns = [row[1] for row in database.execute('PRAGMA table_info(email)')]
assert all(re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', name) for name in columns)
sample = database.execute('SELECT * FROM email ORDER BY id LIMIT 1').fetchone()
assert sample and {'id', 'msg_id', 'text'}.issubset(columns)
before = database.execute('SELECT COUNT(*) FROM email').fetchone()[0]
statement = ('INSERT INTO email (' + ','.join('"' + name + '"' for name in columns)
+ ') VALUES (' + ','.join('?' for _ in columns) + ')')
for index in range(1024):
row = list(sample)
row[columns.index('id')] = None
row[columns.index('msg_id')] = 'interruption-fill-' + str(index) + '@runtime.invalid.internal'
row[columns.index('text')] = 'x' * 16384
database.execute(statement, row)
database.commit()
assert database.execute('SELECT COUNT(*) FROM email').fetchone()[0] == before + 1024
return {'added_rows': 1024, 'text_bytes_per_row': 16384, 'database_bytes': path.stat().st_size}
def assert_copy_uncommitted(root):
assert root_database(root) == 'smail.db'
target = root / 'data/interrupted.db'
journal = root / 'data/interrupted.db-journal'
assert target.stat().st_size > 2 * 1024 * 1024 and journal.stat().st_size > 512
assert target.stat().st_size < (root / 'data/smail.db').stat().st_size
# 页面溢写可能取得排他锁;独立读取被锁定或只见零行,均不能看到已提交复制。
committed_rows = None
target_locked = False
with closing(sqlite3.connect(target.as_uri() + '?mode=ro', uri=True, timeout=0)) as database:
try:
committed_rows = database.execute('SELECT COUNT(*) FROM email').fetchone()[0]
assert committed_rows == 0
except sqlite3.OperationalError as error:
assert 'locked' in str(error).lower()
target_locked = True
with closing(sqlite3.connect((root / 'data/smail.db').as_uri() + '?mode=rw', uri=True, timeout=0)) as database:
try:
database.execute('BEGIN IMMEDIATE')
except sqlite3.OperationalError as error:
assert 'locked' in str(error).lower()
else:
database.rollback()
raise AssertionError('复制期间源库没有写锁')
return {'target_bytes': target.stat().st_size, 'journal_bytes': journal.stat().st_size,
'committed_target_mail_rows': committed_rows, 'target_read_locked': target_locked,
'source_write_locked': True}
def prepare(root, password):
client, _ = login(root, password)
generation = clear_pending_restart(client)
preview = client.success('/api/v1/database/migrations/preview', {
'expectedGeneration': generation, 'target': {'type': 'sqlite', 'localRef': 'interrupted.db'},
})
try:
client.request('/api/v1/database/migrations', {
'id': preview['migration']['id'], 'token': preview['token'],
})
except (OSError, http.client.HTTPException):
return {'request_interrupted': True}
raise AssertionError('请求没有被进程终止中断,不能算作中途故障验收')
def assert_occupied_target(status, payload):
assert status == 409 and isinstance(payload, dict) and payload.get('errorNo') != 0
data = payload.get('data')
assert isinstance(data, dict) and data.get('code') == 'DATABASE_MIGRATION_TARGET_EXISTS'
return {'status': status, 'code': data['code']}
def verify_recovery(root, password):
client, context = login(root, password)
assert root_database(root) == 'smail.db'
status = client.success('/api/v1/config/status', method='GET')
assert not status['pendingRestart']
retained = {path.name: hashlib.sha256(path.read_bytes()).hexdigest()
for path in (root / 'data').glob('interrupted.db*')}
denied, body, _ = client.request('/api/v1/database/migrations/preview', {
'expectedGeneration': status['activeGeneration'],
'target': {'type': 'sqlite', 'localRef': 'interrupted.db'},
})
rejection = assert_occupied_target(denied, body)
assert retained == {path.name: hashlib.sha256(path.read_bytes()).hexdigest()
for path in (root / 'data').glob('interrupted.db*')}
codes = {protocol: client.success('/api/v1/security/credentials/protocols/generate',
{'protocol': protocol})['code'] for protocol in ('smtp', 'imap', 'pop3')}
original = json.loads((root.parent / 'probe-result.json').read_text())['smtp']['message_id']
content = '内部25端口真实收信,检查中文正文和附件。'
attachment = '隔离验收附件内容'.encode('utf-8')
historical = verify_protocol_read(context, codes, original, content, attachment)
message_id = original + '.interruption'
message = EmailMessage()
message['From'], message['To'] = 'interruption@sender.invalid', ADDRESS
message['Subject'] = '迁移中断恢复后的中文邮件验收'
message['Message-ID'] = '<' + message_id + '>'
message.set_content(content)
message.add_attachment(attachment, maintype='text', subtype='plain', filename='验收附件.txt')
with smtp_connect(context) as smtp:
assert not smtp.send_message(message)
received = verify_protocol_read(context, codes, message_id, content, attachment)
with read_database(root / 'data/smail.db') as database:
assert database.execute('SELECT COUNT(*) FROM email WHERE msg_id=?', (message_id,)).fetchone()[0] == 1
assert database.execute('SELECT COUNT(*) FROM config_records').fetchone()[0] > 0
return {'login': True, 'occupied_target_rejected': rejection, 'retained_target_unchanged': True,
'historical_mail': historical,
'new_mail': received, 'source_accepts_writes': True}
def verify_interruption(root, unit, password, result):
assert_scope(root, unit)
details = result['interruption'] = {'passed': False}
before = service_state(unit)
details['seed'] = run_probe(root, unit, password, 'interrupt-seed', probe_script='runtime_interruption.py')
active = service_state(unit)
assert active['MainPID'] != '0' and active['MainPID'] == before['MainPID']
assert Path('/proc/' + active['MainPID'] + '/root/opt/smail/smail').stat().st_ino == (root / 'smail').stat().st_ino
outcome = {}
def request():
try:
outcome['probe'] = run_probe(root, unit, password, 'interrupt-copy', probe_script='runtime_interruption.py')
except Exception as error:
outcome['error'] = type(error).__name__
thread = threading.Thread(target=request, daemon=True)
thread.start()
frozen = False
try:
target = root / 'data/interrupted.db'
journal = root / 'data/interrupted.db-journal'
for _ in range(15000):
if target.exists() and target.stat().st_size > 2 * 1024 * 1024 and journal.exists() and journal.stat().st_size > 512:
subprocess.run(['systemctl', 'kill', '--kill-whom=all', '--signal=SIGSTOP', unit], check=True, timeout=15)
frozen = True
break
assert thread.is_alive(), '复制请求提前退出,未注入中断'
time.sleep(0.005)
else:
raise AssertionError('没有捕捉到未提交复制阶段,不能把普通重启算作中断')
details['copy'] = assert_copy_uncommitted(root)
digest = database_fingerprint(root / 'data/smail.db')
config_digest = hashlib.sha256((root / 'config/config.json').read_bytes()).hexdigest()
snapshot = root.parent / 'interruption-snapshot'
snapshot.mkdir(mode=0o700)
for path in (target, journal):
shutil.copy2(path, snapshot / path.name)
details['snapshot_sha256'] = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in snapshot.iterdir()}
subprocess.run(['systemctl', 'kill', '--kill-whom=all', '--signal=SIGKILL', unit], check=True, timeout=15)
frozen = False
for _ in range(100):
failed = service_state(unit)
if failed['MainPID'] == '0':
break
time.sleep(0.1)
assert failed['MainPID'] == '0' and failed['ExecMainStatus'] == '9'
thread.join(timeout=30)
assert not thread.is_alive() and 'error' not in outcome and outcome['probe']['request_interrupted']
assert database_fingerprint(root / 'data/smail.db') == digest
assert hashlib.sha256((root / 'config/config.json').read_bytes()).hexdigest() == config_digest
details.update(source_unchanged=True, configuration_unchanged=True, killed_state=failed)
finally:
if frozen:
subprocess.run(['systemctl', 'kill', '--kill-whom=all', '--signal=SIGCONT', unit], check=True, timeout=15)
thread.join(timeout=90)
subprocess.run(['systemctl', 'start', unit], check=True, timeout=45)
details['recovery'] = run_probe(root, unit, password, 'interrupt-recovery', probe_script='runtime_interruption.py')
restored = service_state(unit)
assert restored['MainPID'] not in ('0', active['MainPID'])
assert restored['InvocationID'] != active['InvocationID']
details.update(before=active, recovered_state=restored, passed=True)
(root.parent / 'interruption-result.json').write_text(json.dumps(details, ensure_ascii=False, indent=2), encoding='utf-8')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('root', type=Path)
parser.add_argument('phase', choices=('interrupt-seed', 'interrupt-copy', 'interrupt-recovery'))
args = parser.parse_args()
assert os.geteuid() != 0
password = sys.stdin.readline().strip()
assert len(password) == 12
if args.phase == 'interrupt-seed':
result = seed_source(args.root)
elif args.phase == 'interrupt-copy':
result = prepare(args.root, password)
else:
result = verify_recovery(args.root, password)
result['passed'] = True
(args.root.parent / ('migration-' + args.phase + '.json')).write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
print(json.dumps(result, ensure_ascii=False))
if __name__ == '__main__':
main()
+10 -4
View File
@@ -48,17 +48,20 @@ def root_database(root):
return database['localRef']
def migration_integrity(root):
def migration_integrity(root, interrupted=False):
active = root_database(root)
assert active == 'migrated.db'
names = sorted(path.name for path in (root / 'data').glob('*.db'))
assert names == ['migrated.db', 'occupied.db', 'smail.db'], '发现意外数据库文件'
expected = ['migrated.db', 'occupied.db', 'smail.db']
if interrupted:
expected.append('interrupted.db')
assert names == sorted(expected), '发现意外数据库文件'
with read_database(root / 'data' / active) as database:
assert database.execute('PRAGMA integrity_check').fetchone() == ('ok',)
count = database.execute('SELECT COUNT(*) FROM config_records').fetchone()[0]
assert count > 0
return {'active_database': active, 'config_records': count,
'retained_offline_files': ['smail.db', 'occupied.db'], 'integrity': 'ok'}
'retained_offline_files': sorted(set(expected) - {active}), 'integrity': 'ok'}
def wait_ready(client, generation=None):
@@ -194,13 +197,16 @@ def process_uses_target(root, state):
assert '/opt/smail/data/smail.db' not in paths, '进程仍然打开旧业务库'
def run_probe(root, unit, password, phase, sql_settings=None):
def run_probe(root, unit, password, phase, sql_settings=None, probe_script=None):
state = service_state(unit)
pid = state['MainPID']
assert pid.isdigit() and int(pid) > 0
assert Path('/proc/' + pid + '/ns/net').readlink() != Path('/proc/self/ns/net').readlink()
probe = 'smail-migration-probe-' + unit.removeprefix('smail-runtime-review-') + '-' + phase
script = 'runtime_sql_migration.py' if sql_settings else 'runtime_migration.py'
if probe_script is not None:
assert probe_script == 'runtime_interruption.py' and sql_settings is None
script = probe_script
command = ['systemd-run', '--wait', '--pipe', '--unit=' + probe,
'--property=User=smail', '--property=Group=smail',
'--property=NetworkNamespacePath=/proc/' + pid + '/ns/net',
+11 -1
View File
@@ -80,8 +80,11 @@ def main():
parser.add_argument('--memory-max-mib', type=int, choices=(128, 192, 256), default=128)
parser.add_argument('--restore', action='store_true')
parser.add_argument('--migration', action='store_true')
parser.add_argument('--interruption', action='store_true')
parser.add_argument('--database-type', choices=('sqlite', 'mysql', 'postgres'), default='sqlite')
args = parser.parse_args()
if args.interruption and (not args.migration or args.database_type != 'sqlite'):
parser.error('中断验收必须选择 SQLite 迁移独立场景')
if args.migration and (args.nginx_template or args.restore):
parser.error('迁移进程验收仅使用直接 TLS 的独立场景')
if args.restore and args.database_type != 'sqlite':
@@ -131,6 +134,10 @@ def main():
shutil.copy2(Path(__file__).with_name('runtime_sql_migration.py'), target / 'runtime_sql_migration.py')
(target / 'runtime_sql_migration.py').chmod(0o640)
shutil.chown(target / 'runtime_sql_migration.py', user='root', group='smail')
if args.interruption:
shutil.copy2(Path(__file__).with_name('runtime_interruption.py'), target / 'runtime_interruption.py')
(target / 'runtime_interruption.py').chmod(0o640)
shutil.chown(target / 'runtime_interruption.py', user='root', group='smail')
if args.nginx_template:
shutil.copy2(Path(__file__).with_name('runtime_proxy.py'), target / 'runtime_proxy.py')
(target / 'runtime_proxy.py').chmod(0o640)
@@ -267,6 +274,9 @@ def main():
assert database.execute('SELECT COUNT(*) FROM email WHERE msg_id=?', (message_id,)).fetchone()[0] == 0
result['live_message_absent'] = True
if args.migration:
if args.interruption:
from runtime_interruption import verify_interruption
verify_interruption(root, unit, password, result)
if migration_target:
from runtime_sql_migration import verify_migration
verify_migration(root, unit, password, result, sql_fixture, migration_target)
@@ -287,7 +297,7 @@ def main():
result['integrity'] = migration_target.integrity(root)
elif args.migration:
from runtime_migration import migration_integrity
result['integrity'] = migration_integrity(root)
result['integrity'] = migration_integrity(root, interrupted=args.interruption)
else:
result['integrity'] = sql_fixture.integrity(root) if sql_fixture else database_integrity(root)
except (AssertionError, sqlite3.Error, RuntimeError, OSError, ValueError, subprocess.SubprocessError) as error: