补齐SQLite复制中断恢复及云端部署验收
This commit is contained in:
1 parent
fc3c40838c
commit
d7c32d12b2
7 files changed
+405
-5
No files matched your search
@@ -0,0 +1,78 @@
|
||||
"""中断验收的范围、种子数据和误报防护测试,不终止任何真实服务。"""
|
||||
|
||||
from contextlib import closing
|
||||
import http.client
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
|
||||
import runtime_interruption as interruption
|
||||
|
||||
|
||||
class RuntimeInterruptionTest(unittest.TestCase):
|
||||
def test_occupied_target_requires_structured_error_contract(self):
|
||||
payload = {'errorNo': 409, 'data': {'code': 'DATABASE_MIGRATION_TARGET_EXISTS'}}
|
||||
self.assertEqual(interruption.assert_occupied_target(409, payload)['status'], 409)
|
||||
for status, invalid in ((200, payload), (409, None),
|
||||
(409, {'errorNo': 409, 'data': 'DATABASE_MIGRATION_TARGET_EXISTS'}),
|
||||
(409, {'errorNo': 409, 'data': {'code': 'OTHER'}}),
|
||||
(409, {'errorNo': 0, 'data': payload['data']})):
|
||||
with self.assertRaises(AssertionError):
|
||||
interruption.assert_occupied_target(status, invalid)
|
||||
|
||||
def test_formal_instance_rejected_before_service_command(self):
|
||||
with patch.object(interruption, 'service_state') as state, patch.object(interruption.subprocess, 'run') as run:
|
||||
with self.assertRaises(AssertionError):
|
||||
interruption.verify_interruption(Path('/opt/smail'), 'smail.service', 'not-a-secret', {})
|
||||
state.assert_not_called()
|
||||
run.assert_not_called()
|
||||
|
||||
def test_seed_missing_database_is_not_created(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / 'data').mkdir()
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
interruption.seed_source(root)
|
||||
self.assertFalse((root / 'data/smail.db').exists())
|
||||
|
||||
def test_seed_preserves_existing_mail_and_schema(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / 'data').mkdir()
|
||||
path = root / 'data/smail.db'
|
||||
with closing(sqlite3.connect(path)) as database:
|
||||
database.execute('CREATE TABLE email (id INTEGER PRIMARY KEY, msg_id TEXT UNIQUE, text TEXT)')
|
||||
database.execute("INSERT INTO email VALUES (1, 'original', 'body')")
|
||||
database.commit()
|
||||
result = interruption.seed_source(root)
|
||||
self.assertEqual(result['added_rows'], 1024)
|
||||
with closing(sqlite3.connect(path)) as database:
|
||||
self.assertEqual(database.execute('SELECT COUNT(*) FROM email').fetchone()[0], 1025)
|
||||
self.assertEqual(database.execute('SELECT * FROM email WHERE id=1').fetchone(), (1, 'original', 'body'))
|
||||
self.assertEqual(database.execute('SELECT COUNT(*) FROM sqlite_master WHERE type=\'table\'').fetchone()[0], 1)
|
||||
|
||||
def test_completed_request_cannot_count_as_interruption(self):
|
||||
client = Mock()
|
||||
client.success.return_value = {'migration': {'id': 'one'}, 'token': 'not-a-secret'}
|
||||
for response in ((200, {}, {}), (500, {}, {})):
|
||||
client.request.return_value = response
|
||||
with patch.object(interruption, 'login', return_value=(client, None)), \
|
||||
patch.object(interruption, 'clear_pending_restart', return_value=1):
|
||||
with self.assertRaises(AssertionError):
|
||||
interruption.prepare(Path('/unused'), 'not-a-secret')
|
||||
|
||||
def test_disconnected_request_requires_controller_process_evidence(self):
|
||||
client = Mock()
|
||||
client.success.return_value = {'migration': {'id': 'one'}, 'token': 'not-a-secret'}
|
||||
client.request.side_effect = http.client.RemoteDisconnected()
|
||||
with patch.object(interruption, 'login', return_value=(client, None)), \
|
||||
patch.object(interruption, 'clear_pending_restart', return_value=1):
|
||||
self.assertEqual(interruption.prepare(Path('/unused'), 'not-a-secret'), {'request_interrupted': True})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,232 @@
|
||||
"""在隔离 SQLite 实例复制事务未提交时终止进程,保留残留目标并验证恢复。"""
|
||||
|
||||
import argparse
|
||||
from contextlib import closing
|
||||
from email.message import EmailMessage
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
from runtime_migration import (
|
||||
clear_pending_restart, database_fingerprint, login, read_database,
|
||||
root_database, run_probe, service_state,
|
||||
)
|
||||
from runtime_probe import ACCOUNT, ADDRESS, smtp_connect, verify_protocol_read
|
||||
|
||||
|
||||
def assert_scope(root, unit):
|
||||
assert root.parent.parent == Path('/var/tmp')
|
||||
assert re.fullmatch(r'smail-review-runtime-\d{8}-\d{6}', root.parent.name)
|
||||
assert root.name == 'instance' and root.resolve(strict=True) == root
|
||||
assert unit == root.parent.name.replace('smail-review-runtime-', 'smail-runtime-review-')
|
||||
assert (root / 'data').resolve(strict=True) == root / 'data'
|
||||
|
||||
|
||||
def seed_source(root):
|
||||
path = root / 'data/smail.db'
|
||||
assert path.resolve(strict=True) == path and path.is_file()
|
||||
# 使用本轮服务身份和独占写事务扩充已有邮件表,不新增结构,不连接正式数据库。
|
||||
with closing(sqlite3.connect(path.as_uri() + '?mode=rw', uri=True)) as database:
|
||||
database.execute('BEGIN IMMEDIATE')
|
||||
columns = [row[1] for row in database.execute('PRAGMA table_info(email)')]
|
||||
assert all(re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', name) for name in columns)
|
||||
sample = database.execute('SELECT * FROM email ORDER BY id LIMIT 1').fetchone()
|
||||
assert sample and {'id', 'msg_id', 'text'}.issubset(columns)
|
||||
before = database.execute('SELECT COUNT(*) FROM email').fetchone()[0]
|
||||
statement = ('INSERT INTO email (' + ','.join('"' + name + '"' for name in columns)
|
||||
+ ') VALUES (' + ','.join('?' for _ in columns) + ')')
|
||||
for index in range(1024):
|
||||
row = list(sample)
|
||||
row[columns.index('id')] = None
|
||||
row[columns.index('msg_id')] = 'interruption-fill-' + str(index) + '@runtime.invalid.internal'
|
||||
row[columns.index('text')] = 'x' * 16384
|
||||
database.execute(statement, row)
|
||||
database.commit()
|
||||
assert database.execute('SELECT COUNT(*) FROM email').fetchone()[0] == before + 1024
|
||||
return {'added_rows': 1024, 'text_bytes_per_row': 16384, 'database_bytes': path.stat().st_size}
|
||||
|
||||
|
||||
def assert_copy_uncommitted(root):
|
||||
assert root_database(root) == 'smail.db'
|
||||
target = root / 'data/interrupted.db'
|
||||
journal = root / 'data/interrupted.db-journal'
|
||||
assert target.stat().st_size > 2 * 1024 * 1024 and journal.stat().st_size > 512
|
||||
assert target.stat().st_size < (root / 'data/smail.db').stat().st_size
|
||||
# 页面溢写可能取得排他锁;独立读取被锁定或只见零行,均不能看到已提交复制。
|
||||
committed_rows = None
|
||||
target_locked = False
|
||||
with closing(sqlite3.connect(target.as_uri() + '?mode=ro', uri=True, timeout=0)) as database:
|
||||
try:
|
||||
committed_rows = database.execute('SELECT COUNT(*) FROM email').fetchone()[0]
|
||||
assert committed_rows == 0
|
||||
except sqlite3.OperationalError as error:
|
||||
assert 'locked' in str(error).lower()
|
||||
target_locked = True
|
||||
with closing(sqlite3.connect((root / 'data/smail.db').as_uri() + '?mode=rw', uri=True, timeout=0)) as database:
|
||||
try:
|
||||
database.execute('BEGIN IMMEDIATE')
|
||||
except sqlite3.OperationalError as error:
|
||||
assert 'locked' in str(error).lower()
|
||||
else:
|
||||
database.rollback()
|
||||
raise AssertionError('复制期间源库没有写锁')
|
||||
return {'target_bytes': target.stat().st_size, 'journal_bytes': journal.stat().st_size,
|
||||
'committed_target_mail_rows': committed_rows, 'target_read_locked': target_locked,
|
||||
'source_write_locked': True}
|
||||
|
||||
|
||||
def prepare(root, password):
|
||||
client, _ = login(root, password)
|
||||
generation = clear_pending_restart(client)
|
||||
preview = client.success('/api/v1/database/migrations/preview', {
|
||||
'expectedGeneration': generation, 'target': {'type': 'sqlite', 'localRef': 'interrupted.db'},
|
||||
})
|
||||
try:
|
||||
client.request('/api/v1/database/migrations', {
|
||||
'id': preview['migration']['id'], 'token': preview['token'],
|
||||
})
|
||||
except (OSError, http.client.HTTPException):
|
||||
return {'request_interrupted': True}
|
||||
raise AssertionError('请求没有被进程终止中断,不能算作中途故障验收')
|
||||
|
||||
|
||||
def assert_occupied_target(status, payload):
|
||||
assert status == 409 and isinstance(payload, dict) and payload.get('errorNo') != 0
|
||||
data = payload.get('data')
|
||||
assert isinstance(data, dict) and data.get('code') == 'DATABASE_MIGRATION_TARGET_EXISTS'
|
||||
return {'status': status, 'code': data['code']}
|
||||
|
||||
|
||||
def verify_recovery(root, password):
|
||||
client, context = login(root, password)
|
||||
assert root_database(root) == 'smail.db'
|
||||
status = client.success('/api/v1/config/status', method='GET')
|
||||
assert not status['pendingRestart']
|
||||
retained = {path.name: hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
for path in (root / 'data').glob('interrupted.db*')}
|
||||
denied, body, _ = client.request('/api/v1/database/migrations/preview', {
|
||||
'expectedGeneration': status['activeGeneration'],
|
||||
'target': {'type': 'sqlite', 'localRef': 'interrupted.db'},
|
||||
})
|
||||
rejection = assert_occupied_target(denied, body)
|
||||
assert retained == {path.name: hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
for path in (root / 'data').glob('interrupted.db*')}
|
||||
codes = {protocol: client.success('/api/v1/security/credentials/protocols/generate',
|
||||
{'protocol': protocol})['code'] for protocol in ('smtp', 'imap', 'pop3')}
|
||||
original = json.loads((root.parent / 'probe-result.json').read_text())['smtp']['message_id']
|
||||
content = '内部25端口真实收信,检查中文正文和附件。'
|
||||
attachment = '隔离验收附件内容'.encode('utf-8')
|
||||
historical = verify_protocol_read(context, codes, original, content, attachment)
|
||||
message_id = original + '.interruption'
|
||||
message = EmailMessage()
|
||||
message['From'], message['To'] = 'interruption@sender.invalid', ADDRESS
|
||||
message['Subject'] = '迁移中断恢复后的中文邮件验收'
|
||||
message['Message-ID'] = '<' + message_id + '>'
|
||||
message.set_content(content)
|
||||
message.add_attachment(attachment, maintype='text', subtype='plain', filename='验收附件.txt')
|
||||
with smtp_connect(context) as smtp:
|
||||
assert not smtp.send_message(message)
|
||||
received = verify_protocol_read(context, codes, message_id, content, attachment)
|
||||
with read_database(root / 'data/smail.db') as database:
|
||||
assert database.execute('SELECT COUNT(*) FROM email WHERE msg_id=?', (message_id,)).fetchone()[0] == 1
|
||||
assert database.execute('SELECT COUNT(*) FROM config_records').fetchone()[0] > 0
|
||||
return {'login': True, 'occupied_target_rejected': rejection, 'retained_target_unchanged': True,
|
||||
'historical_mail': historical,
|
||||
'new_mail': received, 'source_accepts_writes': True}
|
||||
|
||||
|
||||
def verify_interruption(root, unit, password, result):
|
||||
assert_scope(root, unit)
|
||||
details = result['interruption'] = {'passed': False}
|
||||
before = service_state(unit)
|
||||
details['seed'] = run_probe(root, unit, password, 'interrupt-seed', probe_script='runtime_interruption.py')
|
||||
active = service_state(unit)
|
||||
assert active['MainPID'] != '0' and active['MainPID'] == before['MainPID']
|
||||
assert Path('/proc/' + active['MainPID'] + '/root/opt/smail/smail').stat().st_ino == (root / 'smail').stat().st_ino
|
||||
outcome = {}
|
||||
|
||||
def request():
|
||||
try:
|
||||
outcome['probe'] = run_probe(root, unit, password, 'interrupt-copy', probe_script='runtime_interruption.py')
|
||||
except Exception as error:
|
||||
outcome['error'] = type(error).__name__
|
||||
|
||||
thread = threading.Thread(target=request, daemon=True)
|
||||
thread.start()
|
||||
frozen = False
|
||||
try:
|
||||
target = root / 'data/interrupted.db'
|
||||
journal = root / 'data/interrupted.db-journal'
|
||||
for _ in range(15000):
|
||||
if target.exists() and target.stat().st_size > 2 * 1024 * 1024 and journal.exists() and journal.stat().st_size > 512:
|
||||
subprocess.run(['systemctl', 'kill', '--kill-whom=all', '--signal=SIGSTOP', unit], check=True, timeout=15)
|
||||
frozen = True
|
||||
break
|
||||
assert thread.is_alive(), '复制请求提前退出,未注入中断'
|
||||
time.sleep(0.005)
|
||||
else:
|
||||
raise AssertionError('没有捕捉到未提交复制阶段,不能把普通重启算作中断')
|
||||
details['copy'] = assert_copy_uncommitted(root)
|
||||
digest = database_fingerprint(root / 'data/smail.db')
|
||||
config_digest = hashlib.sha256((root / 'config/config.json').read_bytes()).hexdigest()
|
||||
snapshot = root.parent / 'interruption-snapshot'
|
||||
snapshot.mkdir(mode=0o700)
|
||||
for path in (target, journal):
|
||||
shutil.copy2(path, snapshot / path.name)
|
||||
details['snapshot_sha256'] = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in snapshot.iterdir()}
|
||||
subprocess.run(['systemctl', 'kill', '--kill-whom=all', '--signal=SIGKILL', unit], check=True, timeout=15)
|
||||
frozen = False
|
||||
for _ in range(100):
|
||||
failed = service_state(unit)
|
||||
if failed['MainPID'] == '0':
|
||||
break
|
||||
time.sleep(0.1)
|
||||
assert failed['MainPID'] == '0' and failed['ExecMainStatus'] == '9'
|
||||
thread.join(timeout=30)
|
||||
assert not thread.is_alive() and 'error' not in outcome and outcome['probe']['request_interrupted']
|
||||
assert database_fingerprint(root / 'data/smail.db') == digest
|
||||
assert hashlib.sha256((root / 'config/config.json').read_bytes()).hexdigest() == config_digest
|
||||
details.update(source_unchanged=True, configuration_unchanged=True, killed_state=failed)
|
||||
finally:
|
||||
if frozen:
|
||||
subprocess.run(['systemctl', 'kill', '--kill-whom=all', '--signal=SIGCONT', unit], check=True, timeout=15)
|
||||
thread.join(timeout=90)
|
||||
subprocess.run(['systemctl', 'start', unit], check=True, timeout=45)
|
||||
details['recovery'] = run_probe(root, unit, password, 'interrupt-recovery', probe_script='runtime_interruption.py')
|
||||
restored = service_state(unit)
|
||||
assert restored['MainPID'] not in ('0', active['MainPID'])
|
||||
assert restored['InvocationID'] != active['InvocationID']
|
||||
details.update(before=active, recovered_state=restored, passed=True)
|
||||
(root.parent / 'interruption-result.json').write_text(json.dumps(details, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('root', type=Path)
|
||||
parser.add_argument('phase', choices=('interrupt-seed', 'interrupt-copy', 'interrupt-recovery'))
|
||||
args = parser.parse_args()
|
||||
assert os.geteuid() != 0
|
||||
password = sys.stdin.readline().strip()
|
||||
assert len(password) == 12
|
||||
if args.phase == 'interrupt-seed':
|
||||
result = seed_source(args.root)
|
||||
elif args.phase == 'interrupt-copy':
|
||||
result = prepare(args.root, password)
|
||||
else:
|
||||
result = verify_recovery(args.root, password)
|
||||
result['passed'] = True
|
||||
(args.root.parent / ('migration-' + args.phase + '.json')).write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -48,17 +48,20 @@ def root_database(root):
|
||||
return database['localRef']
|
||||
|
||||
|
||||
def migration_integrity(root):
|
||||
def migration_integrity(root, interrupted=False):
|
||||
active = root_database(root)
|
||||
assert active == 'migrated.db'
|
||||
names = sorted(path.name for path in (root / 'data').glob('*.db'))
|
||||
assert names == ['migrated.db', 'occupied.db', 'smail.db'], '发现意外数据库文件'
|
||||
expected = ['migrated.db', 'occupied.db', 'smail.db']
|
||||
if interrupted:
|
||||
expected.append('interrupted.db')
|
||||
assert names == sorted(expected), '发现意外数据库文件'
|
||||
with read_database(root / 'data' / active) as database:
|
||||
assert database.execute('PRAGMA integrity_check').fetchone() == ('ok',)
|
||||
count = database.execute('SELECT COUNT(*) FROM config_records').fetchone()[0]
|
||||
assert count > 0
|
||||
return {'active_database': active, 'config_records': count,
|
||||
'retained_offline_files': ['smail.db', 'occupied.db'], 'integrity': 'ok'}
|
||||
'retained_offline_files': sorted(set(expected) - {active}), 'integrity': 'ok'}
|
||||
|
||||
|
||||
def wait_ready(client, generation=None):
|
||||
@@ -194,13 +197,16 @@ def process_uses_target(root, state):
|
||||
assert '/opt/smail/data/smail.db' not in paths, '进程仍然打开旧业务库'
|
||||
|
||||
|
||||
def run_probe(root, unit, password, phase, sql_settings=None):
|
||||
def run_probe(root, unit, password, phase, sql_settings=None, probe_script=None):
|
||||
state = service_state(unit)
|
||||
pid = state['MainPID']
|
||||
assert pid.isdigit() and int(pid) > 0
|
||||
assert Path('/proc/' + pid + '/ns/net').readlink() != Path('/proc/self/ns/net').readlink()
|
||||
probe = 'smail-migration-probe-' + unit.removeprefix('smail-runtime-review-') + '-' + phase
|
||||
script = 'runtime_sql_migration.py' if sql_settings else 'runtime_migration.py'
|
||||
if probe_script is not None:
|
||||
assert probe_script == 'runtime_interruption.py' and sql_settings is None
|
||||
script = probe_script
|
||||
command = ['systemd-run', '--wait', '--pipe', '--unit=' + probe,
|
||||
'--property=User=smail', '--property=Group=smail',
|
||||
'--property=NetworkNamespacePath=/proc/' + pid + '/ns/net',
|
||||
|
||||
@@ -80,8 +80,11 @@ def main():
|
||||
parser.add_argument('--memory-max-mib', type=int, choices=(128, 192, 256), default=128)
|
||||
parser.add_argument('--restore', action='store_true')
|
||||
parser.add_argument('--migration', action='store_true')
|
||||
parser.add_argument('--interruption', action='store_true')
|
||||
parser.add_argument('--database-type', choices=('sqlite', 'mysql', 'postgres'), default='sqlite')
|
||||
args = parser.parse_args()
|
||||
if args.interruption and (not args.migration or args.database_type != 'sqlite'):
|
||||
parser.error('中断验收必须选择 SQLite 迁移独立场景')
|
||||
if args.migration and (args.nginx_template or args.restore):
|
||||
parser.error('迁移进程验收仅使用直接 TLS 的独立场景')
|
||||
if args.restore and args.database_type != 'sqlite':
|
||||
@@ -131,6 +134,10 @@ def main():
|
||||
shutil.copy2(Path(__file__).with_name('runtime_sql_migration.py'), target / 'runtime_sql_migration.py')
|
||||
(target / 'runtime_sql_migration.py').chmod(0o640)
|
||||
shutil.chown(target / 'runtime_sql_migration.py', user='root', group='smail')
|
||||
if args.interruption:
|
||||
shutil.copy2(Path(__file__).with_name('runtime_interruption.py'), target / 'runtime_interruption.py')
|
||||
(target / 'runtime_interruption.py').chmod(0o640)
|
||||
shutil.chown(target / 'runtime_interruption.py', user='root', group='smail')
|
||||
if args.nginx_template:
|
||||
shutil.copy2(Path(__file__).with_name('runtime_proxy.py'), target / 'runtime_proxy.py')
|
||||
(target / 'runtime_proxy.py').chmod(0o640)
|
||||
@@ -267,6 +274,9 @@ def main():
|
||||
assert database.execute('SELECT COUNT(*) FROM email WHERE msg_id=?', (message_id,)).fetchone()[0] == 0
|
||||
result['live_message_absent'] = True
|
||||
if args.migration:
|
||||
if args.interruption:
|
||||
from runtime_interruption import verify_interruption
|
||||
verify_interruption(root, unit, password, result)
|
||||
if migration_target:
|
||||
from runtime_sql_migration import verify_migration
|
||||
verify_migration(root, unit, password, result, sql_fixture, migration_target)
|
||||
@@ -287,7 +297,7 @@ def main():
|
||||
result['integrity'] = migration_target.integrity(root)
|
||||
elif args.migration:
|
||||
from runtime_migration import migration_integrity
|
||||
result['integrity'] = migration_integrity(root)
|
||||
result['integrity'] = migration_integrity(root, interrupted=args.interruption)
|
||||
else:
|
||||
result['integrity'] = sql_fixture.integrity(root) if sql_fixture else database_integrity(root)
|
||||
except (AssertionError, sqlite3.Error, RuntimeError, OSError, ValueError, subprocess.SubprocessError) as error:
|
||||
|
||||
Reference in new issue
Block a user