补齐双SQL复制中断实机验收与恢复证据
Security Checks / security (push) Waiting to run

This commit is contained in:
xuans committed 2026-09-21 23:13:33 +08:00
1 parent 8dc040a9ab
commit dd2038dc7f
6 files changed
+441 -5

No files matched your search

@@ -0,0 +1,115 @@
"""SQL 中断验收的范围、真实事务判定与清理回归,不连接真实数据库。"""
import http.client
from pathlib import Path
import sys
import tempfile
import unittest
from unittest.mock import Mock, patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
import runtime_sql_interruption as interruption
from runtime_sql import SQLFixture
import runtime_review as review
class RuntimeSQLInterruptionTest(unittest.TestCase):
def test_same_engine_interruption_only(self):
for kind in ('sqlite', 'mysql', 'postgres'):
self.assertEqual(review.migration_kind(kind, None, True, True), kind)
with self.assertRaises(ValueError):
review.migration_kind('mysql', 'postgres', True, True)
def test_formal_instance_rejected_before_target_creation(self):
with patch.object(interruption, 'MigrationTarget') as target:
with self.assertRaises(AssertionError):
interruption.verify_interruption(Path('/opt/smail'), 'smail.service', 'unused', {}, SQLFixture('mysql'))
target.assert_not_called()
def test_seed_does_not_copy_primary_key_or_change_existing_rows(self):
for kind in ('mysql', 'postgres'):
source = SQLFixture(kind)
with patch.object(source, 'query', side_effect=['id\nmsg_id\ntext\nsubject', '1', '', '1025']) as query:
self.assertEqual(interruption.seed_source(source)['added_rows'], 1024)
sql = query.call_args_list[2].args[0]
self.assertTrue(sql.startswith('INSERT INTO email'))
self.assertNotIn('UPDATE ', sql)
self.assertNotIn('DELETE ', sql)
self.assertNotIn('DROP ', sql)
self.assertIn('REPEAT(\'x\',16384)', sql)
self.assertIn('SELECT 1023 AS i', sql)
self.assertNotIn('id', sql.split(' SELECT ')[0].replace('msg_id', ''))
def test_seed_rejects_unsafe_column_before_write(self):
source = SQLFixture('mysql')
with patch.object(source, 'query', return_value='id\nmsg_id\ntext\nbad;DROP') as query:
with self.assertRaises(AssertionError):
interruption.seed_source(source)
self.assertEqual(query.call_count, 1)
def test_mysql_observation_filters_exact_account_and_database(self):
target = SQLFixture('mysql')
with patch.object(target, 'admin', return_value='150') as admin:
self.assertEqual(interruption.copy_observation(target),
{'open_copy_transactions': 1, 'modified_rows': 150})
sql = admin.call_args.args[0]
for key in ('name', 'user'):
self.assertIn(target.settings[key], sql)
self.assertNotIn(target.settings['password'], sql)
self.assertIn('trx_rows_modified>128', sql)
with patch.object(target, 'admin', return_value=''):
self.assertEqual(interruption.copy_observation(target)['open_copy_transactions'], 0)
def test_postgres_requires_database_pages_xid_and_write_lock(self):
target = SQLFixture('postgres')
with patch.object(target, 'admin', return_value='0') as admin:
self.assertEqual(interruption.copy_observation(target)['open_copy_transactions'], 0)
self.assertEqual(admin.call_count, 1)
for size, expected in (('0', 0), ('32768', 0), ('65536', 1)):
with patch.object(target, 'admin', side_effect=['1', '1', size]) as admin:
self.assertEqual(interruption.copy_observation(target)['open_copy_transactions'], expected)
sql = admin.call_args_list[1].args[0]
for value in (target.settings['name'], target.settings['user'], 'backend_xid', 'RowExclusiveLock'):
self.assertIn(value, sql)
self.assertEqual(admin.call_args_list[1].args[1], target.settings['name'])
def test_committed_or_missing_or_ambiguous_copy_cannot_pass(self):
target = SQLFixture('mysql')
for count, rows in ((0, ['0', '0']), (2, ['0', '0']),
(1, ['1', '0']), (1, ['0', '1'])):
with patch.object(target, 'query', side_effect=rows):
with self.assertRaises(AssertionError):
interruption.assert_uncommitted(target, {'open_copy_transactions': count})
with patch.object(target, 'query', side_effect=['0', '0']):
self.assertEqual(interruption.assert_uncommitted(target, {'open_copy_transactions': 1})['committed_mail_rows'], 0)
def test_normal_response_is_not_interruption(self):
client = Mock()
client.success.return_value = {'migration': {'id': 'one'}, 'token': 'test-token'}
with patch.object(interruption, 'login', return_value=(client, None)), \
patch.object(interruption, 'clear_pending_restart', return_value=1):
for response in ((200, {}, {}), (500, {}, {})):
client.request.return_value = response
with self.assertRaises(AssertionError):
interruption.prepare(Path('/unused'), 'unused', SQLFixture('postgres'))
client.request.side_effect = http.client.RemoteDisconnected()
self.assertEqual(interruption.prepare(Path('/unused'), 'unused', SQLFixture('postgres')),
{'request_interrupted': True})
def test_partial_create_failure_still_locks_account_and_preserves_result(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory) / 'instance'
source, target = SQLFixture('postgres'), Mock()
target.settings = SQLFixture('postgres').settings
target.create.side_effect = RuntimeError('controlled failure')
result = {}
with patch.object(interruption, 'assert_scope'), patch.object(interruption, 'MigrationTarget', return_value=target):
with self.assertRaises(RuntimeError):
interruption.verify_interruption(root, 'unused', 'unused', result, source)
target.lock.assert_called_once_with()
self.assertFalse(result['interruption']['passed'])
self.assertTrue((root.parent / 'interruption-result.json').exists())
if __name__ == '__main__':
unittest.main()
+2 -1
View File
@@ -206,7 +206,8 @@ def run_probe(root, unit, password, phase, sql_settings=None, probe_script=None)
script = 'runtime_sql_migration.py' if sql_settings else 'runtime_migration.py'
if probe_script is not None:
assert ((probe_script == 'runtime_interruption.py' and sql_settings is None)
or (probe_script == 'runtime_mixed_migration.py' and sql_settings is not None))
or (probe_script in ('runtime_mixed_migration.py', 'runtime_sql_interruption.py')
and sql_settings is not None))
script = probe_script
command = ['systemd-run', '--wait', '--pipe', '--unit=' + probe,
'--property=User=smail', '--property=Group=smail',
+12 -4
View File
@@ -130,8 +130,8 @@ def main():
target_kind = migration_kind(args.database_type, args.migration_target_type, args.migration, args.interruption)
except ValueError as error:
parser.error(str(error))
if args.interruption and (not args.migration or args.database_type != 'sqlite'):
parser.error('中断验收必须选择 SQLite 迁移独立场景')
if args.interruption and not args.migration:
parser.error('中断验收必须选择同引擎迁移独立场景')
if args.migration and (args.nginx_template or args.restore):
parser.error('迁移进程验收仅使用直接 TLS 的独立场景')
if args.restore and args.database_type != 'sqlite':
@@ -190,6 +190,10 @@ def main():
shutil.copy2(Path(__file__).with_name('runtime_interruption.py'), target / 'runtime_interruption.py')
(target / 'runtime_interruption.py').chmod(0o640)
shutil.chown(target / 'runtime_interruption.py', user='root', group='smail')
if args.database_type != 'sqlite':
shutil.copy2(Path(__file__).with_name('runtime_sql_interruption.py'), target / 'runtime_sql_interruption.py')
(target / 'runtime_sql_interruption.py').chmod(0o640)
shutil.chown(target / 'runtime_sql_interruption.py', user='root', group='smail')
if args.nginx_template:
shutil.copy2(Path(__file__).with_name('runtime_proxy.py'), target / 'runtime_proxy.py')
(target / 'runtime_proxy.py').chmod(0o640)
@@ -342,8 +346,12 @@ def main():
result['live_message_absent'] = True
if args.migration:
if args.interruption:
from runtime_interruption import verify_interruption
verify_interruption(root, unit, password, result)
if sql_fixture:
from runtime_sql_interruption import verify_interruption
verify_interruption(root, unit, password, result, sql_fixture)
else:
from runtime_interruption import verify_interruption
verify_interruption(root, unit, password, result)
if mixed:
from runtime_mixed_migration import verify_migration
verify_migration(root, unit, password, result, mixed_source, mixed_target)
+222
View File
@@ -0,0 +1,222 @@
"""在随机 SQL 测试库中捕捉未提交复制,只终止本轮隔离应用。"""
import argparse
import hashlib
import http.client
import json
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
from runtime_interruption import assert_occupied_target, assert_scope
from runtime_migration import clear_pending_restart, login, run_probe, service_state
from runtime_sql import SQLDatabase
from runtime_sql_migration import (
MigrationTarget, assert_active, connection_count, fingerprint, identifier,
target_payload, verify_mail,
)
def seed_source(source):
schema = 'DATABASE()' if source.kind == 'mysql' else "'public'"
columns = source.query('SELECT column_name FROM information_schema.columns WHERE table_schema=' +
schema + " AND table_name='email' ORDER BY ordinal_position").splitlines()
assert {'id', 'msg_id', 'text'}.issubset(columns)
quote = '`' if source.kind == 'mysql' else '"'
columns = [identifier(column) for column in columns if column != 'id']
names = ','.join(quote + column + quote for column in columns)
values = []
for column in columns:
if column == 'msg_id':
values.append("CONCAT('interruption-fill-',n.i,'@runtime.invalid.internal')")
elif column == 'text':
values.append("REPEAT('x',16384)")
else:
values.append('e.' + quote + column + quote)
before = int(source.query('SELECT COUNT(*) FROM email'))
assert before > 0
numbers = ' UNION ALL '.join('SELECT ' + str(index) + ' AS i' for index in range(1024))
source.query('INSERT INTO email (' + names + ') SELECT ' + ','.join(values) +
' FROM (SELECT * FROM email ORDER BY id LIMIT 1) e CROSS JOIN (' + numbers + ') n;')
assert int(source.query('SELECT COUNT(*) FROM email')) == before + 1024
return {'added_rows': 1024, 'text_bytes_per_row': 16384}
def copy_observation(target):
name, user = target.settings['name'], target.settings['user']
if target.kind == 'mysql':
raw = target.admin("SELECT t.trx_rows_modified FROM information_schema.innodb_trx t "
"JOIN information_schema.processlist p ON p.ID=t.trx_mysql_thread_id "
"WHERE p.USER='" + user + "' AND p.DB='" + name +
"' AND t.trx_rows_modified>128;")
rows = [int(value) for value in raw.splitlines()]
return {'open_copy_transactions': len(rows), 'modified_rows': max(rows, default=0)}
exists = target.admin("SELECT COUNT(*) FROM pg_database WHERE datname='" + name + "';")
if exists != '1':
return {'open_copy_transactions': 0, 'relation_bytes': 0}
raw = target.admin("SELECT COUNT(*) FROM pg_stat_activity a WHERE a.usename='" + user +
"' AND a.datname='" + name + "' AND a.xact_start IS NOT NULL "
"AND a.backend_xid IS NOT NULL AND EXISTS (SELECT 1 FROM pg_locks l "
"WHERE l.pid=a.pid AND l.locktype='relation' AND l.mode='RowExclusiveLock' "
"AND l.granted AND l.relation=to_regclass('public.email'));", name)
size = target.admin("SELECT COALESCE(pg_relation_size(to_regclass('public.email')),0);", name)
# 只接受真实分配了邮件页、持有写锁且分配了事务号的复制,不把空结构创建算成复制。
count, size = int(raw), int(size)
return {'open_copy_transactions': count if size > 32768 else 0,
'relation_bytes': size}
def assert_uncommitted(target, observation):
assert observation['open_copy_transactions'] == 1, '未捕获唯一未提交复制事务'
assert target.query('SELECT COUNT(*) FROM email') == '0', '目标邮件已经提交,不能算未提交中断'
assert target.query('SELECT COUNT(*) FROM config_records') == '0', '配置历史已经提交'
return dict(observation, committed_mail_rows=0, committed_config_rows=0)
def prepare(root, password, target):
client, _ = login(root, password)
generation = clear_pending_restart(client)
preview = client.success('/api/v1/database/migrations/preview', {
'expectedGeneration': generation, 'target': target_payload(target),
})
try:
client.request('/api/v1/database/migrations', {
'id': preview['migration']['id'], 'token': preview['token'],
})
except (OSError, http.client.HTTPException):
return {'request_interrupted': True}
raise AssertionError('请求正常结束,不能当作进程中断验收')
def verify_recovery(root, password, source, target):
client, _ = login(root, password)
assert_active(root, source)
status = client.success('/api/v1/config/status', method='GET')
assert not status['pendingRestart']
digest = fingerprint(target)
denied, body, _ = client.request('/api/v1/database/migrations/preview', {
'expectedGeneration': status['activeGeneration'], 'target': target_payload(target),
})
rejected = assert_occupied_target(denied, body)
assert fingerprint(target) == digest
# 中断后源库仍是唯一活动库;复用三协议验收,验证新邮件不写入失败目标。
mail = verify_mail(root, password, 'interrupt-recovery', target, source)
return {'occupied_target_rejected': rejected, 'retained_target_unchanged': True,
'source_accepts_writes': True, 'mail': mail}
def verify_interruption(root, unit, password, result, source):
assert_scope(root, unit)
assert isinstance(source, SQLDatabase) and source.kind in ('mysql', 'postgres')
target = MigrationTarget(source.kind)
assert source.settings['name'] != target.settings['name']
details = result['interruption'] = {'passed': False, 'database_type': source.kind,
'retained_target_database': target.settings['name']}
settings = {'source': source.settings, 'target': target.settings}
frozen, thread, outcome = False, None, {}
def probe(phase):
return run_probe(root, unit, password, phase, settings,
probe_script='runtime_sql_interruption.py')
def request():
try:
outcome['probe'] = probe('interrupt-copy')
except Exception as error:
outcome['error'] = type(error).__name__
try:
target.create()
before = service_state(unit)
details['seed'] = probe('interrupt-seed')
active = service_state(unit)
assert active['MainPID'] != '0' and active['MainPID'] == before['MainPID']
assert Path('/proc/' + active['MainPID'] + '/root/opt/smail/smail').stat().st_ino == (root / 'smail').stat().st_ino
thread = threading.Thread(target=request, daemon=True)
thread.start()
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
observation = copy_observation(target)
if observation['open_copy_transactions'] == 1:
subprocess.run(['systemctl', 'kill', '--kill-whom=all', '--signal=SIGSTOP', unit], check=True, timeout=15)
frozen = True
break
assert thread.is_alive(), '复制请求提前退出,未注入中断'
time.sleep(0.02)
assert frozen, '未捕获未提交事务,不能把普通重启算成中断'
details['copy'] = assert_uncommitted(target, copy_observation(target))
assert_active(root, source)
source_digest = fingerprint(source)
config_digest = hashlib.sha256((root / 'config/config.json').read_bytes()).hexdigest()
subprocess.run(['systemctl', 'kill', '--kill-whom=all', '--signal=SIGKILL', unit], check=True, timeout=15)
frozen = False
for _ in range(100):
failed = service_state(unit)
if failed['MainPID'] == '0' and connection_count(target) == 0:
break
time.sleep(0.1)
assert failed['MainPID'] == '0' and failed['ExecMainStatus'] == '9'
assert connection_count(target) == 0
thread.join(timeout=30)
assert not thread.is_alive() and 'error' not in outcome and outcome['probe']['request_interrupted']
assert fingerprint(source) == source_digest
assert hashlib.sha256((root / 'config/config.json').read_bytes()).hexdigest() == config_digest
assert target.query('SELECT COUNT(*) FROM email') == '0'
assert target.query('SELECT COUNT(*) FROM config_records') == '0'
target.revoke_creation()
details.update(source_unchanged=True, configuration_unchanged=True,
target_rolled_back=True, killed_state=failed)
subprocess.run(['systemctl', 'start', unit], check=True, timeout=45)
details['recovery'] = probe('interrupt-recovery')
restored = service_state(unit)
assert restored['MainPID'] not in ('0', active['MainPID'])
assert restored['InvocationID'] != active['InvocationID']
assert connection_count(target) == 0
details.update(before=active, recovered_state=restored, passed=True)
finally:
try:
if frozen:
subprocess.run(['systemctl', 'kill', '--kill-whom=all', '--signal=SIGCONT', unit], check=True, timeout=15)
if thread:
thread.join(timeout=90)
finally:
try:
target.lock()
details['retained_target_account_locked'] = True
except Exception:
details['passed'] = False
raise
finally:
(root.parent / 'interruption-result.json').write_text(
json.dumps(details, ensure_ascii=False, indent=2), encoding='utf-8')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('root', type=Path)
parser.add_argument('phase', choices=('interrupt-seed', 'interrupt-copy', 'interrupt-recovery'))
args = parser.parse_args()
assert os.geteuid() != 0
password = sys.stdin.readline().strip()
assert len(password) == 12
settings = json.loads(sys.stdin.readline())
assert set(settings) == {'source', 'target'}
source, target = (SQLDatabase(settings[key]) for key in ('source', 'target'))
assert source.kind == target.kind and source.settings['name'] != target.settings['name']
if args.phase == 'interrupt-seed':
result = seed_source(source)
elif args.phase == 'interrupt-copy':
result = prepare(args.root, password, target)
else:
result = verify_recovery(args.root, password, source, target)
result['passed'] = True
(args.root.parent / ('migration-' + args.phase + '.json')).write_text(
json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
print(json.dumps(result, ensure_ascii=False))
if __name__ == '__main__':
main()