修复构建窗口异常恢复并完成远程前端生产构建验收
Security Checks / security (push) Waiting to run

This commit is contained in:
xuans committed 2026-09-18 01:37:29 +08:00
1 parent 4f5b05e95c
commit c1dd533c80
4 files changed
+166 -7

No files matched your search

+59 -1
View File
@@ -18,7 +18,7 @@ class RestoreServicesTest(unittest.TestCase):
with patch.object(review.subprocess, 'run') as run:
self.assertEqual(review.restore_services(self.unit), [])
self.assertEqual(run.call_args_list, [
call(['systemctl', 'stop', self.unit], timeout=90, check=False),
call(['systemctl', 'stop', self.unit], timeout=90, check=True),
call(['systemctl', 'start', 'mysql.service'], check=True, timeout=90),
call(['systemctl', 'start', 'Scert.service'], check=True, timeout=90),
])
@@ -45,6 +45,64 @@ class RestoreServicesTest(unittest.TestCase):
review.restore_services(name)
run.assert_not_called()
def test_stop_timeout_does_not_skip_restore(self):
error = subprocess.TimeoutExpired('systemctl', 90)
with patch.object(review.subprocess, 'run', side_effect=[error, None, None, None]) as run:
self.assertEqual(review.restore_services(self.unit), [
{'service': self.unit, 'error': 'TimeoutExpired'},
])
self.assertEqual(run.call_args_list[1].args[0], [
'systemctl', 'kill', '--kill-whom=all', '--signal=SIGKILL', self.unit,
])
self.assertEqual([item.args[0] for item in run.call_args_list[-2:]], [
['systemctl', 'start', 'mysql.service'], ['systemctl', 'start', 'Scert.service'],
])
def test_missing_build_unit_is_not_a_restore_failure(self):
error = subprocess.CalledProcessError(5, 'systemctl')
with patch.object(review.subprocess, 'run', side_effect=[error, None, None]) as run, \
patch.object(review, 'run', return_value='not-found'):
self.assertEqual(review.restore_services(self.unit), [])
self.assertEqual(run.call_count, 3)
def test_existing_unit_stop_failure_is_reported(self):
error = subprocess.CalledProcessError(1, 'systemctl')
with patch.object(review.subprocess, 'run', side_effect=[error, None, None, None]), \
patch.object(review, 'run', return_value='loaded'):
self.assertEqual(review.restore_services(self.unit), [
{'service': self.unit, 'error': 'CalledProcessError'},
])
def test_stop_and_kill_failures_do_not_skip_remaining_restores(self):
failures = [OSError(), subprocess.TimeoutExpired('systemctl', 30),
subprocess.CalledProcessError(1, 'systemctl'), None]
with patch.object(review.subprocess, 'run', side_effect=failures) as run:
self.assertEqual(review.restore_services(self.unit), [
{'service': self.unit, 'error': 'OSError'},
{'service': self.unit, 'error': 'TimeoutExpired', 'action': 'kill'},
{'service': 'mysql.service', 'error': 'CalledProcessError'},
])
self.assertEqual(run.call_args_list[-1].args[0], ['systemctl', 'start', 'Scert.service'])
def test_unknown_unit_state_cannot_hide_stop_failure(self):
error = subprocess.CalledProcessError(1, 'systemctl')
with patch.object(review.subprocess, 'run', side_effect=[error, None, None, None]), \
patch.object(review, 'run', side_effect=OSError()):
self.assertEqual(review.restore_services(self.unit), [
{'service': self.unit, 'error': 'CalledProcessError'},
])
class BuildBudgetTest(unittest.TestCase):
def test_full_budget_is_accepted(self):
review.check_memory_budget(1024 * 1024)
self.assertEqual(review.SYSTEM_RESERVE_MIB, 128)
self.assertLess(review.NODE_HEAP_MIB, review.BUILD_MEMORY_MIB)
def test_insufficient_reserve_is_rejected(self):
with self.assertRaises(RuntimeError):
review.check_memory_budget(1024 * 1024 - 1)
if __name__ == '__main__':
unittest.main()
+36 -6
View File
@@ -13,6 +13,9 @@ from run_frontend_review import dependency_bindings
PAUSED = ('Scert.service', 'mysql.service')
PROTECTED = ('smail.service', 'nginx.service')
BUILD_MEMORY_MIB = 896
NODE_HEAP_MIB = 640
SYSTEM_RESERVE_MIB = 128
def run(*args):
@@ -27,8 +30,24 @@ def states():
def restore_services(build_unit):
assert re.fullmatch(r'smail-frontend-build-[0-9]{8}-[0-9]{6}', build_unit)
subprocess.run(['systemctl', 'stop', build_unit], timeout=90, check=False)
errors = []
try:
subprocess.run(['systemctl', 'stop', build_unit], timeout=90, check=True)
except Exception as error:
# 预检拒绝时构建单元尚不存在;其他停止失败必须记录并尝试终止残留进程。
missing = False
if isinstance(error, subprocess.CalledProcessError):
try:
missing = run('systemctl', 'show', build_unit, '--property=LoadState', '--value') == 'not-found'
except Exception:
pass
if not missing:
errors.append({'service': build_unit, 'error': type(error).__name__})
try:
subprocess.run(['systemctl', 'kill', '--kill-whom=all', '--signal=SIGKILL', build_unit],
timeout=30, check=True)
except Exception as kill_error:
errors.append({'service': build_unit, 'error': type(kill_error).__name__, 'action': 'kill'})
for name in reversed(PAUSED):
try:
subprocess.run(['systemctl', 'start', name], check=True, timeout=90)
@@ -37,6 +56,12 @@ def restore_services(build_unit):
return errors
def check_memory_budget(available_kib):
required_kib = (BUILD_MEMORY_MIB + SYSTEM_RESERVE_MIB) * 1024
if available_kib < required_kib:
raise RuntimeError('暂停服务后仍不足以保留 128 MiB 系统预算')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('directory')
@@ -68,7 +93,7 @@ def main():
# 由独立 systemd 单元负责恢复,SSH 断开或工作进程被终止也不会取消恢复钩子。
return subprocess.run([
'systemd-run', '--wait', '--pipe', '--unit=' + unit,
'-p', 'RuntimeMaxSec=900s', '-p', 'TimeoutStopSec=180s',
'-p', 'RuntimeMaxSec=900s', '-p', 'TimeoutStopSec=480s',
'-p', 'KillMode=control-group', '-p', 'UMask=0077', '-p', 'LimitCORE=0',
'-p', 'ExecStopPost=/usr/bin/python3 ' + str(script) + ' ' + str(root)
+ ' --restore-unit ' + build_unit,
@@ -79,17 +104,19 @@ def main():
stamp = time.strftime('%Y%m%d-%H%M%S')
unit = args.build_unit
assert unit and re.fullmatch(r'smail-frontend-build-[0-9]{8}-[0-9]{6}', unit)
result = {'before': before, 'unit': unit, 'started': time.strftime('%Y-%m-%dT%H:%M:%S%z')}
result = {'before': before, 'unit': unit, 'started': time.strftime('%Y-%m-%dT%H:%M:%S%z'),
'build_memory_mib': BUILD_MEMORY_MIB, 'node_heap_mib': NODE_HEAP_MIB,
'system_reserve_mib': SYSTEM_RESERVE_MIB}
try:
for name in PAUSED:
subprocess.run(['systemctl', 'stop', name], check=True, timeout=90)
memory = dict(line.split(':', 1) for line in Path('/proc/meminfo').read_text().splitlines())
result['available_kib'] = int(memory['MemAvailable'].split()[0])
assert result['available_kib'] >= 1088 * 1024, '暂停服务后仍不足以保留 128 MiB 系统预算'
check_memory_budget(result['available_kib'])
node = root / 'node-v22.23.2-linux-x64/bin/node'
command = [
'systemd-run', '--wait', '--pipe', '--unit=' + unit,
'-p', 'User=smail', '-p', 'Group=smail', '-p', 'MemoryMax=960M', '-p', 'MemorySwapMax=0',
'-p', 'User=smail', '-p', 'Group=smail', '-p', f'MemoryMax={BUILD_MEMORY_MIB}M', '-p', 'MemorySwapMax=0',
'-p', 'CPUQuota=75%', '-p', 'TasksMax=128', '-p', 'RuntimeMaxSec=600s',
'-p', 'TimeoutStopSec=10s', '-p', 'KillMode=control-group', '-p', 'OOMPolicy=stop',
'-p', 'PrivateTmp=yes', '-p', 'PrivateNetwork=yes', '-p', 'ProtectSystem=strict',
@@ -97,7 +124,7 @@ def main():
'-p', 'InaccessiblePaths=/opt/smail /run/smail', '-p', 'BindPaths=' + str(root),
'-p', 'ReadWritePaths=' + str(root), '-p', 'WorkingDirectory=' + str(root / 'frontend'),
*[argument for target in bindings for argument in ('-p', 'BindReadOnlyPaths=' + target)],
'--setenv=NODE_OPTIONS=--max-old-space-size=768', '--setenv=HOME=' + str(root / 'build-home'),
f'--setenv=NODE_OPTIONS=--max-old-space-size={NODE_HEAP_MIB}', '--setenv=HOME=' + str(root / 'build-home'),
'/bin/sh', '-c',
'set -eu\n' + str(node) + ' node_modules/vite/bin/vite.js build\n'
+ str(node) + ' ../deploy/prepare-frontend.mjs\n',
@@ -108,6 +135,9 @@ def main():
result['resources'] = run('systemctl', 'show', unit,
'--property=Result,ExecMainStatus,MemoryPeak,MemorySwapPeak')
assert result['build_code'] == 0, '前端构建失败,保留日志并恢复服务'
except Exception as error:
result['error'] = type(error).__name__
raise
finally:
# 先恢复数据库,再恢复依赖数据库的站点;构建成功与否均执行。
result['restore_errors'] = restore_services(unit)