修复投递结果恢复并隔离远程验收资源
This commit is contained in:
1 parent
5198e2d515
commit
1d202934cf
13 files changed
+511
-34
No files matched your search
@@ -0,0 +1,45 @@
|
||||
param([string]$OutputName = 'full-review-20260917')
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$root = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..\..'))
|
||||
if ($OutputName -notmatch '^[a-z0-9-]+$') { throw '测试产物目录名称无效' }
|
||||
$output = Join-Path $root "out\$OutputName"
|
||||
New-Item -ItemType Directory -Path $output -Force | Out-Null
|
||||
$previous = @{}
|
||||
foreach ($name in @('GOOS', 'GOARCH', 'CGO_ENABLED')) {
|
||||
$previous[$name] = [Environment]::GetEnvironmentVariable($name, 'Process')
|
||||
}
|
||||
try {
|
||||
$env:GOOS = 'linux'
|
||||
$env:GOARCH = 'amd64'
|
||||
$env:CGO_ENABLED = '0'
|
||||
Push-Location (Join-Path $root 'backend')
|
||||
try {
|
||||
$packages = & go list -f '{{if or .TestGoFiles .XTestGoFiles}}{{.ImportPath}}{{end}}' ./...
|
||||
if ($LASTEXITCODE -ne 0) { throw '无法枚举测试包' }
|
||||
$manifest = @()
|
||||
foreach ($package in $packages) {
|
||||
if (-not $package.Trim()) { continue }
|
||||
$relative = $package.Substring('git.pcux.cn/xuans/Smail'.Length).TrimStart('/')
|
||||
$directory = if ($relative) { "backend/$relative" } else { 'backend' }
|
||||
$target = Join-Path $output $directory
|
||||
New-Item -ItemType Directory -Path $target -Force | Out-Null
|
||||
& go test -c -o (Join-Path $target 'review.test') $package
|
||||
if ($LASTEXITCODE -ne 0) { throw "测试编译失败:$package" }
|
||||
$manifest += @{ package = $package; directory = $directory }
|
||||
Write-Output "已编译:$package"
|
||||
}
|
||||
$json = ConvertTo-Json -InputObject $manifest -Depth 4
|
||||
[IO.File]::WriteAllText((Join-Path $output 'manifest.json'), $json, [Text.UTF8Encoding]::new($false))
|
||||
} finally { Pop-Location }
|
||||
Copy-Item -LiteralPath (Join-Path $root 'deploy') -Destination (Join-Path $output 'deploy') -Recurse -Force
|
||||
Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'run_review_tests.py') -Destination $output -Force
|
||||
Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'start_review_tests.py') -Destination $output -Force
|
||||
& tar -czf "$output.tar.gz" -C $output .
|
||||
if ($LASTEXITCODE -ne 0) { throw '测试产物归档失败' }
|
||||
Write-Output "仅完成交叉编译,未在本机运行测试:$output.tar.gz"
|
||||
} finally {
|
||||
foreach ($name in $previous.Keys) {
|
||||
[Environment]::SetEnvironmentVariable($name, $previous[$name], 'Process')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
"""在 47 服务器的独立命名空间恢复备份,不替换当前服务或删除文件。"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import tarfile
|
||||
import time
|
||||
|
||||
|
||||
def run(*args):
|
||||
return subprocess.check_output(args, text=True).strip()
|
||||
|
||||
|
||||
def database_counts(root):
|
||||
counts = {}
|
||||
for name in ('smail.db', 'smail-state.db'):
|
||||
with sqlite3.connect(f'file:{root / "data" / name}?mode=ro', uri=True) as database:
|
||||
assert database.execute('PRAGMA integrity_check').fetchone()[0] == 'ok'
|
||||
tables = database.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").fetchall()
|
||||
counts[name] = {
|
||||
table: database.execute('SELECT COUNT(*) FROM "' + table.replace('"', '""') + '"').fetchone()[0]
|
||||
for (table,) in tables
|
||||
}
|
||||
return counts
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('backup')
|
||||
parser.add_argument('--archive-sha', required=True)
|
||||
args = parser.parse_args()
|
||||
assert os.geteuid() == 0 and run('hostname') == 'ikun2'
|
||||
backup = Path(args.backup).resolve(strict=True)
|
||||
assert backup.parent == Path('/var/backups') and backup.name.startswith('smail-before-')
|
||||
archive = backup / 'smail.tar.gz'
|
||||
with archive.open('rb') as stream:
|
||||
assert hashlib.file_digest(stream, 'sha256').hexdigest() == args.archive_sha, '备份摘要不匹配'
|
||||
memory = dict(line.split(':', 1) for line in Path('/proc/meminfo').read_text().splitlines())
|
||||
assert int(memory['MemAvailable'].split()[0]) >= 256 * 1024, '可用内存不足,暂停恢复演练'
|
||||
stamp = time.strftime('%Y%m%d-%H%M%S')
|
||||
target = Path('/var/tmp') / ('smail-review-restore-' + stamp)
|
||||
target.mkdir(mode=0o700)
|
||||
with tarfile.open(archive) as stream:
|
||||
for member in stream.getmembers():
|
||||
path = Path(member.name)
|
||||
assert not path.is_absolute() and '..' not in path.parts and path.parts[0] == 'smail'
|
||||
assert member.isdir() or member.isfile(), '备份含特殊文件,停止自动恢复'
|
||||
subprocess.run(['tar', '-xzf', str(archive), '-C', str(target)], check=True)
|
||||
restored = target / 'smail'
|
||||
control = target / 'control'
|
||||
control.mkdir(mode=0o750)
|
||||
shutil.chown(control, user='smail', group='smail')
|
||||
before = database_counts(restored)
|
||||
live_pid = run('systemctl', 'show', 'smail.service', '--property=MainPID', '--value')
|
||||
unit = 'smail-restore-review-' + stamp
|
||||
result = {'backup': str(backup), 'restore': str(target), 'unit': unit, 'before': before}
|
||||
try:
|
||||
subprocess.run([
|
||||
'systemd-run', '--unit=' + unit, '--property=User=smail', '--property=Group=smail',
|
||||
'--property=PrivateNetwork=yes', '--property=PrivateTmp=yes',
|
||||
'--property=ProtectSystem=strict', '--property=ProtectHome=yes',
|
||||
'--property=NoNewPrivileges=yes', '--property=CapabilityBoundingSet=CAP_NET_BIND_SERVICE',
|
||||
'--property=AmbientCapabilities=CAP_NET_BIND_SERVICE',
|
||||
'--property=MemoryMax=128M', '--property=MemorySwapMax=0', '--property=CPUQuota=25%',
|
||||
'--property=TasksMax=128', '--property=RuntimeMaxSec=180s', '--property=OOMPolicy=stop',
|
||||
'--property=KillMode=control-group', '--property=TimeoutStopSec=10s',
|
||||
'--property=BindPaths=' + str(restored) + ':/opt/smail ' + str(control) + ':/run/smail',
|
||||
'--property=ReadWritePaths=/opt/smail /run/smail',
|
||||
'--property=WorkingDirectory=/opt/smail', '--setenv=SMAIL_ROOT=/opt/smail',
|
||||
'--setenv=GOMEMLIMIT=96MiB', '--setenv=GOMAXPROCS=1',
|
||||
'/opt/smail/smail',
|
||||
], check=True)
|
||||
ready = False
|
||||
for _ in range(60):
|
||||
pid = run('systemctl', 'show', unit, '--property=MainPID', '--value')
|
||||
if pid != '0':
|
||||
probe = subprocess.run([
|
||||
'nsenter', '--target', pid, '--net', 'curl', '--silent', '--fail', '--max-time', '2',
|
||||
'--noproxy', '*', '--cacert', str(restored / 'ssl/custom.crt'),
|
||||
'--resolve', 'mail.solit.cn:443:127.0.0.1', 'https://mail.solit.cn/api/ping',
|
||||
], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
if probe.returncode == 0:
|
||||
ready = True
|
||||
break
|
||||
time.sleep(1)
|
||||
assert ready, '恢复服务未就绪'
|
||||
assert os.readlink('/proc/' + pid + '/ns/net') != os.readlink('/proc/' + live_pid + '/ns/net')
|
||||
assert os.stat('/proc/' + pid + '/root/opt/smail/data/smail.db').st_ino == (restored / 'data/smail.db').stat().st_ino
|
||||
assert (restored / 'data/smail.db').stat().st_ino != Path('/opt/smail/data/smail.db').stat().st_ino
|
||||
probe = r'''
|
||||
import smtplib,ssl,sys
|
||||
from email.message import EmailMessage
|
||||
message=EmailMessage()
|
||||
message['From']='restore@review.invalid'
|
||||
message['To']='remotecheck@solit.cn'
|
||||
message['Subject']='restore-review'
|
||||
message['Message-ID']='<restore-'+sys.argv[2]+'@review.invalid>'
|
||||
message.set_content('restore database write check')
|
||||
context=ssl.create_default_context(cafile=sys.argv[1])
|
||||
with smtplib.SMTP('127.0.0.1',25,timeout=20) as client:
|
||||
client.ehlo('restore.invalid')
|
||||
client._host='mail.solit.cn'
|
||||
client.starttls(context=context)
|
||||
client.ehlo('restore.invalid')
|
||||
assert not client.send_message(message)
|
||||
'''
|
||||
subprocess.run(['nsenter', '--target', pid, '--net', 'python3', '-c', probe,
|
||||
str(restored / 'ssl/custom.crt'), stamp], check=True)
|
||||
with sqlite3.connect(f'file:{restored / "data/smail.db"}?mode=ro', uri=True) as database:
|
||||
assert database.execute('SELECT COUNT(*) FROM email WHERE msg_id=?', ('restore-' + stamp + '@review.invalid',)).fetchone()[0] == 1
|
||||
with sqlite3.connect('file:/opt/smail/data/smail.db?mode=ro', uri=True) as database:
|
||||
assert database.execute('SELECT COUNT(*) FROM email WHERE msg_id=?', ('restore-' + stamp + '@review.invalid',)).fetchone()[0] == 0
|
||||
result.update(ready=True, isolated=True, restored_smtp=True, live_pid_unchanged=live_pid == run(
|
||||
'systemctl', 'show', 'smail.service', '--property=MainPID', '--value'))
|
||||
finally:
|
||||
subprocess.run(['systemctl', 'stop', unit], check=True)
|
||||
result['after'] = database_counts(restored)
|
||||
result['service_log'] = run('journalctl', '-u', unit, '--no-pager', '-n', '80')
|
||||
(target / 'result.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print(json.dumps({key: value for key, value in result.items() if key not in ('before', 'after', 'service_log')}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,65 @@
|
||||
"""在远程独立目录逐包执行测试,保留全部日志和跳过记录。"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--only', action='append', default=[])
|
||||
parser.add_argument('--run', default='')
|
||||
args = parser.parse_args()
|
||||
root = Path(__file__).resolve().parent
|
||||
assert root.parent == Path('/var/tmp') and root.name.startswith('smail-review-')
|
||||
assert os.geteuid() != 0, '禁止使用 root 执行测试'
|
||||
environment = os.environ.copy()
|
||||
for name in list(environment):
|
||||
if name.startswith('SMAIL_'):
|
||||
environment.pop(name)
|
||||
environment['GOMAXPROCS'] = '1'
|
||||
environment['GOMEMLIMIT'] = '64MiB'
|
||||
environment['GOGC'] = '20'
|
||||
output = root / ('results-' + time.strftime('%Y%m%d-%H%M%S'))
|
||||
output.mkdir(mode=0o700)
|
||||
results = []
|
||||
for item in json.loads((root / 'manifest.json').read_text(encoding='utf-8')):
|
||||
if args.only and item['directory'] not in args.only:
|
||||
continue
|
||||
directory = (root / item['directory']).resolve()
|
||||
assert directory.is_relative_to(root)
|
||||
binary = directory / 'review.test'
|
||||
with binary.open('rb') as stream:
|
||||
assert stream.read(5) == b'\x7fELF\x02'
|
||||
started = time.monotonic()
|
||||
log = output / (item['directory'].replace('/', '-') + '.log')
|
||||
with log.open('w', encoding='utf-8') as stream:
|
||||
command = [str(binary), '-test.v', '-test.timeout=180s']
|
||||
if args.run:
|
||||
command.append('-test.run=' + args.run)
|
||||
process = subprocess.Popen(command, cwd=directory, env=environment,
|
||||
stdout=stream, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
try:
|
||||
code = process.wait(timeout=195)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait()
|
||||
code = 124
|
||||
lines = log.read_text(encoding='utf-8', errors='replace').splitlines()
|
||||
result = dict(item, code=code, log=str(log), seconds=round(time.monotonic() - started, 2),
|
||||
passed=sum(line.startswith('--- PASS:') for line in lines),
|
||||
failed=[line for line in lines if line.lstrip().startswith('--- FAIL:')],
|
||||
skipped=[line for line in lines if line.lstrip().startswith('--- SKIP:')])
|
||||
results.append(result)
|
||||
print(json.dumps(result, ensure_ascii=False), flush=True)
|
||||
(output / 'results.json').write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print('测试记录:' + str(output), flush=True)
|
||||
return int(any(item['code'] != 0 for item in results))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,47 @@
|
||||
"""通过受限 systemd 临时服务执行远程测试,不复用生产服务进程。"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('directory')
|
||||
parser.add_argument('--only', action='append', default=[])
|
||||
parser.add_argument('--run', default='')
|
||||
parser.add_argument('--memory-mib', type=int, choices=(64, 96, 128, 192), default=192)
|
||||
args = parser.parse_args()
|
||||
root = Path(args.directory).resolve(strict=True)
|
||||
assert os.geteuid() == 0
|
||||
assert root.parent == Path('/var/tmp') and root.name.startswith('smail-review-')
|
||||
memory = dict(line.split(':', 1) for line in Path('/proc/meminfo').read_text().splitlines())
|
||||
assert int(memory['MemAvailable'].split()[0]) >= (args.memory_mib + 128) * 1024, '可用内存不足,停止增加测试负载'
|
||||
unit = 'smail-tests-' + time.strftime('%Y%m%d-%H%M%S')
|
||||
command = [
|
||||
'systemd-run', '--wait', '--pipe', '--unit=' + unit,
|
||||
'--property=User=smail', '--property=Group=smail',
|
||||
'--property=MemoryMax=' + str(args.memory_mib) + 'M', '--property=MemorySwapMax=0',
|
||||
'--property=TasksMax=128', '--property=CPUQuota=25%',
|
||||
'--property=RuntimeMaxSec=1800s', '--property=TimeoutStopSec=10s',
|
||||
'--property=KillMode=control-group', '--property=OOMPolicy=stop',
|
||||
'--property=PrivateTmp=yes', '--property=PrivateNetwork=yes',
|
||||
'--property=BindPaths=' + str(root),
|
||||
'--property=ProtectSystem=strict', '--property=ProtectHome=yes',
|
||||
'--property=InaccessiblePaths=/opt/smail /run/smail',
|
||||
'--property=NoNewPrivileges=yes', '--property=ReadWritePaths=' + str(root),
|
||||
'--property=WorkingDirectory=' + str(root),
|
||||
'python3', str(root / 'run_review_tests.py'),
|
||||
]
|
||||
for directory in args.only:
|
||||
command.extend(['--only', directory])
|
||||
if args.run:
|
||||
command.extend(['--run', args.run])
|
||||
print('测试服务:' + unit, flush=True)
|
||||
return subprocess.run(command).returncode
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user