修复投递结果恢复并隔离远程验收资源
This commit is contained in:
1 parent
5198e2d515
commit
1d202934cf
13 files changed
+511
-34
No files matched your search
@@ -24,6 +24,13 @@ type Runtime struct {
|
||||
done chan struct{}
|
||||
once sync.Once
|
||||
deliver func(*context.Context, *parsemail.Email, string) error
|
||||
pending *pendingResult
|
||||
}
|
||||
|
||||
type pendingResult struct {
|
||||
job models.OutboundDelivery
|
||||
ctx *context.Context
|
||||
message *parsemail.Email
|
||||
}
|
||||
|
||||
func Start(engine *xorm.Engine) (*Runtime, error) {
|
||||
@@ -58,20 +65,8 @@ func (r *Runtime) loop() {
|
||||
return
|
||||
default:
|
||||
}
|
||||
var jobs []models.OutboundDelivery
|
||||
err := r.engine.Where("state = ? AND next_attempt <= ?", queued, time.Now().Unix()).Asc("id").Limit(20).Find(&jobs)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("读取持久投递队列失败")
|
||||
}
|
||||
for _, job := range jobs {
|
||||
select {
|
||||
case <-r.stop:
|
||||
return
|
||||
default:
|
||||
}
|
||||
if err := r.process(job); err != nil {
|
||||
log.WithError(err).WithField("delivery_id", job.ID).Error("处理持久投递队列失败")
|
||||
}
|
||||
if err := r.tick(); err != nil {
|
||||
log.WithError(err).Error("处理持久投递队列失败")
|
||||
}
|
||||
select {
|
||||
case <-r.stop:
|
||||
@@ -81,12 +76,37 @@ func (r *Runtime) loop() {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runtime) tick() error {
|
||||
// 单工作器最多保留一个未落库结果,恢复前不领取新任务,避免内存积压和重复投递。
|
||||
if r.pending != nil {
|
||||
return dbmigration.WhileWritable(r.flushResult)
|
||||
}
|
||||
var jobs []models.OutboundDelivery
|
||||
if err := r.engine.Where("state = ? AND next_attempt <= ?", queued, time.Now().Unix()).Asc("id").Limit(20).Find(&jobs); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, job := range jobs {
|
||||
select {
|
||||
case <-r.stop:
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
if err := r.process(job); err != nil {
|
||||
return fmt.Errorf("投递任务 %d: %w", job.ID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runtime) process(job models.OutboundDelivery) error {
|
||||
// 迁移必须等待 SMTP 及结果落库结束,冻结源库后不再领取任务。
|
||||
return dbmigration.WhileWritable(func() error { return r.processWritable(job) })
|
||||
}
|
||||
|
||||
func (r *Runtime) processWritable(job models.OutboundDelivery) error {
|
||||
if r.pending != nil {
|
||||
return r.flushResult()
|
||||
}
|
||||
// 读取失败时尚未开始投递,保留排队状态供数据库恢复后继续处理。
|
||||
var stored models.Email
|
||||
emailFound, err := r.engine.ID(job.EmailID).Get(&stored)
|
||||
@@ -126,20 +146,23 @@ func (r *Runtime) processWritable(job models.OutboundDelivery) error {
|
||||
}
|
||||
return r.deliver(ctx, message, job.Recipient)
|
||||
}()
|
||||
// 只重试结果落库,绝不因数据库故障再次调用 SMTP。
|
||||
var terminal bool
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
terminal, err = finish(r.engine, job, deliveryErr)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
// 固定首次结果及退避时间,后续只重试落库,绝不再次调用 SMTP。
|
||||
r.pending = &pendingResult{job: deliveryResult(job, deliveryErr), ctx: ctx, message: message}
|
||||
return r.flushResult()
|
||||
}
|
||||
|
||||
func (r *Runtime) flushResult() error {
|
||||
result := r.pending
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
terminal, err := persistResult(r.engine, result.job)
|
||||
if err != nil {
|
||||
return fmt.Errorf("投递结果写入失败,保留在途记录并禁止自动重投: %w", err)
|
||||
return fmt.Errorf("投递任务 %d 结果写入失败,已暂停领取任务并等待数据库恢复: %w", result.job.ID, err)
|
||||
}
|
||||
if terminal && message != nil {
|
||||
r.notify(ctx, message, job.EmailID)
|
||||
r.pending = nil
|
||||
if terminal && result.message != nil {
|
||||
r.notify(result.ctx, result.message, result.job.EmailID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -151,6 +151,10 @@ func claim(engine *xorm.Engine, job *models.OutboundDelivery) (bool, error) {
|
||||
}
|
||||
|
||||
func finish(engine *xorm.Engine, job models.OutboundDelivery, deliveryErr error) (bool, error) {
|
||||
return persistResult(engine, deliveryResult(job, deliveryErr))
|
||||
}
|
||||
|
||||
func deliveryResult(job models.OutboundDelivery, deliveryErr error) models.OutboundDelivery {
|
||||
job.State, job.LastError, job.NextAttempt = accepted, "", 0
|
||||
if deliveryErr != nil {
|
||||
// 不保存远端响应原文,避免凭据或恶意超长内容进入数据库和页面。
|
||||
@@ -163,6 +167,10 @@ func finish(engine *xorm.Engine, job models.OutboundDelivery, deliveryErr error)
|
||||
job.NextAttempt = time.Now().Add(time.Minute * time.Duration(1<<(job.Attempts-1))).Unix()
|
||||
}
|
||||
}
|
||||
return job
|
||||
}
|
||||
|
||||
func persistResult(engine *xorm.Engine, job models.OutboundDelivery) (bool, error) {
|
||||
terminal := false
|
||||
err := db.Transaction(engine, func(session *xorm.Session) error {
|
||||
count, err := session.ID(job.ID).Where("state = ? AND attempts = ?", running, job.Attempts).Cols("state", "last_error", "next_attempt").Update(&job)
|
||||
@@ -170,7 +178,17 @@ func finish(engine *xorm.Engine, job models.OutboundDelivery, deliveryErr error)
|
||||
return err
|
||||
}
|
||||
if count != 1 {
|
||||
return errors.New("投递任务状态已发生变化")
|
||||
// 提交响应丢失时,确认同一次尝试的相同结果已落库,不覆盖后续尝试。
|
||||
var stored models.OutboundDelivery
|
||||
found, readErr := session.ID(job.ID).Get(&stored)
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
if !found || stored.EmailID != job.EmailID || stored.Recipient != job.Recipient ||
|
||||
stored.Attempts != job.Attempts || stored.State != job.State ||
|
||||
stored.LastError != job.LastError || stored.NextAttempt != job.NextAttempt {
|
||||
return errors.New("投递任务状态已发生变化")
|
||||
}
|
||||
}
|
||||
terminal, err = syncStatus(session, job.EmailID)
|
||||
return err
|
||||
|
||||
@@ -173,8 +173,8 @@ func TestResultFailureRollsBackAndNeverRedelivers(t *testing.T) {
|
||||
if err := r.process(jobs[0]); err == nil {
|
||||
t.Fatal("result failure was ignored")
|
||||
}
|
||||
if err := r.process(jobs[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
if err := r.tick(); err == nil {
|
||||
t.Fatal("result retry failure was ignored")
|
||||
}
|
||||
if calls.Load() != 1 {
|
||||
t.Fatalf("SMTP called %d times", calls.Load())
|
||||
@@ -187,6 +187,77 @@ func TestResultFailureRollsBackAndNeverRedelivers(t *testing.T) {
|
||||
if job.State != running {
|
||||
t.Fatalf("state=%s", job.State)
|
||||
}
|
||||
if _, err := engine.Exec("DROP TRIGGER reject_result"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.tick(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.pending != nil || calls.Load() != 1 {
|
||||
t.Fatal("result was not recovered without another SMTP attempt")
|
||||
}
|
||||
var persisted models.OutboundDelivery
|
||||
if found, err := engine.ID(job.ID).Get(&persisted); err != nil || !found || persisted.State != accepted {
|
||||
t.Fatalf("result not persisted: %+v %v", persisted, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistResultAfterLostCommitResponse(t *testing.T) {
|
||||
for _, deliveryErr := range []error{nil, &textproto.Error{Code: 451, Msg: "later"}, &smtp.DeliveryUncertainError{Cause: io.EOF}} {
|
||||
engine, _, jobs := queueFixture(t)
|
||||
job := jobs[0]
|
||||
if ok, err := claim(engine, &job); err != nil || !ok {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := deliveryResult(job, deliveryErr)
|
||||
if _, err := persistResult(engine, result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := persistResult(engine, result); err != nil {
|
||||
t.Fatalf("same result must be idempotent: %v", err)
|
||||
}
|
||||
result.Attempts++
|
||||
if _, err := persistResult(engine, result); err == nil {
|
||||
t.Fatal("different attempt must not overwrite persisted result")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPendingResultStopsNewClaimsAndSurvivesRestartSafely(t *testing.T) {
|
||||
engine, _, jobs := queueFixture(t)
|
||||
if _, err := engine.Exec("CREATE TRIGGER reject_result BEFORE UPDATE ON user_email BEGIN SELECT RAISE(ABORT, 'injected failure'); END"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var calls int
|
||||
r := &Runtime{engine: engine, deliver: func(_ *context.Context, _ *parsemail.Email, _ string) error {
|
||||
calls++
|
||||
return nil
|
||||
}}
|
||||
for i := 0; i < 4; i++ {
|
||||
if err := r.tick(); err == nil {
|
||||
t.Fatal("persistence failure ignored")
|
||||
}
|
||||
}
|
||||
var second models.OutboundDelivery
|
||||
if _, err := engine.ID(jobs[1].ID).Get(&second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 1 || second.State != queued || second.Attempts != 0 {
|
||||
t.Fatalf("new delivery started during outage: calls=%d job=%+v", calls, second)
|
||||
}
|
||||
if _, err := engine.Exec("DROP TRIGGER reject_result"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recoverInterrupted(engine); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var first models.OutboundDelivery
|
||||
if _, err := engine.ID(jobs[0].ID).Get(&first); err != nil || first.State != uncertain {
|
||||
t.Fatalf("lost in-memory result must remain uncertain: %+v %v", first, err)
|
||||
}
|
||||
if _, err := persistResult(engine, r.pending.job); err == nil {
|
||||
t.Fatal("stale runtime overwrote restart recovery")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueuedJobsSurviveRestartAndDrain(t *testing.T) {
|
||||
|
||||
@@ -203,6 +203,10 @@ func isAbsoluteUserCertificatePath(value string) bool {
|
||||
if filepath.IsAbs(value) {
|
||||
return true
|
||||
}
|
||||
// 非 Windows 主机也要识别盘符,避免把异平台路径静默拼到项目目录。
|
||||
if len(value) >= 2 && value[1] == ':' && ((value[0] >= 'A' && value[0] <= 'Z') || (value[0] >= 'a' && value[0] <= 'z')) {
|
||||
return true
|
||||
}
|
||||
volume := filepath.VolumeName(value)
|
||||
return volume != "" || strings.HasPrefix(value, `\\`) || strings.HasPrefix(value, "//")
|
||||
}
|
||||
@@ -218,8 +222,8 @@ func validateUserCertificatePaths(privatePath, publicPath string) (string, strin
|
||||
}
|
||||
path := strings.TrimSpace(value)
|
||||
if isAbsoluteUserCertificatePath(path) {
|
||||
if filepath.VolumeName(path) != "" && !filepath.IsAbs(path) {
|
||||
return "", nil, newUserCertificateError("SSL 路径必须是完整的绝对路径")
|
||||
if !filepath.IsAbs(path) {
|
||||
return "", nil, newUserCertificateError("SSL 路径必须是当前服务器支持的完整绝对路径")
|
||||
}
|
||||
} else {
|
||||
path = filepath.Join(rootPath, path)
|
||||
|
||||
@@ -54,12 +54,13 @@ func TestIsAbsoluteUserCertificatePathRecognizesWindowsPaths(t *testing.T) {
|
||||
cases := []string{
|
||||
`C:\outside\private.key`,
|
||||
`D:/outside/public.crt`,
|
||||
`C:private.key`,
|
||||
`\\server\share\private.key`,
|
||||
`//server/share/public.crt`,
|
||||
}
|
||||
for _, value := range cases {
|
||||
if !isAbsoluteUserCertificatePath(value) {
|
||||
t.Errorf("path %q was accepted", value)
|
||||
t.Errorf("path %q was not recognized", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -73,6 +74,17 @@ func TestIsAbsoluteUserCertificatePathRejectsEmptyAndAcceptsRelative(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUserCertificatePathsRejectsNonNativeAbsolutePath(t *testing.T) {
|
||||
for _, path := range []string{`C:private.key`, `C:\outside\private.key`, `\\server\share\private.key`} {
|
||||
if filepath.IsAbs(path) {
|
||||
continue
|
||||
}
|
||||
if _, _, err := validateUserCertificatePaths(path, path); err == nil || err.Error() != "SSL 路径必须是当前服务器支持的完整绝对路径" {
|
||||
t.Fatalf("non-native path %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyManagedSSLReferenceDoesNotPersistLegacyPaths(t *testing.T) {
|
||||
cfg := &config.Config{
|
||||
SSLPrivateKeyPath: "config/ssl/private.key",
|
||||
|
||||
Reference in new issue
Block a user