修复生产审查凭据泄露风险并增加持久投递队列
This commit is contained in:
1 parent
071460ed9a
commit
6b009197bc
70 files changed
+1581
-262
No files matched your search
@@ -0,0 +1,93 @@
|
||||
package send
|
||||
|
||||
import (
|
||||
stdcontext "context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/textproto"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/utils/smtp"
|
||||
)
|
||||
|
||||
var errNullMX = errors.New("收件域名声明不接收邮件")
|
||||
|
||||
type mxLookup func(stdcontext.Context, string) ([]*net.MX, error)
|
||||
|
||||
func resolveMX(domain string, lookup mxLookup) ([]string, error) {
|
||||
ctx, cancel := stdcontext.WithTimeout(stdcontext.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
records, err := lookup(ctx, domain)
|
||||
if err != nil {
|
||||
var dnsErr *net.DNSError
|
||||
if !errors.As(err, &dnsErr) || !dnsErr.IsNotFound || dnsErr.IsTemporary || dnsErr.IsTimeout {
|
||||
return nil, fmt.Errorf("查询收件域名 MX 失败: %w", err)
|
||||
}
|
||||
// 无 MX 时使用域名自身的地址记录,不猜测 smtp 子域名。
|
||||
return []string{domain}, nil
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return []string{domain}, nil
|
||||
}
|
||||
records = append([]*net.MX(nil), records...)
|
||||
for _, record := range records {
|
||||
if record == nil || record.Host == "" {
|
||||
return nil, errors.New("收件域名 MX 记录无效")
|
||||
}
|
||||
if record.Host == "." {
|
||||
return nil, errNullMX
|
||||
}
|
||||
}
|
||||
sort.SliceStable(records, func(i, j int) bool { return records[i].Pref < records[j].Pref })
|
||||
hosts := make([]string, 0, len(records))
|
||||
seen := make(map[string]bool)
|
||||
for _, record := range records {
|
||||
host := strings.ToLower(strings.TrimSuffix(record.Host, "."))
|
||||
if !seen[host] {
|
||||
hosts = append(hosts, host)
|
||||
seen[host] = true
|
||||
}
|
||||
}
|
||||
return hosts, nil
|
||||
}
|
||||
|
||||
func stopFallback(err error) bool {
|
||||
return smtp.IsDeliveryUncertain(err) || isCertificateVerificationError(err) || isPermanentSMTPResponse(err) || errors.Is(err, smtp.ErrUnsafeRemoteAddress)
|
||||
}
|
||||
|
||||
// deliverMX 只在确认本次未接收时切换安全通道,DATA 不确定结果不进入下一条路由。
|
||||
func deliverMX(hosts []string, attempt func(host, port string) error) error {
|
||||
var lastErr error
|
||||
for _, host := range hosts {
|
||||
for _, port := range []string{"25", "587", "465"} {
|
||||
lastErr = attempt(host, port)
|
||||
if lastErr == nil || stopFallback(lastErr) {
|
||||
return lastErr
|
||||
}
|
||||
}
|
||||
}
|
||||
if lastErr == nil {
|
||||
lastErr = errors.New("收件域名没有可用邮件服务器")
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// Retryable 仅允许有明确临时失败语义的结果进入持久队列重试。
|
||||
func Retryable(err error) bool {
|
||||
if err == nil || stopFallback(err) {
|
||||
return false
|
||||
}
|
||||
var response *textproto.Error
|
||||
if errors.As(err, &response) {
|
||||
return response.Code >= 400 && response.Code <= 499
|
||||
}
|
||||
var dnsErr *net.DNSError
|
||||
if errors.As(err, &dnsErr) {
|
||||
return dnsErr.IsTimeout || dnsErr.IsTemporary
|
||||
}
|
||||
var networkErr net.Error
|
||||
return errors.As(err, &networkErr)
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package send
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/textproto"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"git.pcux.cn/xuans/Smail/utils/smtp"
|
||||
)
|
||||
|
||||
func TestResolveMX(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
records []*net.MX
|
||||
err error
|
||||
want []string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "priority and duplicate", records: []*net.MX{{Host: "backup.example.", Pref: 20}, {Host: "primary.example.", Pref: 10}, {Host: "PRIMARY.example.", Pref: 30}}, want: []string{"primary.example", "backup.example"}},
|
||||
{name: "implicit MX", want: []string{"example.com"}},
|
||||
{name: "no MX record", err: &net.DNSError{IsNotFound: true}, want: []string{"example.com"}},
|
||||
{name: "temporary DNS", err: &net.DNSError{IsTemporary: true}, wantErr: true},
|
||||
{name: "timeout DNS", err: &net.DNSError{IsTimeout: true}, wantErr: true},
|
||||
{name: "null MX", records: []*net.MX{{Host: "."}}, wantErr: true},
|
||||
{name: "invalid MX", records: []*net.MX{nil}, wantErr: true},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
hosts, err := resolveMX("example.com", func(ctx context.Context, domain string) ([]*net.MX, error) {
|
||||
if _, ok := ctx.Deadline(); !ok || domain != "example.com" {
|
||||
t.Fatal("unbounded or wrong lookup")
|
||||
}
|
||||
return tt.records, tt.err
|
||||
})
|
||||
if (err != nil) != tt.wantErr || !reflect.DeepEqual(hosts, tt.want) {
|
||||
t.Fatalf("hosts=%v error=%v", hosts, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliverMXFallbackAndStop(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
err error
|
||||
calls int
|
||||
}{
|
||||
{"network fallback", &net.OpError{Op: "dial", Err: io.EOF}, 4},
|
||||
{"temporary refusal fallback", &textproto.Error{Code: 451, Msg: "later"}, 4},
|
||||
{"permanent refusal", &textproto.Error{Code: 550, Msg: "rejected"}, 1},
|
||||
{"certificate failure", &tls.CertificateVerificationError{Err: io.EOF}, 1},
|
||||
{"uncertain DATA", &smtp.DeliveryUncertainError{Cause: io.EOF}, 1},
|
||||
{"private MX", smtp.ErrUnsafeRemoteAddress, 1},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var calls []string
|
||||
err := deliverMX([]string{"primary.example", "backup.example"}, func(host, port string) error {
|
||||
calls = append(calls, host+":"+port)
|
||||
if host == "backup.example" {
|
||||
return nil
|
||||
}
|
||||
return tt.err
|
||||
})
|
||||
if len(calls) != tt.calls {
|
||||
t.Fatalf("calls=%v", calls)
|
||||
}
|
||||
if tt.calls == 4 && err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tt.calls == 1 && !errors.Is(err, tt.err) {
|
||||
t.Fatalf("error=%v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryable(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{&textproto.Error{Code: 451}, true}, {&net.DNSError{IsTemporary: true}, true},
|
||||
{&net.DNSError{IsNotFound: true}, false}, {&textproto.Error{Code: 550}, false},
|
||||
{&smtp.DeliveryUncertainError{Cause: &net.DNSError{IsTimeout: true}}, false},
|
||||
{&tls.CertificateVerificationError{Err: io.EOF}, false}, {nil, false},
|
||||
} {
|
||||
if got := Retryable(tt.err); got != tt.want {
|
||||
t.Fatalf("Retryable(%v)=%v", tt.err, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
+51
-107
@@ -9,7 +9,6 @@ import (
|
||||
"git.pcux.cn/xuans/Smail/models"
|
||||
"git.pcux.cn/xuans/Smail/services/logstore"
|
||||
"git.pcux.cn/xuans/Smail/utils/array"
|
||||
"git.pcux.cn/xuans/Smail/utils/async"
|
||||
"git.pcux.cn/xuans/Smail/utils/consts"
|
||||
"git.pcux.cn/xuans/Smail/utils/context"
|
||||
"git.pcux.cn/xuans/Smail/utils/smtp"
|
||||
@@ -89,6 +88,23 @@ func Send(ctx *context.Context, e *parsemail.Email) (error, map[string]error) {
|
||||
|
||||
// SendTriggered 显式标记业务发起方,不从邮件主题或内容推断来源。
|
||||
func SendTriggered(ctx *context.Context, e *parsemail.Email, trigger string) (error, map[string]error) {
|
||||
return sendRecipients(ctx, e, nil, trigger)
|
||||
}
|
||||
|
||||
// SendRecipientTriggered 保留原邮件头,只向当前队列收件人发送 SMTP 信封。
|
||||
func SendRecipientTriggered(ctx *context.Context, e *parsemail.Email, recipient string) error {
|
||||
err, failures := sendRecipients(ctx, e, []*parsemail.User{{EmailAddress: recipient}}, "网页写信")
|
||||
if err != nil {
|
||||
for _, cause := range failures {
|
||||
if cause != nil {
|
||||
return cause
|
||||
}
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func sendRecipients(ctx *context.Context, e *parsemail.Email, recipients []*parsemail.User, trigger string) (error, map[string]error) {
|
||||
_, fromDomain := e.From.GetDomainAccount()
|
||||
|
||||
b, err := e.BuildBytes(ctx, true)
|
||||
@@ -104,6 +120,9 @@ func SendTriggered(ctx *context.Context, e *parsemail.Email, trigger string) (er
|
||||
|
||||
var to []*parsemail.User
|
||||
to = append(append(append(to, e.To...), e.Cc...), e.Bcc...)
|
||||
if recipients != nil {
|
||||
to = recipients
|
||||
}
|
||||
|
||||
return doSend(ctx, fromDomain, b, to, e.From.EmailAddress, trigger)
|
||||
|
||||
@@ -176,8 +195,7 @@ func doSend(ctx *context.Context, fromDomain string, data []byte, to []*parsemai
|
||||
invalidErrors := make(map[string]error)
|
||||
|
||||
// 按域名整理
|
||||
toByDomain := map[mxDomain][]*parsemail.User{}
|
||||
mxLookupErrors := map[mxDomain]error{}
|
||||
toByDomain := map[string][]*parsemail.User{}
|
||||
for _, s := range to {
|
||||
address, domain, addressErr := parseRecipientAddress(s)
|
||||
if addressErr != nil {
|
||||
@@ -189,41 +207,7 @@ func doSend(ctx *context.Context, fromDomain string, data []byte, to []*parsemai
|
||||
log.WithContext(ctx).WithError(addressErr).Error("invalid recipient address")
|
||||
continue
|
||||
}
|
||||
args := []string{address, domain}
|
||||
if len(args) == 2 {
|
||||
if args[1] == consts.TEST_DOMAIN {
|
||||
// 测试使用
|
||||
address := mxDomain{
|
||||
domain: "localhost",
|
||||
mxHost: "127.0.0.1",
|
||||
localTest: true,
|
||||
}
|
||||
toByDomain[address] = append(toByDomain[address], s)
|
||||
} else {
|
||||
//查询dns mx记录
|
||||
mxInfo, lookupErr := net.LookupMX(args[1])
|
||||
address := mxDomain{
|
||||
domain: "smtp." + args[1],
|
||||
mxHost: "smtp." + args[1],
|
||||
}
|
||||
if lookupErr != nil {
|
||||
log.WithContext(ctx).Errorf("%s 域名mx记录查询失败,检查邮箱是否存在!", s.EmailAddress)
|
||||
}
|
||||
if len(mxInfo) > 0 {
|
||||
address = mxDomain{
|
||||
domain: args[1],
|
||||
mxHost: mxInfo[0].Host,
|
||||
}
|
||||
}
|
||||
if lookupErr != nil {
|
||||
mxLookupErrors[address] = lookupErr
|
||||
}
|
||||
toByDomain[address] = append(toByDomain[address], s)
|
||||
}
|
||||
} else {
|
||||
log.WithContext(ctx).Errorf("邮箱地址解析错误! %s", s)
|
||||
continue
|
||||
}
|
||||
toByDomain[domain] = append(toByDomain[domain], s)
|
||||
}
|
||||
|
||||
errEmailAddress := append([]string(nil), invalidAddresses...)
|
||||
@@ -234,14 +218,14 @@ func doSend(ctx *context.Context, fromDomain string, data []byte, to []*parsemai
|
||||
errMap.Store(address, err)
|
||||
}
|
||||
|
||||
as := async.New(ctx)
|
||||
var workers sync.WaitGroup
|
||||
for domain, tos := range toByDomain {
|
||||
domain := domain
|
||||
tos := tos
|
||||
mxLookupErr := mxLookupErrors[domain]
|
||||
as.WaitProcess(func(p any) {
|
||||
workers.Add(1)
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
recordFailure := func(err error) {
|
||||
err = deliveryFailureCause(mxLookupErr, err)
|
||||
log.WithContext(ctx).Errorf("%v 邮件投递失败%+v", tos, err)
|
||||
|
||||
errEmailAddressMu.Lock()
|
||||
@@ -250,11 +234,16 @@ func doSend(ctx *context.Context, fromDomain string, data []byte, to []*parsemai
|
||||
}
|
||||
errEmailAddressMu.Unlock()
|
||||
|
||||
errMap.Store(domain.domain, err)
|
||||
errMap.Store(domain, err)
|
||||
}
|
||||
defer func() {
|
||||
if value := recover(); value != nil {
|
||||
recordFailure(&smtp.DeliveryUncertainError{Cause: fmt.Errorf("投递任务异常: %v", value)})
|
||||
}
|
||||
}()
|
||||
|
||||
if allowUnsafeSMTP(domain) {
|
||||
err := smtp.SendMailUnsafe("localhost", domain.mxHost+":25", nil, from, fromDomain, buildAddress(tos), data)
|
||||
if domain == consts.TEST_DOMAIN {
|
||||
err := smtp.SendMailUnsafe("localhost", "127.0.0.1:25", nil, from, fromDomain, buildAddress(tos), data)
|
||||
if err != nil {
|
||||
recordFailure(err)
|
||||
} else {
|
||||
@@ -263,71 +252,26 @@ func doSend(ctx *context.Context, fromDomain string, data []byte, to []*parsemai
|
||||
return
|
||||
}
|
||||
|
||||
// 优先尝试25端口,starttls方式投递
|
||||
err := smtp.SendMail("", domain.mxHost+":25", nil, from, fromDomain, buildAddress(tos), data)
|
||||
if err == nil {
|
||||
hosts, err := resolveMX(domain, net.DefaultResolver.LookupMX)
|
||||
if err != nil {
|
||||
recordFailure(err)
|
||||
return
|
||||
}
|
||||
err = deliverMX(hosts, func(host, port string) error {
|
||||
address := net.JoinHostPort(host, port)
|
||||
if port == "465" {
|
||||
return smtp.SendMailWithTls(host, address, nil, from, fromDomain, buildAddress(tos), data)
|
||||
}
|
||||
return smtp.SendMail(host, address, nil, from, fromDomain, buildAddress(tos), data)
|
||||
})
|
||||
if err != nil {
|
||||
recordFailure(err)
|
||||
} else {
|
||||
accepted.Add(int64(len(tos)))
|
||||
return
|
||||
}
|
||||
if isCertificateVerificationError(err) {
|
||||
recordFailure(err)
|
||||
return
|
||||
}
|
||||
if isPermanentSMTPResponse(err) {
|
||||
recordFailure(err)
|
||||
return
|
||||
}
|
||||
log.WithContext(ctx).WithFields(log.Fields{
|
||||
"action": "SMTP 邮件投递",
|
||||
"result": "failure",
|
||||
"object": domain.mxHost + ":25",
|
||||
}).Infof("SMTP 25 端口 STARTTLS 投递失败: %s", err.Error())
|
||||
|
||||
// 587 使用 STARTTLS,只有 465 使用隐式 TLS。
|
||||
err = smtp.SendMail(strings.TrimSuffix(domain.mxHost, "."), domain.mxHost+":587", nil, from, fromDomain, buildAddress(tos), data)
|
||||
if err == nil {
|
||||
accepted.Add(int64(len(tos)))
|
||||
return
|
||||
}
|
||||
if isCertificateVerificationError(err) {
|
||||
recordFailure(err)
|
||||
return
|
||||
}
|
||||
if isPermanentSMTPResponse(err) {
|
||||
recordFailure(err)
|
||||
return
|
||||
}
|
||||
log.WithContext(ctx).WithFields(log.Fields{
|
||||
"action": "SMTP 邮件投递",
|
||||
"result": "failure",
|
||||
"object": domain.mxHost + ":587",
|
||||
}).Infof("SMTP 587 端口投递失败: %s", err.Error())
|
||||
|
||||
// 再次尝试465投递
|
||||
err = smtp.SendMailWithTls(strings.TrimSuffix(domain.mxHost, "."), domain.mxHost+":465", nil, from, fromDomain, buildAddress(tos), data)
|
||||
if err == nil {
|
||||
accepted.Add(int64(len(tos)))
|
||||
return
|
||||
}
|
||||
if isCertificateVerificationError(err) {
|
||||
recordFailure(err)
|
||||
return
|
||||
}
|
||||
if isPermanentSMTPResponse(err) {
|
||||
recordFailure(err)
|
||||
return
|
||||
}
|
||||
log.WithContext(ctx).WithFields(log.Fields{
|
||||
"action": "SMTP 邮件投递",
|
||||
"result": "failure",
|
||||
"object": domain.mxHost + ":465",
|
||||
}).Infof("SMTP 465 端口投递失败: %s", err.Error())
|
||||
|
||||
// 所有安全投递方式均失败后记录错误,不允许回退到明文连接。
|
||||
recordFailure(err)
|
||||
}, nil)
|
||||
}()
|
||||
}
|
||||
as.Wait()
|
||||
workers.Wait()
|
||||
|
||||
orgMap := map[string]error{}
|
||||
errMap.Range(func(key, value any) bool {
|
||||
|
||||
Reference in new issue
Block a user