mail-tester/backend/checks.go
Jonas Leder 5e37ac0f59 Add Go backend: JMAP poller, mail checks, scoring API, swagger docs
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-16 11:18:49 +01:00

405 lines
12 KiB
Go

package main
import (
"bytes"
"context"
"fmt"
"io"
"net"
"net/mail"
"regexp"
"strings"
"sync"
"time"
"blitiri.com.ar/go/spf"
"github.com/emersion/go-msgauth/dkim"
"github.com/emersion/go-msgauth/dmarc"
"golang.org/x/net/publicsuffix"
)
var dnsbls = []string{"zen.spamhaus.org", "b.barracudacentral.org", "bl.spamcop.net", "psbl.surgemail.net", "dnsbl-1.uceprotect.net", "bl.mailspike.net", "dnsbl.dronebl.org"}
var (
recvIPRe = regexp.MustCompile(`\[(?:IPv6:)?([0-9a-fA-F:.]+)\]`)
recvHeloRe = regexp.MustCompile(`(?i)^from\s+(\S+)`)
)
func mk(id, name string, weight float64) Check {
return Check{ID: id, Name: name, Weight: weight}
}
// finish derives status from the earned fraction (0..1).
func (c Check) finish(frac float64, summary string, details ...string) Check {
c.Score, c.Summary, c.Details = c.Weight*frac, summary, details
switch {
case frac >= 0.999:
c.Status = "pass"
case frac > 0.4:
c.Status = "warn"
default:
c.Status = "fail"
}
return c
}
func skipped(id, name, why string) Check {
return Check{ID: id, Name: name, Status: "skipped", Summary: why}
}
func domainOf(addr string) string {
if i := strings.LastIndex(addr, "@"); i >= 0 {
return strings.ToLower(strings.Trim(addr[i+1:], "<> "))
}
return ""
}
func orgDomain(d string) string {
if o, err := publicsuffix.EffectiveTLDPlusOne(d); err == nil {
return o
}
return d
}
// Analyze runs every check on a raw RFC822 message and returns the score out of 10.
func Analyze(raw []byte, spamd string) (Sender, string, []Check, float64) {
msg, err := mail.ReadMessage(bytes.NewReader(raw))
if err != nil {
return Sender{}, "", []Check{mk("message", "Message format", 1).finish(0, "Message could not be parsed: "+err.Error())}, 0
}
h := msg.Header
var s Sender
if from, err := mail.ParseAddress(h.Get("From")); err == nil {
s.From = from.Address
}
envelope := strings.Trim(h.Get("Return-Path"), "<> ")
if envelope == "" {
envelope = s.From
}
// Stalwart prepends the Received header of the connection that delivered the mail.
if rec := h["Received"]; len(rec) > 0 {
first := strings.Join(strings.Fields(rec[0]), " ")
if m := recvIPRe.FindStringSubmatch(first); m != nil {
s.IP = m[1]
}
if m := recvHeloRe.FindStringSubmatch(first); m != nil {
s.Helo = strings.Trim(m[1], "[]")
}
}
ip := net.ParseIP(s.IP)
fromDom, envDom := domainOf(s.From), domainOf(envelope)
if envDom == "" {
envDom = fromDom
}
var checks []Check
var spfPass bool
var dkimDomains []string
if ip == nil {
checks = append(checks, mk("sender-ip", "Sender IP", 1).finish(0, "Could not determine the sending IP from the Received header"))
} else {
var c Check
c, spfPass = checkSPF(ip, s.Helo, envelope)
checks = append(checks, c)
}
c, dd := checkDKIM(raw)
dkimDomains = dd
checks = append(checks, c, checkDMARC(fromDom, envDom, spfPass, dkimDomains))
if ip != nil {
checks = append(checks, checkRDNS(ip, s.Helo), checkBlacklists(ip))
}
checks = append(checks, checkMX(envDom)...)
checks = append(checks, checkSpamAssassin(raw, spamd), checkHeaders(msg, raw), checkTLS(h))
var earned, total float64
for _, c := range checks {
if c.Status != "skipped" {
earned, total = earned+c.Score, total+c.Weight
}
}
score := 0.0
if total > 0 {
score = float64(int(earned/total*100+0.5)) / 10
}
return s, h.Get("Subject"), checks, score
}
func checkSPF(ip net.IP, helo, sender string) (Check, bool) {
c := mk("spf", "SPF", 2)
res, err := spf.CheckHostWithSender(ip, helo, sender)
d := []string{fmt.Sprintf("ip=%s helo=%s envelope-from=%s", ip, helo, sender)}
if err != nil {
d = append(d, err.Error())
}
switch res {
case spf.Pass:
return c.finish(1, "SPF passed", d...), true
case spf.SoftFail, spf.Neutral:
return c.finish(0.5, "SPF "+string(res)+": this IP is not explicitly authorised", d...), false
case spf.None:
return c.finish(0, "No SPF record found for "+domainOf(sender), d...), false
default:
return c.finish(0, "SPF "+string(res), d...), false
}
}
func checkDKIM(raw []byte) (Check, []string) {
c := mk("dkim", "DKIM", 2)
vs, err := dkim.Verify(bytes.NewReader(raw))
if err != nil && len(vs) == 0 {
return c.finish(0, "DKIM verification failed: "+err.Error()), nil
}
if len(vs) == 0 {
return c.finish(0, "No DKIM signature found"), nil
}
var ok []string
var d []string
for _, v := range vs {
if v.Err == nil {
ok = append(ok, strings.ToLower(v.Domain))
d = append(d, "valid signature for "+v.Domain)
} else {
d = append(d, "invalid signature for "+v.Domain+": "+v.Err.Error())
}
}
if len(ok) == 0 {
return c.finish(0, "DKIM signature present but invalid", d...), nil
}
if len(ok) < len(vs) {
return c.finish(0.75, "Some DKIM signatures are invalid", d...), ok
}
return c.finish(1, "DKIM signature valid", d...), ok
}
func checkDMARC(fromDom, envDom string, spfPass bool, dkimDomains []string) Check {
c := mk("dmarc", "DMARC", 1.5)
if fromDom == "" {
return c.finish(0, "No usable From domain")
}
rec, err := dmarc.Lookup(fromDom)
if err != nil && orgDomain(fromDom) != fromDom {
rec, err = dmarc.Lookup(orgDomain(fromDom))
}
if err != nil {
return c.finish(0, "No valid DMARC record for "+fromDom, err.Error())
}
aligned := spfPass && orgDomain(envDom) == orgDomain(fromDom)
for _, d := range dkimDomains {
aligned = aligned || orgDomain(d) == orgDomain(fromDom)
}
d := []string{"policy=" + string(rec.Policy)}
if !aligned {
return c.finish(0, "DMARC record found but neither SPF nor DKIM is aligned with the From domain", d...)
}
if rec.Policy == dmarc.PolicyNone {
return c.finish(0.75, "DMARC aligned, but policy is p=none (monitoring only)", d...)
}
return c.finish(1, "DMARC aligned, policy p="+string(rec.Policy), d...)
}
func checkRDNS(ip net.IP, helo string) Check {
c := mk("rdns", "Reverse DNS (PTR)", 1)
names, _ := net.LookupAddr(ip.String())
if len(names) == 0 {
return c.finish(0, "No PTR record for "+ip.String())
}
d := []string{"PTR: " + strings.Join(names, ", "), "HELO: " + helo}
for _, n := range names {
addrs, _ := net.LookupHost(strings.TrimSuffix(n, "."))
for _, a := range addrs {
if p := net.ParseIP(a); p != nil && p.Equal(ip) {
if !strings.EqualFold(strings.TrimSuffix(n, "."), helo) {
return c.finish(0.9, "PTR is valid but does not match the HELO name", d...)
}
return c.finish(1, "PTR "+n+" resolves back to "+ip.String(), d...)
}
}
}
return c.finish(0.5, "PTR exists but does not resolve back to the sending IP (no forward-confirmed rDNS)", d...)
}
func reverseIP(ip net.IP) string {
if v4 := ip.To4(); v4 != nil {
return fmt.Sprintf("%d.%d.%d.%d", v4[3], v4[2], v4[1], v4[0])
}
var b strings.Builder
for i := 15; i >= 0; i-- {
fmt.Fprintf(&b, "%x.%x.", ip[i]&0xf, ip[i]>>4)
}
return strings.TrimSuffix(b.String(), ".")
}
func checkBlacklists(ip net.IP) Check {
c := mk("blacklist", "Blacklists", 1.5)
rev := reverseIP(ip)
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)
defer cancel()
var mu sync.Mutex
var wg sync.WaitGroup
var listed, details []string
checked := 0
for _, bl := range dnsbls {
wg.Add(1)
go func() {
defer wg.Done()
addrs, err := net.DefaultResolver.LookupHost(ctx, rev+"."+bl)
mu.Lock()
defer mu.Unlock()
if len(addrs) > 0 {
// 127.255.255.x = "query refused" (public resolver / quota), not a listing
if strings.HasPrefix(addrs[0], "127.255.255.") {
details = append(details, bl+": query refused, use your own resolver")
return
}
checked++
listed = append(listed, bl)
details = append(details, bl+": LISTED ("+addrs[0]+")")
} else if dnsErr, ok := err.(*net.DNSError); ok && dnsErr.IsNotFound {
checked++
details = append(details, bl+": not listed")
} else {
details = append(details, bl+": lookup failed")
}
}()
}
wg.Wait()
if checked == 0 {
return skipped(c.ID, c.Name, "No blacklist could be queried")
}
if len(listed) == 0 {
return c.finish(1, fmt.Sprintf("%s is not listed on %d blacklists", ip, checked), details...)
}
frac := 1 - float64(len(listed))/2
if frac < 0 {
frac = 0
}
return c.finish(frac, fmt.Sprintf("%s is listed on %d blacklist(s): %s", ip, len(listed), strings.Join(listed, ", ")), details...)
}
func checkMX(dom string) []Check {
mx, a := mk("mx", "MX record", 0.5), mk("mx-address", "MX host A/AAAA", 0.5)
if dom == "" {
return []Check{mx.finish(0, "No sender domain"), a.finish(0, "No sender domain")}
}
recs, _ := net.LookupMX(dom)
if len(recs) == 0 {
return []Check{mx.finish(0, "No MX record for "+dom+" (bounces and replies will fail)"), a.finish(0, "No MX hosts to check")}
}
var d []string
ok := 0
for _, r := range recs {
host := strings.TrimSuffix(r.Host, ".")
ips, _ := net.LookupIP(host)
var v4, v6 bool
for _, i := range ips {
if i.To4() != nil {
v4 = true
} else {
v6 = true
}
}
if len(ips) > 0 {
ok++
}
d = append(d, fmt.Sprintf("%s (pref %d): A=%t AAAA=%t", host, r.Pref, v4, v6))
}
ac := a.finish(float64(ok)/float64(len(recs)), fmt.Sprintf("%d of %d MX hosts resolve to an address", ok, len(recs)), d...)
return []Check{mx.finish(1, fmt.Sprintf("%d MX record(s) for %s", len(recs), dom)), ac}
}
var saRuleRe = regexp.MustCompile(`^\s*(-?\d+\.\d+)\s+([A-Z0-9_]+)\s+(.*)$`)
func checkSpamAssassin(raw []byte, addr string) Check {
c := mk("spamassassin", "SpamAssassin", 2.5)
if addr == "" {
return skipped(c.ID, c.Name, "SPAMD_ADDR not configured")
}
conn, err := net.DialTimeout("tcp", addr, 5*time.Second)
if err != nil {
return skipped(c.ID, c.Name, "spamd unreachable: "+err.Error())
}
defer conn.Close()
conn.SetDeadline(time.Now().Add(30 * time.Second))
fmt.Fprintf(conn, "REPORT SPAMC/1.5\r\nContent-length: %d\r\n\r\n", len(raw))
conn.Write(raw)
conn.(*net.TCPConn).CloseWrite()
out, _ := io.ReadAll(conn)
var score, thr float64
// "Spam: True ; 15.0 / 5.0"
parts := strings.FieldsFunc(extract(string(out), "Spam:"), func(r rune) bool { return r == ';' || r == '/' || r == ' ' })
if len(parts) < 3 {
return skipped(c.ID, c.Name, "unexpected spamd response")
}
fmt.Sscan(parts[1], &score)
fmt.Sscan(parts[2], &thr)
var d []string
for _, l := range strings.Split(string(out), "\n") {
if m := saRuleRe.FindStringSubmatch(strings.TrimRight(l, "\r")); m != nil && m[1] != "0.0" {
d = append(d, fmt.Sprintf("%5s %s %s", m[1], m[2], m[3]))
}
}
if thr <= 0 {
thr = 5
}
frac := 1.0
if score > 0 {
frac = 1 - score/thr
}
if frac < 0 {
frac = 0
}
return c.finish(frac, fmt.Sprintf("SpamAssassin score %.1f (spam threshold %.1f)", score, thr), d...)
}
func extract(resp, key string) string {
for _, l := range strings.Split(resp, "\n") {
if strings.HasPrefix(l, key) {
return strings.TrimSpace(strings.TrimPrefix(l, key))
}
}
return ""
}
func checkHeaders(msg *mail.Message, raw []byte) Check {
c := mk("headers", "Message format", 1)
var bad []string
h := msg.Header
for _, k := range []string{"From", "To", "Subject", "Message-Id"} {
if h.Get(k) == "" {
bad = append(bad, "missing header "+k)
}
}
if _, err := h.Date(); err != nil {
bad = append(bad, "missing or invalid Date header")
} else if d, _ := h.Date(); time.Since(d) > 24*time.Hour || time.Until(d) > time.Hour {
bad = append(bad, "Date header is far from the current time")
}
ct := strings.ToLower(h.Get("Content-Type"))
if h.Get("MIME-Version") == "" {
bad = append(bad, "missing MIME-Version header")
}
if strings.Contains(ct, "text/html") && !strings.Contains(ct, "multipart/alternative") && !bytes.Contains(bytes.ToLower(raw), []byte("content-type: text/plain")) {
bad = append(bad, "HTML-only message, add a text/plain alternative")
}
if len(bad) == 0 {
return c.finish(1, "Required headers present and well-formed")
}
frac := 1 - float64(len(bad))/4
if frac < 0 {
frac = 0
}
return c.finish(frac, fmt.Sprintf("%d problem(s) with the message format", len(bad)), bad...)
}
func checkTLS(h mail.Header) Check {
c := mk("tls", "Transport encryption", 0.5)
rec := h["Received"]
if len(rec) == 0 {
return skipped(c.ID, c.Name, "No Received header")
}
first := strings.ToUpper(rec[0])
if strings.Contains(first, "ESMTPS") || strings.Contains(first, "TLS") {
return c.finish(1, "Message was delivered over TLS")
}
return c.finish(0, "Message was delivered without TLS (STARTTLS)")
}