fix mailimport

This commit is contained in:
Jonas Leder 2026-08-26 21:39:04 +02:00
commit 498e96b377

View file

@ -1,9 +1,11 @@
package main
import (
"bytes"
"errors"
"flag"
"fmt"
"net/mail"
"os"
"path/filepath"
"strings"
@ -47,6 +49,45 @@ func collectEmlFiles(paths []string) ([]string, error) {
return files, nil
}
// recipientHeaders are tried in order when auto-detecting which mailbox to
// file an imported mail under. To is the obvious first choice, but a lot of
// real-world archives (BCC/journal-based archiving in particular - e.g.
// Proxmox Mail Gateway/piler dumps) deliver mail with no To header at all,
// or one that only names the archiving system's own mailbox. The delivery
// headers an MTA adds for its own routing (Delivered-To, X-Original-To,
// Envelope-To) usually still carry the real final recipient in that case.
var recipientHeaders = []string{"To", "Delivered-To", "X-Original-To", "X-Envelope-To", "Envelope-To"}
// mailboxFromHeaders derives a mailbox address from a raw message, trying
// recipientHeaders in order and returning the first usable address found.
// Only the first address of a multi-recipient header is used - if a
// message was sent to multiple recipients, one still has to be picked as
// "the" mailbox it's archived under.
func mailboxFromHeaders(raw []byte) (string, error) {
msg, err := mail.ReadMessage(bytes.NewReader(raw))
if err != nil {
return "", fmt.Errorf("parsing message: %w", err)
}
for _, h := range recipientHeaders {
v := msg.Header.Get(h)
if v == "" {
continue
}
if addrs, err := msg.Header.AddressList(h); err == nil && len(addrs) > 0 {
return strings.ToLower(addrs[0].Address), nil
}
// Delivered-To/X-Original-To/Envelope-To are often written by MTAs
// as a bare address with no RFC 5322 address syntax (no <angle
// brackets>, no display name) - fall back to using the raw value
// itself rather than discarding a perfectly good address just
// because it failed strict parsing.
if h != "To" {
return strings.ToLower(strings.TrimSpace(v)), nil
}
}
return "", fmt.Errorf("no usable recipient header found (tried %s)", strings.Join(recipientHeaders, ", "))
}
// pushWithRetry retries transient failures (network errors, 5xx, 429) with
// exponential backoff. 4xx responses are permanent (bad mailbox, oversized
// message, ...) and fail immediately - retrying those would just burn
@ -82,7 +123,8 @@ func runImport(args []string) error {
fs := flag.NewFlagSet("import", flag.ExitOnError)
apiURL := fs.String("api-url", envOr("MAIL_ARCHIVER_API_URL", "http://localhost:8080"), "mail-archiver API base URL")
apiKey := fs.String("api-key", os.Getenv("MAIL_ARCHIVER_API_KEY"), "ingester API key")
mailbox := fs.String("mailbox", "", "mailbox address to archive imported mail under (required)")
mailbox := fs.String("mailbox", "", "mailbox address to archive imported mail under; if omitted, it's auto-detected per mail from its recipient headers (To, Delivered-To, X-Original-To, ...)")
fallbackMailbox := fs.String("fallback-mailbox", "", "with --mailbox omitted, mailbox to use for a mail whose recipient can't be auto-detected, instead of failing it")
direction := fs.String("direction", "inbound", "direction to record for imported mail: inbound or outbound")
concurrency := fs.Int("concurrency", 32, "number of mails to import in parallel")
retries := fs.Int("retries", 3, "retries for transient failures (network errors, 5xx, 429) per mail")
@ -96,8 +138,8 @@ func runImport(args []string) error {
if *apiKey == "" {
return fmt.Errorf("--api-key (or MAIL_ARCHIVER_API_KEY) is required")
}
if *mailbox == "" {
return fmt.Errorf("--mailbox is required")
if strings.ContainsAny(*mailbox, `/\`) {
return fmt.Errorf("--mailbox %q looks like a file path, not a mailbox address - the file/directory to import goes at the end of the command, after the flags (e.g. mailctl import --mailbox you@example.com --api-key ... /path/to/emls)", *mailbox)
}
if *direction != "inbound" && *direction != "outbound" {
return fmt.Errorf("--direction must be \"inbound\" or \"outbound\"")
@ -107,7 +149,7 @@ func runImport(args []string) error {
}
paths := fs.Args()
if len(paths) == 0 {
return fmt.Errorf("at least one .eml file or directory is required")
return fmt.Errorf("at least one .eml file or directory is required - it goes at the end of the command, after all flags (e.g. mailctl import --mailbox you@example.com --api-key ... /path/to/emls)")
}
fmt.Println("scanning for .eml files...")
@ -143,9 +185,26 @@ func runImport(args []string) error {
failedMu.Unlock()
continue
}
if err := pushWithRetry(c, *mailbox, *direction, raw, *retries, *retryBackoff); err != nil {
targetMailbox := *mailbox
if targetMailbox == "" {
targetMailbox, err = mailboxFromHeaders(raw)
if err != nil {
if *fallbackMailbox == "" {
stats.failed.Add(1)
fmt.Fprintf(os.Stderr, "skipping %s: could not detect mailbox: %v\n", f, err)
failedMu.Lock()
failedFiles = append(failedFiles, f)
failedMu.Unlock()
continue
}
targetMailbox = *fallbackMailbox
}
}
if err := pushWithRetry(c, targetMailbox, *direction, raw, *retries, *retryBackoff); err != nil {
stats.failed.Add(1)
fmt.Fprintf(os.Stderr, "failed importing %s: %v\n", f, err)
fmt.Fprintf(os.Stderr, "failed importing %s (mailbox=%s): %v\n", f, targetMailbox, err)
failedMu.Lock()
failedFiles = append(failedFiles, f)
failedMu.Unlock()
@ -153,7 +212,7 @@ func runImport(args []string) error {
}
stats.done.Add(1)
if !*quiet {
fmt.Printf("imported %s\n", f)
fmt.Printf("imported %s -> %s\n", f, targetMailbox)
}
}
}()
@ -186,7 +245,11 @@ func runImport(args []string) error {
if err := os.WriteFile(*failLog, []byte(strings.Join(failedFiles, "\n")+"\n"), 0o644); err != nil {
fmt.Fprintf(os.Stderr, "warning: failed writing fail-log %s: %v\n", *failLog, err)
} else {
fmt.Printf("failed files listed in %s (re-run: mailctl import --mailbox %s ... $(cat %s))\n", *failLog, *mailbox, *failLog)
mailboxFlag := "--mailbox " + *mailbox
if *mailbox == "" {
mailboxFlag = "(auto-detect)"
}
fmt.Printf("failed files listed in %s (re-run: mailctl import %s ... $(cat %s))\n", *failLog, mailboxFlag, *failLog)
}
}