fix mailimport
This commit is contained in:
parent
057f7df378
commit
498e96b377
1 changed files with 71 additions and 8 deletions
|
|
@ -1,9 +1,11 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/mail"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
|
@ -47,6 +49,45 @@ func collectEmlFiles(paths []string) ([]string, error) {
|
|||
return files, nil
|
||||
}
|
||||
|
||||
// recipientHeaders are tried in order when auto-detecting which mailbox to
|
||||
// file an imported mail under. To is the obvious first choice, but a lot of
|
||||
// real-world archives (BCC/journal-based archiving in particular - e.g.
|
||||
// Proxmox Mail Gateway/piler dumps) deliver mail with no To header at all,
|
||||
// or one that only names the archiving system's own mailbox. The delivery
|
||||
// headers an MTA adds for its own routing (Delivered-To, X-Original-To,
|
||||
// Envelope-To) usually still carry the real final recipient in that case.
|
||||
var recipientHeaders = []string{"To", "Delivered-To", "X-Original-To", "X-Envelope-To", "Envelope-To"}
|
||||
|
||||
// mailboxFromHeaders derives a mailbox address from a raw message, trying
|
||||
// recipientHeaders in order and returning the first usable address found.
|
||||
// Only the first address of a multi-recipient header is used - if a
|
||||
// message was sent to multiple recipients, one still has to be picked as
|
||||
// "the" mailbox it's archived under.
|
||||
func mailboxFromHeaders(raw []byte) (string, error) {
|
||||
msg, err := mail.ReadMessage(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parsing message: %w", err)
|
||||
}
|
||||
for _, h := range recipientHeaders {
|
||||
v := msg.Header.Get(h)
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
if addrs, err := msg.Header.AddressList(h); err == nil && len(addrs) > 0 {
|
||||
return strings.ToLower(addrs[0].Address), nil
|
||||
}
|
||||
// Delivered-To/X-Original-To/Envelope-To are often written by MTAs
|
||||
// as a bare address with no RFC 5322 address syntax (no <angle
|
||||
// brackets>, no display name) - fall back to using the raw value
|
||||
// itself rather than discarding a perfectly good address just
|
||||
// because it failed strict parsing.
|
||||
if h != "To" {
|
||||
return strings.ToLower(strings.TrimSpace(v)), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("no usable recipient header found (tried %s)", strings.Join(recipientHeaders, ", "))
|
||||
}
|
||||
|
||||
// pushWithRetry retries transient failures (network errors, 5xx, 429) with
|
||||
// exponential backoff. 4xx responses are permanent (bad mailbox, oversized
|
||||
// message, ...) and fail immediately - retrying those would just burn
|
||||
|
|
@ -82,7 +123,8 @@ func runImport(args []string) error {
|
|||
fs := flag.NewFlagSet("import", flag.ExitOnError)
|
||||
apiURL := fs.String("api-url", envOr("MAIL_ARCHIVER_API_URL", "http://localhost:8080"), "mail-archiver API base URL")
|
||||
apiKey := fs.String("api-key", os.Getenv("MAIL_ARCHIVER_API_KEY"), "ingester API key")
|
||||
mailbox := fs.String("mailbox", "", "mailbox address to archive imported mail under (required)")
|
||||
mailbox := fs.String("mailbox", "", "mailbox address to archive imported mail under; if omitted, it's auto-detected per mail from its recipient headers (To, Delivered-To, X-Original-To, ...)")
|
||||
fallbackMailbox := fs.String("fallback-mailbox", "", "with --mailbox omitted, mailbox to use for a mail whose recipient can't be auto-detected, instead of failing it")
|
||||
direction := fs.String("direction", "inbound", "direction to record for imported mail: inbound or outbound")
|
||||
concurrency := fs.Int("concurrency", 32, "number of mails to import in parallel")
|
||||
retries := fs.Int("retries", 3, "retries for transient failures (network errors, 5xx, 429) per mail")
|
||||
|
|
@ -96,8 +138,8 @@ func runImport(args []string) error {
|
|||
if *apiKey == "" {
|
||||
return fmt.Errorf("--api-key (or MAIL_ARCHIVER_API_KEY) is required")
|
||||
}
|
||||
if *mailbox == "" {
|
||||
return fmt.Errorf("--mailbox is required")
|
||||
if strings.ContainsAny(*mailbox, `/\`) {
|
||||
return fmt.Errorf("--mailbox %q looks like a file path, not a mailbox address - the file/directory to import goes at the end of the command, after the flags (e.g. mailctl import --mailbox you@example.com --api-key ... /path/to/emls)", *mailbox)
|
||||
}
|
||||
if *direction != "inbound" && *direction != "outbound" {
|
||||
return fmt.Errorf("--direction must be \"inbound\" or \"outbound\"")
|
||||
|
|
@ -107,7 +149,7 @@ func runImport(args []string) error {
|
|||
}
|
||||
paths := fs.Args()
|
||||
if len(paths) == 0 {
|
||||
return fmt.Errorf("at least one .eml file or directory is required")
|
||||
return fmt.Errorf("at least one .eml file or directory is required - it goes at the end of the command, after all flags (e.g. mailctl import --mailbox you@example.com --api-key ... /path/to/emls)")
|
||||
}
|
||||
|
||||
fmt.Println("scanning for .eml files...")
|
||||
|
|
@ -143,9 +185,26 @@ func runImport(args []string) error {
|
|||
failedMu.Unlock()
|
||||
continue
|
||||
}
|
||||
if err := pushWithRetry(c, *mailbox, *direction, raw, *retries, *retryBackoff); err != nil {
|
||||
|
||||
targetMailbox := *mailbox
|
||||
if targetMailbox == "" {
|
||||
targetMailbox, err = mailboxFromHeaders(raw)
|
||||
if err != nil {
|
||||
if *fallbackMailbox == "" {
|
||||
stats.failed.Add(1)
|
||||
fmt.Fprintf(os.Stderr, "skipping %s: could not detect mailbox: %v\n", f, err)
|
||||
failedMu.Lock()
|
||||
failedFiles = append(failedFiles, f)
|
||||
failedMu.Unlock()
|
||||
continue
|
||||
}
|
||||
targetMailbox = *fallbackMailbox
|
||||
}
|
||||
}
|
||||
|
||||
if err := pushWithRetry(c, targetMailbox, *direction, raw, *retries, *retryBackoff); err != nil {
|
||||
stats.failed.Add(1)
|
||||
fmt.Fprintf(os.Stderr, "failed importing %s: %v\n", f, err)
|
||||
fmt.Fprintf(os.Stderr, "failed importing %s (mailbox=%s): %v\n", f, targetMailbox, err)
|
||||
failedMu.Lock()
|
||||
failedFiles = append(failedFiles, f)
|
||||
failedMu.Unlock()
|
||||
|
|
@ -153,7 +212,7 @@ func runImport(args []string) error {
|
|||
}
|
||||
stats.done.Add(1)
|
||||
if !*quiet {
|
||||
fmt.Printf("imported %s\n", f)
|
||||
fmt.Printf("imported %s -> %s\n", f, targetMailbox)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
|
@ -186,7 +245,11 @@ func runImport(args []string) error {
|
|||
if err := os.WriteFile(*failLog, []byte(strings.Join(failedFiles, "\n")+"\n"), 0o644); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "warning: failed writing fail-log %s: %v\n", *failLog, err)
|
||||
} else {
|
||||
fmt.Printf("failed files listed in %s (re-run: mailctl import --mailbox %s ... $(cat %s))\n", *failLog, *mailbox, *failLog)
|
||||
mailboxFlag := "--mailbox " + *mailbox
|
||||
if *mailbox == "" {
|
||||
mailboxFlag = "(auto-detect)"
|
||||
}
|
||||
fmt.Printf("failed files listed in %s (re-run: mailctl import %s ... $(cat %s))\n", *failLog, mailboxFlag, *failLog)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue