mail-archiver/cli/cmd_import.go
2026-08-26 21:39:08 +02:00

287 lines
9.6 KiB
Go

package main
import (
"bytes"
"errors"
"flag"
"fmt"
"net/mail"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"jonasled.dev/jonasled/mail-achiver/cli/client"
)
// collectEmlFiles resolves each of paths to a list of .eml files: a file
// path is used as-is (regardless of extension - explicit means explicit), a
// directory is walked recursively for files ending in .eml.
func collectEmlFiles(paths []string) ([]string, error) {
var files []string
for _, p := range paths {
info, err := os.Stat(p)
if err != nil {
return nil, fmt.Errorf("%s: %w", p, err)
}
if !info.IsDir() {
files = append(files, p)
continue
}
err = filepath.WalkDir(p, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
if strings.EqualFold(filepath.Ext(path), ".eml") {
files = append(files, path)
}
return nil
})
if err != nil {
return nil, fmt.Errorf("walking %s: %w", p, err)
}
}
return files, nil
}
// recipientHeaders are tried in order when auto-detecting which mailbox to
// file an imported mail under. To is the obvious first choice, but a lot of
// real-world archives (BCC/journal-based archiving in particular - e.g.
// Proxmox Mail Gateway/piler dumps) deliver mail with no To header at all,
// or one that only names the archiving system's own mailbox. The delivery
// headers an MTA adds for its own routing (Delivered-To, X-Original-To,
// Envelope-To) usually still carry the real final recipient in that case.
var recipientHeaders = []string{"To", "Delivered-To", "X-Original-To", "X-Envelope-To", "Envelope-To"}
// mailboxFromHeaders derives a mailbox address from a raw message, trying
// recipientHeaders in order and returning the first usable address found.
// Only the first address of a multi-recipient header is used - if a
// message was sent to multiple recipients, one still has to be picked as
// "the" mailbox it's archived under.
func mailboxFromHeaders(raw []byte) (string, error) {
msg, err := mail.ReadMessage(bytes.NewReader(raw))
if err != nil {
return "", fmt.Errorf("parsing message: %w", err)
}
for _, h := range recipientHeaders {
v := msg.Header.Get(h)
if v == "" {
continue
}
if addrs, err := msg.Header.AddressList(h); err == nil && len(addrs) > 0 {
return strings.ToLower(addrs[0].Address), nil
}
// Delivered-To/X-Original-To/Envelope-To are often written by MTAs
// as a bare address with no RFC 5322 address syntax (no <angle
// brackets>, no display name) - fall back to using the raw value
// itself rather than discarding a perfectly good address just
// because it failed strict parsing.
if h != "To" {
return strings.ToLower(strings.TrimSpace(v)), nil
}
}
return "", fmt.Errorf("no usable recipient header found (tried %s)", strings.Join(recipientHeaders, ", "))
}
// pushWithRetry retries transient failures (network errors, 5xx, 429) with
// exponential backoff. 4xx responses are permanent (bad mailbox, oversized
// message, ...) and fail immediately - retrying those would just burn
// through the retry budget on every one of 200k files for no benefit.
func pushWithRetry(c *client.Client, mailbox, direction string, raw []byte, retries int, backoff time.Duration) error {
var lastErr error
for attempt := 0; ; attempt++ {
err := c.PushMail(mailbox, direction, raw)
if err == nil {
return nil
}
lastErr = err
var apiErr *client.APIError
if errors.As(err, &apiErr) && apiErr.StatusCode >= 400 && apiErr.StatusCode < 500 && apiErr.StatusCode != 429 {
return err
}
if attempt >= retries {
return lastErr
}
time.Sleep(backoff * time.Duration(1<<uint(attempt)))
}
}
type importStats struct {
total int64
done atomic.Int64
failed atomic.Int64
skipped atomic.Int64
}
func runImport(args []string) error {
fs := flag.NewFlagSet("import", flag.ExitOnError)
apiURL := fs.String("api-url", envOr("MAIL_ARCHIVER_API_URL", "http://localhost:8080"), "mail-archiver API base URL")
apiKey := fs.String("api-key", os.Getenv("MAIL_ARCHIVER_API_KEY"), "ingester API key")
mailbox := fs.String("mailbox", "", "mailbox address to archive imported mail under; if omitted, it's auto-detected per mail from its recipient headers (To, Delivered-To, X-Original-To, ...)")
fallbackMailbox := fs.String("fallback-mailbox", "", "with --mailbox omitted, mailbox to use for a mail whose recipient can't be auto-detected, instead of failing it")
direction := fs.String("direction", "inbound", "direction to record for imported mail: inbound or outbound")
concurrency := fs.Int("concurrency", 32, "number of mails to import in parallel")
retries := fs.Int("retries", 3, "retries for transient failures (network errors, 5xx, 429) per mail")
retryBackoff := fs.Duration("retry-backoff", 500*time.Millisecond, "base delay between retries (doubles each attempt)")
failLog := fs.String("fail-log", "", "optional path to write the list of files that failed to import, one per line")
quiet := fs.Bool("quiet", false, "only print progress and the final summary, not each imported file")
if err := fs.Parse(args); err != nil {
return err
}
if *apiKey == "" {
return fmt.Errorf("--api-key (or MAIL_ARCHIVER_API_KEY) is required")
}
if strings.ContainsAny(*mailbox, `/\`) {
return fmt.Errorf("--mailbox %q looks like a file path, not a mailbox address - the file/directory to import goes at the end of the command, after the flags (e.g. mailctl import --mailbox you@example.com --api-key ... /path/to/emls)", *mailbox)
}
if *direction != "inbound" && *direction != "outbound" {
return fmt.Errorf("--direction must be \"inbound\" or \"outbound\"")
}
if *concurrency < 1 {
*concurrency = 1
}
paths := fs.Args()
if len(paths) == 0 {
return fmt.Errorf("at least one .eml file or directory is required - it goes at the end of the command, after all flags (e.g. mailctl import --mailbox you@example.com --api-key ... /path/to/emls)")
}
fmt.Println("scanning for .eml files...")
files, err := collectEmlFiles(paths)
if err != nil {
return err
}
if len(files) == 0 {
return fmt.Errorf("no .eml files found in %v", paths)
}
fmt.Printf("found %d file(s), importing with concurrency=%d\n", len(files), *concurrency)
c := client.NewConcurrent(*apiURL, *concurrency)
c.APIKey = *apiKey
stats := &importStats{total: int64(len(files))}
var failedMu sync.Mutex
var failedFiles []string
jobs := make(chan string, *concurrency*2)
var wg sync.WaitGroup
for range *concurrency {
wg.Add(1)
go func() {
defer wg.Done()
for f := range jobs {
raw, err := os.ReadFile(f)
if err != nil {
stats.failed.Add(1)
fmt.Fprintf(os.Stderr, "skipping %s: %v\n", f, err)
failedMu.Lock()
failedFiles = append(failedFiles, f)
failedMu.Unlock()
continue
}
targetMailbox := *mailbox
if targetMailbox == "" {
targetMailbox, err = mailboxFromHeaders(raw)
if err != nil {
if *fallbackMailbox == "" {
stats.failed.Add(1)
fmt.Fprintf(os.Stderr, "skipping %s: could not detect mailbox: %v\n", f, err)
failedMu.Lock()
failedFiles = append(failedFiles, f)
failedMu.Unlock()
continue
}
targetMailbox = *fallbackMailbox
}
}
if err := pushWithRetry(c, targetMailbox, *direction, raw, *retries, *retryBackoff); err != nil {
stats.failed.Add(1)
fmt.Fprintf(os.Stderr, "failed importing %s (mailbox=%s): %v\n", f, targetMailbox, err)
failedMu.Lock()
failedFiles = append(failedFiles, f)
failedMu.Unlock()
continue
}
stats.done.Add(1)
if !*quiet {
fmt.Printf("imported %s -> %s\n", f, targetMailbox)
}
}
}()
}
stopProgress := make(chan struct{})
var progressWg sync.WaitGroup
progressWg.Add(1)
go func() {
defer progressWg.Done()
reportProgress(stats, stopProgress)
}()
start := time.Now()
for _, f := range files {
jobs <- f
}
close(jobs)
wg.Wait()
close(stopProgress)
progressWg.Wait()
elapsed := time.Since(start)
ok := stats.done.Load()
failed := stats.failed.Load()
rate := float64(ok+failed) / elapsed.Seconds()
fmt.Printf("done: %d imported, %d failed (of %d total) in %s (%.1f/s)\n", ok, failed, len(files), elapsed.Round(time.Second), rate)
if *failLog != "" && len(failedFiles) > 0 {
if err := os.WriteFile(*failLog, []byte(strings.Join(failedFiles, "\n")+"\n"), 0o644); err != nil {
fmt.Fprintf(os.Stderr, "warning: failed writing fail-log %s: %v\n", *failLog, err)
} else {
mailboxFlag := "--mailbox " + *mailbox
if *mailbox == "" {
mailboxFlag = "(auto-detect)"
}
fmt.Printf("failed files listed in %s (re-run: mailctl import %s ... $(cat %s))\n", *failLog, mailboxFlag, *failLog)
}
}
if failed > 0 {
return fmt.Errorf("%d file(s) failed to import", failed)
}
return nil
}
// reportProgress prints a periodic one-line progress update until stop is
// closed. Per-file logging alone is either far too noisy (200k lines) or,
// with --quiet, gives no feedback at all during a run that can take
// minutes to hours - this gives a steady heartbeat either way.
func reportProgress(stats *importStats, stop <-chan struct{}) {
ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop()
start := time.Now()
for {
select {
case <-stop:
return
case <-ticker.C:
done := stats.done.Load()
failed := stats.failed.Load()
processed := done + failed
elapsed := time.Since(start)
rate := float64(processed) / elapsed.Seconds()
var eta time.Duration
if rate > 0 {
eta = time.Duration(float64(stats.total-processed)/rate) * time.Second
}
fmt.Printf("progress: %d/%d (failed=%d) %.1f/s eta=%s\n", processed, stats.total, failed, rate, eta.Round(time.Second))
}
}
}