287 lines
9.6 KiB
Go
287 lines
9.6 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"net/mail"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"jonasled.dev/jonasled/mail-achiver/cli/client"
|
|
)
|
|
|
|
// collectEmlFiles resolves each of paths to a list of .eml files: a file
|
|
// path is used as-is (regardless of extension - explicit means explicit), a
|
|
// directory is walked recursively for files ending in .eml.
|
|
func collectEmlFiles(paths []string) ([]string, error) {
|
|
var files []string
|
|
for _, p := range paths {
|
|
info, err := os.Stat(p)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s: %w", p, err)
|
|
}
|
|
if !info.IsDir() {
|
|
files = append(files, p)
|
|
continue
|
|
}
|
|
err = filepath.WalkDir(p, func(path string, d os.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if d.IsDir() {
|
|
return nil
|
|
}
|
|
if strings.EqualFold(filepath.Ext(path), ".eml") {
|
|
files = append(files, path)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("walking %s: %w", p, err)
|
|
}
|
|
}
|
|
return files, nil
|
|
}
|
|
|
|
// recipientHeaders are tried in order when auto-detecting which mailbox to
|
|
// file an imported mail under. To is the obvious first choice, but a lot of
|
|
// real-world archives (BCC/journal-based archiving in particular - e.g.
|
|
// Proxmox Mail Gateway/piler dumps) deliver mail with no To header at all,
|
|
// or one that only names the archiving system's own mailbox. The delivery
|
|
// headers an MTA adds for its own routing (Delivered-To, X-Original-To,
|
|
// Envelope-To) usually still carry the real final recipient in that case.
|
|
var recipientHeaders = []string{"To", "Delivered-To", "X-Original-To", "X-Envelope-To", "Envelope-To"}
|
|
|
|
// mailboxFromHeaders derives a mailbox address from a raw message, trying
|
|
// recipientHeaders in order and returning the first usable address found.
|
|
// Only the first address of a multi-recipient header is used - if a
|
|
// message was sent to multiple recipients, one still has to be picked as
|
|
// "the" mailbox it's archived under.
|
|
func mailboxFromHeaders(raw []byte) (string, error) {
|
|
msg, err := mail.ReadMessage(bytes.NewReader(raw))
|
|
if err != nil {
|
|
return "", fmt.Errorf("parsing message: %w", err)
|
|
}
|
|
for _, h := range recipientHeaders {
|
|
v := msg.Header.Get(h)
|
|
if v == "" {
|
|
continue
|
|
}
|
|
if addrs, err := msg.Header.AddressList(h); err == nil && len(addrs) > 0 {
|
|
return strings.ToLower(addrs[0].Address), nil
|
|
}
|
|
// Delivered-To/X-Original-To/Envelope-To are often written by MTAs
|
|
// as a bare address with no RFC 5322 address syntax (no <angle
|
|
// brackets>, no display name) - fall back to using the raw value
|
|
// itself rather than discarding a perfectly good address just
|
|
// because it failed strict parsing.
|
|
if h != "To" {
|
|
return strings.ToLower(strings.TrimSpace(v)), nil
|
|
}
|
|
}
|
|
return "", fmt.Errorf("no usable recipient header found (tried %s)", strings.Join(recipientHeaders, ", "))
|
|
}
|
|
|
|
// pushWithRetry retries transient failures (network errors, 5xx, 429) with
|
|
// exponential backoff. 4xx responses are permanent (bad mailbox, oversized
|
|
// message, ...) and fail immediately - retrying those would just burn
|
|
// through the retry budget on every one of 200k files for no benefit.
|
|
func pushWithRetry(c *client.Client, mailbox, direction string, raw []byte, retries int, backoff time.Duration) error {
|
|
var lastErr error
|
|
for attempt := 0; ; attempt++ {
|
|
err := c.PushMail(mailbox, direction, raw)
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
lastErr = err
|
|
|
|
var apiErr *client.APIError
|
|
if errors.As(err, &apiErr) && apiErr.StatusCode >= 400 && apiErr.StatusCode < 500 && apiErr.StatusCode != 429 {
|
|
return err
|
|
}
|
|
if attempt >= retries {
|
|
return lastErr
|
|
}
|
|
time.Sleep(backoff * time.Duration(1<<uint(attempt)))
|
|
}
|
|
}
|
|
|
|
type importStats struct {
|
|
total int64
|
|
done atomic.Int64
|
|
failed atomic.Int64
|
|
skipped atomic.Int64
|
|
}
|
|
|
|
func runImport(args []string) error {
|
|
fs := flag.NewFlagSet("import", flag.ExitOnError)
|
|
apiURL := fs.String("api-url", envOr("MAIL_ARCHIVER_API_URL", "http://localhost:8080"), "mail-archiver API base URL")
|
|
apiKey := fs.String("api-key", os.Getenv("MAIL_ARCHIVER_API_KEY"), "ingester API key")
|
|
mailbox := fs.String("mailbox", "", "mailbox address to archive imported mail under; if omitted, it's auto-detected per mail from its recipient headers (To, Delivered-To, X-Original-To, ...)")
|
|
fallbackMailbox := fs.String("fallback-mailbox", "", "with --mailbox omitted, mailbox to use for a mail whose recipient can't be auto-detected, instead of failing it")
|
|
direction := fs.String("direction", "inbound", "direction to record for imported mail: inbound or outbound")
|
|
concurrency := fs.Int("concurrency", 32, "number of mails to import in parallel")
|
|
retries := fs.Int("retries", 3, "retries for transient failures (network errors, 5xx, 429) per mail")
|
|
retryBackoff := fs.Duration("retry-backoff", 500*time.Millisecond, "base delay between retries (doubles each attempt)")
|
|
failLog := fs.String("fail-log", "", "optional path to write the list of files that failed to import, one per line")
|
|
quiet := fs.Bool("quiet", false, "only print progress and the final summary, not each imported file")
|
|
if err := fs.Parse(args); err != nil {
|
|
return err
|
|
}
|
|
|
|
if *apiKey == "" {
|
|
return fmt.Errorf("--api-key (or MAIL_ARCHIVER_API_KEY) is required")
|
|
}
|
|
if strings.ContainsAny(*mailbox, `/\`) {
|
|
return fmt.Errorf("--mailbox %q looks like a file path, not a mailbox address - the file/directory to import goes at the end of the command, after the flags (e.g. mailctl import --mailbox you@example.com --api-key ... /path/to/emls)", *mailbox)
|
|
}
|
|
if *direction != "inbound" && *direction != "outbound" {
|
|
return fmt.Errorf("--direction must be \"inbound\" or \"outbound\"")
|
|
}
|
|
if *concurrency < 1 {
|
|
*concurrency = 1
|
|
}
|
|
paths := fs.Args()
|
|
if len(paths) == 0 {
|
|
return fmt.Errorf("at least one .eml file or directory is required - it goes at the end of the command, after all flags (e.g. mailctl import --mailbox you@example.com --api-key ... /path/to/emls)")
|
|
}
|
|
|
|
fmt.Println("scanning for .eml files...")
|
|
files, err := collectEmlFiles(paths)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(files) == 0 {
|
|
return fmt.Errorf("no .eml files found in %v", paths)
|
|
}
|
|
fmt.Printf("found %d file(s), importing with concurrency=%d\n", len(files), *concurrency)
|
|
|
|
c := client.NewConcurrent(*apiURL, *concurrency)
|
|
c.APIKey = *apiKey
|
|
|
|
stats := &importStats{total: int64(len(files))}
|
|
var failedMu sync.Mutex
|
|
var failedFiles []string
|
|
|
|
jobs := make(chan string, *concurrency*2)
|
|
var wg sync.WaitGroup
|
|
for range *concurrency {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for f := range jobs {
|
|
raw, err := os.ReadFile(f)
|
|
if err != nil {
|
|
stats.failed.Add(1)
|
|
fmt.Fprintf(os.Stderr, "skipping %s: %v\n", f, err)
|
|
failedMu.Lock()
|
|
failedFiles = append(failedFiles, f)
|
|
failedMu.Unlock()
|
|
continue
|
|
}
|
|
|
|
targetMailbox := *mailbox
|
|
if targetMailbox == "" {
|
|
targetMailbox, err = mailboxFromHeaders(raw)
|
|
if err != nil {
|
|
if *fallbackMailbox == "" {
|
|
stats.failed.Add(1)
|
|
fmt.Fprintf(os.Stderr, "skipping %s: could not detect mailbox: %v\n", f, err)
|
|
failedMu.Lock()
|
|
failedFiles = append(failedFiles, f)
|
|
failedMu.Unlock()
|
|
continue
|
|
}
|
|
targetMailbox = *fallbackMailbox
|
|
}
|
|
}
|
|
|
|
if err := pushWithRetry(c, targetMailbox, *direction, raw, *retries, *retryBackoff); err != nil {
|
|
stats.failed.Add(1)
|
|
fmt.Fprintf(os.Stderr, "failed importing %s (mailbox=%s): %v\n", f, targetMailbox, err)
|
|
failedMu.Lock()
|
|
failedFiles = append(failedFiles, f)
|
|
failedMu.Unlock()
|
|
continue
|
|
}
|
|
stats.done.Add(1)
|
|
if !*quiet {
|
|
fmt.Printf("imported %s -> %s\n", f, targetMailbox)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
stopProgress := make(chan struct{})
|
|
var progressWg sync.WaitGroup
|
|
progressWg.Add(1)
|
|
go func() {
|
|
defer progressWg.Done()
|
|
reportProgress(stats, stopProgress)
|
|
}()
|
|
|
|
start := time.Now()
|
|
for _, f := range files {
|
|
jobs <- f
|
|
}
|
|
close(jobs)
|
|
wg.Wait()
|
|
close(stopProgress)
|
|
progressWg.Wait()
|
|
|
|
elapsed := time.Since(start)
|
|
ok := stats.done.Load()
|
|
failed := stats.failed.Load()
|
|
rate := float64(ok+failed) / elapsed.Seconds()
|
|
fmt.Printf("done: %d imported, %d failed (of %d total) in %s (%.1f/s)\n", ok, failed, len(files), elapsed.Round(time.Second), rate)
|
|
|
|
if *failLog != "" && len(failedFiles) > 0 {
|
|
if err := os.WriteFile(*failLog, []byte(strings.Join(failedFiles, "\n")+"\n"), 0o644); err != nil {
|
|
fmt.Fprintf(os.Stderr, "warning: failed writing fail-log %s: %v\n", *failLog, err)
|
|
} else {
|
|
mailboxFlag := "--mailbox " + *mailbox
|
|
if *mailbox == "" {
|
|
mailboxFlag = "(auto-detect)"
|
|
}
|
|
fmt.Printf("failed files listed in %s (re-run: mailctl import %s ... $(cat %s))\n", *failLog, mailboxFlag, *failLog)
|
|
}
|
|
}
|
|
|
|
if failed > 0 {
|
|
return fmt.Errorf("%d file(s) failed to import", failed)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// reportProgress prints a periodic one-line progress update until stop is
|
|
// closed. Per-file logging alone is either far too noisy (200k lines) or,
|
|
// with --quiet, gives no feedback at all during a run that can take
|
|
// minutes to hours - this gives a steady heartbeat either way.
|
|
func reportProgress(stats *importStats, stop <-chan struct{}) {
|
|
ticker := time.NewTicker(2 * time.Second)
|
|
defer ticker.Stop()
|
|
start := time.Now()
|
|
for {
|
|
select {
|
|
case <-stop:
|
|
return
|
|
case <-ticker.C:
|
|
done := stats.done.Load()
|
|
failed := stats.failed.Load()
|
|
processed := done + failed
|
|
elapsed := time.Since(start)
|
|
rate := float64(processed) / elapsed.Seconds()
|
|
var eta time.Duration
|
|
if rate > 0 {
|
|
eta = time.Duration(float64(stats.total-processed)/rate) * time.Second
|
|
}
|
|
fmt.Printf("progress: %d/%d (failed=%d) %.1f/s eta=%s\n", processed, stats.total, failed, rate, eta.Round(time.Second))
|
|
}
|
|
}
|
|
}
|