Add API server and ingester services

API server (Go/gin): JWT auth with admin/auditor/receiver roles and
mailbox-scoped access, file/S3 storage backends with optional
encryption-at-rest, Meilisearch search integration, mail retention
expiry, and a reindex job that rebuilds the database purely from
storage metadata sidecars for disaster recovery.

Ingester (Go): SMTP listener for direct mail push and IMAP puller for
mailbox pull, both forwarding to the API server over a REST ingest
endpoint. Configuration is fetched dynamically from the API so the
ingester itself stays stateless.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Jonas Leder 2026-08-26 15:04:06 +02:00
commit e1d132d0dd
37 changed files with 4473 additions and 2 deletions

18
.gitignore vendored
View file

@ -1,2 +1,16 @@
api/database.db
.env
# Go
api/data/
*.db
api/*.db
ingester/*.db
# Node
frontend/node_modules/
frontend/dist/
# Env
.env
*.env.local
# OS
.DS_Store

173
api/archive/ingest.go Normal file
View file

@ -0,0 +1,173 @@
// Package archive contains the core "store an incoming mail" logic shared
// by the ingest API endpoint and the disaster-recovery reindex job. It is
// the only place that writes to the storage backend, the database, and the
// search index for a mail, so those three stay consistent.
package archive
import (
"bytes"
"context"
"encoding/json"
"fmt"
"regexp"
"strings"
"time"
"github.com/google/uuid"
"jonasled.dev/jonasled/go-libs/log"
"jonasled.dev/jonasled/mail-achiver/api/database"
"jonasled.dev/jonasled/mail-achiver/api/mailparse"
"jonasled.dev/jonasled/mail-achiver/api/search"
"jonasled.dev/jonasled/mail-achiver/api/storage"
)
var unsafePathChars = regexp.MustCompile(`[^a-zA-Z0-9.@_-]+`)
func sanitizeAddress(addr string) string {
addr = strings.ToLower(strings.TrimSpace(addr))
if addr == "" {
addr = "unknown"
}
return unsafePathChars.ReplaceAllString(addr, "_")
}
// KeyFor builds the storage key prefix (without .eml/.json suffix) for a
// mail belonging to mailboxAddress sent/received on sentDate.
func KeyFor(mailboxAddress string, sentDate time.Time, id string) string {
return fmt.Sprintf("mails/%04d/%02d/%s/%s", sentDate.Year(), sentDate.Month(), sanitizeAddress(mailboxAddress), id)
}
// StoreMail persists a raw RFC 5322 message for mailboxAddress: it writes
// the .eml + metadata sidecar to the storage backend, creates the Mailbox
// if needed, inserts the Mail/Attachment rows, and best-effort indexes the
// mail for search. Storage + DB row creation happen before search indexing
// so a search-index failure never loses the archived mail itself.
func StoreMail(ctx context.Context, mailboxAddress string, direction database.MailDirection, raw []byte) (*database.Mail, error) {
mailbox, err := findOrCreateMailbox(mailboxAddress)
if err != nil {
return nil, fmt.Errorf("resolving mailbox: %w", err)
}
parsed, err := mailparse.Parse(raw)
if err != nil {
return nil, fmt.Errorf("parsing message: %w", err)
}
if parsed.Date.IsZero() {
parsed.Date = time.Now().UTC()
}
id := uuid.NewString()
key := KeyFor(mailboxAddress, parsed.Date, id)
backend, err := storage.Current()
if err != nil {
return nil, fmt.Errorf("loading storage backend: %w", err)
}
expiresAt := computeExpiry(mailbox, parsed.Date)
attachmentNames := make([]string, 0, len(parsed.Attachments))
for _, a := range parsed.Attachments {
attachmentNames = append(attachmentNames, a.Filename)
}
sidecar := storage.MetaSidecar{
SchemaVersion: storage.CurrentSchemaVersion,
MailboxAddress: mailboxAddress,
MessageID: parsed.MessageID,
Direction: string(direction),
From: parsed.From,
To: parsed.To,
Cc: parsed.Cc,
Subject: parsed.Subject,
SentDate: parsed.Date.UTC().Format(time.RFC3339),
Size: int64(len(raw)),
Attachments: attachmentNames,
ExpiresAt: expiresAt.UTC().Format(time.RFC3339),
}
sidecarBytes, err := json.Marshal(sidecar)
if err != nil {
return nil, err
}
if err := backend.Put(ctx, storage.EmlKey(key), bytes.NewReader(raw), int64(len(raw))); err != nil {
return nil, fmt.Errorf("writing eml to storage: %w", err)
}
if err := backend.Put(ctx, storage.MetaKey(key), bytes.NewReader(sidecarBytes), int64(len(sidecarBytes))); err != nil {
return nil, fmt.Errorf("writing metadata sidecar to storage: %w", err)
}
mail := database.Mail{
MailboxID: mailbox.ID,
MessageID: parsed.MessageID,
Direction: direction,
FromAddress: parsed.From,
ToAddress: parsed.To,
CcAddress: parsed.Cc,
Subject: parsed.Subject,
SentDate: parsed.Date,
Size: int64(len(raw)),
StorageKey: key,
HasAttachments: len(parsed.Attachments) > 0,
ExpiresAt: expiresAt,
}
for i, a := range parsed.Attachments {
mail.Attachments = append(mail.Attachments, database.Attachment{
Filename: a.Filename,
ContentType: a.ContentType,
Size: int64(len(a.Content)),
PartIndex: i,
})
}
if err := database.Db.Create(&mail).Error; err != nil {
return nil, fmt.Errorf("saving mail record: %w", err)
}
if err := indexMail(mail, parsed, mailboxAddress); err != nil {
log.Log.Warn("Failed indexing mail into search engine (will retry later): ", err.Error())
} else {
database.Db.Model(&mail).Update("indexed", true)
}
return &mail, nil
}
func indexMail(mail database.Mail, parsed *mailparse.Parsed, mailboxAddress string) error {
body := parsed.TextBody
if body == "" {
body = parsed.HTMLBody
}
return search.IndexMail(search.MailDocument{
ID: mail.ID,
MailboxID: mail.MailboxID,
MailboxAddress: mailboxAddress,
From: mail.FromAddress,
To: mail.ToAddress,
Cc: mail.CcAddress,
Subject: mail.Subject,
Body: body,
SentDate: mail.SentDate.Unix(),
HasAttachments: mail.HasAttachments,
})
}
func findOrCreateMailbox(address string) (*database.Mailbox, error) {
address = strings.ToLower(strings.TrimSpace(address))
var mailbox database.Mailbox
err := database.Db.Where("address = ?", address).FirstOrCreate(&mailbox, database.Mailbox{Address: address}).Error
return &mailbox, err
}
func computeExpiry(mailbox *database.Mailbox, sentDate time.Time) time.Time {
if mailbox.RetentionDays > 0 {
return sentDate.AddDate(0, 0, mailbox.RetentionDays)
}
var retention database.RetentionSettings
database.Db.First(&retention, 1)
if !retention.Enabled || retention.DefaultRetentionDays <= 0 {
return time.Time{} // never expires
}
return sentDate.AddDate(0, 0, retention.DefaultRetentionDays)
}

140
api/archive/reindex.go Normal file
View file

@ -0,0 +1,140 @@
package archive
import (
"context"
"encoding/json"
"io"
"strings"
"time"
"jonasled.dev/jonasled/go-libs/log"
"jonasled.dev/jonasled/mail-achiver/api/database"
"jonasled.dev/jonasled/mail-achiver/api/mailparse"
"jonasled.dev/jonasled/mail-achiver/api/search"
"jonasled.dev/jonasled/mail-achiver/api/storage"
)
type ReindexStats struct {
SidecarsFound int
MailsRestored int
Errors int
}
// Reindex walks the entire storage backend and rebuilds the Mail (and
// Mailbox) database rows plus the search index purely from the .json
// metadata sidecars written alongside every .eml. This is the disaster
// recovery path when the database has been lost: point a fresh API server
// at the same storage backend, run this once, and the archive (minus
// users/settings, which are not required to read mail) is fully restored.
// Existing rows (matched by StorageKey) are left untouched.
func Reindex(ctx context.Context, backend storage.Backend) (*ReindexStats, error) {
stats := &ReindexStats{}
err := backend.List(ctx, "mails/", func(obj storage.ObjectInfo) error {
if !strings.HasSuffix(obj.Key, ".json") {
return nil
}
stats.SidecarsFound++
key := strings.TrimSuffix(obj.Key, ".json")
var existing database.Mail
if err := database.Db.Where("storage_key = ?", key).First(&existing).Error; err == nil {
return nil // already have this mail
}
if err := restoreOne(ctx, backend, key); err != nil {
stats.Errors++
log.Log.Warn("Reindex: failed restoring ", key, ": ", err.Error())
return nil // keep walking; one bad object shouldn't abort the whole restore
}
stats.MailsRestored++
return nil
})
if err != nil {
return stats, err
}
return stats, nil
}
func restoreOne(ctx context.Context, backend storage.Backend, key string) error {
metaRC, err := backend.Get(ctx, storage.MetaKey(key))
if err != nil {
return err
}
defer metaRC.Close()
metaBytes, err := io.ReadAll(metaRC)
if err != nil {
return err
}
var sidecar storage.MetaSidecar
if err := json.Unmarshal(metaBytes, &sidecar); err != nil {
return err
}
mailbox, err := findOrCreateMailbox(sidecar.MailboxAddress)
if err != nil {
return err
}
sentDate, _ := time.Parse(time.RFC3339, sidecar.SentDate)
expiresAt, _ := time.Parse(time.RFC3339, sidecar.ExpiresAt)
mail := database.Mail{
MailboxID: mailbox.ID,
MessageID: sidecar.MessageID,
Direction: database.MailDirection(sidecar.Direction),
FromAddress: sidecar.From,
ToAddress: sidecar.To,
CcAddress: sidecar.Cc,
Subject: sidecar.Subject,
SentDate: sentDate,
Size: sidecar.Size,
StorageKey: key,
HasAttachments: len(sidecar.Attachments) > 0,
ExpiresAt: expiresAt,
}
for i, name := range sidecar.Attachments {
mail.Attachments = append(mail.Attachments, database.Attachment{
Filename: name,
PartIndex: i,
})
}
if err := database.Db.Create(&mail).Error; err != nil {
return err
}
body := ""
if emlRC, err := backend.Get(ctx, storage.EmlKey(key)); err == nil {
if raw, err := io.ReadAll(emlRC); err == nil {
if parsed, err := mailparse.Parse(raw); err == nil {
body = parsed.TextBody
if body == "" {
body = parsed.HTMLBody
}
}
}
emlRC.Close()
}
if err := search.IndexMail(search.MailDocument{
ID: mail.ID,
MailboxID: mail.MailboxID,
MailboxAddress: sidecar.MailboxAddress,
From: mail.FromAddress,
To: mail.ToAddress,
Cc: mail.CcAddress,
Subject: mail.Subject,
Body: body,
SentDate: mail.SentDate.Unix(),
HasAttachments: mail.HasAttachments,
}); err != nil {
log.Log.Warn("Reindex: failed search-indexing restored mail ", key, ": ", err.Error())
return nil
}
database.Db.Model(&mail).Update("indexed", true)
return nil
}

78
api/auth/jwt.go Normal file
View file

@ -0,0 +1,78 @@
// Package auth implements stateless JWT authentication so the API server
// can run as multiple replicas with no shared session store.
package auth
import (
"crypto/sha256"
"encoding/hex"
"errors"
"time"
"github.com/golang-jwt/jwt/v5"
"golang.org/x/crypto/bcrypt"
"jonasled.dev/jonasled/go-libs/config"
"jonasled.dev/jonasled/go-libs/log"
"jonasled.dev/jonasled/mail-achiver/api/database"
)
var jwtSecret []byte
func Init() {
secret := config.GetOrDefault("JWT_SECRET", "")
if secret == "" {
log.Log.Fatal("JWT_SECRET must be set (required for stateless multi-replica auth)")
}
jwtSecret = []byte(secret)
}
type Claims struct {
UserID uint `json:"uid"`
Role database.UserRole `json:"role"`
jwt.RegisteredClaims
}
const tokenTTL = 12 * time.Hour
func IssueToken(user database.User) (string, error) {
claims := Claims{
UserID: user.ID,
Role: user.Role,
RegisteredClaims: jwt.RegisteredClaims{
Subject: user.Email,
ExpiresAt: jwt.NewNumericDate(time.Now().Add(tokenTTL)),
IssuedAt: jwt.NewNumericDate(time.Now()),
},
}
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
return token.SignedString(jwtSecret)
}
func ParseToken(tokenString string) (*Claims, error) {
claims := &Claims{}
token, err := jwt.ParseWithClaims(tokenString, claims, func(t *jwt.Token) (interface{}, error) {
if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
return nil, errors.New("unexpected signing method")
}
return jwtSecret, nil
})
if err != nil || !token.Valid {
return nil, errors.New("invalid token")
}
return claims, nil
}
func HashPassword(password string) (string, error) {
b, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
return string(b), err
}
func CheckPassword(hash, password string) bool {
return bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)) == nil
}
// HashAPIKey/GenerateAPIKey support ingester service authentication: the
// plaintext key is shown once at creation and only its hash is stored.
func HashAPIKey(key string) string {
sum := sha256.Sum256([]byte(key))
return hex.EncodeToString(sum[:])
}

135
api/database/gorm.go Normal file
View file

@ -0,0 +1,135 @@
package database
import (
"fmt"
"net/url"
"time"
"github.com/glebarez/sqlite"
gormloggerlogrus "github.com/nekomeowww/gorm-logger-logrus"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/bcrypt"
"gorm.io/driver/mysql"
"gorm.io/driver/postgres"
"gorm.io/driver/sqlserver"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"jonasled.dev/jonasled/go-libs/config"
"jonasled.dev/jonasled/go-libs/helper"
"jonasled.dev/jonasled/go-libs/log"
)
var Db *gorm.DB
func Init() {
var err error
var dbdriver gorm.Dialector
switch config.GetOrDefault("DB_DRIVER", "sqlite") {
case "sqlite":
dbdriver = sqlite.Open(config.GetOrDefault("DB_NAME", "./database.db") + "?_foreign_keys=on")
case "mysql":
dbdriver = mysql.Open(fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8mb4&parseTime=True&loc=Local",
config.GetOrFatal("DB_USER"),
config.GetOrFatal("DB_PASSWORD"),
config.GetOrFatal("DB_HOST"),
config.GetOrDefault("DB_PORT", "3306"),
config.GetOrFatal("DB_NAME")))
case "postgres":
dbdriver = postgres.Open(fmt.Sprintf(
"host=%s user=%s password=%s dbname=%s port=%s sslmode=%s TimeZone=%s",
config.GetOrFatal("DB_HOST"),
config.GetOrFatal("DB_USER"),
config.GetOrFatal("DB_PASSWORD"),
config.GetOrFatal("DB_NAME"),
config.GetOrDefault("DB_PORT", "5432"),
config.GetOrDefault("DB_SSLMODE", "disable"),
config.GetOrDefault("DB_TIMEZONE", "Local"),
))
case "mssql":
query := url.Values{}
query.Set("database", config.GetOrFatal("DB_NAME"))
query.Set("encrypt", config.GetOrDefault("DB_ENCRYPT", "disable"))
query.Set("TrustServerCertificate", config.GetOrDefault("DB_TRUST_SERVER_CERTIFICATE", "true"))
dbdriver = sqlserver.Open((&url.URL{
Scheme: "sqlserver",
User: url.UserPassword(config.GetOrFatal("DB_USER"), config.GetOrFatal("DB_PASSWORD")),
Host: fmt.Sprintf("%s:%s", config.GetOrFatal("DB_HOST"), config.GetOrDefault("DB_PORT", "1433")),
RawQuery: query.Encode(),
}).String())
default:
log.Log.Fatal("Please set DB_DRIVER to a valid value")
}
logLevel := logger.Error
if log.Log.Level >= logrus.DebugLevel {
logLevel = logger.Info
}
Db, err = gorm.Open(dbdriver, &gorm.Config{
Logger: gormloggerlogrus.New(gormloggerlogrus.Options{
Logger: logrus.NewEntry(log.Log),
LogLevel: logLevel,
IgnoreRecordNotFoundError: true,
SlowThreshold: time.Millisecond * 200,
FileWithLineNumField: "file",
}),
})
if err != nil {
log.Log.Fatal("Failed initializing database: ", err.Error())
}
if config.GetOrDefault("EXECUTE_MIGRATIONS", "true") == "true" {
log.Log.Info("Executing database migrations")
if err := Db.AutoMigrate(AllModels()...); err != nil {
log.Log.Fatal("Failed executing database migrations: ", err.Error())
}
}
seedDefaults()
}
// seedDefaults ensures the singleton settings rows and an initial admin user
// exist. Settings rows are safe to lose (see StorageSettings doc comment);
// this just avoids nil-config panics on a fresh or restored database.
func seedDefaults() {
var storageSettings StorageSettings
if err := Db.FirstOrCreate(&storageSettings, StorageSettings{ID: 1}).Error; err != nil {
log.Log.Fatal("Failed seeding storage settings: ", err.Error())
}
if storageSettings.Backend == "" {
storageSettings.Backend = StorageBackendFile
storageSettings.FileBasePath = config.GetOrDefault("STORAGE_FILE_PATH", "./data/mails")
Db.Save(&storageSettings)
}
var ingesterSettings IngesterSettings
Db.FirstOrCreate(&ingesterSettings, IngesterSettings{ID: 1})
var retentionSettings RetentionSettings
Db.FirstOrCreate(&retentionSettings, RetentionSettings{ID: 1})
var userCount int64
Db.Model(&User{}).Count(&userCount)
if userCount == 0 {
email := config.GetOrDefault("INITIAL_ADMIN_EMAIL", "admin@localhost")
password := config.GetOrDefault("INITIAL_ADMIN_PASSWORD", "")
if password == "" {
password = helper.GenerateRandomString(20)
log.Log.Warnf("No INITIAL_ADMIN_PASSWORD set, generated admin password for %s: %s", email, password)
}
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil {
log.Log.Fatal("Failed hashing initial admin password: ", err.Error())
}
admin := User{
Email: email,
Name: "Administrator",
Role: UserRoleAdmin,
Password: string(hash),
}
if err := Db.Create(&admin).Error; err != nil {
log.Log.Fatal("Failed creating initial admin user: ", err.Error())
}
log.Log.Infof("Created initial admin user %s", email)
}
}

203
api/database/models.go Normal file
View file

@ -0,0 +1,203 @@
package database
import (
"time"
"gorm.io/gorm"
)
type UserRole string
const (
UserRoleAdmin UserRole = "admin" // full access: config, users, all mail
UserRoleAuditor UserRole = "auditor" // read-only access to all mail, no config/user changes
UserRoleReceiver UserRole = "receiver" // scoped to their own mailbox(es) only
)
type User struct {
ID uint `gorm:"primaryKey" json:"id"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
Email string `gorm:"uniqueIndex;size:191;not null" json:"email"`
Name string `gorm:"size:191;not null" json:"name"`
Role UserRole `gorm:"size:16;not null;default:receiver" json:"role"`
Password string `gorm:"not null" json:"-"`
Disabled bool `gorm:"not null;default:false" json:"disabled"`
Mailboxes []Mailbox `gorm:"many2many:user_mailboxes;" json:"mailboxes,omitempty"`
}
// Mailbox represents an archived mail address (a "receiver" scope).
type Mailbox struct {
ID uint `gorm:"primaryKey" json:"id"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
Address string `gorm:"uniqueIndex;size:255;not null" json:"address"`
Description string `gorm:"size:255" json:"description"`
// RetentionDays overrides the global retention if > 0.
RetentionDays int `gorm:"default:0" json:"retentionDays"`
Users []User `gorm:"many2many:user_mailboxes;" json:"-"`
}
type MailDirection string
const (
MailDirectionInbound MailDirection = "inbound"
MailDirectionOutbound MailDirection = "outbound"
)
// Mail holds metadata only. The authoritative content lives in the storage
// backend as a raw .eml file plus a JSON metadata sidecar, so the archive
// can be fully rebuilt (see storage/reindex) if this table is lost.
type Mail struct {
ID uint `gorm:"primaryKey" json:"id"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
MailboxID uint `gorm:"index;not null" json:"mailboxId"`
Mailbox Mailbox `json:"-"`
MessageID string `gorm:"index;size:998" json:"messageId"`
Direction MailDirection `gorm:"size:16;not null" json:"direction"`
FromAddress string `gorm:"size:998" json:"from"`
ToAddress string `gorm:"size:2048" json:"to"`
CcAddress string `gorm:"size:2048" json:"cc"`
Subject string `gorm:"size:998" json:"subject"`
SentDate time.Time `gorm:"index" json:"sentDate"`
Size int64 `json:"size"`
// StorageKey is the path/object key of the raw .eml in the storage backend.
StorageKey string `gorm:"size:1024;not null" json:"-"`
HasAttachments bool `json:"hasAttachments"`
ExpiresAt time.Time `gorm:"index" json:"expiresAt"`
Indexed bool `gorm:"index;default:false" json:"-"`
Attachments []Attachment `json:"attachments,omitempty"`
}
type Attachment struct {
ID uint `gorm:"primaryKey" json:"id"`
CreatedAt time.Time `json:"createdAt"`
MailID uint `gorm:"index;not null" json:"mailId"`
Filename string `gorm:"size:512" json:"filename"`
ContentType string `gorm:"size:255" json:"contentType"`
Size int64 `json:"size"`
PartIndex int `json:"-"`
}
type StorageBackendType string
const (
StorageBackendFile StorageBackendType = "file"
StorageBackendS3 StorageBackendType = "s3"
)
// StorageSettings is a singleton row (ID always 1) configuring which storage
// backend is active. Losing this row is acceptable: files remain on disk/S3
// and can be reattached by reconfiguring this row and running a reindex.
type StorageSettings struct {
ID uint `gorm:"primaryKey" json:"id"`
Backend StorageBackendType `gorm:"size:16;not null;default:file" json:"backend"`
FileBasePath string `gorm:"size:1024" json:"fileBasePath"`
S3Endpoint string `gorm:"size:255" json:"s3Endpoint"`
S3Region string `gorm:"size:64" json:"s3Region"`
S3Bucket string `gorm:"size:255" json:"s3Bucket"`
S3AccessKey string `gorm:"size:255" json:"-"`
S3SecretKey string `gorm:"size:255" json:"-"`
S3UsePathStyle bool `gorm:"default:false" json:"s3UsePathStyle"`
UpdatedAt time.Time `json:"updatedAt"`
}
type IngesterIMAPSource struct {
ID uint `gorm:"primaryKey" json:"id"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
Name string `gorm:"size:191;not null" json:"name"`
Host string `gorm:"size:255;not null" json:"host"`
Port int `gorm:"not null;default:993" json:"port"`
TLS bool `gorm:"not null;default:true" json:"tls"`
Username string `gorm:"size:255;not null" json:"username"`
Password string `gorm:"size:255;not null" json:"-"`
Folder string `gorm:"size:255;not null;default:INBOX" json:"folder"`
// MailboxAddress ties fetched mail to an archive Mailbox scope.
MailboxAddress string `gorm:"size:255;not null" json:"mailboxAddress"`
PollIntervalSeconds int `gorm:"not null;default:60" json:"pollIntervalSeconds"`
DeleteAfterFetch bool `gorm:"not null;default:false" json:"deleteAfterFetch"`
Enabled bool `gorm:"not null;default:true" json:"enabled"`
}
// IngesterSettings holds global config for the SMTP ingest listener.
type IngesterSettings struct {
ID uint `gorm:"primaryKey" json:"id"`
SMTPEnabled bool `gorm:"not null;default:false" json:"smtpEnabled"`
SMTPAddress string `gorm:"size:255;default::2525" json:"smtpAddress"`
SMTPHostname string `gorm:"size:255;default:mail-archiver" json:"smtpHostname"`
UpdatedAt time.Time `json:"updatedAt"`
}
// APIKey authenticates the ingester service (or other trusted service
// clients) against the API server's ingest endpoints.
type APIKey struct {
ID uint `gorm:"primaryKey" json:"id"`
CreatedAt time.Time `json:"createdAt"`
DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
Name string `gorm:"size:191;not null" json:"name"`
KeyHash string `gorm:"size:191;not null;uniqueIndex" json:"-"`
LastUsed time.Time `json:"lastUsed"`
Disabled bool `gorm:"not null;default:false" json:"disabled"`
}
// RetentionSettings configures the default expiry applied to archived mail.
type RetentionSettings struct {
ID uint `gorm:"primaryKey" json:"id"`
DefaultRetentionDays int `gorm:"not null;default:3650" json:"defaultRetentionDays"` // ~10 years
Enabled bool `gorm:"not null;default:true" json:"enabled"`
UpdatedAt time.Time `json:"updatedAt"`
}
// SchedulerLease implements a simple DB-based leader election so that only
// one API replica runs background jobs (retention sweep, IMAP polling
// trigger) at a time.
type SchedulerLease struct {
Name string `gorm:"primaryKey;size:64" json:"name"`
Owner string `gorm:"size:191" json:"owner"`
ExpiresAt time.Time `json:"expiresAt"`
}
func AllModels() []interface{} {
return []interface{}{
&User{},
&Mailbox{},
&Mail{},
&Attachment{},
&StorageSettings{},
&IngesterIMAPSource{},
&IngesterSettings{},
&APIKey{},
&RetentionSettings{},
&SchedulerLease{},
}
}

303
api/docs/docs.go Normal file
View file

@ -0,0 +1,303 @@
// Code generated by swaggo/swag. DO NOT EDIT.
package docs
import "github.com/swaggo/swag"
const docTemplate = `{
"schemes": {{ marshal .Schemes }},
"swagger": "2.0",
"info": {
"description": "{{escape .Description}}",
"title": "{{.Title}}",
"contact": {},
"version": "{{.Version}}"
},
"host": "{{.Host}}",
"basePath": "{{.BasePath}}",
"paths": {
"/api/admin/reindex": {
"post": {
"responses": {}
}
},
"/api/auth/login": {
"post": {
"summary": "Login",
"parameters": [
{
"description": "credentials",
"name": "body",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/handlers.loginRequest"
}
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/handlers.loginResponse"
}
}
}
}
},
"/api/auth/me": {
"get": {
"summary": "Current user",
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/database.User"
}
}
}
}
},
"/api/ingest/config": {
"get": {
"responses": {}
}
},
"/api/ingest/mail": {
"post": {
"responses": {}
}
},
"/api/mailboxes": {
"get": {
"responses": {}
},
"post": {
"responses": {}
}
},
"/api/mailboxes/{id}": {
"put": {
"responses": {}
},
"delete": {
"responses": {}
}
},
"/api/mails": {
"get": {
"responses": {}
}
},
"/api/mails/{id}": {
"get": {
"responses": {}
},
"delete": {
"responses": {}
}
},
"/api/mails/{id}/attachments/{attachmentId}": {
"get": {
"responses": {}
}
},
"/api/mails/{id}/eml": {
"get": {
"responses": {}
}
},
"/api/mails/{id}/html": {
"get": {
"responses": {}
}
},
"/api/settings/apikeys": {
"get": {
"responses": {}
},
"post": {
"responses": {}
}
},
"/api/settings/apikeys/{id}": {
"delete": {
"responses": {}
}
},
"/api/settings/ingester": {
"get": {
"responses": {}
},
"put": {
"responses": {}
}
},
"/api/settings/ingester/imap": {
"get": {
"responses": {}
},
"post": {
"responses": {}
}
},
"/api/settings/ingester/imap/{id}": {
"put": {
"responses": {}
},
"delete": {
"responses": {}
}
},
"/api/settings/retention": {
"get": {
"responses": {}
},
"put": {
"responses": {}
}
},
"/api/settings/storage": {
"get": {
"responses": {}
},
"put": {
"responses": {}
}
},
"/api/users": {
"get": {
"responses": {}
},
"post": {
"responses": {}
}
},
"/api/users/{id}": {
"put": {
"responses": {}
},
"delete": {
"responses": {}
}
}
},
"definitions": {
"database.Mailbox": {
"type": "object",
"properties": {
"address": {
"type": "string"
},
"createdAt": {
"type": "string"
},
"description": {
"type": "string"
},
"id": {
"type": "integer"
},
"retentionDays": {
"description": "RetentionDays overrides the global retention if \u003e 0.",
"type": "integer"
},
"updatedAt": {
"type": "string"
}
}
},
"database.User": {
"type": "object",
"properties": {
"createdAt": {
"type": "string"
},
"disabled": {
"type": "boolean"
},
"email": {
"type": "string"
},
"id": {
"type": "integer"
},
"mailboxes": {
"type": "array",
"items": {
"$ref": "#/definitions/database.Mailbox"
}
},
"name": {
"type": "string"
},
"role": {
"$ref": "#/definitions/database.UserRole"
},
"updatedAt": {
"type": "string"
}
}
},
"database.UserRole": {
"type": "string",
"enum": [
"admin",
"auditor",
"receiver"
],
"x-enum-comments": {
"UserRoleAdmin": "full access: config, users, all mail",
"UserRoleAuditor": "read-only access to all mail, no config/user changes",
"UserRoleReceiver": "scoped to their own mailbox(es) only"
},
"x-enum-varnames": [
"UserRoleAdmin",
"UserRoleAuditor",
"UserRoleReceiver"
]
},
"handlers.loginRequest": {
"type": "object",
"required": [
"email",
"password"
],
"properties": {
"email": {
"type": "string"
},
"password": {
"type": "string"
}
}
},
"handlers.loginResponse": {
"type": "object",
"properties": {
"token": {
"type": "string"
},
"user": {
"$ref": "#/definitions/database.User"
}
}
}
}
}`
// SwaggerInfo holds exported Swagger Info so clients can modify it
var SwaggerInfo = &swag.Spec{
Version: "1.0",
Host: "",
BasePath: "/",
Schemes: []string{},
Title: "Mail Archive API",
Description: "API for Mail archiver",
InfoInstanceName: "swagger",
SwaggerTemplate: docTemplate,
}
func init() {
swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo)
}

279
api/docs/swagger.json Normal file
View file

@ -0,0 +1,279 @@
{
"swagger": "2.0",
"info": {
"description": "API for Mail archiver",
"title": "Mail Archive API",
"contact": {},
"version": "1.0"
},
"basePath": "/",
"paths": {
"/api/admin/reindex": {
"post": {
"responses": {}
}
},
"/api/auth/login": {
"post": {
"summary": "Login",
"parameters": [
{
"description": "credentials",
"name": "body",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/handlers.loginRequest"
}
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/handlers.loginResponse"
}
}
}
}
},
"/api/auth/me": {
"get": {
"summary": "Current user",
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/database.User"
}
}
}
}
},
"/api/ingest/config": {
"get": {
"responses": {}
}
},
"/api/ingest/mail": {
"post": {
"responses": {}
}
},
"/api/mailboxes": {
"get": {
"responses": {}
},
"post": {
"responses": {}
}
},
"/api/mailboxes/{id}": {
"put": {
"responses": {}
},
"delete": {
"responses": {}
}
},
"/api/mails": {
"get": {
"responses": {}
}
},
"/api/mails/{id}": {
"get": {
"responses": {}
},
"delete": {
"responses": {}
}
},
"/api/mails/{id}/attachments/{attachmentId}": {
"get": {
"responses": {}
}
},
"/api/mails/{id}/eml": {
"get": {
"responses": {}
}
},
"/api/mails/{id}/html": {
"get": {
"responses": {}
}
},
"/api/settings/apikeys": {
"get": {
"responses": {}
},
"post": {
"responses": {}
}
},
"/api/settings/apikeys/{id}": {
"delete": {
"responses": {}
}
},
"/api/settings/ingester": {
"get": {
"responses": {}
},
"put": {
"responses": {}
}
},
"/api/settings/ingester/imap": {
"get": {
"responses": {}
},
"post": {
"responses": {}
}
},
"/api/settings/ingester/imap/{id}": {
"put": {
"responses": {}
},
"delete": {
"responses": {}
}
},
"/api/settings/retention": {
"get": {
"responses": {}
},
"put": {
"responses": {}
}
},
"/api/settings/storage": {
"get": {
"responses": {}
},
"put": {
"responses": {}
}
},
"/api/users": {
"get": {
"responses": {}
},
"post": {
"responses": {}
}
},
"/api/users/{id}": {
"put": {
"responses": {}
},
"delete": {
"responses": {}
}
}
},
"definitions": {
"database.Mailbox": {
"type": "object",
"properties": {
"address": {
"type": "string"
},
"createdAt": {
"type": "string"
},
"description": {
"type": "string"
},
"id": {
"type": "integer"
},
"retentionDays": {
"description": "RetentionDays overrides the global retention if \u003e 0.",
"type": "integer"
},
"updatedAt": {
"type": "string"
}
}
},
"database.User": {
"type": "object",
"properties": {
"createdAt": {
"type": "string"
},
"disabled": {
"type": "boolean"
},
"email": {
"type": "string"
},
"id": {
"type": "integer"
},
"mailboxes": {
"type": "array",
"items": {
"$ref": "#/definitions/database.Mailbox"
}
},
"name": {
"type": "string"
},
"role": {
"$ref": "#/definitions/database.UserRole"
},
"updatedAt": {
"type": "string"
}
}
},
"database.UserRole": {
"type": "string",
"enum": [
"admin",
"auditor",
"receiver"
],
"x-enum-comments": {
"UserRoleAdmin": "full access: config, users, all mail",
"UserRoleAuditor": "read-only access to all mail, no config/user changes",
"UserRoleReceiver": "scoped to their own mailbox(es) only"
},
"x-enum-varnames": [
"UserRoleAdmin",
"UserRoleAuditor",
"UserRoleReceiver"
]
},
"handlers.loginRequest": {
"type": "object",
"required": [
"email",
"password"
],
"properties": {
"email": {
"type": "string"
},
"password": {
"type": "string"
}
}
},
"handlers.loginResponse": {
"type": "object",
"properties": {
"token": {
"type": "string"
},
"user": {
"$ref": "#/definitions/database.User"
}
}
}
}
}

179
api/docs/swagger.yaml Normal file
View file

@ -0,0 +1,179 @@
basePath: /
definitions:
database.Mailbox:
properties:
address:
type: string
createdAt:
type: string
description:
type: string
id:
type: integer
retentionDays:
description: RetentionDays overrides the global retention if > 0.
type: integer
updatedAt:
type: string
type: object
database.User:
properties:
createdAt:
type: string
disabled:
type: boolean
email:
type: string
id:
type: integer
mailboxes:
items:
$ref: '#/definitions/database.Mailbox'
type: array
name:
type: string
role:
$ref: '#/definitions/database.UserRole'
updatedAt:
type: string
type: object
database.UserRole:
enum:
- admin
- auditor
- receiver
type: string
x-enum-comments:
UserRoleAdmin: 'full access: config, users, all mail'
UserRoleAuditor: read-only access to all mail, no config/user changes
UserRoleReceiver: scoped to their own mailbox(es) only
x-enum-varnames:
- UserRoleAdmin
- UserRoleAuditor
- UserRoleReceiver
handlers.loginRequest:
properties:
email:
type: string
password:
type: string
required:
- email
- password
type: object
handlers.loginResponse:
properties:
token:
type: string
user:
$ref: '#/definitions/database.User'
type: object
info:
contact: {}
description: API for Mail archiver
title: Mail Archive API
version: "1.0"
paths:
/api/admin/reindex:
post:
responses: {}
/api/auth/login:
post:
parameters:
- description: credentials
in: body
name: body
required: true
schema:
$ref: '#/definitions/handlers.loginRequest'
responses:
"200":
description: OK
schema:
$ref: '#/definitions/handlers.loginResponse'
summary: Login
/api/auth/me:
get:
responses:
"200":
description: OK
schema:
$ref: '#/definitions/database.User'
summary: Current user
/api/ingest/config:
get:
responses: {}
/api/ingest/mail:
post:
responses: {}
/api/mailboxes:
get:
responses: {}
post:
responses: {}
/api/mailboxes/{id}:
delete:
responses: {}
put:
responses: {}
/api/mails:
get:
responses: {}
/api/mails/{id}:
delete:
responses: {}
get:
responses: {}
/api/mails/{id}/attachments/{attachmentId}:
get:
responses: {}
/api/mails/{id}/eml:
get:
responses: {}
/api/mails/{id}/html:
get:
responses: {}
/api/settings/apikeys:
get:
responses: {}
post:
responses: {}
/api/settings/apikeys/{id}:
delete:
responses: {}
/api/settings/ingester:
get:
responses: {}
put:
responses: {}
/api/settings/ingester/imap:
get:
responses: {}
post:
responses: {}
/api/settings/ingester/imap/{id}:
delete:
responses: {}
put:
responses: {}
/api/settings/retention:
get:
responses: {}
put:
responses: {}
/api/settings/storage:
get:
responses: {}
put:
responses: {}
/api/users:
get:
responses: {}
post:
responses: {}
/api/users/{id}:
delete:
responses: {}
put:
responses: {}
swagger: "2.0"

102
api/go.mod Normal file
View file

@ -0,0 +1,102 @@
module jonasled.dev/jonasled/mail-achiver/api
go 1.26.5
require (
github.com/emersion/go-message v0.18.1
github.com/gin-contrib/cors v1.7.2
github.com/gin-gonic/gin v1.12.0
github.com/glebarez/sqlite v1.11.0
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/google/uuid v1.6.0
github.com/joho/godotenv v1.5.1
github.com/meilisearch/meilisearch-go v0.36.3
github.com/microcosm-cc/bluemonday v1.0.27
github.com/nekomeowww/gorm-logger-logrus v1.0.8
github.com/sirupsen/logrus v1.10.1
github.com/swaggo/files v1.0.1
github.com/swaggo/gin-swagger v1.6.1
github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f
golang.org/x/crypto v0.54.0
gorm.io/driver/mysql v1.6.0
gorm.io/driver/postgres v1.6.2
gorm.io/driver/sqlserver v1.6.4
gorm.io/gorm v1.31.2
jonasled.dev/jonasled/go-libs v0.0.3
)
require (
github.com/go-ini/ini v1.67.0 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/rs/xid v1.6.0 // indirect
)
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/andybalholm/brotli v1.1.1 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/bytedance/gopkg v0.1.3 // indirect
github.com/bytedance/sonic v1.15.0 // indirect
github.com/bytedance/sonic/loader v0.5.0 // indirect
github.com/cloudwego/base64x v0.1.6 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/gabriel-vasile/mimetype v1.4.12 // indirect
github.com/gin-contrib/sse v1.1.0 // indirect
github.com/glebarez/go-sqlite v1.21.2 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.6 // indirect
github.com/go-openapi/spec v0.20.4 // indirect
github.com/go-openapi/swag v0.19.15 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.30.1 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/goccy/go-yaml v1.19.2 // indirect
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.10.0 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/microsoft/go-mssqldb v1.9.6 // indirect
github.com/minio/minio-go/v7 v7.0.85
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/quic-go/qpack v0.6.0 // indirect
github.com/quic-go/quic-go v0.59.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/swaggo/swag v1.16.6 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.3.1 // indirect
go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect
golang.org/x/arch v0.22.0 // indirect
golang.org/x/mod v0.38.0 // indirect
golang.org/x/net v0.57.0 // indirect
golang.org/x/sync v0.22.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.41.0 // indirect
golang.org/x/tools v0.48.0 // indirect
google.golang.org/protobuf v1.36.10 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
modernc.org/libc v1.22.5 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.5.0 // indirect
modernc.org/sqlite v1.23.1 // indirect
)

294
api/go.sum Normal file
View file

@ -0,0 +1,294 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1/go.mod h1:JdM5psgjfBf5fo2uWOZhflPWyDBZ/O/CNAH9CtsuZE4=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.1 h1:Wgf5rZba3YZqeTNJPtvqZoBu1sBN/L4sry+u2U3Y75w=
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.1/go.mod h1:xxCBG/f/4Vbmh2XQJBsOmNdxWUY5j/s27jujKPbQf14=
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.1 h1:bFWuoEKg+gImo7pvkiQEFAc8ocibADgXeiLAxWhWmkI=
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.1/go.mod h1:Vih/3yc6yac2JzU4hzpaDupBJP0Flaia9rXXrU8xyww=
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M=
github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE=
github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k=
github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE=
github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo=
github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/emersion/go-message v0.18.1 h1:tfTxIoXFSFRwWaZsgnqS1DSZuGpYGzSmCZD8SK3QA2E=
github.com/emersion/go-message v0.18.1/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA=
github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw=
github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/gin-contrib/cors v1.7.2 h1:oLDHxdg8W/XDoN/8zamqk/Drgt4oVZDvaV0YmvVICQw=
github.com/gin-contrib/cors v1.7.2/go.mod h1:SUJVARKgQ40dmrzgXEVxj2m7Ig1v1qIboQkPDTQ9t2E=
github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4=
github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk=
github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8=
github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc=
github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs=
github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns=
github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M=
github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w=
github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/meilisearch/meilisearch-go v0.36.3 h1:Yx1aTY5jDgtbStPVkhJTDoLnZTy5sejQSPyjfNMy6e4=
github.com/meilisearch/meilisearch-go v0.36.3/go.mod h1:hWcR0MuWLSzHfbz9GGzIr3s9rnXLm1jqkmHkJPbUSvM=
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
github.com/microsoft/go-mssqldb v1.9.6 h1:1MNQg5UiSsokiPz3++K2KPx4moKrwIqly1wv+RyCKTw=
github.com/microsoft/go-mssqldb v1.9.6/go.mod h1:yYMPDufyoF2vVuVCUGtZARr06DKFIhMrluTcgWlXpr4=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.85 h1:9psTLS/NTvC3MWoyjhjXpwcKoNbkongaCSF3PNpSuXo=
github.com/minio/minio-go/v7 v7.0.85/go.mod h1:57YXpvc5l3rjPdhqNrDsvVlY0qPI6UTk1bflAe+9doY=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/nekomeowww/gorm-logger-logrus v1.0.8 h1:/XdgNrSBSrTwiZ7fgf8Y9zW9juW1rVmRoY7OkX/tPnY=
github.com/nekomeowww/gorm-logger-logrus v1.0.8/go.mod h1:V266BSFFzJF1jnKWVAucFsh4U+ib3pEqJCzQY5zm7iA=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
github.com/sirupsen/logrus v1.10.1 h1:xi4336Zh11WpU14fXR6I67V3yaTPQYwRx2WEtHbRg4Q=
github.com/sirupsen/logrus v1.10.1/go.mod h1:vsQHnG7xzNsxk3NrwboUiWPnIC3dmbjcGPykD7+tiHk=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.12.0 h1:K6Mr6jO9JICuend/5xzTM03ydSV3vdNRYAdPSukj8uI=
github.com/stretchr/testify v1.12.0/go.mod h1:bOYBZb5qJ00vPzWfIqBUZPaxK8jWiXc6d3ErP4Ca9Gw=
github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE=
github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg=
github.com/swaggo/gin-swagger v1.6.1 h1:Ri06G4gc9N4t4k8hekMigJ9zKTFSlqj/9paAQCQs7cY=
github.com/swaggo/gin-swagger v1.6.1/go.mod h1:LQ+hJStHakCWRiK/YNYtJOu4mR2FP+pxLnILT/qNiTw=
github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI=
github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg=
github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f h1:oqdnd6OGlOUu1InG37hWcCB3a+Jy3fwjylyVboaNMwY=
github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f/go.mod h1:X3Dd1SB8Gt1V968NTzpKFjMM6O8ccta2NPC6MprOxZQ=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY=
github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE=
go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0=
go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI=
golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg=
gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo=
gorm.io/driver/postgres v1.6.2 h1:BvXQ/cNUg63q5TFNg672DmDcowZSFrNLkkA3Xe6GXq4=
gorm.io/driver/postgres v1.6.2/go.mod h1:0c4fQA44XhOklXDkgtuKqysHCycTa5i9e3EIpDGCwXk=
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
gorm.io/driver/sqlserver v1.6.4 h1:kGA9Z0D7dnIz7yVvWp18qLBSPFpUQWGqMA4rnxkScdQ=
gorm.io/driver/sqlserver v1.6.4/go.mod h1:oRtXDKFRYj8MqyMq+JFEdaA+StSQKC4zupU6blIdB0s=
gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo=
gorm.io/gorm v1.31.2/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=
jonasled.dev/jonasled/go-libs v0.0.3 h1:2Tg1pYJMufBMPkyYVVAvsUJe/R+TRRtRVW9MCbc24bM=
jonasled.dev/jonasled/go-libs v0.0.3/go.mod h1:E8Sev4obR0x6y7x0HaIN1tImeLnVJgwJXt++jzTwiFM=
modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=

59
api/handlers/auth.go Normal file
View file

@ -0,0 +1,59 @@
package handlers
import (
"net/http"
"github.com/gin-gonic/gin"
"jonasled.dev/jonasled/mail-achiver/api/auth"
"jonasled.dev/jonasled/mail-achiver/api/database"
"jonasled.dev/jonasled/mail-achiver/api/middleware"
)
type loginRequest struct {
Email string `json:"email" binding:"required,email"`
Password string `json:"password" binding:"required"`
}
type loginResponse struct {
Token string `json:"token"`
User database.User `json:"user"`
}
// Login authenticates a user and issues a JWT.
// @Summary Login
// @Param body body loginRequest true "credentials"
// @Success 200 {object} loginResponse
// @Router /api/auth/login [post]
func Login(c *gin.Context) {
var req loginRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
var user database.User
if err := database.Db.Where("email = ?", req.Email).First(&user).Error; err != nil {
c.JSON(http.StatusUnauthorized, gin.H{"error": "invalid email or password"})
return
}
if user.Disabled || !auth.CheckPassword(user.Password, req.Password) {
c.JSON(http.StatusUnauthorized, gin.H{"error": "invalid email or password"})
return
}
token, err := auth.IssueToken(user)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed issuing token"})
return
}
c.JSON(http.StatusOK, loginResponse{Token: token, User: user})
}
// Me returns the currently authenticated user.
// @Summary Current user
// @Success 200 {object} database.User
// @Router /api/auth/me [get]
func Me(c *gin.Context) {
c.JSON(http.StatusOK, middleware.CurrentUser(c))
}

23
api/handlers/health.go Normal file
View file

@ -0,0 +1,23 @@
package handlers
import (
"net/http"
"github.com/gin-gonic/gin"
"jonasled.dev/jonasled/mail-achiver/api/database"
)
// Healthz is a liveness probe: process is up.
func Healthz(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"status": "ok"})
}
// Readyz is a readiness probe: dependencies (database) are reachable.
func Readyz(c *gin.Context) {
sqlDB, err := database.Db.DB()
if err != nil || sqlDB.Ping() != nil {
c.JSON(http.StatusServiceUnavailable, gin.H{"status": "database unreachable"})
return
}
c.JSON(http.StatusOK, gin.H{"status": "ok"})
}

82
api/handlers/ingest.go Normal file
View file

@ -0,0 +1,82 @@
package handlers
import (
"io"
"net/http"
"github.com/gin-gonic/gin"
"jonasled.dev/jonasled/mail-achiver/api/archive"
"jonasled.dev/jonasled/mail-achiver/api/database"
"jonasled.dev/jonasled/mail-achiver/api/storage"
)
const maxMailSize = 50 << 20 // 50MB
// IngestMail accepts a raw RFC 5322 message from the ingester service
// (SMTP listener or IMAP puller) and archives it. Authenticated via
// APIKey, not user JWT, since it's a service-to-service call.
// @Router /api/ingest/mail [post]
func IngestMail(c *gin.Context) {
mailboxAddress := c.Query("mailbox")
if mailboxAddress == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "mailbox query parameter is required"})
return
}
direction := database.MailDirection(c.DefaultQuery("direction", string(database.MailDirectionInbound)))
raw, err := io.ReadAll(io.LimitReader(c.Request.Body, maxMailSize+1))
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "failed reading request body"})
return
}
if len(raw) == 0 {
c.JSON(http.StatusBadRequest, gin.H{"error": "empty message body"})
return
}
if len(raw) > maxMailSize {
c.JSON(http.StatusRequestEntityTooLarge, gin.H{"error": "message exceeds maximum size"})
return
}
mail, err := archive.StoreMail(c.Request.Context(), mailboxAddress, direction, raw)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed archiving mail: " + err.Error()})
return
}
c.JSON(http.StatusCreated, mail)
}
// GetIngesterConfig lets the ingester service fetch its SMTP/IMAP config
// from the API, keeping the ingester itself stateless/configless.
// @Router /api/ingest/config [get]
func GetIngesterConfig(c *gin.Context) {
var settings database.IngesterSettings
database.Db.First(&settings, 1)
var imapSources []database.IngesterIMAPSource
database.Db.Where("enabled = ?", true).Find(&imapSources)
c.JSON(http.StatusOK, gin.H{
"smtp": settings,
"imap": imapSources,
})
}
// Reindex triggers a full rebuild of the database (and search index) from
// the storage backend's metadata sidecars. Intended for disaster recovery
// after a database loss - point a fresh database at the same storage
// backend, then call this once.
// @Router /api/admin/reindex [post]
func Reindex(c *gin.Context) {
backend, err := storage.Current()
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "storage backend unavailable"})
return
}
stats, err := archive.Reindex(c.Request.Context(), backend)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error(), "stats": stats})
return
}
c.JSON(http.StatusOK, stats)
}

98
api/handlers/mailboxes.go Normal file
View file

@ -0,0 +1,98 @@
package handlers
import (
"net/http"
"strconv"
"github.com/gin-gonic/gin"
"jonasled.dev/jonasled/mail-achiver/api/database"
)
// ListMailboxes returns mailboxes visible to the current user.
// @Router /api/mailboxes [get]
func ListMailboxes(c *gin.Context) {
ids, unrestricted := allowedMailboxIDs(c)
var mailboxes []database.Mailbox
q := database.Db.Model(&database.Mailbox{})
if !unrestricted {
q = q.Where("id IN ?", ids)
}
q.Find(&mailboxes)
c.JSON(http.StatusOK, mailboxes)
}
type createMailboxRequest struct {
Address string `json:"address" binding:"required,email"`
Description string `json:"description"`
RetentionDays int `json:"retentionDays"`
}
// CreateMailbox creates a new archived mailbox (admin only).
// @Router /api/mailboxes [post]
func CreateMailbox(c *gin.Context) {
var req createMailboxRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
mailbox := database.Mailbox{
Address: req.Address,
Description: req.Description,
RetentionDays: req.RetentionDays,
}
if err := database.Db.Create(&mailbox).Error; err != nil {
c.JSON(http.StatusConflict, gin.H{"error": "failed creating mailbox (address already exists?)"})
return
}
c.JSON(http.StatusCreated, mailbox)
}
type updateMailboxRequest struct {
Description *string `json:"description"`
RetentionDays *int `json:"retentionDays"`
}
// UpdateMailbox updates a mailbox (admin only).
// @Router /api/mailboxes/{id} [put]
func UpdateMailbox(c *gin.Context) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"})
return
}
var mailbox database.Mailbox
if err := database.Db.First(&mailbox, id).Error; err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "mailbox not found"})
return
}
var req updateMailboxRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
if req.Description != nil {
mailbox.Description = *req.Description
}
if req.RetentionDays != nil {
mailbox.RetentionDays = *req.RetentionDays
}
database.Db.Save(&mailbox)
c.JSON(http.StatusOK, mailbox)
}
// DeleteMailbox deletes a mailbox record (admin only). Archived mail files
// are left untouched in storage; only the DB row (and its mail rows) go.
// @Router /api/mailboxes/{id} [delete]
func DeleteMailbox(c *gin.Context) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"})
return
}
database.Db.Delete(&database.Mailbox{}, id)
c.Status(http.StatusNoContent)
}

223
api/handlers/mails.go Normal file
View file

@ -0,0 +1,223 @@
package handlers
import (
"io"
"net/http"
"strconv"
"github.com/gin-gonic/gin"
"github.com/microcosm-cc/bluemonday"
"jonasled.dev/jonasled/mail-achiver/api/database"
"jonasled.dev/jonasled/mail-achiver/api/mailparse"
"jonasled.dev/jonasled/mail-achiver/api/search"
"jonasled.dev/jonasled/mail-achiver/api/storage"
)
// SearchMails full-text searches mail via Meilisearch, scoped to the
// caller's allowed mailboxes.
// @Router /api/mails [get]
func SearchMails(c *gin.Context) {
ids, unrestricted := allowedMailboxIDs(c)
if !unrestricted && len(ids) == 0 {
c.JSON(http.StatusOK, gin.H{"hits": []search.MailDocument{}, "total": 0})
return
}
limit, _ := strconv.ParseInt(c.DefaultQuery("limit", "25"), 10, 64)
offset, _ := strconv.ParseInt(c.DefaultQuery("offset", "0"), 10, 64)
if mailboxIDParam := c.Query("mailboxId"); mailboxIDParam != "" {
mid, err := strconv.ParseUint(mailboxIDParam, 10, 64)
if err == nil {
if !mailboxAllowed(c, uint(mid)) {
c.JSON(http.StatusForbidden, gin.H{"error": "mailbox not accessible"})
return
}
ids = []uint{uint(mid)}
unrestricted = false
}
}
res, err := search.Search(c.Request.Context(), search.SearchParams{
Query: c.Query("q"),
MailboxIDs: ids,
Limit: limit,
Offset: offset,
})
if err != nil {
c.JSON(http.StatusBadGateway, gin.H{"error": "search backend unavailable: " + err.Error()})
return
}
_ = unrestricted
c.JSON(http.StatusOK, gin.H{"hits": res.Hits, "total": res.EstimatedTotal})
}
func loadMail(c *gin.Context) (*database.Mail, bool) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"})
return nil, false
}
var mail database.Mail
if err := database.Db.Preload("Attachments").First(&mail, id).Error; err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "mail not found"})
return nil, false
}
if !mailboxAllowed(c, mail.MailboxID) {
c.JSON(http.StatusForbidden, gin.H{"error": "mail not accessible"})
return nil, false
}
return &mail, true
}
// GetMail returns mail metadata.
// @Router /api/mails/{id} [get]
func GetMail(c *gin.Context) {
mail, ok := loadMail(c)
if !ok {
return
}
c.JSON(http.StatusOK, mail)
}
func fetchRaw(c *gin.Context, mail *database.Mail) ([]byte, bool) {
backend, err := storage.Current()
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "storage backend unavailable"})
return nil, false
}
rc, err := backend.Get(c.Request.Context(), storage.EmlKey(mail.StorageKey))
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "mail content not found in storage"})
return nil, false
}
defer rc.Close()
raw, err := io.ReadAll(rc)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed reading mail content"})
return nil, false
}
return raw, true
}
// DownloadEml streams the raw, byte-identical .eml file.
// @Router /api/mails/{id}/eml [get]
func DownloadEml(c *gin.Context) {
mail, ok := loadMail(c)
if !ok {
return
}
raw, ok := fetchRaw(c, mail)
if !ok {
return
}
c.Header("Content-Disposition", `attachment; filename="`+strconv.FormatUint(uint64(mail.ID), 10)+`.eml"`)
c.Data(http.StatusOK, "message/rfc822", raw)
}
var htmlSanitizer = newSanitizer()
func newSanitizer() *bluemonday.Policy {
// UGCPolicy allows common formatting/structure but strips <script>,
// event handlers (onclick, onerror, ...), <iframe>, <object>, forms,
// and disallows javascript:/data: URLs - mail HTML is one of the most
// common script-injection vectors so this must stay strict.
p := bluemonday.UGCPolicy()
p.AllowStyling()
p.RequireNoFollowOnLinks(true)
p.AddTargetBlankToFullyQualifiedLinks(true)
return p
}
// ViewHtml returns a sanitized HTML rendering of the mail body, safe to
// embed in the frontend (e.g. inside a sandboxed iframe).
// @Router /api/mails/{id}/html [get]
func ViewHtml(c *gin.Context) {
mail, ok := loadMail(c)
if !ok {
return
}
raw, ok := fetchRaw(c, mail)
if !ok {
return
}
parsed, err := mailparse.Parse(raw)
if err != nil {
c.JSON(http.StatusUnprocessableEntity, gin.H{"error": "failed parsing mail content"})
return
}
body := parsed.HTMLBody
if body == "" {
// No HTML part: render plain text safely escaped inside <pre>.
c.Header("Content-Type", "text/html; charset=utf-8")
c.String(http.StatusOK, "<pre>%s</pre>", htmlSanitizer.Sanitize(parsed.TextBody))
return
}
sanitized := htmlSanitizer.Sanitize(body)
c.Header("Content-Type", "text/html; charset=utf-8")
c.String(http.StatusOK, sanitized)
}
// DownloadAttachment streams a single attachment, re-extracted on demand
// from the stored .eml (attachments are not duplicated in storage).
// @Router /api/mails/{id}/attachments/{attachmentId} [get]
func DownloadAttachment(c *gin.Context) {
mail, ok := loadMail(c)
if !ok {
return
}
attachmentID, err := strconv.ParseUint(c.Param("attachmentId"), 10, 64)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid attachment id"})
return
}
var attachment database.Attachment
if err := database.Db.Where("id = ? AND mail_id = ?", attachmentID, mail.ID).First(&attachment).Error; err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "attachment not found"})
return
}
raw, ok := fetchRaw(c, mail)
if !ok {
return
}
parsed, err := mailparse.Parse(raw)
if err != nil || attachment.PartIndex >= len(parsed.Attachments) {
c.JSON(http.StatusUnprocessableEntity, gin.H{"error": "failed extracting attachment"})
return
}
part := parsed.Attachments[attachment.PartIndex]
c.Header("Content-Disposition", `attachment; filename="`+attachment.Filename+`"`)
c.Data(http.StatusOK, orDefault(part.ContentType, "application/octet-stream"), part.Content)
}
func orDefault(v, def string) string {
if v == "" {
return def
}
return v
}
// DeleteMail deletes a mail's DB row, storage objects and search entry
// (admin only). Use for legal takedowns; ordinary lifecycle is via
// retention expiry.
// @Router /api/mails/{id} [delete]
func DeleteMail(c *gin.Context) {
mail, ok := loadMail(c)
if !ok {
return
}
backend, err := storage.Current()
if err == nil {
backend.Delete(c.Request.Context(), storage.EmlKey(mail.StorageKey))
backend.Delete(c.Request.Context(), storage.MetaKey(mail.StorageKey))
}
search.DeleteMail(mail.ID)
database.Db.Delete(&database.Attachment{}, "mail_id = ?", mail.ID)
database.Db.Delete(mail)
c.Status(http.StatusNoContent)
}

38
api/handlers/scope.go Normal file
View file

@ -0,0 +1,38 @@
package handlers
import (
"github.com/gin-gonic/gin"
"jonasled.dev/jonasled/mail-achiver/api/database"
"jonasled.dev/jonasled/mail-achiver/api/middleware"
)
// allowedMailboxIDs returns the mailbox IDs a user is allowed to see.
// admin and auditor get unrestricted=true (no filtering); receiver gets
// the explicit list of mailboxes assigned to them.
func allowedMailboxIDs(c *gin.Context) (ids []uint, unrestricted bool) {
user := middleware.CurrentUser(c)
if user.Role == database.UserRoleAdmin || user.Role == database.UserRoleAuditor {
return nil, true
}
var mailboxes []database.Mailbox
database.Db.Model(&user).Association("Mailboxes").Find(&mailboxes)
ids = make([]uint, 0, len(mailboxes))
for _, m := range mailboxes {
ids = append(ids, m.ID)
}
return ids, false
}
func mailboxAllowed(c *gin.Context, mailboxID uint) bool {
ids, unrestricted := allowedMailboxIDs(c)
if unrestricted {
return true
}
for _, id := range ids {
if id == mailboxID {
return true
}
}
return false
}

276
api/handlers/settings.go Normal file
View file

@ -0,0 +1,276 @@
package handlers
import (
"crypto/rand"
"encoding/hex"
"net/http"
"strconv"
"github.com/gin-gonic/gin"
"jonasled.dev/jonasled/mail-achiver/api/auth"
"jonasled.dev/jonasled/mail-achiver/api/database"
"jonasled.dev/jonasled/mail-achiver/api/storage"
)
// GetStorageSettings returns the active storage backend configuration.
// @Router /api/settings/storage [get]
func GetStorageSettings(c *gin.Context) {
var s database.StorageSettings
database.Db.First(&s, 1)
c.JSON(http.StatusOK, s)
}
type storageSettingsRequest struct {
Backend database.StorageBackendType `json:"backend" binding:"required"`
FileBasePath string `json:"fileBasePath"`
S3Endpoint string `json:"s3Endpoint"`
S3Region string `json:"s3Region"`
S3Bucket string `json:"s3Bucket"`
S3AccessKey string `json:"s3AccessKey"`
S3SecretKey string `json:"s3SecretKey"`
S3UsePathStyle bool `json:"s3UsePathStyle"`
}
// UpdateStorageSettings updates the storage backend configuration and
// reloads the active backend. This row is safe to lose (see
// database.StorageSettings doc comment) - it only controls where the API
// looks, not what has been archived.
// @Router /api/settings/storage [put]
func UpdateStorageSettings(c *gin.Context) {
var req storageSettingsRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
var s database.StorageSettings
database.Db.First(&s, 1)
s.Backend = req.Backend
s.FileBasePath = req.FileBasePath
s.S3Endpoint = req.S3Endpoint
s.S3Region = req.S3Region
s.S3Bucket = req.S3Bucket
s.S3UsePathStyle = req.S3UsePathStyle
if req.S3AccessKey != "" {
s.S3AccessKey = req.S3AccessKey
}
if req.S3SecretKey != "" {
s.S3SecretKey = req.S3SecretKey
}
database.Db.Save(&s)
if _, err := storage.Reload(); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "settings saved but failed to activate: " + err.Error()})
return
}
c.JSON(http.StatusOK, s)
}
// GetRetentionSettings returns global retention config.
// @Router /api/settings/retention [get]
func GetRetentionSettings(c *gin.Context) {
var s database.RetentionSettings
database.Db.First(&s, 1)
c.JSON(http.StatusOK, s)
}
type retentionSettingsRequest struct {
DefaultRetentionDays int `json:"defaultRetentionDays" binding:"required,min=1"`
Enabled bool `json:"enabled"`
}
// UpdateRetentionSettings updates the global default retention period
// (e.g. 3650 days ~= 10 years). Individual mailboxes may override this.
// @Router /api/settings/retention [put]
func UpdateRetentionSettings(c *gin.Context) {
var req retentionSettingsRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
var s database.RetentionSettings
database.Db.First(&s, 1)
s.DefaultRetentionDays = req.DefaultRetentionDays
s.Enabled = req.Enabled
database.Db.Save(&s)
c.JSON(http.StatusOK, s)
}
// GetIngesterSettings returns the global SMTP ingest listener config.
// @Router /api/settings/ingester [get]
func GetIngesterSettings(c *gin.Context) {
var s database.IngesterSettings
database.Db.First(&s, 1)
c.JSON(http.StatusOK, s)
}
type ingesterSettingsRequest struct {
SMTPEnabled bool `json:"smtpEnabled"`
SMTPAddress string `json:"smtpAddress"`
SMTPHostname string `json:"smtpHostname"`
}
// UpdateIngesterSettings updates the global SMTP ingest listener config.
// The ingester service polls this via GET /api/ingest/config.
// @Router /api/settings/ingester [put]
func UpdateIngesterSettings(c *gin.Context) {
var req ingesterSettingsRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
var s database.IngesterSettings
database.Db.First(&s, 1)
s.SMTPEnabled = req.SMTPEnabled
s.SMTPAddress = req.SMTPAddress
s.SMTPHostname = req.SMTPHostname
database.Db.Save(&s)
c.JSON(http.StatusOK, s)
}
// ListImapSources lists configured IMAP pull sources.
// @Router /api/settings/ingester/imap [get]
func ListImapSources(c *gin.Context) {
var sources []database.IngesterIMAPSource
database.Db.Find(&sources)
c.JSON(http.StatusOK, sources)
}
type imapSourceRequest struct {
Name string `json:"name" binding:"required"`
Host string `json:"host" binding:"required"`
Port int `json:"port"`
TLS bool `json:"tls"`
Username string `json:"username" binding:"required"`
Password string `json:"password"`
Folder string `json:"folder"`
MailboxAddress string `json:"mailboxAddress" binding:"required,email"`
PollIntervalSeconds int `json:"pollIntervalSeconds"`
DeleteAfterFetch bool `json:"deleteAfterFetch"`
Enabled bool `json:"enabled"`
}
// CreateImapSource adds a new IMAP pull source.
// @Router /api/settings/ingester/imap [post]
func CreateImapSource(c *gin.Context) {
var req imapSourceRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
source := database.IngesterIMAPSource{
Name: req.Name, Host: req.Host, Port: req.Port, TLS: req.TLS,
Username: req.Username, Password: req.Password, Folder: req.Folder,
MailboxAddress: req.MailboxAddress,
PollIntervalSeconds: req.PollIntervalSeconds,
DeleteAfterFetch: req.DeleteAfterFetch,
Enabled: req.Enabled,
}
if source.Port == 0 {
source.Port = 993
}
if source.Folder == "" {
source.Folder = "INBOX"
}
if source.PollIntervalSeconds == 0 {
source.PollIntervalSeconds = 60
}
if err := database.Db.Create(&source).Error; err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed creating imap source"})
return
}
c.JSON(http.StatusCreated, source)
}
// UpdateImapSource updates an IMAP pull source.
// @Router /api/settings/ingester/imap/{id} [put]
func UpdateImapSource(c *gin.Context) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"})
return
}
var source database.IngesterIMAPSource
if err := database.Db.First(&source, id).Error; err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "imap source not found"})
return
}
var req imapSourceRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
source.Name, source.Host, source.Port, source.TLS = req.Name, req.Host, req.Port, req.TLS
source.Username, source.Folder = req.Username, req.Folder
if req.Password != "" {
source.Password = req.Password
}
source.MailboxAddress = req.MailboxAddress
source.PollIntervalSeconds = req.PollIntervalSeconds
source.DeleteAfterFetch = req.DeleteAfterFetch
source.Enabled = req.Enabled
database.Db.Save(&source)
c.JSON(http.StatusOK, source)
}
// DeleteImapSource removes an IMAP pull source.
// @Router /api/settings/ingester/imap/{id} [delete]
func DeleteImapSource(c *gin.Context) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"})
return
}
database.Db.Delete(&database.IngesterIMAPSource{}, id)
c.Status(http.StatusNoContent)
}
// ListApiKeys lists issued ingester API keys (hashes only, never plaintext).
// @Router /api/settings/apikeys [get]
func ListApiKeys(c *gin.Context) {
var keys []database.APIKey
database.Db.Find(&keys)
c.JSON(http.StatusOK, keys)
}
type createApiKeyRequest struct {
Name string `json:"name" binding:"required"`
}
// CreateApiKey generates a new API key for the ingester service. The
// plaintext key is returned exactly once and never stored.
// @Router /api/settings/apikeys [post]
func CreateApiKey(c *gin.Context) {
var req createApiKeyRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
raw := make([]byte, 32)
if _, err := rand.Read(raw); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed generating key"})
return
}
plaintext := hex.EncodeToString(raw)
key := database.APIKey{Name: req.Name, KeyHash: auth.HashAPIKey(plaintext)}
if err := database.Db.Create(&key).Error; err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed creating api key"})
return
}
c.JSON(http.StatusCreated, gin.H{"id": key.ID, "name": key.Name, "key": plaintext})
}
// DeleteApiKey revokes an API key.
// @Router /api/settings/apikeys/{id} [delete]
func DeleteApiKey(c *gin.Context) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"})
return
}
database.Db.Delete(&database.APIKey{}, id)
c.Status(http.StatusNoContent)
}

128
api/handlers/users.go Normal file
View file

@ -0,0 +1,128 @@
package handlers
import (
"net/http"
"strconv"
"github.com/gin-gonic/gin"
"jonasled.dev/jonasled/mail-achiver/api/auth"
"jonasled.dev/jonasled/mail-achiver/api/database"
)
// ListUsers returns all users (admin only).
// @Router /api/users [get]
func ListUsers(c *gin.Context) {
var users []database.User
database.Db.Preload("Mailboxes").Find(&users)
c.JSON(http.StatusOK, users)
}
type createUserRequest struct {
Email string `json:"email" binding:"required,email"`
Name string `json:"name" binding:"required"`
Password string `json:"password" binding:"required,min=8"`
Role database.UserRole `json:"role" binding:"required"`
MailboxIDs []uint `json:"mailboxIds"`
}
// CreateUser creates a user (admin only).
// @Router /api/users [post]
func CreateUser(c *gin.Context) {
var req createUserRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
hash, err := auth.HashPassword(req.Password)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed hashing password"})
return
}
user := database.User{
Email: req.Email,
Name: req.Name,
Role: req.Role,
Password: hash,
}
if len(req.MailboxIDs) > 0 {
database.Db.Where("id IN ?", req.MailboxIDs).Find(&user.Mailboxes)
}
if err := database.Db.Create(&user).Error; err != nil {
c.JSON(http.StatusConflict, gin.H{"error": "failed creating user (email already exists?)"})
return
}
c.JSON(http.StatusCreated, user)
}
type updateUserRequest struct {
Name *string `json:"name"`
Password *string `json:"password"`
Role *database.UserRole `json:"role"`
Disabled *bool `json:"disabled"`
MailboxIDs *[]uint `json:"mailboxIds"`
}
// UpdateUser updates a user (admin only).
// @Router /api/users/{id} [put]
func UpdateUser(c *gin.Context) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"})
return
}
var user database.User
if err := database.Db.First(&user, id).Error; err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "user not found"})
return
}
var req updateUserRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
if req.Name != nil {
user.Name = *req.Name
}
if req.Role != nil {
user.Role = *req.Role
}
if req.Disabled != nil {
user.Disabled = *req.Disabled
}
if req.Password != nil && *req.Password != "" {
hash, err := auth.HashPassword(*req.Password)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed hashing password"})
return
}
user.Password = hash
}
database.Db.Save(&user)
if req.MailboxIDs != nil {
var mailboxes []database.Mailbox
database.Db.Where("id IN ?", *req.MailboxIDs).Find(&mailboxes)
database.Db.Model(&user).Association("Mailboxes").Replace(mailboxes)
}
c.JSON(http.StatusOK, user)
}
// DeleteUser deletes a user (admin only).
// @Router /api/users/{id} [delete]
func DeleteUser(c *gin.Context) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"})
return
}
database.Db.Delete(&database.User{}, id)
c.Status(http.StatusNoContent)
}

159
api/mailparse/parse.go Normal file
View file

@ -0,0 +1,159 @@
// Package mailparse turns a raw RFC 5322 message into the metadata the
// archive needs for its database row, search index document, and
// attachment list, without mutating the original bytes (those are stored
// verbatim so the .eml download is always byte-identical to what arrived).
package mailparse
import (
"bytes"
"fmt"
"io"
"mime"
"net/mail"
"strings"
"time"
"github.com/emersion/go-message"
"github.com/emersion/go-message/charset"
)
type AttachmentPart struct {
Filename string
ContentType string
Content []byte
}
type Parsed struct {
MessageID string
From string
To string
Cc string
Subject string
Date time.Time
TextBody string
HTMLBody string
Attachments []AttachmentPart
}
// Parse reads raw RFC 5322 bytes and extracts headers, text/html bodies and
// attachments. It is tolerant of malformed messages: on structural parse
// failure it falls back to header-only extraction via net/mail so ingestion
// never hard-fails on a slightly malformed message.
func Parse(raw []byte) (*Parsed, error) {
entity, err := message.Read(bytes.NewReader(raw))
if err != nil && entity == nil {
return parseHeadersOnly(raw)
}
p := &Parsed{}
h := entity.Header
p.MessageID = strings.Trim(h.Get("Message-Id"), "<>")
p.From = decodeHeader(h.Get("From"))
p.To = decodeHeader(h.Get("To"))
p.Cc = decodeHeader(h.Get("Cc"))
p.Subject = decodeHeader(h.Get("Subject"))
if date, err := mail.ParseDate(h.Get("Date")); err == nil {
p.Date = date
} else {
p.Date = time.Now().UTC()
}
if err := walkParts(entity, p); err != nil {
return p, nil // best-effort: keep whatever headers/parts we got
}
return p, nil
}
func walkParts(entity *message.Entity, p *Parsed) error {
if mr := entity.MultipartReader(); mr != nil {
for {
part, err := mr.NextPart()
if err == io.EOF {
break
}
if err != nil {
return err
}
if err := walkParts(part, p); err != nil {
return err
}
}
return nil
}
contentType, params, _ := entity.Header.ContentType()
disposition, dispParams, _ := parseDisposition(entity.Header.Get("Content-Disposition"))
body, err := io.ReadAll(entity.Body)
if err != nil {
return err
}
filename := dispParams["filename"]
if filename == "" {
filename = params["name"]
}
isAttachment := disposition == "attachment" || (filename != "" && !strings.HasPrefix(contentType, "text/"))
switch {
case isAttachment:
p.Attachments = append(p.Attachments, AttachmentPart{
Filename: filename,
ContentType: contentType,
Content: body,
})
case contentType == "text/html":
p.HTMLBody += string(body)
case contentType == "text/plain" || contentType == "":
p.TextBody += string(body)
default:
if filename != "" {
p.Attachments = append(p.Attachments, AttachmentPart{
Filename: filename,
ContentType: contentType,
Content: body,
})
}
}
return nil
}
func parseDisposition(header string) (string, map[string]string, error) {
if header == "" {
return "", map[string]string{}, nil
}
return mime.ParseMediaType(header)
}
func decodeHeader(v string) string {
dec := mime.WordDecoder{CharsetReader: charset.Reader}
out, err := dec.DecodeHeader(v)
if err != nil {
return v
}
return out
}
func parseHeadersOnly(raw []byte) (*Parsed, error) {
msg, err := mail.ReadMessage(bytes.NewReader(raw))
if err != nil {
return nil, fmt.Errorf("mailparse: unable to parse message: %w", err)
}
p := &Parsed{
MessageID: strings.Trim(msg.Header.Get("Message-Id"), "<>"),
From: decodeHeader(msg.Header.Get("From")),
To: decodeHeader(msg.Header.Get("To")),
Cc: decodeHeader(msg.Header.Get("Cc")),
Subject: decodeHeader(msg.Header.Get("Subject")),
}
if date, err := msg.Header.Date(); err == nil {
p.Date = date
} else {
p.Date = time.Now().UTC()
}
body, _ := io.ReadAll(msg.Body)
p.TextBody = string(body)
return p, nil
}

33
api/main.go Normal file
View file

@ -0,0 +1,33 @@
// @title Mail Archive API
// @version 1.0
// @description API for Mail archiver
// @BasePath /
package main
import (
_ "github.com/joho/godotenv/autoload"
"jonasled.dev/jonasled/go-libs/log"
"jonasled.dev/jonasled/mail-achiver/api/auth"
"jonasled.dev/jonasled/mail-achiver/api/database"
_ "jonasled.dev/jonasled/mail-achiver/api/docs"
"jonasled.dev/jonasled/mail-achiver/api/search"
"jonasled.dev/jonasled/mail-achiver/api/server"
"jonasled.dev/jonasled/mail-achiver/api/storage"
"jonasled.dev/jonasled/mail-achiver/api/worker"
)
func main() {
log.Init()
database.Init()
auth.Init()
search.Init()
if _, err := storage.Reload(); err != nil {
log.Log.Fatal("Failed initializing storage backend: ", err.Error())
}
worker.Init()
server.Init()
server.Run()
}

91
api/middleware/auth.go Normal file
View file

@ -0,0 +1,91 @@
package middleware
import (
"net/http"
"strings"
"time"
"github.com/gin-gonic/gin"
"jonasled.dev/jonasled/mail-achiver/api/auth"
"jonasled.dev/jonasled/mail-achiver/api/database"
)
const (
CtxUserID = "userID"
CtxRole = "role"
)
// RequireAuth validates the Bearer JWT and stores the user id/role in the
// gin context for downstream handlers and scope filtering.
func RequireAuth() gin.HandlerFunc {
return func(c *gin.Context) {
header := c.GetHeader("Authorization")
if !strings.HasPrefix(header, "Bearer ") {
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "missing bearer token"})
return
}
token := strings.TrimPrefix(header, "Bearer ")
claims, err := auth.ParseToken(token)
if err != nil {
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid or expired token"})
return
}
var user database.User
if err := database.Db.First(&user, claims.UserID).Error; err != nil || user.Disabled {
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "user not found or disabled"})
return
}
c.Set(CtxUserID, user.ID)
c.Set(CtxRole, user.Role)
c.Set("currentUser", user)
c.Next()
}
}
// RequireRole restricts a route to one or more roles.
func RequireRole(roles ...database.UserRole) gin.HandlerFunc {
allowed := make(map[database.UserRole]bool, len(roles))
for _, r := range roles {
allowed[r] = true
}
return func(c *gin.Context) {
role, _ := c.Get(CtxRole)
r, _ := role.(database.UserRole)
if !allowed[r] {
c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "insufficient permissions"})
return
}
c.Next()
}
}
func CurrentUser(c *gin.Context) database.User {
u, _ := c.Get("currentUser")
user, _ := u.(database.User)
return user
}
// RequireAPIKey authenticates trusted service clients (the ingester) via a
// static API key hashed and compared against the APIKey table.
func RequireAPIKey() gin.HandlerFunc {
return func(c *gin.Context) {
header := c.GetHeader("Authorization")
if !strings.HasPrefix(header, "ApiKey ") {
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "missing api key"})
return
}
key := strings.TrimPrefix(header, "ApiKey ")
hash := auth.HashAPIKey(key)
var apiKey database.APIKey
if err := database.Db.Where("key_hash = ? AND disabled = ?", hash, false).First(&apiKey).Error; err != nil {
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid api key"})
return
}
database.Db.Model(&apiKey).Update("last_used", time.Now())
c.Next()
}
}

111
api/search/meilisearch.go Normal file
View file

@ -0,0 +1,111 @@
// Package search integrates Meilisearch for full-text search over archived
// mail. The search index is purely derived (rebuildable from the database
// or from storage sidecars), so it is never treated as a source of truth.
package search
import (
"context"
"encoding/json"
"strconv"
"github.com/meilisearch/meilisearch-go"
"jonasled.dev/jonasled/go-libs/config"
"jonasled.dev/jonasled/go-libs/log"
)
func itoa(v uint) string { return strconv.FormatUint(uint64(v), 10) }
const IndexName = "mails"
type MailDocument struct {
ID uint `json:"id"`
MailboxID uint `json:"mailboxId"`
MailboxAddress string `json:"mailboxAddress"`
From string `json:"from"`
To string `json:"to"`
Cc string `json:"cc"`
Subject string `json:"subject"`
Body string `json:"body"`
SentDate int64 `json:"sentDate"` // unix seconds, for filter/sort
HasAttachments bool `json:"hasAttachments"`
}
var client meilisearch.ServiceManager
func Init() {
host := config.GetOrDefault("MEILISEARCH_HOST", "http://localhost:7700")
apiKey := config.GetOrDefault("MEILISEARCH_API_KEY", "")
client = meilisearch.New(host, meilisearch.WithAPIKey(apiKey))
index := client.Index(IndexName)
filterable := []interface{}{"mailboxId", "sentDate", "hasAttachments"}
if _, err := index.UpdateFilterableAttributes(&filterable); err != nil {
log.Log.Warn("Failed configuring meilisearch filterable attributes: ", err.Error())
}
if _, err := index.UpdateSortableAttributes(&[]string{"sentDate"}); err != nil {
log.Log.Warn("Failed configuring meilisearch sortable attributes: ", err.Error())
}
if _, err := index.UpdateSearchableAttributes(&[]string{"subject", "from", "to", "cc", "body"}); err != nil {
log.Log.Warn("Failed configuring meilisearch searchable attributes: ", err.Error())
}
}
var primaryKey = "id"
func IndexMail(doc MailDocument) error {
_, err := client.Index(IndexName).AddDocuments([]MailDocument{doc}, &meilisearch.DocumentOptions{PrimaryKey: &primaryKey})
return err
}
func DeleteMail(id uint) error {
_, err := client.Index(IndexName).DeleteDocument(itoa(id), nil)
return err
}
type SearchParams struct {
Query string
MailboxIDs []uint // nil/empty => no mailbox restriction (admin/auditor)
Limit int64
Offset int64
}
type SearchResult struct {
Hits []MailDocument
EstimatedTotal int64
}
func Search(ctx context.Context, params SearchParams) (*SearchResult, error) {
req := &meilisearch.SearchRequest{
Limit: params.Limit,
Offset: params.Offset,
}
if len(params.MailboxIDs) > 0 {
filter := "mailboxId IN ["
for i, id := range params.MailboxIDs {
if i > 0 {
filter += ","
}
filter += itoa(id)
}
filter += "]"
req.Filter = filter
}
res, err := client.Index(IndexName).SearchWithContext(ctx, params.Query, req)
if err != nil {
return nil, err
}
hits := make([]MailDocument, 0, len(res.Hits))
for _, h := range res.Hits {
var doc MailDocument
if b, err := json.Marshal(h); err == nil {
if err := json.Unmarshal(b, &doc); err == nil {
hits = append(hits, doc)
}
}
}
return &SearchResult{Hits: hits, EstimatedTotal: res.EstimatedTotalHits}, nil
}

107
api/server/server.go Normal file
View file

@ -0,0 +1,107 @@
// Package server wires the gin HTTP engine: middleware, CORS, and route
// registration. Kept free of business logic (see handlers/*) so it stays a
// readable map of the API surface.
package server
import (
"github.com/gin-contrib/cors"
"github.com/gin-gonic/gin"
swaggerfiles "github.com/swaggo/files"
ginswagger "github.com/swaggo/gin-swagger"
"github.com/toorop/gin-logrus"
"jonasled.dev/jonasled/go-libs/config"
"jonasled.dev/jonasled/go-libs/log"
"jonasled.dev/jonasled/mail-achiver/api/database"
"jonasled.dev/jonasled/mail-achiver/api/handlers"
"jonasled.dev/jonasled/mail-achiver/api/middleware"
)
var engine *gin.Engine
func Init() {
if log.Log.Level < 5 {
gin.SetMode(gin.ReleaseMode)
}
engine = gin.New()
engine.Use(ginlogrus.Logger(log.Log), gin.Recovery())
corsConfig := cors.DefaultConfig()
corsConfig.AllowAllOrigins = true
corsConfig.AllowHeaders = append(corsConfig.AllowHeaders, "Authorization")
engine.Use(cors.New(corsConfig))
engine.GET("/healthz", handlers.Healthz)
engine.GET("/readyz", handlers.Readyz)
engine.GET("/swagger/*any", ginswagger.WrapHandler(swaggerfiles.Handler))
api := engine.Group("/api")
// Public.
api.POST("/auth/login", handlers.Login)
// Service-to-service (ingester -> API), authenticated via static key.
ingest := api.Group("/ingest")
ingest.Use(middleware.RequireAPIKey())
{
ingest.POST("/mail", handlers.IngestMail)
ingest.GET("/config", handlers.GetIngesterConfig)
}
// Authenticated user routes.
authed := api.Group("")
authed.Use(middleware.RequireAuth())
{
authed.GET("/auth/me", handlers.Me)
authed.GET("/mailboxes", handlers.ListMailboxes)
authed.GET("/mails", handlers.SearchMails)
authed.GET("/mails/:id", handlers.GetMail)
authed.GET("/mails/:id/eml", handlers.DownloadEml)
authed.GET("/mails/:id/html", handlers.ViewHtml)
authed.GET("/mails/:id/attachments/:attachmentId", handlers.DownloadAttachment)
admin := authed.Group("")
admin.Use(middleware.RequireRole(database.UserRoleAdmin))
{
admin.POST("/mailboxes", handlers.CreateMailbox)
admin.PUT("/mailboxes/:id", handlers.UpdateMailbox)
admin.DELETE("/mailboxes/:id", handlers.DeleteMailbox)
admin.DELETE("/mails/:id", handlers.DeleteMail)
admin.GET("/users", handlers.ListUsers)
admin.POST("/users", handlers.CreateUser)
admin.PUT("/users/:id", handlers.UpdateUser)
admin.DELETE("/users/:id", handlers.DeleteUser)
admin.GET("/settings/storage", handlers.GetStorageSettings)
admin.PUT("/settings/storage", handlers.UpdateStorageSettings)
admin.GET("/settings/retention", handlers.GetRetentionSettings)
admin.PUT("/settings/retention", handlers.UpdateRetentionSettings)
admin.GET("/settings/ingester", handlers.GetIngesterSettings)
admin.PUT("/settings/ingester", handlers.UpdateIngesterSettings)
admin.GET("/settings/ingester/imap", handlers.ListImapSources)
admin.POST("/settings/ingester/imap", handlers.CreateImapSource)
admin.PUT("/settings/ingester/imap/:id", handlers.UpdateImapSource)
admin.DELETE("/settings/ingester/imap/:id", handlers.DeleteImapSource)
admin.GET("/settings/apikeys", handlers.ListApiKeys)
admin.POST("/settings/apikeys", handlers.CreateApiKey)
admin.DELETE("/settings/apikeys/:id", handlers.DeleteApiKey)
admin.POST("/admin/reindex", handlers.Reindex)
}
}
}
func Run() {
addr := config.GetOrDefault("LISTEN_ADDRESS", ":8080")
log.Log.Infof("Starting API server on %s", addr)
if err := engine.Run(addr); err != nil {
log.Log.Fatal("Failed starting server: ", err.Error())
}
}

59
api/storage/backend.go Normal file
View file

@ -0,0 +1,59 @@
// Package storage abstracts the archive's object store. Every mail is
// persisted as two objects under the same key prefix: a raw ".eml" file
// (the RFC 5322 message exactly as received) and a ".json" metadata
// sidecar. Because the sidecar carries everything needed to rebuild the
// database row (mailbox address, from/to/subject/date/message-id,
// attachment list), the full archive can be restored purely by walking the
// storage backend and re-importing sidecars — no database backup required.
// Losing the database only loses derived state: users, settings, and search
// index, none of which are needed to recover the mail content itself.
package storage
import (
"context"
"errors"
"io"
)
var ErrNotFound = errors.New("storage: object not found")
type ObjectInfo struct {
Key string
Size int64
}
// Backend is the storage interface implemented by the file and S3 backends.
type Backend interface {
// Put writes an object, creating or overwriting it.
Put(ctx context.Context, key string, r io.Reader, size int64) error
// Get opens an object for reading. Caller must close it.
Get(ctx context.Context, key string) (io.ReadCloser, error)
// Delete removes an object. Not returning an error if it doesn't exist.
Delete(ctx context.Context, key string) error
// List enumerates all object keys under prefix, calling fn for each.
// Used by the reindex/restore job to walk the whole archive.
List(ctx context.Context, prefix string, fn func(ObjectInfo) error) error
}
// MetaSidecar is the JSON document stored alongside every raw .eml so the
// archive can be reindexed into the database and search engine from files
// alone.
type MetaSidecar struct {
SchemaVersion int `json:"schemaVersion"`
MailboxAddress string `json:"mailboxAddress"`
MessageID string `json:"messageId"`
Direction string `json:"direction"`
From string `json:"from"`
To string `json:"to"`
Cc string `json:"cc"`
Subject string `json:"subject"`
SentDate string `json:"sentDate"` // RFC3339
Size int64 `json:"size"`
Attachments []string `json:"attachments"`
ExpiresAt string `json:"expiresAt"` // RFC3339
}
const CurrentSchemaVersion = 1
func EmlKey(key string) string { return key + ".eml" }
func MetaKey(key string) string { return key + ".json" }

103
api/storage/encryption.go Normal file
View file

@ -0,0 +1,103 @@
package storage
import (
"bytes"
"context"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"errors"
"io"
)
// encryptionMagic prefixes every ciphertext object so Get can tell an
// encrypted object from a plaintext one written before encryption was
// enabled (or with a different key), and so a restore run never needs to
// guess.
var encryptionMagic = []byte("MAENC1")
// EncryptingBackend wraps another Backend and transparently AES-256-GCM
// encrypts/decrypts objects. The key comes from an environment variable
// (see storage.NewEncryptingBackendFromEnv), never from the database: the
// archive's storage backend is the disaster-recovery source of truth, so
// the key that unlocks it must live independently of the database that is
// allowed to be lost.
type EncryptingBackend struct {
inner Backend
gcm cipher.AEAD
}
func NewEncryptingBackend(inner Backend, rawKey string) (*EncryptingBackend, error) {
key := sha256.Sum256([]byte(rawKey))
block, err := aes.NewCipher(key[:])
if err != nil {
return nil, err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
return &EncryptingBackend{inner: inner, gcm: gcm}, nil
}
func (e *EncryptingBackend) Put(ctx context.Context, key string, r io.Reader, _ int64) error {
plaintext, err := io.ReadAll(r)
if err != nil {
return err
}
nonce := make([]byte, e.gcm.NonceSize())
if _, err := rand.Read(nonce); err != nil {
return err
}
ciphertext := e.gcm.Seal(nil, nonce, plaintext, nil)
buf := bytes.NewBuffer(nil)
buf.Write(encryptionMagic)
buf.Write(nonce)
buf.Write(ciphertext)
return e.inner.Put(ctx, key, buf, int64(buf.Len()))
}
func (e *EncryptingBackend) Get(ctx context.Context, key string) (io.ReadCloser, error) {
rc, err := e.inner.Get(ctx, key)
if err != nil {
return nil, err
}
defer rc.Close()
data, err := io.ReadAll(rc)
if err != nil {
return nil, err
}
if !bytes.HasPrefix(data, encryptionMagic) {
// Not encrypted (written before encryption was enabled, or by a
// deployment without a key) - return as-is.
return io.NopCloser(bytes.NewReader(data)), nil
}
rest := data[len(encryptionMagic):]
nonceSize := e.gcm.NonceSize()
if len(rest) < nonceSize {
return nil, errors.New("storage: encrypted object truncated")
}
nonce, ciphertext := rest[:nonceSize], rest[nonceSize:]
plaintext, err := e.gcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return nil, errors.New("storage: failed to decrypt object (wrong key?): " + err.Error())
}
return io.NopCloser(bytes.NewReader(plaintext)), nil
}
func (e *EncryptingBackend) Delete(ctx context.Context, key string) error {
return e.inner.Delete(ctx, key)
}
func (e *EncryptingBackend) List(ctx context.Context, prefix string, fn func(ObjectInfo) error) error {
return e.inner.List(ctx, prefix, fn)
}

77
api/storage/factory.go Normal file
View file

@ -0,0 +1,77 @@
package storage
import (
"context"
"fmt"
"sync"
"jonasled.dev/jonasled/go-libs/config"
"jonasled.dev/jonasled/go-libs/log"
"jonasled.dev/jonasled/mail-achiver/api/database"
)
var (
mu sync.RWMutex
current Backend
)
// Current returns the currently configured backend, building it from the
// StorageSettings row if not yet initialized or if settings changed.
func Current() (Backend, error) {
mu.RLock()
if current != nil {
defer mu.RUnlock()
return current, nil
}
mu.RUnlock()
return Reload()
}
// Reload rebuilds the active backend from the current StorageSettings row.
// Call after settings are updated via the API.
func Reload() (Backend, error) {
mu.Lock()
defer mu.Unlock()
var settings database.StorageSettings
if err := database.Db.First(&settings, 1).Error; err != nil {
return nil, fmt.Errorf("loading storage settings: %w", err)
}
var backend Backend
switch settings.Backend {
case database.StorageBackendS3:
b, err := NewS3Backend(context.Background(), S3Config{
Endpoint: settings.S3Endpoint,
Region: settings.S3Region,
Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey,
SecretKey: settings.S3SecretKey,
UsePathStyle: settings.S3UsePathStyle,
})
if err != nil {
return nil, fmt.Errorf("initializing s3 backend: %w", err)
}
backend = b
default:
path := settings.FileBasePath
if path == "" {
path = "./data/mails"
}
backend = NewFileBackend(path)
}
if key := config.GetOrDefault("STORAGE_ENCRYPTION_KEY", ""); key != "" {
enc, err := NewEncryptingBackend(backend, key)
if err != nil {
return nil, fmt.Errorf("initializing encrypting backend: %w", err)
}
log.Log.Info("Storage encryption enabled (STORAGE_ENCRYPTION_KEY set)")
backend = enc
} else {
log.Log.Warn("STORAGE_ENCRYPTION_KEY not set: mail files will be stored in cleartext")
}
current = backend
return current, nil
}

92
api/storage/file.go Normal file
View file

@ -0,0 +1,92 @@
package storage
import (
"context"
"io"
"os"
"path/filepath"
"strings"
)
// FileBackend stores objects as plain files under BasePath. Suitable for a
// single replica with a local disk, or multiple replicas sharing a
// ReadWriteMany volume (e.g. NFS-backed PVC on Kubernetes).
type FileBackend struct {
BasePath string
}
func NewFileBackend(basePath string) *FileBackend {
return &FileBackend{BasePath: basePath}
}
func (f *FileBackend) resolve(key string) string {
return filepath.Join(f.BasePath, filepath.FromSlash(key))
}
func (f *FileBackend) Put(_ context.Context, key string, r io.Reader, _ int64) error {
path := f.resolve(key)
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
return err
}
tmp := path + ".tmp"
out, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o640)
if err != nil {
return err
}
if _, err := io.Copy(out, r); err != nil {
out.Close()
os.Remove(tmp)
return err
}
if err := out.Close(); err != nil {
os.Remove(tmp)
return err
}
return os.Rename(tmp, path)
}
func (f *FileBackend) Get(_ context.Context, key string) (io.ReadCloser, error) {
file, err := os.Open(f.resolve(key))
if err != nil {
if os.IsNotExist(err) {
return nil, ErrNotFound
}
return nil, err
}
return file, nil
}
func (f *FileBackend) Delete(_ context.Context, key string) error {
err := os.Remove(f.resolve(key))
if err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
func (f *FileBackend) List(_ context.Context, prefix string, fn func(ObjectInfo) error) error {
root := f.resolve(prefix)
return filepath.Walk(f.BasePath, func(path string, info os.FileInfo, err error) error {
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
if info.IsDir() {
return nil
}
if !strings.HasPrefix(path, root) && prefix != "" {
return nil
}
rel, err := filepath.Rel(f.BasePath, path)
if err != nil {
return err
}
key := filepath.ToSlash(rel)
if strings.HasSuffix(key, ".tmp") {
return nil
}
return fn(ObjectInfo{Key: key, Size: info.Size()})
})
}

112
api/storage/s3.go Normal file
View file

@ -0,0 +1,112 @@
package storage
import (
"context"
"io"
"strings"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
// S3Backend stores objects in an S3-compatible bucket (AWS S3, MinIO,
// Ceph RGW, etc). Recommended for multi-replica deployments since it
// requires no shared filesystem. Uses minio-go rather than the AWS SDK:
// it's a much smaller dependency and speaks the same S3 API.
type S3Backend struct {
client *minio.Client
bucket string
}
type S3Config struct {
Endpoint string
Region string
Bucket string
AccessKey string
SecretKey string
UsePathStyle bool
}
func NewS3Backend(ctx context.Context, cfg S3Config) (*S3Backend, error) {
endpoint := cfg.Endpoint
secure := true
switch {
case strings.HasPrefix(endpoint, "http://"):
secure = false
endpoint = strings.TrimPrefix(endpoint, "http://")
case strings.HasPrefix(endpoint, "https://"):
endpoint = strings.TrimPrefix(endpoint, "https://")
}
lookup := minio.BucketLookupAuto
if cfg.UsePathStyle {
lookup = minio.BucketLookupPath
}
client, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
Secure: secure,
Region: cfg.Region,
BucketLookup: lookup,
})
if err != nil {
return nil, err
}
exists, err := client.BucketExists(ctx, cfg.Bucket)
if err != nil {
return nil, err
}
if !exists {
if err := client.MakeBucket(ctx, cfg.Bucket, minio.MakeBucketOptions{Region: cfg.Region}); err != nil {
return nil, err
}
}
return &S3Backend{client: client, bucket: cfg.Bucket}, nil
}
func (s *S3Backend) Put(ctx context.Context, key string, r io.Reader, size int64) error {
_, err := s.client.PutObject(ctx, s.bucket, key, r, size, minio.PutObjectOptions{
ContentType: "message/rfc822",
})
return err
}
func (s *S3Backend) Get(ctx context.Context, key string) (io.ReadCloser, error) {
obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
if err != nil {
return nil, err
}
// minio-go's GetObject is lazy: the request only happens on first
// read, so check existence up front to return ErrNotFound reliably.
if _, err := obj.Stat(); err != nil {
obj.Close()
if isNotFound(err) {
return nil, ErrNotFound
}
return nil, err
}
return obj, nil
}
func (s *S3Backend) Delete(ctx context.Context, key string) error {
return s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
}
func (s *S3Backend) List(ctx context.Context, prefix string, fn func(ObjectInfo) error) error {
for obj := range s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {
if obj.Err != nil {
return obj.Err
}
if err := fn(ObjectInfo{Key: obj.Key, Size: obj.Size}); err != nil {
return err
}
}
return nil
}
func isNotFound(err error) bool {
resp := minio.ToErrorResponse(err)
return resp.Code == "NoSuchKey" || resp.Code == "NotFound"
}

139
api/worker/scheduler.go Normal file
View file

@ -0,0 +1,139 @@
// Package worker runs background jobs (retention sweep, search reindex
// retry). Because the API runs as multiple Kubernetes replicas, jobs are
// gated by a simple database-backed lease so only one replica is active at
// a time; every replica still serves HTTP traffic normally.
package worker
import (
"context"
"os"
"time"
"jonasled.dev/jonasled/go-libs/config"
"jonasled.dev/jonasled/go-libs/helper"
"jonasled.dev/jonasled/go-libs/log"
"jonasled.dev/jonasled/mail-achiver/api/database"
"jonasled.dev/jonasled/mail-achiver/api/search"
"jonasled.dev/jonasled/mail-achiver/api/storage"
)
const (
leaseName = "background-jobs"
leaseDuration = 30 * time.Second
)
var ownerID = hostnameOr(helper.GenerateRandomString(8))
func hostnameOr(fallback string) string {
if h, err := os.Hostname(); err == nil && h != "" {
return h
}
return fallback
}
func Init() {
if config.GetOrDefault("ENABLE_SCHEDULER", "true") != "true" {
log.Log.Info("Background scheduler disabled on this replica (ENABLE_SCHEDULER=false)")
return
}
go run()
}
func run() {
ticker := time.NewTicker(1 * time.Minute)
defer ticker.Stop()
for {
if acquireLease() {
runRetentionSweep()
runIndexRetry()
}
<-ticker.C
}
}
// acquireLease implements a minimal lease: try to (re)claim the row if it
// is unowned, expired, or already owned by us.
func acquireLease() bool {
now := time.Now()
var lease database.SchedulerLease
err := database.Db.Where("name = ?", leaseName).First(&lease).Error
if err != nil {
database.Db.Create(&database.SchedulerLease{Name: leaseName, Owner: ownerID, ExpiresAt: now.Add(leaseDuration)})
return true
}
if lease.Owner != ownerID && lease.ExpiresAt.After(now) {
return false // another replica holds the lease
}
res := database.Db.Model(&database.SchedulerLease{}).
Where("name = ? AND (owner = ? OR expires_at <= ?)", leaseName, ownerID, now).
Updates(map[string]interface{}{"owner": ownerID, "expires_at": now.Add(leaseDuration)})
return res.RowsAffected > 0
}
func runRetentionSweep() {
var retention database.RetentionSettings
database.Db.First(&retention, 1)
if !retention.Enabled {
return
}
var expired []database.Mail
now := time.Now()
if err := database.Db.Where("expires_at < ? AND expires_at != ?", now, time.Time{}).Find(&expired).Error; err != nil {
log.Log.Warn("Retention sweep: failed loading expired mails: ", err.Error())
return
}
if len(expired) == 0 {
return
}
backend, err := storage.Current()
if err != nil {
log.Log.Warn("Retention sweep: failed loading storage backend: ", err.Error())
return
}
ctx := context.Background()
for _, mail := range expired {
if err := backend.Delete(ctx, storage.EmlKey(mail.StorageKey)); err != nil {
log.Log.Warn("Retention sweep: failed deleting eml for mail ", mail.ID, ": ", err.Error())
continue
}
backend.Delete(ctx, storage.MetaKey(mail.StorageKey))
search.DeleteMail(mail.ID)
database.Db.Delete(&database.Attachment{}, "mail_id = ?", mail.ID)
database.Db.Delete(&mail)
}
log.Log.Infof("Retention sweep: expired and removed %d mail(s)", len(expired))
}
// runIndexRetry re-attempts search indexing for mails that were stored
// successfully but failed to index (e.g. Meilisearch was briefly down).
func runIndexRetry() {
var pending []database.Mail
if err := database.Db.Where("indexed = ?", false).Limit(200).Find(&pending).Error; err != nil {
return
}
for _, mail := range pending {
var mailbox database.Mailbox
if err := database.Db.First(&mailbox, mail.MailboxID).Error; err != nil {
continue
}
err := search.IndexMail(search.MailDocument{
ID: mail.ID,
MailboxID: mail.MailboxID,
MailboxAddress: mailbox.Address,
From: mail.FromAddress,
To: mail.ToAddress,
Cc: mail.CcAddress,
Subject: mail.Subject,
SentDate: mail.SentDate.Unix(),
HasAttachments: mail.HasAttachments,
})
if err == nil {
database.Db.Model(&mail).Update("indexed", true)
}
}
}

View file

@ -0,0 +1,103 @@
// Package apiclient is the ingester's only way of reaching the API server:
// it pushes archived mail and pulls the ingester's own dynamic
// configuration (SMTP/IMAP settings configured by an admin in the web UI).
package apiclient
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"time"
)
type Client struct {
baseURL string
apiKey string
http *http.Client
}
func New(baseURL, apiKey string) *Client {
return &Client{
baseURL: baseURL,
apiKey: apiKey,
http: &http.Client{Timeout: 30 * time.Second},
}
}
// PushMail sends a raw RFC 5322 message to the API server for archiving.
func (c *Client) PushMail(mailboxAddress, direction string, raw []byte) error {
u := fmt.Sprintf("%s/api/ingest/mail?mailbox=%s&direction=%s",
c.baseURL, url.QueryEscape(mailboxAddress), url.QueryEscape(direction))
req, err := http.NewRequest(http.MethodPost, u, bytes.NewReader(raw))
if err != nil {
return err
}
req.Header.Set("Content-Type", "message/rfc822")
req.Header.Set("Authorization", "ApiKey "+c.apiKey)
resp, err := c.http.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 300 {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("api server returned %d: %s", resp.StatusCode, string(body))
}
return nil
}
type IngesterConfig struct {
SMTP struct {
SMTPEnabled bool `json:"smtpEnabled"`
SMTPAddress string `json:"smtpAddress"`
SMTPHostname string `json:"smtpHostname"`
} `json:"smtp"`
IMAP []IMAPSource `json:"imap"`
}
type IMAPSource struct {
ID uint `json:"id"`
Name string `json:"name"`
Host string `json:"host"`
Port int `json:"port"`
TLS bool `json:"tls"`
Username string `json:"username"`
Password string `json:"password"`
Folder string `json:"folder"`
MailboxAddress string `json:"mailboxAddress"`
PollIntervalSeconds int `json:"pollIntervalSeconds"`
DeleteAfterFetch bool `json:"deleteAfterFetch"`
Enabled bool `json:"enabled"`
}
// FetchConfig retrieves the ingester's dynamic configuration from the API.
func (c *Client) FetchConfig() (*IngesterConfig, error) {
req, err := http.NewRequest(http.MethodGet, c.baseURL+"/api/ingest/config", nil)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "ApiKey "+c.apiKey)
resp, err := c.http.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode >= 300 {
body, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("api server returned %d: %s", resp.StatusCode, string(body))
}
var cfg IngesterConfig
if err := json.NewDecoder(resp.Body).Decode(&cfg); err != nil {
return nil, err
}
return &cfg, nil
}

31
ingester/config/config.go Normal file
View file

@ -0,0 +1,31 @@
// Package config holds the ingester's own bootstrap configuration (how to
// reach the API server). Everything else - whether the SMTP listener is
// enabled, which IMAP mailboxes to poll - is fetched dynamically from the
// API server (GET /api/ingest/config) so the ingester itself stays
// stateless and can be reconfigured entirely from the web UI.
package config
import (
"time"
golibsconfig "jonasled.dev/jonasled/go-libs/config"
)
type Bootstrap struct {
APIBaseURL string
APIKey string
RefreshInterval time.Duration
}
func Load() Bootstrap {
refreshSeconds := golibsconfig.GetOrDefault("CONFIG_REFRESH_SECONDS", "30")
interval, err := time.ParseDuration(refreshSeconds + "s")
if err != nil {
interval = 30 * time.Second
}
return Bootstrap{
APIBaseURL: golibsconfig.GetOrFatal("API_BASE_URL"),
APIKey: golibsconfig.GetOrFatal("API_KEY"),
RefreshInterval: interval,
}
}

18
ingester/go.mod Normal file
View file

@ -0,0 +1,18 @@
module jonasled.dev/jonasled/mail-achiver/ingester
go 1.26.5
require (
github.com/emersion/go-imap/v2 v2.0.0-beta.8
github.com/emersion/go-smtp v0.21.3
github.com/joho/godotenv v1.5.1
jonasled.dev/jonasled/go-libs v0.0.3
)
require (
github.com/emersion/go-message v0.18.2 // indirect
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
golang.org/x/sys v0.5.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
)

61
ingester/go.sum Normal file
View file

@ -0,0 +1,61 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/emersion/go-imap/v2 v2.0.0-beta.8 h1:5IXZK1E33DyeP526320J3RS7eFlCYGFgtbrfapqDPug=
github.com/emersion/go-imap/v2 v2.0.0-beta.8/go.mod h1:dhoFe2Q0PwLrMD7oZw8ODuaD0vLYPe5uj2wcOMnvh48=
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA=
github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3DfFi4EXdT+B+gTtzx8GC9xsc26Znk=
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
github.com/emersion/go-smtp v0.21.3 h1:7uVwagE8iPYE48WhNsng3RRpCUpFvNl39JGNSIyGVMY=
github.com/emersion/go-smtp v0.21.3/go.mod h1:qm27SGYgoIPRot6ubfQ/GpiPy/g3PaZAVRxiO/sDUgQ=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
jonasled.dev/jonasled/go-libs v0.0.3 h1:2Tg1pYJMufBMPkyYVVAvsUJe/R+TRRtRVW9MCbc24bM=
jonasled.dev/jonasled/go-libs v0.0.3/go.mod h1:E8Sev4obR0x6y7x0HaIN1tImeLnVJgwJXt++jzTwiFM=

View file

@ -0,0 +1,179 @@
// Package imappuller implements the "pull" ingestion mode: periodically
// connect to an IMAP mailbox, fetch messages (optionally only unseen
// ones), push each to the API server, then mark them seen or delete them
// per source configuration.
package imappuller
import (
"context"
"fmt"
"sync"
"time"
"github.com/emersion/go-imap/v2"
"github.com/emersion/go-imap/v2/imapclient"
"jonasled.dev/jonasled/go-libs/log"
"jonasled.dev/jonasled/mail-achiver/ingester/apiclient"
)
type Pusher interface {
PushMail(mailboxAddress, direction string, raw []byte) error
}
// Manager keeps one polling goroutine per configured IMAP source, started
// and stopped as the API-supplied configuration changes.
type Manager struct {
mu sync.Mutex
running map[uint]context.CancelFunc
pusher Pusher
}
func NewManager(pusher Pusher) *Manager {
return &Manager{running: map[uint]context.CancelFunc{}, pusher: pusher}
}
// Sync reconciles the running pollers with the desired set of sources:
// starts new/changed ones, stops removed or disabled ones.
func (m *Manager) Sync(sources []apiclient.IMAPSource) {
m.mu.Lock()
defer m.mu.Unlock()
desired := map[uint]apiclient.IMAPSource{}
for _, s := range sources {
if s.Enabled {
desired[s.ID] = s
}
}
for id, cancel := range m.running {
if _, ok := desired[id]; !ok {
cancel()
delete(m.running, id)
}
}
for id, source := range desired {
if _, ok := m.running[id]; ok {
continue
}
ctx, cancel := context.WithCancel(context.Background())
m.running[id] = cancel
go m.run(ctx, source)
}
}
func (m *Manager) run(ctx context.Context, source apiclient.IMAPSource) {
interval := time.Duration(source.PollIntervalSeconds) * time.Second
if interval <= 0 {
interval = time.Minute
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
log.Log.Infof("IMAP poller started for %s (%s)", source.Name, source.MailboxAddress)
for {
if err := m.pollOnce(ctx, source); err != nil {
log.Log.Warn("IMAP poller (", source.Name, "): ", err.Error())
}
select {
case <-ctx.Done():
log.Log.Infof("IMAP poller stopped for %s", source.Name)
return
case <-ticker.C:
}
}
}
func (m *Manager) pollOnce(ctx context.Context, source apiclient.IMAPSource) error {
addr := fmt.Sprintf("%s:%d", source.Host, source.Port)
var client *imapclient.Client
var err error
if source.TLS {
client, err = imapclient.DialTLS(addr, nil)
} else {
client, err = imapclient.DialInsecure(addr, nil)
}
if err != nil {
return fmt.Errorf("connecting: %w", err)
}
defer client.Close()
if err := client.Login(source.Username, source.Password).Wait(); err != nil {
return fmt.Errorf("login: %w", err)
}
folder := source.Folder
if folder == "" {
folder = "INBOX"
}
if _, err := client.Select(folder, nil).Wait(); err != nil {
return fmt.Errorf("selecting folder %s: %w", folder, err)
}
searchData, err := client.UIDSearch(&imap.SearchCriteria{
NotFlag: []imap.Flag{imap.FlagSeen},
}, nil).Wait()
if err != nil {
return fmt.Errorf("searching unseen: %w", err)
}
uids := searchData.AllUIDs()
if len(uids) == 0 {
return nil
}
uidSet := imap.UIDSetNum(uids...)
fetchOptions := &imap.FetchOptions{
UID: true,
BodySection: []*imap.FetchItemBodySection{{}},
}
fetchCmd := client.Fetch(uidSet, fetchOptions)
messages, err := fetchCmd.Collect()
if err != nil {
return fmt.Errorf("fetching: %w", err)
}
var succeeded []imap.UID
for _, msg := range messages {
var raw []byte
for _, buf := range msg.BodySection {
raw = buf.Bytes
break
}
if len(raw) == 0 {
continue
}
if err := m.pusher.PushMail(source.MailboxAddress, "inbound", raw); err != nil {
log.Log.Warn("IMAP poller (", source.Name, "): failed pushing message uid ", msg.UID, ": ", err.Error())
continue
}
succeeded = append(succeeded, msg.UID)
}
if len(succeeded) == 0 {
return nil
}
successSet := imap.UIDSetNum(succeeded...)
if source.DeleteAfterFetch {
if _, err := client.Store(successSet, &imap.StoreFlags{
Op: imap.StoreFlagsAdd,
Flags: []imap.Flag{imap.FlagDeleted},
}, nil).Collect(); err != nil {
return fmt.Errorf("marking deleted: %w", err)
}
if err := client.Expunge().Close(); err != nil {
return fmt.Errorf("expunging: %w", err)
}
return nil
}
if _, err := client.Store(successSet, &imap.StoreFlags{
Op: imap.StoreFlagsAdd,
Flags: []imap.Flag{imap.FlagSeen},
}, nil).Collect(); err != nil {
return fmt.Errorf("marking seen: %w", err)
}
return nil
}

61
ingester/main.go Normal file
View file

@ -0,0 +1,61 @@
package main
import (
"context"
"time"
_ "github.com/joho/godotenv/autoload"
"jonasled.dev/jonasled/go-libs/log"
"jonasled.dev/jonasled/mail-achiver/ingester/apiclient"
"jonasled.dev/jonasled/mail-achiver/ingester/config"
"jonasled.dev/jonasled/mail-achiver/ingester/imappuller"
"jonasled.dev/jonasled/mail-achiver/ingester/smtpserver"
)
func main() {
log.Init()
bootstrap := config.Load()
client := apiclient.New(bootstrap.APIBaseURL, bootstrap.APIKey)
imapManager := imappuller.NewManager(client)
smtpSrv := &smtpserver.Server{}
var smtpCancel context.CancelFunc
smtpRunning := false
var lastAddr string
refresh := func() {
cfg, err := client.FetchConfig()
if err != nil {
log.Log.Warn("Failed fetching ingester config from API: ", err.Error())
return
}
imapManager.Sync(cfg.IMAP)
switch {
case cfg.SMTP.SMTPEnabled && (!smtpRunning || lastAddr != cfg.SMTP.SMTPAddress):
if smtpRunning {
smtpSrv.Stop()
smtpCancel()
}
ctx, cancel := context.WithCancel(context.Background())
smtpCancel = cancel
smtpSrv.Start(ctx, cfg.SMTP.SMTPAddress, cfg.SMTP.SMTPHostname, client)
smtpRunning = true
lastAddr = cfg.SMTP.SMTPAddress
case !cfg.SMTP.SMTPEnabled && smtpRunning:
smtpSrv.Stop()
smtpCancel()
smtpRunning = false
}
}
refresh()
ticker := time.NewTicker(bootstrap.RefreshInterval)
defer ticker.Stop()
for range ticker.C {
refresh()
}
}

View file

@ -0,0 +1,108 @@
// Package smtpserver implements the "push" ingestion mode: an SMTP server
// that a mail server (or MTA relay rule) can deliver mail to directly. Each
// RCPT TO recipient is archived under that address as its own mailbox
// scope; a message with multiple recipients is archived once per
// recipient.
package smtpserver
import (
"bytes"
"context"
"io"
"sync"
"github.com/emersion/go-smtp"
"jonasled.dev/jonasled/go-libs/log"
"jonasled.dev/jonasled/mail-achiver/ingester/apiclient"
)
type Pusher interface {
PushMail(mailboxAddress, direction string, raw []byte) error
}
type backend struct {
pusher Pusher
}
func (b *backend) NewSession(_ *smtp.Conn) (smtp.Session, error) {
return &session{pusher: b.pusher}, nil
}
type session struct {
pusher Pusher
recipients []string
}
func (s *session) AuthMechanisms() []string { return nil }
func (s *session) Mail(_ string, _ *smtp.MailOptions) error {
s.recipients = nil
return nil
}
func (s *session) Rcpt(to string, _ *smtp.RcptOptions) error {
s.recipients = append(s.recipients, to)
return nil
}
func (s *session) Data(r io.Reader) error {
raw, err := io.ReadAll(r)
if err != nil {
return err
}
for _, rcpt := range s.recipients {
if err := s.pusher.PushMail(rcpt, "inbound", bytes.Clone(raw)); err != nil {
log.Log.Error("smtp ingest: failed archiving mail for ", rcpt, ": ", err.Error())
return &smtp.SMTPError{Code: 451, Message: "temporary archiving failure"}
}
}
return nil
}
func (s *session) Reset() {
s.recipients = nil
}
func (s *session) Logout() error { return nil }
// Server runs the SMTP ingest listener. It is safe to Stop and re-Start
// when configuration changes (address, enabled flag).
type Server struct {
mu sync.Mutex
server *smtp.Server
}
func (s *Server) Start(ctx context.Context, addr, hostname string, client *apiclient.Client) {
s.mu.Lock()
defer s.mu.Unlock()
srv := smtp.NewServer(&backend{pusher: client})
srv.Addr = addr
srv.Domain = hostname
srv.AllowInsecureAuth = true
srv.MaxMessageBytes = 50 << 20
srv.MaxRecipients = 100
s.server = srv
go func() {
log.Log.Infof("SMTP ingest listener starting on %s", addr)
if err := srv.ListenAndServe(); err != nil {
log.Log.Warn("SMTP ingest listener stopped: ", err.Error())
}
}()
go func() {
<-ctx.Done()
srv.Close()
}()
}
func (s *Server) Stop() {
s.mu.Lock()
defer s.mu.Unlock()
if s.server != nil {
s.server.Close()
s.server = nil
}
}