Multi-replica backend: atomic claim per test, replicaCountBackend in Helm

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Jonas Leder 2026-11-16 11:45:26 +01:00
commit cbbeb0cd1b
7 changed files with 73 additions and 6 deletions

View file

@ -18,7 +18,7 @@ SpamAssassin runs as a sidecar container next to the backend (the backend image
docker compose up --build # frontend :8080, backend, spamassassin
```
Kubernetes: `helm install mail-tester ./helm -f my-values.yaml` (frontend and backend are separate deployments,
SpamAssassin is a sidecar of the backend; backend is deliberately a single replica).
SpamAssassin is a sidecar of the backend; backend scales with `replicaCountBackend` on mysql/postgres; sqlite is limited to 1).
Database: `DB_DRIVER=sqlite|mysql|postgres` with `DB_HOST`, `DB_PORT`, `DB_USER`, `DB_PASSWORD`, `DB_NAME`
(`DB_SSLMODE` for postgres; for sqlite `DB_NAME` is the file path). See `.env.example`.

View file

@ -26,5 +26,12 @@ func migrate(db *gorm.DB) error {
return tx.AutoMigrate(&Test{})
},
Rollback: func(tx *gorm.DB) error { return tx.Migrator().DropTable("tests") },
}, {
ID: "202611160002_add_claimed_at",
Migrate: func(tx *gorm.DB) error {
type Test struct{ ClaimedAt *time.Time }
return tx.AutoMigrate(&Test{})
},
Rollback: func(tx *gorm.DB) error { return tx.Migrator().DropColumn("tests", "claimed_at") },
}}).Migrate()
}

View file

@ -32,6 +32,8 @@ type Test struct {
Checks []Check `json:"checks" gorm:"serializer:json"`
CreatedAt time.Time `json:"created_at"`
ExpiresAt time.Time `json:"expires_at"`
// ClaimedAt marks a replica working on this test, so replicas never analyse the same mail twice.
ClaimedAt *time.Time `json:"-"`
}
type ErrorResponse struct {

View file

@ -17,6 +17,7 @@ import (
const (
waitWindow = 30 * time.Minute // how long we wait for the mail to arrive
retention = 24 * time.Hour // how long reports and raw mail are kept
claimTTL = 2 * time.Minute // a claim older than this is considered dead (replica crashed)
)
func openDB() (*gorm.DB, error) {
@ -62,10 +63,16 @@ func runWorker(db *gorm.DB, j *JMAP, spamd string) {
}
}
func claim(db *gorm.DB, id string) bool {
now := time.Now()
res := db.Model(&Test{}).Where("id = ? AND status = ? AND (claimed_at IS NULL OR claimed_at < ?)", id, "waiting", now.Add(-claimTTL)).Update("claimed_at", now)
return res.Error == nil && res.RowsAffected == 1
}
func process(db *gorm.DB, j *JMAP, spamd string) {
db.Model(&Test{}).Where("status = ? AND created_at < ?", "waiting", time.Now().Add(-waitWindow)).Update("status", "expired")
var tests []Test
db.Where("status = ?", "waiting").Limit(100).Find(&tests)
db.Where("status = ? AND (claimed_at IS NULL OR claimed_at < ?)", "waiting", time.Now().Add(-claimTTL)).Limit(100).Find(&tests)
if len(tests) == 0 {
return
}
@ -83,11 +90,17 @@ func process(db *gorm.DB, j *JMAP, spamd string) {
if !ok {
continue
}
// Atomic claim: with several replicas only the one whose UPDATE matches a row does the work.
if !claim(db, t.ID) {
continue
}
raw, err := j.Download(blob)
if err != nil {
log.Log.Warn("jmap download: ", err)
db.Model(&Test{}).Where("id = ?", t.ID).Update("claimed_at", nil)
continue
}
t.ClaimedAt = nil
t.Sender, t.Subject, t.Checks, t.Score = Analyze(raw, spamd)
t.Status = "done"
if err := db.Save(&t).Error; err != nil {

39
backend/worker_test.go Normal file
View file

@ -0,0 +1,39 @@
package main
import (
"path/filepath"
"sync"
"sync/atomic"
"testing"
"time"
)
func TestClaimIsExclusive(t *testing.T) {
t.Setenv("DB_NAME", filepath.Join(t.TempDir(), "t.db"))
db, err := openDB()
if err != nil {
t.Fatal(err)
}
db.Create(&Test{ID: "a", Status: "waiting", Checks: []Check{}, CreatedAt: time.Now()})
var won atomic.Int32
var wg sync.WaitGroup
for i := 0; i < 20; i++ {
wg.Add(1)
go func() {
defer wg.Done()
if claim(db, "a") {
won.Add(1)
}
}()
}
wg.Wait()
if won.Load() != 1 {
t.Fatalf("%d replicas claimed the same test", won.Load())
}
// a dead claim is taken over after claimTTL
old := time.Now().Add(-2 * claimTTL)
db.Model(&Test{}).Where("id = ?", "a").Update("claimed_at", old)
if !claim(db, "a") {
t.Fatal("stale claim not taken over")
}
}

View file

@ -5,9 +5,14 @@ metadata:
labels:
{{- include "mail-tester.labels" . | nindent 4 }}
spec:
replicas: 1
{{- if and (gt (int .Values.replicaCountBackend) 1) (eq (include "mail-tester.sqlite" .) "true") }}
{{- fail "replicaCountBackend > 1 needs db.driver mysql or postgres (sqlite is a single-writer file)" }}
{{- end }}
replicas: {{ .Values.replicaCountBackend }}
{{- if eq (include "mail-tester.sqlite" .) "true" }}
strategy:
type: Recreate # sqlite volume is ReadWriteOnce; two pollers would double-process mail
type: Recreate # the sqlite volume is ReadWriteOnce
{{- end }}
selector:
matchLabels:
{{- include "mail-tester.selectorLabels" . | nindent 6 }}

View file

@ -20,8 +20,9 @@ podAnnotations: {}
podSecurityContext: {}
securityContext: {}
# The backend polls Stalwart, so it must run as a single replica
# (several replicas would analyse the same mails twice).
# Several backend replicas share the work (each mail is claimed atomically in the database),
# which needs db.driver mysql or postgres. sqlite is limited to 1.
replicaCountBackend: 1
replicaCountFrontend: 1
service: