Multi-replica backend: atomic claim per test, replicaCountBackend in Helm
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
71efa8c0a9
commit
cbbeb0cd1b
7 changed files with 73 additions and 6 deletions
|
|
@ -18,7 +18,7 @@ SpamAssassin runs as a sidecar container next to the backend (the backend image
|
|||
docker compose up --build # frontend :8080, backend, spamassassin
|
||||
```
|
||||
Kubernetes: `helm install mail-tester ./helm -f my-values.yaml` (frontend and backend are separate deployments,
|
||||
SpamAssassin is a sidecar of the backend; backend is deliberately a single replica).
|
||||
SpamAssassin is a sidecar of the backend; backend scales with `replicaCountBackend` on mysql/postgres; sqlite is limited to 1).
|
||||
|
||||
Database: `DB_DRIVER=sqlite|mysql|postgres` with `DB_HOST`, `DB_PORT`, `DB_USER`, `DB_PASSWORD`, `DB_NAME`
|
||||
(`DB_SSLMODE` for postgres; for sqlite `DB_NAME` is the file path). See `.env.example`.
|
||||
|
|
|
|||
|
|
@ -26,5 +26,12 @@ func migrate(db *gorm.DB) error {
|
|||
return tx.AutoMigrate(&Test{})
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error { return tx.Migrator().DropTable("tests") },
|
||||
}, {
|
||||
ID: "202611160002_add_claimed_at",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
type Test struct{ ClaimedAt *time.Time }
|
||||
return tx.AutoMigrate(&Test{})
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error { return tx.Migrator().DropColumn("tests", "claimed_at") },
|
||||
}}).Migrate()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ type Test struct {
|
|||
Checks []Check `json:"checks" gorm:"serializer:json"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
// ClaimedAt marks a replica working on this test, so replicas never analyse the same mail twice.
|
||||
ClaimedAt *time.Time `json:"-"`
|
||||
}
|
||||
|
||||
type ErrorResponse struct {
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import (
|
|||
const (
|
||||
waitWindow = 30 * time.Minute // how long we wait for the mail to arrive
|
||||
retention = 24 * time.Hour // how long reports and raw mail are kept
|
||||
claimTTL = 2 * time.Minute // a claim older than this is considered dead (replica crashed)
|
||||
)
|
||||
|
||||
func openDB() (*gorm.DB, error) {
|
||||
|
|
@ -62,10 +63,16 @@ func runWorker(db *gorm.DB, j *JMAP, spamd string) {
|
|||
}
|
||||
}
|
||||
|
||||
func claim(db *gorm.DB, id string) bool {
|
||||
now := time.Now()
|
||||
res := db.Model(&Test{}).Where("id = ? AND status = ? AND (claimed_at IS NULL OR claimed_at < ?)", id, "waiting", now.Add(-claimTTL)).Update("claimed_at", now)
|
||||
return res.Error == nil && res.RowsAffected == 1
|
||||
}
|
||||
|
||||
func process(db *gorm.DB, j *JMAP, spamd string) {
|
||||
db.Model(&Test{}).Where("status = ? AND created_at < ?", "waiting", time.Now().Add(-waitWindow)).Update("status", "expired")
|
||||
var tests []Test
|
||||
db.Where("status = ?", "waiting").Limit(100).Find(&tests)
|
||||
db.Where("status = ? AND (claimed_at IS NULL OR claimed_at < ?)", "waiting", time.Now().Add(-claimTTL)).Limit(100).Find(&tests)
|
||||
if len(tests) == 0 {
|
||||
return
|
||||
}
|
||||
|
|
@ -83,11 +90,17 @@ func process(db *gorm.DB, j *JMAP, spamd string) {
|
|||
if !ok {
|
||||
continue
|
||||
}
|
||||
// Atomic claim: with several replicas only the one whose UPDATE matches a row does the work.
|
||||
if !claim(db, t.ID) {
|
||||
continue
|
||||
}
|
||||
raw, err := j.Download(blob)
|
||||
if err != nil {
|
||||
log.Log.Warn("jmap download: ", err)
|
||||
db.Model(&Test{}).Where("id = ?", t.ID).Update("claimed_at", nil)
|
||||
continue
|
||||
}
|
||||
t.ClaimedAt = nil
|
||||
t.Sender, t.Subject, t.Checks, t.Score = Analyze(raw, spamd)
|
||||
t.Status = "done"
|
||||
if err := db.Save(&t).Error; err != nil {
|
||||
|
|
|
|||
39
backend/worker_test.go
Normal file
39
backend/worker_test.go
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestClaimIsExclusive(t *testing.T) {
|
||||
t.Setenv("DB_NAME", filepath.Join(t.TempDir(), "t.db"))
|
||||
db, err := openDB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.Create(&Test{ID: "a", Status: "waiting", Checks: []Check{}, CreatedAt: time.Now()})
|
||||
var won atomic.Int32
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 20; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if claim(db, "a") {
|
||||
won.Add(1)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if won.Load() != 1 {
|
||||
t.Fatalf("%d replicas claimed the same test", won.Load())
|
||||
}
|
||||
// a dead claim is taken over after claimTTL
|
||||
old := time.Now().Add(-2 * claimTTL)
|
||||
db.Model(&Test{}).Where("id = ?", "a").Update("claimed_at", old)
|
||||
if !claim(db, "a") {
|
||||
t.Fatal("stale claim not taken over")
|
||||
}
|
||||
}
|
||||
|
|
@ -5,9 +5,14 @@ metadata:
|
|||
labels:
|
||||
{{- include "mail-tester.labels" . | nindent 4 }}
|
||||
spec:
|
||||
replicas: 1
|
||||
{{- if and (gt (int .Values.replicaCountBackend) 1) (eq (include "mail-tester.sqlite" .) "true") }}
|
||||
{{- fail "replicaCountBackend > 1 needs db.driver mysql or postgres (sqlite is a single-writer file)" }}
|
||||
{{- end }}
|
||||
replicas: {{ .Values.replicaCountBackend }}
|
||||
{{- if eq (include "mail-tester.sqlite" .) "true" }}
|
||||
strategy:
|
||||
type: Recreate # sqlite volume is ReadWriteOnce; two pollers would double-process mail
|
||||
type: Recreate # the sqlite volume is ReadWriteOnce
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "mail-tester.selectorLabels" . | nindent 6 }}
|
||||
|
|
|
|||
|
|
@ -20,8 +20,9 @@ podAnnotations: {}
|
|||
podSecurityContext: {}
|
||||
securityContext: {}
|
||||
|
||||
# The backend polls Stalwart, so it must run as a single replica
|
||||
# (several replicas would analyse the same mails twice).
|
||||
# Several backend replicas share the work (each mail is claimed atomically in the database),
|
||||
# which needs db.driver mysql or postgres. sqlite is limited to 1.
|
||||
replicaCountBackend: 1
|
||||
replicaCountFrontend: 1
|
||||
|
||||
service:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue