haste_server/main.go

774 lines
19 KiB
Go

package main
import (
"context"
"crypto/rand"
"embed"
"encoding/json"
"errors"
"fmt"
"io"
"log"
mathrand "math/rand"
"mime"
"net"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/redis/go-redis/v9"
)
//go:embed static/*
var embeddedStatic embed.FS
type Config struct {
Host string
Port int
KeyLength int
MaxLength int
StaticMaxAge int
KeyGenerator KeyGeneratorConfig
RateLimits RateLimitsConfig
Storage StorageConfig
Documents map[string]string
}
type KeyGeneratorConfig struct {
Type string
Keyspace string
Path string
}
type RateLimitsConfig struct {
Whitelist []string
Blacklist []string
Categories map[string]RateLimitBucket
}
type RateLimitBucket struct {
TotalRequests int
Every int
}
type StorageConfig struct {
Type string
Host string
Port int
Expire int
DB int
Password string
}
type DocumentStore interface {
Get(ctx context.Context, key string, skipExpire bool) (string, bool, error)
Set(ctx context.Context, key, value string, skipExpire bool) error
}
type RedisStore struct {
client *redis.Client
expire time.Duration
}
func NewRedisStore(cfg StorageConfig) (*RedisStore, error) {
if rawURL := os.Getenv("REDISTOGO_URL"); rawURL != "" {
opts, err := redis.ParseURL(rawURL)
if err != nil {
return nil, err
}
client := redis.NewClient(opts)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := client.Ping(ctx).Err(); err != nil {
return nil, err
}
store := &RedisStore{
client: client,
expire: time.Duration(cfg.Expire) * time.Second,
}
log.Printf("connected to redis via REDISTOGO_URL")
return store, nil
}
host := cfg.Host
if host == "" {
host = "127.0.0.1"
}
port := cfg.Port
if port == 0 {
port = 6379
}
client := redis.NewClient(&redis.Options{
Addr: fmt.Sprintf("%s:%d", host, port),
Password: cfg.Password,
DB: cfg.DB,
})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := client.Ping(ctx).Err(); err != nil {
return nil, err
}
store := &RedisStore{
client: client,
expire: time.Duration(cfg.Expire) * time.Second,
}
log.Printf("connected to redis on %s:%d/%d", host, port, cfg.DB)
return store, nil
}
func (s *RedisStore) Get(ctx context.Context, key string, skipExpire bool) (string, bool, error) {
val, err := s.client.Get(ctx, key).Result()
if errors.Is(err, redis.Nil) {
return "", false, nil
}
if err != nil {
return "", false, err
}
if !skipExpire && s.expire > 0 {
go s.client.Expire(context.Background(), key, s.expire)
}
return val, true, nil
}
func (s *RedisStore) Set(ctx context.Context, key, value string, skipExpire bool) error {
if err := s.client.Set(ctx, key, value, 0).Err(); err != nil {
return err
}
if !skipExpire && s.expire > 0 {
go s.client.Expire(context.Background(), key, s.expire)
}
return nil
}
type KeyGenerator interface {
CreateKey(length int) string
}
type RandomKeyGenerator struct {
keyspace string
}
func (g RandomKeyGenerator) CreateKey(length int) string {
if g.keyspace == "" {
g.keyspace = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
}
var b strings.Builder
for i := 0; i < length; i++ {
b.WriteByte(g.keyspace[mathrand.Intn(len(g.keyspace))])
}
return b.String()
}
type PhoneticKeyGenerator struct{}
func (g PhoneticKeyGenerator) CreateKey(length int) string {
const vowels = "aeiou"
const consonants = "bcdfghjklmnpqrstvwxyz"
start := mathrand.Intn(2)
var b strings.Builder
for i := 0; i < length; i++ {
if i%2 == start {
b.WriteByte(consonants[mathrand.Intn(len(consonants))])
continue
}
b.WriteByte(vowels[mathrand.Intn(len(vowels))])
}
return b.String()
}
type DictionaryKeyGenerator struct {
words []string
}
func (g DictionaryKeyGenerator) CreateKey(length int) string {
var b strings.Builder
for i := 0; i < length; i++ {
b.WriteString(g.words[mathrand.Intn(len(g.words))])
}
return b.String()
}
type Server struct {
cfg Config
store DocumentStore
keyGenerator KeyGenerator
rateLimiter *RateLimiter
staticDocs map[string]struct{}
staticDir string
}
func main() {
seedMathRand()
cfg := loadConfig()
if cfg.Storage.Type == "" {
cfg.Storage.Type = "redis"
}
if cfg.Storage.Type != "redis" {
log.Fatalf("unsupported storage type %q: only redis is implemented in the Go backend", cfg.Storage.Type)
}
store, err := NewRedisStore(cfg.Storage)
if err != nil {
log.Fatalf("connect redis: %v", err)
}
keyGenerator, err := newKeyGenerator(cfg.KeyGenerator)
if err != nil {
log.Fatalf("configure key generator: %v", err)
}
srv := &Server{
cfg: cfg,
store: store,
keyGenerator: keyGenerator,
rateLimiter: NewRateLimiter(cfg.RateLimits),
staticDocs: make(map[string]struct{}, len(cfg.Documents)),
staticDir: filepath.Join(".", "static"),
}
if err := srv.loadStaticDocuments(); err != nil {
log.Fatalf("load static documents: %v", err)
}
address := net.JoinHostPort(cfg.Host, strconv.Itoa(cfg.Port))
log.Printf("listening on %s", address)
serve := &http.Server{
Addr: address,
Handler: srv.routes(),
ReadHeaderTimeout: 5 * time.Second,
}
if err := serve.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Fatalf("listen: %v", err)
}
}
func seedMathRand() {
var seedBytes [8]byte
if _, err := rand.Read(seedBytes[:]); err == nil {
var seed int64
for _, b := range seedBytes {
seed = (seed << 8) | int64(b)
}
mathrand.Seed(seed)
return
}
mathrand.Seed(time.Now().UnixNano())
}
func loadConfig() Config {
cfg := Config{
Host: "localhost",
Port: 7777,
KeyLength: 10,
MaxLength: 400000,
StaticMaxAge: 86400,
KeyGenerator: KeyGeneratorConfig{Type: "random"},
RateLimits: RateLimitsConfig{Categories: map[string]RateLimitBucket{}},
Storage: StorageConfig{Type: "redis", Host: "localhost", Port: 6379},
Documents: map[string]string{},
}
applyEnvOverrides(&cfg)
return cfg
}
func applyEnvOverrides(cfg *Config) {
if v := os.Getenv("HOST"); v != "" {
cfg.Host = v
}
if v := os.Getenv("PORT"); v != "" {
cfg.Port = atoiDefault(v, cfg.Port)
}
if v := os.Getenv("KEY_LENGTH"); v != "" {
cfg.KeyLength = atoiDefault(v, cfg.KeyLength)
}
if v := os.Getenv("MAX_LENGTH"); v != "" {
cfg.MaxLength = atoiDefault(v, cfg.MaxLength)
}
if v := os.Getenv("STATIC_MAX_AGE"); v != "" {
cfg.StaticMaxAge = atoiDefault(v, cfg.StaticMaxAge)
}
if v := os.Getenv("KEYGENERATOR_TYPE"); v != "" {
cfg.KeyGenerator.Type = v
}
if v := os.Getenv("KEY_GENERATOR_KEYSPACE"); v != "" {
cfg.KeyGenerator.Keyspace = v
}
if v := os.Getenv("STORAGE_TYPE"); v != "" {
cfg.Storage.Type = v
}
if v := os.Getenv("STORAGE_HOST"); v != "" {
cfg.Storage.Host = v
}
if v := os.Getenv("STORAGE_PORT"); v != "" {
cfg.Storage.Port = atoiDefault(v, cfg.Storage.Port)
}
if v := os.Getenv("STORAGE_EXPIRE_SECONDS"); v != "" {
cfg.Storage.Expire = atoiDefault(v, cfg.Storage.Expire)
}
if v := os.Getenv("STORAGE_DB"); v != "" {
cfg.Storage.DB = atoiDefault(v, cfg.Storage.DB)
}
if v := os.Getenv("STORAGE_PASSWORD"); v != "" {
cfg.Storage.Password = v
}
if v := os.Getenv("DOCUMENTS"); v != "" {
cfg.Documents = parseDocuments(v)
}
applyRateLimitEnv(cfg)
}
func applyRateLimitEnv(cfg *Config) {
normal := cfg.RateLimits.Categories["normal"]
if v := os.Getenv("RATE_LIMITS_NORMAL_TOTAL_REQUESTS"); v != "" {
normal.TotalRequests = atoiDefault(v, normal.TotalRequests)
}
if v := os.Getenv("RATE_LIMITS_NORMAL_EVERY_MILLISECONDS"); v != "" {
normal.Every = atoiDefault(v, normal.Every)
}
if normal.TotalRequests > 0 && normal.Every > 0 {
cfg.RateLimits.Categories["normal"] = normal
}
if v := os.Getenv("RATE_LIMITS_WHITELIST"); v != "" {
cfg.RateLimits.Whitelist = splitTrimmed(v)
}
if v := os.Getenv("RATE_LIMITS_BLACKLIST"); v != "" {
cfg.RateLimits.Blacklist = splitTrimmed(v)
}
whitelist := cfg.RateLimits.Categories["whitelist"]
if v := os.Getenv("RATE_LIMITS_WHITELIST_TOTAL_REQUESTS"); v != "" {
whitelist.TotalRequests = atoiDefault(v, whitelist.TotalRequests)
}
if v := os.Getenv("RATE_LIMITS_WHITELIST_EVERY_MILLISECONDS"); v != "" {
whitelist.Every = atoiDefault(v, whitelist.Every)
}
if whitelist.TotalRequests > 0 && whitelist.Every > 0 {
cfg.RateLimits.Categories["whitelist"] = whitelist
}
blacklist := cfg.RateLimits.Categories["blacklist"]
if v := os.Getenv("RATE_LIMITS_BLACKLIST_TOTAL_REQUESTS"); v != "" {
blacklist.TotalRequests = atoiDefault(v, blacklist.TotalRequests)
}
if v := os.Getenv("RATE_LIMITS_BLACKLIST_EVERY_MILLISECONDS"); v != "" {
blacklist.Every = atoiDefault(v, blacklist.Every)
}
if blacklist.TotalRequests > 0 && blacklist.Every > 0 {
cfg.RateLimits.Categories["blacklist"] = blacklist
}
}
func newKeyGenerator(cfg KeyGeneratorConfig) (KeyGenerator, error) {
switch cfg.Type {
case "", "random":
return RandomKeyGenerator{keyspace: cfg.Keyspace}, nil
case "phonetic":
return PhoneticKeyGenerator{}, nil
case "dictionary":
if cfg.Path == "" {
return nil, errors.New("dictionary key generator requires path")
}
data, err := os.ReadFile(cfg.Path)
if err != nil {
return nil, err
}
words := strings.Fields(string(data))
if len(words) == 0 {
return nil, errors.New("dictionary file is empty")
}
return DictionaryKeyGenerator{words: words}, nil
default:
return nil, fmt.Errorf("unsupported key generator %q", cfg.Type)
}
}
func (s *Server) loadStaticDocuments() error {
ctx := context.Background()
for name, path := range s.cfg.Documents {
data, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("%s: %w", path, err)
}
if err := s.store.Set(ctx, name, string(data), true); err != nil {
return fmt.Errorf("store %s: %w", name, err)
}
s.staticDocs[name] = struct{}{}
log.Printf("loaded static document %s from %s", name, path)
}
return nil
}
func (s *Server) routes() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/documents", s.handleDocumentsPost)
mux.HandleFunc("/documents/", s.handleDocumentsGet)
mux.HandleFunc("/raw/", s.handleRawGet)
mux.Handle("/", s)
if s.rateLimiter != nil {
return s.rateLimiter.Middleware(mux)
}
return mux
}
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet && r.Method != http.MethodHead {
http.NotFound(w, r)
return
}
path := strings.TrimPrefix(r.URL.Path, "/")
if path == "" {
s.serveIndex(w, r)
return
}
if s.tryServeStatic(w, r, path) {
return
}
if !strings.Contains(path, "/") {
s.serveIndex(w, r)
return
}
http.NotFound(w, r)
}
func (s *Server) tryServeStatic(w http.ResponseWriter, r *http.Request, path string) bool {
cleanPath := filepath.Clean(path)
if strings.HasPrefix(cleanPath, "..") {
http.NotFound(w, r)
return true
}
localPath := filepath.Join(s.staticDir, cleanPath)
if info, err := os.Stat(localPath); err == nil && !info.IsDir() {
s.serveDiskFile(w, r, localPath)
return true
}
if file, err := embeddedStatic.Open(filepath.ToSlash(filepath.Join("static", cleanPath))); err == nil {
_ = file.Close()
s.serveEmbeddedFile(w, r, filepath.ToSlash(filepath.Join("static", cleanPath)))
return true
}
return false
}
func (s *Server) serveIndex(w http.ResponseWriter, r *http.Request) {
indexPath := filepath.Join(s.staticDir, "index.html")
if _, err := os.Stat(indexPath); err == nil {
s.serveDiskFile(w, r, indexPath)
return
}
s.serveEmbeddedFile(w, r, "static/index.html")
}
func (s *Server) serveDiskFile(w http.ResponseWriter, r *http.Request, path string) {
file, err := os.Open(path)
if err != nil {
http.NotFound(w, r)
return
}
defer file.Close()
info, err := file.Stat()
if err != nil {
http.NotFound(w, r)
return
}
setStaticHeaders(w, path, s.cfg.StaticMaxAge)
http.ServeContent(w, r, info.Name(), info.ModTime(), file)
}
func (s *Server) serveEmbeddedFile(w http.ResponseWriter, r *http.Request, path string) {
data, err := embeddedStatic.ReadFile(path)
if err != nil {
http.NotFound(w, r)
return
}
setStaticHeaders(w, path, s.cfg.StaticMaxAge)
if ctype := mime.TypeByExtension(filepath.Ext(path)); ctype != "" {
w.Header().Set("Content-Type", ctype)
}
http.ServeContent(w, r, filepath.Base(path), time.Time{}, strings.NewReader(string(data)))
}
func setStaticHeaders(w http.ResponseWriter, path string, maxAge int) {
if maxAge > 0 {
w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", maxAge))
}
if strings.HasSuffix(path, ".css") {
w.Header().Set("Content-Type", "text/css; charset=utf-8")
}
if strings.HasSuffix(path, ".js") {
w.Header().Set("Content-Type", "application/javascript; charset=utf-8")
}
}
func (s *Server) handleDocumentsPost(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/documents" {
http.NotFound(w, r)
return
}
if r.Method != http.MethodPost {
http.NotFound(w, r)
return
}
body, err := s.readPostBody(r)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"message": "Connection error."})
return
}
if s.cfg.MaxLength > 0 && len(body) > s.cfg.MaxLength {
writeJSON(w, http.StatusBadRequest, map[string]string{"message": "Document exceeds maximum length."})
return
}
key, err := s.chooseKey(r.Context())
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"message": "Error adding document."})
return
}
if err := s.store.Set(r.Context(), key, body, false); err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"message": "Error adding document."})
return
}
writeJSON(w, http.StatusOK, map[string]string{"key": key})
}
func (s *Server) handleDocumentsGet(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet && r.Method != http.MethodHead {
http.NotFound(w, r)
return
}
key := documentKey(strings.TrimPrefix(r.URL.Path, "/documents/"))
skipExpire := s.isStaticDoc(key)
val, ok, err := s.store.Get(r.Context(), key, skipExpire)
if err != nil || !ok {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusNotFound)
if r.Method != http.MethodHead {
_ = json.NewEncoder(w).Encode(map[string]string{"message": "Document not found."})
}
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if r.Method == http.MethodHead {
return
}
_ = json.NewEncoder(w).Encode(map[string]string{"data": val, "key": key})
}
func (s *Server) handleRawGet(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet && r.Method != http.MethodHead {
http.NotFound(w, r)
return
}
key := documentKey(strings.TrimPrefix(r.URL.Path, "/raw/"))
skipExpire := s.isStaticDoc(key)
val, ok, err := s.store.Get(r.Context(), key, skipExpire)
if err != nil || !ok {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusNotFound)
if r.Method != http.MethodHead {
_ = json.NewEncoder(w).Encode(map[string]string{"message": "Document not found."})
}
return
}
w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
w.WriteHeader(http.StatusOK)
if r.Method != http.MethodHead {
_, _ = io.WriteString(w, val)
}
}
func (s *Server) readPostBody(r *http.Request) (string, error) {
contentType := r.Header.Get("Content-Type")
mediaType, _, _ := mime.ParseMediaType(contentType)
if mediaType == "multipart/form-data" {
if err := r.ParseMultipartForm(int64(s.cfg.MaxLength) + 1024); err != nil {
return "", err
}
return r.FormValue("data"), nil
}
data, err := io.ReadAll(r.Body)
if err != nil {
return "", err
}
return string(data), nil
}
func (s *Server) chooseKey(ctx context.Context) (string, error) {
for i := 0; i < 64; i++ {
key := s.keyGenerator.CreateKey(s.cfg.KeyLength)
if _, exists, err := s.store.Get(ctx, key, true); err != nil {
return "", err
} else if !exists {
return key, nil
}
}
return "", errors.New("failed to allocate unique key")
}
func (s *Server) isStaticDoc(key string) bool {
_, ok := s.staticDocs[key]
return ok
}
func documentKey(path string) string {
parts := strings.SplitN(path, ".", 2)
return parts[0]
}
func writeJSON(w http.ResponseWriter, status int, payload any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(payload)
}
type RateLimiter struct {
cfg RateLimitsConfig
mu sync.Mutex
counters map[string]*rateCounter
}
type rateCounter struct {
count int
expires time.Time
}
func NewRateLimiter(cfg RateLimitsConfig) *RateLimiter {
if len(cfg.Categories) == 0 {
return nil
}
return &RateLimiter{
cfg: cfg,
counters: map[string]*rateCounter{},
}
}
func (r *RateLimiter) Middleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
category, bucket, enabled := r.bucketFor(req)
if enabled {
ip := clientIP(req)
if !r.allow(category, ip, bucket) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusTooManyRequests)
_ = json.NewEncoder(w).Encode(map[string]string{"message": "Rate limit exceeded."})
return
}
}
next.ServeHTTP(w, req)
})
}
func (r *RateLimiter) bucketFor(req *http.Request) (string, RateLimitBucket, bool) {
ip := clientIP(req)
if contains(r.cfg.Blacklist, ip) {
if bucket, ok := r.cfg.Categories["blacklist"]; ok && bucket.TotalRequests > 0 && bucket.Every > 0 {
return "blacklist", bucket, true
}
}
if contains(r.cfg.Whitelist, ip) {
if bucket, ok := r.cfg.Categories["whitelist"]; ok && bucket.TotalRequests > 0 && bucket.Every > 0 {
return "whitelist", bucket, true
}
}
bucket, ok := r.cfg.Categories["normal"]
if !ok || bucket.TotalRequests <= 0 || bucket.Every <= 0 {
return "", RateLimitBucket{}, false
}
return "normal", bucket, true
}
func (r *RateLimiter) allow(category, ip string, bucket RateLimitBucket) bool {
now := time.Now()
key := category + ":" + ip
window := time.Duration(bucket.Every) * time.Millisecond
r.mu.Lock()
defer r.mu.Unlock()
counter, ok := r.counters[key]
if !ok || now.After(counter.expires) {
r.counters[key] = &rateCounter{count: 1, expires: now.Add(window)}
return true
}
if counter.count >= bucket.TotalRequests {
return false
}
counter.count++
return true
}
func clientIP(r *http.Request) string {
host, _, err := net.SplitHostPort(r.RemoteAddr)
if err == nil {
return host
}
return r.RemoteAddr
}
func atoiDefault(v string, fallback int) int {
parsed, err := strconv.Atoi(v)
if err != nil {
return fallback
}
return parsed
}
func splitTrimmed(v string) []string {
parts := strings.Split(v, ",")
out := make([]string, 0, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part != "" {
out = append(out, part)
}
}
return out
}
func contains(values []string, target string) bool {
for _, value := range values {
if value == target {
return true
}
}
return false
}
func parseDocuments(v string) map[string]string {
docs := map[string]string{}
for _, entry := range strings.Split(v, ",") {
entry = strings.TrimSpace(entry)
if entry == "" {
continue
}
parts := strings.SplitN(entry, "=", 2)
if len(parts) != 2 {
continue
}
key := strings.TrimSpace(parts[0])
value := strings.TrimSpace(parts[1])
if key != "" && value != "" {
docs[key] = value
}
}
return docs
}