From d02de1ca4f1763b1e29d1000f67971847c1c8692 Mon Sep 17 00:00:00 2001 From: Jonas Leder <jonas@jonasled.de> Date: Sun, 26 Jan 2025 16:14:50 +0100 Subject: [PATCH 01/11] configure client to use local database connection --- main.go | 12 +++++++----- messageworker/{main.go => client.go} | 17 +++++++++++++---- 2 files changed, 20 insertions(+), 9 deletions(-) rename messageworker/{main.go => client.go} (88%) diff --git a/main.go b/main.go index 1c83f09..c9c67ba 100644 --- a/main.go +++ b/main.go @@ -15,7 +15,6 @@ import ( func main() { log.Init() - queue.Init() if os.Getenv("OUTPUT_DATABSE") != "" { @@ -27,10 +26,13 @@ func main() { log.Log.Info("Starting embedded MQTT server") mqttserver.Start() } - mqttclient.Init() - go messageworker.Run() - for { - time.Sleep(time.Second) // reduce CPU usage by adding a short sleep here instead of a empty for loop + + if os.Getenv("LOGGER_CLIENT") != "false" && os.Getenv("LOGGER_SERVER") != "true" { + mqttclient.Init() + go messageworker.RunClient() + for { + time.Sleep(time.Second) // reduce CPU usage by adding a short sleep here instead of a empty for loop + } } } diff --git a/messageworker/main.go b/messageworker/client.go similarity index 88% rename from messageworker/main.go rename to messageworker/client.go index 3b0cebb..8ebe615 100644 --- a/messageworker/main.go +++ b/messageworker/client.go @@ -2,6 +2,7 @@ package messageworker import ( "encoding/json" + "os" "time" "jonasled.dev/jonasled/ems-esp-logger/database" @@ -12,7 +13,15 @@ import ( "jonasled.dev/jonasled/ems-esp-logger/types" ) -func Run() { +func RunClient() { + if os.Getenv("CLIENT_USE_SERVER") != "true" { + + log.Log.Info("Intialized local client with direct datbase connection") + runLocalDbClient() + } +} + +func runLocalDbClient() { for { taskData, taskId, err := queue.MainQueue.Dequeue() if err != nil { @@ -24,14 +33,14 @@ func Run() { err = json.Unmarshal([]byte(taskData), &task) if err != nil { log.Log.Error("Failed decoding task as JSON: ", err.Error()) - return + continue } var instance tables.Instance err = database.Db.Where("name = ?", task.Instance).First(&instance).Error if err != nil { log.Log.Error("Failed retreiving instance from database, pushing task back to queue: ", err.Error()) queue.MainQueue.Enqueue(taskData, 60) - return + continue } var jsonData map[string]interface{} @@ -39,7 +48,7 @@ func Run() { if err != nil { log.Log.Error("Failed decoding boiler JSON: ", err.Error()) queue.MainQueue.Enqueue(taskData, 60) - return + continue } valuesToInsert := []tables.Value{} for key, value := range jsonData { -- GitLab From 264a39a61b65af9d0ec9ae25b221c3b1a25d3363 Mon Sep 17 00:00:00 2001 From: Jonas Leder <jonas@jonasled.de> Date: Sun, 26 Jan 2025 16:20:20 +0100 Subject: [PATCH 02/11] init webserver --- .env.example | 4 ++- go.mod | 23 ++++++++++++++ go.sum | 68 +++++++++++++++++++++++++++++++++++++++++ main.go | 7 +++++ server/generalRoutes.go | 16 ++++++++++ server/main.go | 22 +++++++++++++ 6 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 server/generalRoutes.go create mode 100644 server/main.go diff --git a/.env.example b/.env.example index 415106b..fb668f4 100644 --- a/.env.example +++ b/.env.example @@ -21,4 +21,6 @@ OUTPUT_DATABASE_TYPE=mysql OUTPUT_DATABASE_EXECUTE_MIGRATIONS=true INSTANCE_NAME=test -INSTANCE_DESCRIPTION="My test instance" \ No newline at end of file +INSTANCE_DESCRIPTION="My test instance" + +GIN_MODE=release \ No newline at end of file diff --git a/go.mod b/go.mod index b350c1c..a268147 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,13 @@ go 1.23.4 require ( github.com/eclipse/paho.mqtt.golang v1.5.0 + github.com/gin-gonic/gin v1.10.0 github.com/glebarez/sqlite v1.11.0 github.com/google/uuid v1.3.0 github.com/joho/godotenv v1.5.1 github.com/nekomeowww/gorm-logger-logrus v1.0.8 github.com/sirupsen/logrus v1.9.3 + github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f github.com/wind-c/comqtt/v2 v2.6.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gorm.io/driver/mysql v1.5.7 @@ -17,19 +19,40 @@ require ( require ( filippo.io/edwards25519 v1.1.0 // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect github.com/glebarez/go-sqlite v1.21.2 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/go-sql-driver/mysql v1.8.1 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rs/xid v1.5.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/crypto v0.25.0 // indirect golang.org/x/net v0.27.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/libc v1.22.5 // indirect modernc.org/mathutil v1.5.0 // indirect diff --git a/go.sum b/go.sum index 2713a23..f6dff72 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,13 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -7,13 +15,32 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o= github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -28,10 +55,25 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/nekomeowww/gorm-logger-logrus v1.0.8 h1:/XdgNrSBSrTwiZ7fgf8Y9zW9juW1rVmRoY7OkX/tPnY= github.com/nekomeowww/gorm-logger-logrus v1.0.8/go.mod h1:V266BSFFzJF1jnKWVAucFsh4U+ib3pEqJCzQY5zm7iA= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= @@ -42,21 +84,45 @@ github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f h1:oqdnd6OGlOUu1InG37hWcCB3a+Jy3fwjylyVboaNMwY= +github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f/go.mod h1:X3Dd1SB8Gt1V968NTzpKFjMM6O8ccta2NPC6MprOxZQ= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/wind-c/comqtt/v2 v2.6.0 h1:huLdOwYDrwMTrEwH7+mSs1GftHZ/tDqJw8nOz3iX7kc= github.com/wind-c/comqtt/v2 v2.6.0/go.mod h1:6O4VilBrTQ/cNIcmIgNdMLCK9DTiLRxkal00t0DLN64= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= @@ -77,3 +143,5 @@ modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/main.go b/main.go index c9c67ba..bde7bd2 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ import ( "jonasled.dev/jonasled/ems-esp-logger/mqttclient" "jonasled.dev/jonasled/ems-esp-logger/mqttserver" "jonasled.dev/jonasled/ems-esp-logger/queue" + "jonasled.dev/jonasled/ems-esp-logger/server" ) func main() { @@ -34,5 +35,11 @@ func main() { time.Sleep(time.Second) // reduce CPU usage by adding a short sleep here instead of a empty for loop } } + if os.Getenv("LOGGER_SERVER") == "true" { + server.Init() + server.Run() + } else { + log.Log.Fatal("Either LOGGER_CLIENT or LOGGER_SERVER has to be enabled in config") + } } diff --git a/server/generalRoutes.go b/server/generalRoutes.go new file mode 100644 index 0000000..990a8c2 --- /dev/null +++ b/server/generalRoutes.go @@ -0,0 +1,16 @@ +package server + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +func initGeneralRoutes() { + R.GET("/ready", func(ctx *gin.Context) { + ctx.String(http.StatusOK, "OK") + }) + R.GET("/", func(ctx *gin.Context) { + ctx.String(http.StatusOK, "EMS-ESP server") + }) +} diff --git a/server/main.go b/server/main.go new file mode 100644 index 0000000..1c76636 --- /dev/null +++ b/server/main.go @@ -0,0 +1,22 @@ +package server + +import ( + "github.com/gin-gonic/gin" + ginlogrus "github.com/toorop/gin-logrus" + "jonasled.dev/jonasled/ems-esp-logger/log" +) + +var R *gin.Engine + +func Init() { + log.Log.Info("Initializing webserver") + R = gin.New() + R.Use(ginlogrus.Logger(log.Log), gin.Recovery()) + + initGeneralRoutes() +} + +func Run() { + log.Log.Info("starting webserver") + R.Run() +} -- GitLab From 77844083faeedadb6ca9dabad0ec7dcf6468070a Mon Sep 17 00:00:00 2001 From: Jonas Leder <jonas@jonasled.de> Date: Sun, 26 Jan 2025 16:38:57 +0100 Subject: [PATCH 03/11] add HTTP endpoint for creating new instances --- server/createInstanceRoute.go | 73 +++++++++++++++++++++++++++++++++++ server/main.go | 1 + 2 files changed, 74 insertions(+) create mode 100644 server/createInstanceRoute.go diff --git a/server/createInstanceRoute.go b/server/createInstanceRoute.go new file mode 100644 index 0000000..baf290f --- /dev/null +++ b/server/createInstanceRoute.go @@ -0,0 +1,73 @@ +package server + +import ( + "encoding/json" + "net/http" + "os" + + "github.com/gin-gonic/gin" + "gorm.io/gorm" + "jonasled.dev/jonasled/ems-esp-logger/database" + "jonasled.dev/jonasled/ems-esp-logger/database/tables" + "jonasled.dev/jonasled/ems-esp-logger/log" +) + +func initCreateInstanceRoute() { + apiKey := os.Getenv("MASTER_APIKEY") + if apiKey == "" { + log.Log.Fatal("MASTER_APIKEY is not specified") + } + + R.POST("/api/instance", func(c *gin.Context) { + authHeader := c.GetHeader("Authentication") + if authHeader != apiKey { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"}) + return + } + contentType := c.GetHeader("Content-Type") + if contentType == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "Content-Type header missing"}) + return + } + if contentType != "application/json" { + c.JSON(http.StatusBadRequest, gin.H{"error": "expected application/json as content type"}) + return + } + var instance tables.Instance + body, _ := c.GetRawData() + err := json.Unmarshal(body, &instance) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "failed decoding json", "message": err.Error()}) + return + } + if instance.Name == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "the name cannot be empty"}) + return + } + + var existingInstance tables.Instance + err = database.Db.Where("name = ?", instance.Name).First(&existingInstance).Error + + if err != nil { + if err == gorm.ErrRecordNotFound { + if err := database.Db.Create(&instance).Error; err != nil { + log.Log.Error("Failed to create instance: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed creating new instance definition"}) + } + c.JSON(http.StatusCreated, gin.H{"message": "Created new entry"}) + } else { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed creating new instance definition"}) + log.Log.Error("Failed to query database, while trying to create new instance: %v", err) + } + } else { + existingInstance.Description = instance.Description + if err := database.Db.Save(&existingInstance).Error; err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed updating instance definition"}) + log.Log.Error("Failed to query database, while trying to update instance: %v", err) + } + c.JSON(http.StatusCreated, gin.H{"message": "Updated entry"}) + } + + }) + +} diff --git a/server/main.go b/server/main.go index 1c76636..be39cf1 100644 --- a/server/main.go +++ b/server/main.go @@ -14,6 +14,7 @@ func Init() { R.Use(ginlogrus.Logger(log.Log), gin.Recovery()) initGeneralRoutes() + initCreateInstanceRoute() } func Run() { -- GitLab From b43c3567240c22e6767761e9b6a10a28df74d61c Mon Sep 17 00:00:00 2001 From: Jonas Leder <jonas@jonasled.de> Date: Sun, 26 Jan 2025 16:40:25 +0100 Subject: [PATCH 04/11] extend table with apikey --- database/tables/instance.go | 1 + 1 file changed, 1 insertion(+) diff --git a/database/tables/instance.go b/database/tables/instance.go index 1ecfb7f..0effe70 100644 --- a/database/tables/instance.go +++ b/database/tables/instance.go @@ -4,4 +4,5 @@ type Instance struct { ID uint `gorm:"primaryKey"` Name string Description string + Apikey string } -- GitLab From c8574eb1f6fc9ac0ab4abab6e7f1946b6fffdfa0 Mon Sep 17 00:00:00 2001 From: Jonas Leder <jonas@jonasled.de> Date: Sun, 26 Jan 2025 16:41:41 +0100 Subject: [PATCH 05/11] set correct status code for updated --- server/createInstanceRoute.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/createInstanceRoute.go b/server/createInstanceRoute.go index baf290f..f71b7ae 100644 --- a/server/createInstanceRoute.go +++ b/server/createInstanceRoute.go @@ -65,7 +65,7 @@ func initCreateInstanceRoute() { c.JSON(http.StatusInternalServerError, gin.H{"error": "failed updating instance definition"}) log.Log.Error("Failed to query database, while trying to update instance: %v", err) } - c.JSON(http.StatusCreated, gin.H{"message": "Updated entry"}) + c.JSON(http.StatusOK, gin.H{"message": "Updated entry"}) } }) -- GitLab From 244e9e5ce7f80ce9303ccc84ec82cb787d6b5fa4 Mon Sep 17 00:00:00 2001 From: Jonas Leder <jonas@jonasled.de> Date: Sun, 26 Jan 2025 17:12:40 +0100 Subject: [PATCH 06/11] add API endpoint for data upload --- server/main.go | 1 + server/uploadDataRoute.go | 61 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 server/uploadDataRoute.go diff --git a/server/main.go b/server/main.go index be39cf1..5a3ada7 100644 --- a/server/main.go +++ b/server/main.go @@ -15,6 +15,7 @@ func Init() { initGeneralRoutes() initCreateInstanceRoute() + initUploadDataRouter() } func Run() { diff --git a/server/uploadDataRoute.go b/server/uploadDataRoute.go new file mode 100644 index 0000000..b9a7137 --- /dev/null +++ b/server/uploadDataRoute.go @@ -0,0 +1,61 @@ +package server + +import ( + "encoding/json" + "net/http" + + "github.com/gin-gonic/gin" + "jonasled.dev/jonasled/ems-esp-logger/database" + "jonasled.dev/jonasled/ems-esp-logger/database/tables" + "jonasled.dev/jonasled/ems-esp-logger/queue" + "jonasled.dev/jonasled/ems-esp-logger/types" +) + +func initUploadDataRouter() { + R.POST("/api/data", func(c *gin.Context) { + instanceHeader := c.GetHeader("X-Instance") + if instanceHeader == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "Missing X-Instance header"}) + return + } + var instance tables.Instance + err := database.Db.Where("name = ?", instanceHeader).First(&instance).Error + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "No matching instance found"}) + return + } + + authHeader := c.GetHeader("Authentication") + if authHeader != instance.Apikey { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"}) + return + } + contentType := c.GetHeader("Content-Type") + if contentType == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "Content-Type header missing"}) + return + } + if contentType != "application/json" { + c.JSON(http.StatusBadRequest, gin.H{"error": "expected application/json as content type"}) + return + } + + var uploadedTask types.Task + body, _ := c.GetRawData() + err = json.Unmarshal(body, &uploadedTask) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "failed decoding json", "message": err.Error()}) + return + } + if uploadedTask.Data == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "the data field cannot be empty"}) + return + } + if uploadedTask.Instance == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "the instance field cannot be empty"}) + return + } + queue.MainQueue.Enqueue(string(body), 0) + c.JSON(http.StatusOK, gin.H{"message": "Accepted new data"}) + }) +} -- GitLab From 71f683232fc88d7b676522baf835b5decb525a12 Mon Sep 17 00:00:00 2001 From: Jonas Leder <jonas@jonasled.de> Date: Sun, 26 Jan 2025 17:14:16 +0100 Subject: [PATCH 07/11] run message worker for local queue --- main.go | 1 + messageworker/client.go | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index bde7bd2..fa13f6a 100644 --- a/main.go +++ b/main.go @@ -37,6 +37,7 @@ func main() { } if os.Getenv("LOGGER_SERVER") == "true" { server.Init() + go messageworker.RunServer() server.Run() } else { log.Log.Fatal("Either LOGGER_CLIENT or LOGGER_SERVER has to be enabled in config") diff --git a/messageworker/client.go b/messageworker/client.go index 8ebe615..17b1c85 100644 --- a/messageworker/client.go +++ b/messageworker/client.go @@ -17,11 +17,15 @@ func RunClient() { if os.Getenv("CLIENT_USE_SERVER") != "true" { log.Log.Info("Intialized local client with direct datbase connection") - runLocalDbClient() + runDbClient() } } -func runLocalDbClient() { +func RunServer() { + runDbClient() +} + +func runDbClient() { for { taskData, taskId, err := queue.MainQueue.Dequeue() if err != nil { -- GitLab From 6f10e2d51ff88378459437f653803efd40bd08c3 Mon Sep 17 00:00:00 2001 From: Jonas Leder <jonas@jonasled.de> Date: Sun, 26 Jan 2025 17:25:25 +0100 Subject: [PATCH 08/11] implement HTTP upload for new log messages --- go.mod | 11 +++++---- go.sum | 24 +++++++++++-------- messageworker/client.go | 52 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index a268147..8de657f 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/eclipse/paho.mqtt.golang v1.5.0 github.com/gin-gonic/gin v1.10.0 github.com/glebarez/sqlite v1.11.0 + github.com/go-resty/resty/v2 v2.16.5 github.com/google/uuid v1.3.0 github.com/joho/godotenv v1.5.1 github.com/nekomeowww/gorm-logger-logrus v1.0.8 @@ -47,11 +48,11 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/crypto v0.25.0 // indirect - golang.org/x/net v0.27.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/libc v1.22.5 // indirect diff --git a/go.sum b/go.sum index f6dff72..38a6f3b 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM= +github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= @@ -106,19 +108,21 @@ github.com/wind-c/comqtt/v2 v2.6.0/go.mod h1:6O4VilBrTQ/cNIcmIgNdMLCK9DTiLRxkal0 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= -golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= diff --git a/messageworker/client.go b/messageworker/client.go index 17b1c85..4dfb814 100644 --- a/messageworker/client.go +++ b/messageworker/client.go @@ -5,6 +5,7 @@ import ( "os" "time" + "github.com/go-resty/resty/v2" "jonasled.dev/jonasled/ems-esp-logger/database" "jonasled.dev/jonasled/ems-esp-logger/database/tables" "jonasled.dev/jonasled/ems-esp-logger/helper" @@ -18,6 +19,9 @@ func RunClient() { log.Log.Info("Intialized local client with direct datbase connection") runDbClient() + } else { + log.Log.Info("Using remote server as data storage") + runHttpClient() } } @@ -25,6 +29,48 @@ func RunServer() { runDbClient() } +func runHttpClient() { + client := resty.New() + client.SetBaseURL(os.Getenv("API_ENDPOINT")) + client.SetHeader("Authentication", os.Getenv("INSTANCE_APIKEY")) + client.SetHeader("X-Instance", os.Getenv("INSTANCE_NAME")) + + for { + taskData, taskId, err := queue.MainQueue.Dequeue() + if err != nil { + time.Sleep(time.Second) + continue + } + log.Log.Debug("Received new task: ", taskData) + var task types.Task + err = json.Unmarshal([]byte(taskData), &task) + if err != nil { + log.Log.Error("Failed reading task data, discarding task: ", err.Error()) + continue + } + resp, err := client.R().SetBody(task).Post("/api/instance") + if err != nil { + log.Log.Error("Failed uploading task to server: ", err.Error()) + queue.MainQueue.Enqueue(taskData, 60) + queue.MainQueue.MarkDone(taskId) + continue + } + + if resp.StatusCode() != 200 { + log.Log.Error("Received invalid status code from server ", resp.StatusCode(), " response: ", string(resp.Body())) + queue.MainQueue.Enqueue(taskData, 60) + queue.MainQueue.MarkDone(taskId) + continue + } + + err = queue.MainQueue.MarkDone(taskId) + if err != nil { + log.Log.Error("Failed marking element in queue as done: ", err.Error()) + } + log.Log.Info("Successfully uploaded dataset to server") + } +} + func runDbClient() { for { taskData, taskId, err := queue.MainQueue.Dequeue() @@ -36,14 +82,17 @@ func runDbClient() { var task types.Task err = json.Unmarshal([]byte(taskData), &task) if err != nil { - log.Log.Error("Failed decoding task as JSON: ", err.Error()) + log.Log.Error("Failed reading task data, discarding task: ", err.Error()) + queue.MainQueue.MarkDone(taskId) continue } + var instance tables.Instance err = database.Db.Where("name = ?", task.Instance).First(&instance).Error if err != nil { log.Log.Error("Failed retreiving instance from database, pushing task back to queue: ", err.Error()) queue.MainQueue.Enqueue(taskData, 60) + queue.MainQueue.MarkDone(taskId) continue } @@ -52,6 +101,7 @@ func runDbClient() { if err != nil { log.Log.Error("Failed decoding boiler JSON: ", err.Error()) queue.MainQueue.Enqueue(taskData, 60) + queue.MainQueue.MarkDone(taskId) continue } valuesToInsert := []tables.Value{} -- GitLab From fb427276c3557f960958f1d9549e155ce8136650 Mon Sep 17 00:00:00 2001 From: Jonas Leder <jonas@jonasled.de> Date: Sun, 26 Jan 2025 17:29:27 +0100 Subject: [PATCH 09/11] fix using wrong api endpoint --- messageworker/client.go | 2 +- server/uploadDataRoute.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/messageworker/client.go b/messageworker/client.go index 4dfb814..b86403e 100644 --- a/messageworker/client.go +++ b/messageworker/client.go @@ -48,7 +48,7 @@ func runHttpClient() { log.Log.Error("Failed reading task data, discarding task: ", err.Error()) continue } - resp, err := client.R().SetBody(task).Post("/api/instance") + resp, err := client.R().SetBody(task).Post("/api/data") if err != nil { log.Log.Error("Failed uploading task to server: ", err.Error()) queue.MainQueue.Enqueue(taskData, 60) diff --git a/server/uploadDataRoute.go b/server/uploadDataRoute.go index b9a7137..6f4e9e4 100644 --- a/server/uploadDataRoute.go +++ b/server/uploadDataRoute.go @@ -7,6 +7,7 @@ import ( "github.com/gin-gonic/gin" "jonasled.dev/jonasled/ems-esp-logger/database" "jonasled.dev/jonasled/ems-esp-logger/database/tables" + "jonasled.dev/jonasled/ems-esp-logger/log" "jonasled.dev/jonasled/ems-esp-logger/queue" "jonasled.dev/jonasled/ems-esp-logger/types" ) @@ -26,6 +27,7 @@ func initUploadDataRouter() { } authHeader := c.GetHeader("Authentication") + log.Log.Debug("Provided apikey: ", authHeader, " expected apikey: ", instance.Apikey) if authHeader != instance.Apikey { c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"}) return -- GitLab From 1f421bff2e5e79509562abacb628c993f0bc2aa2 Mon Sep 17 00:00:00 2001 From: Jonas Leder <jonas@jonasled.de> Date: Sun, 26 Jan 2025 17:30:54 +0100 Subject: [PATCH 10/11] only establish db connection, when not running in client / server setup --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index fa13f6a..e61cbe0 100644 --- a/main.go +++ b/main.go @@ -18,7 +18,7 @@ func main() { log.Init() queue.Init() - if os.Getenv("OUTPUT_DATABSE") != "" { + if os.Getenv("OUTPUT_DATABSE") != "" && os.Getenv("CLIENT_USE_SERVER") != "true" { database.Init() database.CreateInstance() } -- GitLab From f88fca859cbe58090fdd7ea37d3201217db6e8e4 Mon Sep 17 00:00:00 2001 From: Jonas Leder <jonas@jonasled.de> Date: Sun, 26 Jan 2025 17:33:34 +0100 Subject: [PATCH 11/11] don't call CreateInstance when running as server --- main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index e61cbe0..05162d3 100644 --- a/main.go +++ b/main.go @@ -20,7 +20,9 @@ func main() { if os.Getenv("OUTPUT_DATABSE") != "" && os.Getenv("CLIENT_USE_SERVER") != "true" { database.Init() - database.CreateInstance() + if os.Getenv("LOGGER_SERVER") != "true" { + database.CreateInstance() + } } if os.Getenv("MQTT_SERVER_ENABLED") == "true" { -- GitLab