From d02de1ca4f1763b1e29d1000f67971847c1c8692 Mon Sep 17 00:00:00 2001
From: Jonas Leder <jonas@jonasled.de>
Date: Sun, 26 Jan 2025 16:14:50 +0100
Subject: [PATCH 01/11] configure client to use local database connection
---
main.go | 12 +++++++-----
messageworker/{main.go => client.go} | 17 +++++++++++++----
2 files changed, 20 insertions(+), 9 deletions(-)
rename messageworker/{main.go => client.go} (88%)
diff --git a/main.go b/main.go
index 1c83f09..c9c67ba 100644
--- a/main.go
+++ b/main.go
@@ -15,7 +15,6 @@ import (
func main() {
log.Init()
-
queue.Init()
if os.Getenv("OUTPUT_DATABSE") != "" {
@@ -27,10 +26,13 @@ func main() {
log.Log.Info("Starting embedded MQTT server")
mqttserver.Start()
}
- mqttclient.Init()
- go messageworker.Run()
- for {
- time.Sleep(time.Second) // reduce CPU usage by adding a short sleep here instead of a empty for loop
+
+ if os.Getenv("LOGGER_CLIENT") != "false" && os.Getenv("LOGGER_SERVER") != "true" {
+ mqttclient.Init()
+ go messageworker.RunClient()
+ for {
+ time.Sleep(time.Second) // reduce CPU usage by adding a short sleep here instead of a empty for loop
+ }
}
}
diff --git a/messageworker/main.go b/messageworker/client.go
similarity index 88%
rename from messageworker/main.go
rename to messageworker/client.go
index 3b0cebb..8ebe615 100644
--- a/messageworker/main.go
+++ b/messageworker/client.go
@@ -2,6 +2,7 @@ package messageworker
import (
"encoding/json"
+ "os"
"time"
"jonasled.dev/jonasled/ems-esp-logger/database"
@@ -12,7 +13,15 @@ import (
"jonasled.dev/jonasled/ems-esp-logger/types"
)
-func Run() {
+func RunClient() {
+ if os.Getenv("CLIENT_USE_SERVER") != "true" {
+
+ log.Log.Info("Intialized local client with direct datbase connection")
+ runLocalDbClient()
+ }
+}
+
+func runLocalDbClient() {
for {
taskData, taskId, err := queue.MainQueue.Dequeue()
if err != nil {
@@ -24,14 +33,14 @@ func Run() {
err = json.Unmarshal([]byte(taskData), &task)
if err != nil {
log.Log.Error("Failed decoding task as JSON: ", err.Error())
- return
+ continue
}
var instance tables.Instance
err = database.Db.Where("name = ?", task.Instance).First(&instance).Error
if err != nil {
log.Log.Error("Failed retreiving instance from database, pushing task back to queue: ", err.Error())
queue.MainQueue.Enqueue(taskData, 60)
- return
+ continue
}
var jsonData map[string]interface{}
@@ -39,7 +48,7 @@ func Run() {
if err != nil {
log.Log.Error("Failed decoding boiler JSON: ", err.Error())
queue.MainQueue.Enqueue(taskData, 60)
- return
+ continue
}
valuesToInsert := []tables.Value{}
for key, value := range jsonData {
--
GitLab
From 264a39a61b65af9d0ec9ae25b221c3b1a25d3363 Mon Sep 17 00:00:00 2001
From: Jonas Leder <jonas@jonasled.de>
Date: Sun, 26 Jan 2025 16:20:20 +0100
Subject: [PATCH 02/11] init webserver
---
.env.example | 4 ++-
go.mod | 23 ++++++++++++++
go.sum | 68 +++++++++++++++++++++++++++++++++++++++++
main.go | 7 +++++
server/generalRoutes.go | 16 ++++++++++
server/main.go | 22 +++++++++++++
6 files changed, 139 insertions(+), 1 deletion(-)
create mode 100644 server/generalRoutes.go
create mode 100644 server/main.go
diff --git a/.env.example b/.env.example
index 415106b..fb668f4 100644
--- a/.env.example
+++ b/.env.example
@@ -21,4 +21,6 @@ OUTPUT_DATABASE_TYPE=mysql
OUTPUT_DATABASE_EXECUTE_MIGRATIONS=true
INSTANCE_NAME=test
-INSTANCE_DESCRIPTION="My test instance"
\ No newline at end of file
+INSTANCE_DESCRIPTION="My test instance"
+
+GIN_MODE=release
\ No newline at end of file
diff --git a/go.mod b/go.mod
index b350c1c..a268147 100644
--- a/go.mod
+++ b/go.mod
@@ -4,11 +4,13 @@ go 1.23.4
require (
github.com/eclipse/paho.mqtt.golang v1.5.0
+ github.com/gin-gonic/gin v1.10.0
github.com/glebarez/sqlite v1.11.0
github.com/google/uuid v1.3.0
github.com/joho/godotenv v1.5.1
github.com/nekomeowww/gorm-logger-logrus v1.0.8
github.com/sirupsen/logrus v1.9.3
+ github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f
github.com/wind-c/comqtt/v2 v2.6.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gorm.io/driver/mysql v1.5.7
@@ -17,19 +19,40 @@ require (
require (
filippo.io/edwards25519 v1.1.0 // indirect
+ github.com/bytedance/sonic v1.11.6 // indirect
+ github.com/bytedance/sonic/loader v0.1.1 // indirect
+ github.com/cloudwego/base64x v0.1.4 // indirect
+ github.com/cloudwego/iasm v0.2.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
+ github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+ github.com/gin-contrib/sse v0.1.0 // indirect
github.com/glebarez/go-sqlite v1.21.2 // indirect
+ github.com/go-playground/locales v0.14.1 // indirect
+ github.com/go-playground/universal-translator v0.18.1 // indirect
+ github.com/go-playground/validator/v10 v10.20.0 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
+ github.com/goccy/go-json v0.10.2 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
+ github.com/json-iterator/go v1.1.12 // indirect
+ github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+ github.com/leodido/go-urn v1.4.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
+ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+ github.com/modern-go/reflect2 v1.0.2 // indirect
+ github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rs/xid v1.5.0 // indirect
+ github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+ github.com/ugorji/go/codec v1.2.12 // indirect
+ golang.org/x/arch v0.8.0 // indirect
+ golang.org/x/crypto v0.25.0 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/text v0.16.0 // indirect
+ google.golang.org/protobuf v1.34.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
modernc.org/libc v1.22.5 // indirect
modernc.org/mathutil v1.5.0 // indirect
diff --git a/go.sum b/go.sum
index 2713a23..f6dff72 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,13 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
+github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
+github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
+github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
+github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
+github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
+github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
+github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -7,13 +15,32 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o=
github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk=
+github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
+github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
+github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
+github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
+github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
+github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
+github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
+github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
+github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
+github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
@@ -28,10 +55,25 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
+github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
+github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
+github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/nekomeowww/gorm-logger-logrus v1.0.8 h1:/XdgNrSBSrTwiZ7fgf8Y9zW9juW1rVmRoY7OkX/tPnY=
github.com/nekomeowww/gorm-logger-logrus v1.0.8/go.mod h1:V266BSFFzJF1jnKWVAucFsh4U+ib3pEqJCzQY5zm7iA=
+github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
+github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
@@ -42,21 +84,45 @@ github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f h1:oqdnd6OGlOUu1InG37hWcCB3a+Jy3fwjylyVboaNMwY=
+github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f/go.mod h1:X3Dd1SB8Gt1V968NTzpKFjMM6O8ccta2NPC6MprOxZQ=
+github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
+github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
+github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
+github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/wind-c/comqtt/v2 v2.6.0 h1:huLdOwYDrwMTrEwH7+mSs1GftHZ/tDqJw8nOz3iX7kc=
github.com/wind-c/comqtt/v2 v2.6.0/go.mod h1:6O4VilBrTQ/cNIcmIgNdMLCK9DTiLRxkal00t0DLN64=
+golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
+golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
+golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
+golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
+google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
@@ -77,3 +143,5 @@ modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
+nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
diff --git a/main.go b/main.go
index c9c67ba..bde7bd2 100644
--- a/main.go
+++ b/main.go
@@ -11,6 +11,7 @@ import (
"jonasled.dev/jonasled/ems-esp-logger/mqttclient"
"jonasled.dev/jonasled/ems-esp-logger/mqttserver"
"jonasled.dev/jonasled/ems-esp-logger/queue"
+ "jonasled.dev/jonasled/ems-esp-logger/server"
)
func main() {
@@ -34,5 +35,11 @@ func main() {
time.Sleep(time.Second) // reduce CPU usage by adding a short sleep here instead of a empty for loop
}
}
+ if os.Getenv("LOGGER_SERVER") == "true" {
+ server.Init()
+ server.Run()
+ } else {
+ log.Log.Fatal("Either LOGGER_CLIENT or LOGGER_SERVER has to be enabled in config")
+ }
}
diff --git a/server/generalRoutes.go b/server/generalRoutes.go
new file mode 100644
index 0000000..990a8c2
--- /dev/null
+++ b/server/generalRoutes.go
@@ -0,0 +1,16 @@
+package server
+
+import (
+ "net/http"
+
+ "github.com/gin-gonic/gin"
+)
+
+func initGeneralRoutes() {
+ R.GET("/ready", func(ctx *gin.Context) {
+ ctx.String(http.StatusOK, "OK")
+ })
+ R.GET("/", func(ctx *gin.Context) {
+ ctx.String(http.StatusOK, "EMS-ESP server")
+ })
+}
diff --git a/server/main.go b/server/main.go
new file mode 100644
index 0000000..1c76636
--- /dev/null
+++ b/server/main.go
@@ -0,0 +1,22 @@
+package server
+
+import (
+ "github.com/gin-gonic/gin"
+ ginlogrus "github.com/toorop/gin-logrus"
+ "jonasled.dev/jonasled/ems-esp-logger/log"
+)
+
+var R *gin.Engine
+
+func Init() {
+ log.Log.Info("Initializing webserver")
+ R = gin.New()
+ R.Use(ginlogrus.Logger(log.Log), gin.Recovery())
+
+ initGeneralRoutes()
+}
+
+func Run() {
+ log.Log.Info("starting webserver")
+ R.Run()
+}
--
GitLab
From 77844083faeedadb6ca9dabad0ec7dcf6468070a Mon Sep 17 00:00:00 2001
From: Jonas Leder <jonas@jonasled.de>
Date: Sun, 26 Jan 2025 16:38:57 +0100
Subject: [PATCH 03/11] add HTTP endpoint for creating new instances
---
server/createInstanceRoute.go | 73 +++++++++++++++++++++++++++++++++++
server/main.go | 1 +
2 files changed, 74 insertions(+)
create mode 100644 server/createInstanceRoute.go
diff --git a/server/createInstanceRoute.go b/server/createInstanceRoute.go
new file mode 100644
index 0000000..baf290f
--- /dev/null
+++ b/server/createInstanceRoute.go
@@ -0,0 +1,73 @@
+package server
+
+import (
+ "encoding/json"
+ "net/http"
+ "os"
+
+ "github.com/gin-gonic/gin"
+ "gorm.io/gorm"
+ "jonasled.dev/jonasled/ems-esp-logger/database"
+ "jonasled.dev/jonasled/ems-esp-logger/database/tables"
+ "jonasled.dev/jonasled/ems-esp-logger/log"
+)
+
+func initCreateInstanceRoute() {
+ apiKey := os.Getenv("MASTER_APIKEY")
+ if apiKey == "" {
+ log.Log.Fatal("MASTER_APIKEY is not specified")
+ }
+
+ R.POST("/api/instance", func(c *gin.Context) {
+ authHeader := c.GetHeader("Authentication")
+ if authHeader != apiKey {
+ c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"})
+ return
+ }
+ contentType := c.GetHeader("Content-Type")
+ if contentType == "" {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Content-Type header missing"})
+ return
+ }
+ if contentType != "application/json" {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "expected application/json as content type"})
+ return
+ }
+ var instance tables.Instance
+ body, _ := c.GetRawData()
+ err := json.Unmarshal(body, &instance)
+ if err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "failed decoding json", "message": err.Error()})
+ return
+ }
+ if instance.Name == "" {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "the name cannot be empty"})
+ return
+ }
+
+ var existingInstance tables.Instance
+ err = database.Db.Where("name = ?", instance.Name).First(&existingInstance).Error
+
+ if err != nil {
+ if err == gorm.ErrRecordNotFound {
+ if err := database.Db.Create(&instance).Error; err != nil {
+ log.Log.Error("Failed to create instance: %v", err)
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "failed creating new instance definition"})
+ }
+ c.JSON(http.StatusCreated, gin.H{"message": "Created new entry"})
+ } else {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "failed creating new instance definition"})
+ log.Log.Error("Failed to query database, while trying to create new instance: %v", err)
+ }
+ } else {
+ existingInstance.Description = instance.Description
+ if err := database.Db.Save(&existingInstance).Error; err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "failed updating instance definition"})
+ log.Log.Error("Failed to query database, while trying to update instance: %v", err)
+ }
+ c.JSON(http.StatusCreated, gin.H{"message": "Updated entry"})
+ }
+
+ })
+
+}
diff --git a/server/main.go b/server/main.go
index 1c76636..be39cf1 100644
--- a/server/main.go
+++ b/server/main.go
@@ -14,6 +14,7 @@ func Init() {
R.Use(ginlogrus.Logger(log.Log), gin.Recovery())
initGeneralRoutes()
+ initCreateInstanceRoute()
}
func Run() {
--
GitLab
From b43c3567240c22e6767761e9b6a10a28df74d61c Mon Sep 17 00:00:00 2001
From: Jonas Leder <jonas@jonasled.de>
Date: Sun, 26 Jan 2025 16:40:25 +0100
Subject: [PATCH 04/11] extend table with apikey
---
database/tables/instance.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/database/tables/instance.go b/database/tables/instance.go
index 1ecfb7f..0effe70 100644
--- a/database/tables/instance.go
+++ b/database/tables/instance.go
@@ -4,4 +4,5 @@ type Instance struct {
ID uint `gorm:"primaryKey"`
Name string
Description string
+ Apikey string
}
--
GitLab
From c8574eb1f6fc9ac0ab4abab6e7f1946b6fffdfa0 Mon Sep 17 00:00:00 2001
From: Jonas Leder <jonas@jonasled.de>
Date: Sun, 26 Jan 2025 16:41:41 +0100
Subject: [PATCH 05/11] set correct status code for updated
---
server/createInstanceRoute.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/server/createInstanceRoute.go b/server/createInstanceRoute.go
index baf290f..f71b7ae 100644
--- a/server/createInstanceRoute.go
+++ b/server/createInstanceRoute.go
@@ -65,7 +65,7 @@ func initCreateInstanceRoute() {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed updating instance definition"})
log.Log.Error("Failed to query database, while trying to update instance: %v", err)
}
- c.JSON(http.StatusCreated, gin.H{"message": "Updated entry"})
+ c.JSON(http.StatusOK, gin.H{"message": "Updated entry"})
}
})
--
GitLab
From 244e9e5ce7f80ce9303ccc84ec82cb787d6b5fa4 Mon Sep 17 00:00:00 2001
From: Jonas Leder <jonas@jonasled.de>
Date: Sun, 26 Jan 2025 17:12:40 +0100
Subject: [PATCH 06/11] add API endpoint for data upload
---
server/main.go | 1 +
server/uploadDataRoute.go | 61 +++++++++++++++++++++++++++++++++++++++
2 files changed, 62 insertions(+)
create mode 100644 server/uploadDataRoute.go
diff --git a/server/main.go b/server/main.go
index be39cf1..5a3ada7 100644
--- a/server/main.go
+++ b/server/main.go
@@ -15,6 +15,7 @@ func Init() {
initGeneralRoutes()
initCreateInstanceRoute()
+ initUploadDataRouter()
}
func Run() {
diff --git a/server/uploadDataRoute.go b/server/uploadDataRoute.go
new file mode 100644
index 0000000..b9a7137
--- /dev/null
+++ b/server/uploadDataRoute.go
@@ -0,0 +1,61 @@
+package server
+
+import (
+ "encoding/json"
+ "net/http"
+
+ "github.com/gin-gonic/gin"
+ "jonasled.dev/jonasled/ems-esp-logger/database"
+ "jonasled.dev/jonasled/ems-esp-logger/database/tables"
+ "jonasled.dev/jonasled/ems-esp-logger/queue"
+ "jonasled.dev/jonasled/ems-esp-logger/types"
+)
+
+func initUploadDataRouter() {
+ R.POST("/api/data", func(c *gin.Context) {
+ instanceHeader := c.GetHeader("X-Instance")
+ if instanceHeader == "" {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Missing X-Instance header"})
+ return
+ }
+ var instance tables.Instance
+ err := database.Db.Where("name = ?", instanceHeader).First(&instance).Error
+ if err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "No matching instance found"})
+ return
+ }
+
+ authHeader := c.GetHeader("Authentication")
+ if authHeader != instance.Apikey {
+ c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"})
+ return
+ }
+ contentType := c.GetHeader("Content-Type")
+ if contentType == "" {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Content-Type header missing"})
+ return
+ }
+ if contentType != "application/json" {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "expected application/json as content type"})
+ return
+ }
+
+ var uploadedTask types.Task
+ body, _ := c.GetRawData()
+ err = json.Unmarshal(body, &uploadedTask)
+ if err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "failed decoding json", "message": err.Error()})
+ return
+ }
+ if uploadedTask.Data == "" {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "the data field cannot be empty"})
+ return
+ }
+ if uploadedTask.Instance == "" {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "the instance field cannot be empty"})
+ return
+ }
+ queue.MainQueue.Enqueue(string(body), 0)
+ c.JSON(http.StatusOK, gin.H{"message": "Accepted new data"})
+ })
+}
--
GitLab
From 71f683232fc88d7b676522baf835b5decb525a12 Mon Sep 17 00:00:00 2001
From: Jonas Leder <jonas@jonasled.de>
Date: Sun, 26 Jan 2025 17:14:16 +0100
Subject: [PATCH 07/11] run message worker for local queue
---
main.go | 1 +
messageworker/client.go | 8 ++++++--
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/main.go b/main.go
index bde7bd2..fa13f6a 100644
--- a/main.go
+++ b/main.go
@@ -37,6 +37,7 @@ func main() {
}
if os.Getenv("LOGGER_SERVER") == "true" {
server.Init()
+ go messageworker.RunServer()
server.Run()
} else {
log.Log.Fatal("Either LOGGER_CLIENT or LOGGER_SERVER has to be enabled in config")
diff --git a/messageworker/client.go b/messageworker/client.go
index 8ebe615..17b1c85 100644
--- a/messageworker/client.go
+++ b/messageworker/client.go
@@ -17,11 +17,15 @@ func RunClient() {
if os.Getenv("CLIENT_USE_SERVER") != "true" {
log.Log.Info("Intialized local client with direct datbase connection")
- runLocalDbClient()
+ runDbClient()
}
}
-func runLocalDbClient() {
+func RunServer() {
+ runDbClient()
+}
+
+func runDbClient() {
for {
taskData, taskId, err := queue.MainQueue.Dequeue()
if err != nil {
--
GitLab
From 6f10e2d51ff88378459437f653803efd40bd08c3 Mon Sep 17 00:00:00 2001
From: Jonas Leder <jonas@jonasled.de>
Date: Sun, 26 Jan 2025 17:25:25 +0100
Subject: [PATCH 08/11] implement HTTP upload for new log messages
---
go.mod | 11 +++++----
go.sum | 24 +++++++++++--------
messageworker/client.go | 52 ++++++++++++++++++++++++++++++++++++++++-
3 files changed, 71 insertions(+), 16 deletions(-)
diff --git a/go.mod b/go.mod
index a268147..8de657f 100644
--- a/go.mod
+++ b/go.mod
@@ -6,6 +6,7 @@ require (
github.com/eclipse/paho.mqtt.golang v1.5.0
github.com/gin-gonic/gin v1.10.0
github.com/glebarez/sqlite v1.11.0
+ github.com/go-resty/resty/v2 v2.16.5
github.com/google/uuid v1.3.0
github.com/joho/godotenv v1.5.1
github.com/nekomeowww/gorm-logger-logrus v1.0.8
@@ -47,11 +48,11 @@ require (
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
golang.org/x/arch v0.8.0 // indirect
- golang.org/x/crypto v0.25.0 // indirect
- golang.org/x/net v0.27.0 // indirect
- golang.org/x/sync v0.7.0 // indirect
- golang.org/x/sys v0.22.0 // indirect
- golang.org/x/text v0.16.0 // indirect
+ golang.org/x/crypto v0.31.0 // indirect
+ golang.org/x/net v0.33.0 // indirect
+ golang.org/x/sync v0.10.0 // indirect
+ golang.org/x/sys v0.28.0 // indirect
+ golang.org/x/text v0.21.0 // indirect
google.golang.org/protobuf v1.34.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
modernc.org/libc v1.22.5 // indirect
diff --git a/go.sum b/go.sum
index f6dff72..38a6f3b 100644
--- a/go.sum
+++ b/go.sum
@@ -33,6 +33,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
+github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
+github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
@@ -106,19 +108,21 @@ github.com/wind-c/comqtt/v2 v2.6.0/go.mod h1:6O4VilBrTQ/cNIcmIgNdMLCK9DTiLRxkal0
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
-golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
-golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
-golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
-golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
-golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
+golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
diff --git a/messageworker/client.go b/messageworker/client.go
index 17b1c85..4dfb814 100644
--- a/messageworker/client.go
+++ b/messageworker/client.go
@@ -5,6 +5,7 @@ import (
"os"
"time"
+ "github.com/go-resty/resty/v2"
"jonasled.dev/jonasled/ems-esp-logger/database"
"jonasled.dev/jonasled/ems-esp-logger/database/tables"
"jonasled.dev/jonasled/ems-esp-logger/helper"
@@ -18,6 +19,9 @@ func RunClient() {
log.Log.Info("Intialized local client with direct datbase connection")
runDbClient()
+ } else {
+ log.Log.Info("Using remote server as data storage")
+ runHttpClient()
}
}
@@ -25,6 +29,48 @@ func RunServer() {
runDbClient()
}
+func runHttpClient() {
+ client := resty.New()
+ client.SetBaseURL(os.Getenv("API_ENDPOINT"))
+ client.SetHeader("Authentication", os.Getenv("INSTANCE_APIKEY"))
+ client.SetHeader("X-Instance", os.Getenv("INSTANCE_NAME"))
+
+ for {
+ taskData, taskId, err := queue.MainQueue.Dequeue()
+ if err != nil {
+ time.Sleep(time.Second)
+ continue
+ }
+ log.Log.Debug("Received new task: ", taskData)
+ var task types.Task
+ err = json.Unmarshal([]byte(taskData), &task)
+ if err != nil {
+ log.Log.Error("Failed reading task data, discarding task: ", err.Error())
+ continue
+ }
+ resp, err := client.R().SetBody(task).Post("/api/instance")
+ if err != nil {
+ log.Log.Error("Failed uploading task to server: ", err.Error())
+ queue.MainQueue.Enqueue(taskData, 60)
+ queue.MainQueue.MarkDone(taskId)
+ continue
+ }
+
+ if resp.StatusCode() != 200 {
+ log.Log.Error("Received invalid status code from server ", resp.StatusCode(), " response: ", string(resp.Body()))
+ queue.MainQueue.Enqueue(taskData, 60)
+ queue.MainQueue.MarkDone(taskId)
+ continue
+ }
+
+ err = queue.MainQueue.MarkDone(taskId)
+ if err != nil {
+ log.Log.Error("Failed marking element in queue as done: ", err.Error())
+ }
+ log.Log.Info("Successfully uploaded dataset to server")
+ }
+}
+
func runDbClient() {
for {
taskData, taskId, err := queue.MainQueue.Dequeue()
@@ -36,14 +82,17 @@ func runDbClient() {
var task types.Task
err = json.Unmarshal([]byte(taskData), &task)
if err != nil {
- log.Log.Error("Failed decoding task as JSON: ", err.Error())
+ log.Log.Error("Failed reading task data, discarding task: ", err.Error())
+ queue.MainQueue.MarkDone(taskId)
continue
}
+
var instance tables.Instance
err = database.Db.Where("name = ?", task.Instance).First(&instance).Error
if err != nil {
log.Log.Error("Failed retreiving instance from database, pushing task back to queue: ", err.Error())
queue.MainQueue.Enqueue(taskData, 60)
+ queue.MainQueue.MarkDone(taskId)
continue
}
@@ -52,6 +101,7 @@ func runDbClient() {
if err != nil {
log.Log.Error("Failed decoding boiler JSON: ", err.Error())
queue.MainQueue.Enqueue(taskData, 60)
+ queue.MainQueue.MarkDone(taskId)
continue
}
valuesToInsert := []tables.Value{}
--
GitLab
From fb427276c3557f960958f1d9549e155ce8136650 Mon Sep 17 00:00:00 2001
From: Jonas Leder <jonas@jonasled.de>
Date: Sun, 26 Jan 2025 17:29:27 +0100
Subject: [PATCH 09/11] fix using wrong api endpoint
---
messageworker/client.go | 2 +-
server/uploadDataRoute.go | 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/messageworker/client.go b/messageworker/client.go
index 4dfb814..b86403e 100644
--- a/messageworker/client.go
+++ b/messageworker/client.go
@@ -48,7 +48,7 @@ func runHttpClient() {
log.Log.Error("Failed reading task data, discarding task: ", err.Error())
continue
}
- resp, err := client.R().SetBody(task).Post("/api/instance")
+ resp, err := client.R().SetBody(task).Post("/api/data")
if err != nil {
log.Log.Error("Failed uploading task to server: ", err.Error())
queue.MainQueue.Enqueue(taskData, 60)
diff --git a/server/uploadDataRoute.go b/server/uploadDataRoute.go
index b9a7137..6f4e9e4 100644
--- a/server/uploadDataRoute.go
+++ b/server/uploadDataRoute.go
@@ -7,6 +7,7 @@ import (
"github.com/gin-gonic/gin"
"jonasled.dev/jonasled/ems-esp-logger/database"
"jonasled.dev/jonasled/ems-esp-logger/database/tables"
+ "jonasled.dev/jonasled/ems-esp-logger/log"
"jonasled.dev/jonasled/ems-esp-logger/queue"
"jonasled.dev/jonasled/ems-esp-logger/types"
)
@@ -26,6 +27,7 @@ func initUploadDataRouter() {
}
authHeader := c.GetHeader("Authentication")
+ log.Log.Debug("Provided apikey: ", authHeader, " expected apikey: ", instance.Apikey)
if authHeader != instance.Apikey {
c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"})
return
--
GitLab
From 1f421bff2e5e79509562abacb628c993f0bc2aa2 Mon Sep 17 00:00:00 2001
From: Jonas Leder <jonas@jonasled.de>
Date: Sun, 26 Jan 2025 17:30:54 +0100
Subject: [PATCH 10/11] only establish db connection, when not running in
client / server setup
---
main.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/main.go b/main.go
index fa13f6a..e61cbe0 100644
--- a/main.go
+++ b/main.go
@@ -18,7 +18,7 @@ func main() {
log.Init()
queue.Init()
- if os.Getenv("OUTPUT_DATABSE") != "" {
+ if os.Getenv("OUTPUT_DATABSE") != "" && os.Getenv("CLIENT_USE_SERVER") != "true" {
database.Init()
database.CreateInstance()
}
--
GitLab
From f88fca859cbe58090fdd7ea37d3201217db6e8e4 Mon Sep 17 00:00:00 2001
From: Jonas Leder <jonas@jonasled.de>
Date: Sun, 26 Jan 2025 17:33:34 +0100
Subject: [PATCH 11/11] don't call CreateInstance when running as server
---
main.go | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/main.go b/main.go
index e61cbe0..05162d3 100644
--- a/main.go
+++ b/main.go
@@ -20,7 +20,9 @@ func main() {
if os.Getenv("OUTPUT_DATABSE") != "" && os.Getenv("CLIENT_USE_SERVER") != "true" {
database.Init()
- database.CreateInstance()
+ if os.Getenv("LOGGER_SERVER") != "true" {
+ database.CreateInstance()
+ }
}
if os.Getenv("MQTT_SERVER_ENABLED") == "true" {
--
GitLab