diff --git a/.env.example b/.env.example
index b7fe1737c86f1a88c714c5fe23c3e24da0dceaef..415106b81edc18a09d5c460675b2a004ff4ec75b 100644
--- a/.env.example
+++ b/.env.example
@@ -1,10 +1,24 @@
-MQTT_SERVER_ENABLED=false
+MQTT_SERVER_ENABLED=true
MQTT_LISTEN=0.0.0.0:1883 # only required for MQTT server
-MQTT_HOST=1.2.3.4
+MQTT_HOST=127.0.0.1
MQTT_PORT=1883
MQTT_USERNAME=mqtt
-MQTT_PASSWORD=password
+MQTT_PASSWORD=mqtt
MQTT_TOPIC_BOILER=ems-esp/boiler_data
+
LOG_LEVEL=INFO
+
OUTPUT_FILE_NAME="boiler.csv"
-OUTPUT_FILE_NAME_RAW="boiler.txt"
\ No newline at end of file
+OUTPUT_FILE_NAME_RAW="boiler.txt"
+
+QUEUE_STATE_FOLDER="./queues"
+
+OUTPUT_DATABSE=ems_esp_logger
+OUTPUT_DATABASE_USER=root
+OUTPUT_DATABASE_PASSWORD=password
+OUTPUT_DATABASE_HOST=localhost
+OUTPUT_DATABASE_TYPE=mysql
+OUTPUT_DATABASE_EXECUTE_MIGRATIONS=true
+
+INSTANCE_NAME=test
+INSTANCE_DESCRIPTION="My test instance"
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 0400075052efd799519ba894abd902d40b3cec96..68f09f4e59c1daf80444590c9f0eab52ab3c6eec 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
+!.gitkeep
.env
boiler.csv
-boiler.txt
\ No newline at end of file
+boiler.txt
+queues/
\ No newline at end of file
diff --git a/database/createInstance.go b/database/createInstance.go
new file mode 100644
index 0000000000000000000000000000000000000000..509ac931526a115686d4f03e7a2f51de10d82491
--- /dev/null
+++ b/database/createInstance.go
@@ -0,0 +1,42 @@
+package database
+
+import (
+ "os"
+
+ "gorm.io/gorm"
+ "jonasled.dev/jonasled/ems-esp-logger/database/tables"
+ "jonasled.dev/jonasled/ems-esp-logger/log"
+)
+
+func CreateInstance() {
+ instanceName := os.Getenv("INSTANCE_NAME")
+ instanceDescription := os.Getenv("INSTANCE_DESCRIPTION")
+ if instanceName == "" || instanceDescription == "" {
+ log.Log.Fatal("INSTANCE_NAME and INSTANCE_DESCRIPTION must be set")
+ }
+ var instance tables.Instance
+ err := Db.Where("name = ?", instanceName).First(&instance).Error
+
+ if err != nil {
+ if err == gorm.ErrRecordNotFound {
+ // Create a new instance if it doesn't exist
+ instance = tables.Instance{
+ Name: instanceName,
+ Description: instanceDescription,
+ }
+ if err := Db.Create(&instance).Error; err != nil {
+ log.Log.Fatalf("Failed to create instance: %v", err)
+ }
+ log.Log.Infof("Created new instance: %+v\n", instance)
+ } else {
+ log.Log.Fatalf("Failed to query database: %v", err)
+ }
+ } else {
+ // Update the description if the instance exists
+ instance.Description = instanceDescription
+ if err := Db.Save(&instance).Error; err != nil {
+ log.Log.Fatalf("Failed to update instance description: %v", err)
+ }
+ log.Log.Infof("Updated instance description: %+v\n", instance)
+ }
+}
diff --git a/database/getOrCreateValueType.go b/database/getOrCreateValueType.go
new file mode 100644
index 0000000000000000000000000000000000000000..9e51c1a67ae9fca45bb9af7dc4c35220a243b4f7
--- /dev/null
+++ b/database/getOrCreateValueType.go
@@ -0,0 +1,30 @@
+package database
+
+import (
+ "gorm.io/gorm"
+ "jonasled.dev/jonasled/ems-esp-logger/database/tables"
+ "jonasled.dev/jonasled/ems-esp-logger/log"
+)
+
+func GetOrCreateValueType(name string) tables.ValueType {
+ var valueType tables.ValueType
+ err := Db.Where("name = ?", name).First(&valueType).Error
+
+ if err != nil {
+ if err == gorm.ErrRecordNotFound {
+ // Create a new instance if it doesn't exist
+ valueType = tables.ValueType{
+ Name: name,
+ }
+ if err := Db.Create(&valueType).Error; err != nil {
+ log.Log.Fatalf("Failed to create value type: %v", err)
+ }
+ log.Log.Infof("Created new value type: %+v\n", valueType)
+ } else {
+ log.Log.Errorf("Failed to query database: %v", err)
+ }
+ }
+
+ return valueType
+
+}
diff --git a/database/main.go b/database/main.go
new file mode 100644
index 0000000000000000000000000000000000000000..93b69595482730be07f903adb3dcfc808e7a9480
--- /dev/null
+++ b/database/main.go
@@ -0,0 +1,55 @@
+package database
+
+import (
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/glebarez/sqlite"
+ gormloggerlogrus "github.com/nekomeowww/gorm-logger-logrus"
+ "github.com/sirupsen/logrus"
+ "gorm.io/driver/mysql"
+ "gorm.io/gorm"
+ "gorm.io/gorm/logger"
+ "jonasled.dev/jonasled/ems-esp-logger/database/tables"
+ "jonasled.dev/jonasled/ems-esp-logger/log"
+)
+
+var Db *gorm.DB
+
+func Init() {
+ var err error
+ var dbdriver gorm.Dialector
+ switch os.Getenv("OUTPUT_DATABASE_TYPE") {
+ case "sqlite":
+ dbdriver = sqlite.Open(os.Getenv("OUTPUT_DATABSE"))
+ case "mysql":
+ dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local",
+ os.Getenv("OUTPUT_DATABASE_USER"),
+ os.Getenv("OUTPUT_DATABASE_PASSWORD"),
+ os.Getenv("OUTPUT_DATABASE_HOST"),
+ os.Getenv("OUTPUT_DATABSE"),
+ )
+ dbdriver = mysql.Open(dsn)
+ default:
+ log.Log.Fatal("Please set OUTPUT_DATABASE_TYPE to a valid value")
+ }
+ Db, err = gorm.Open(dbdriver, &gorm.Config{
+ Logger: gormloggerlogrus.New(gormloggerlogrus.Options{
+ Logger: logrus.NewEntry(log.Log),
+ LogLevel: logger.Error,
+ IgnoreRecordNotFoundError: false,
+ SlowThreshold: time.Millisecond * 200,
+ FileWithLineNumField: "file",
+ }),
+ })
+
+ if err != nil {
+ log.Log.Fatal("Failed initializing database: ", err.Error())
+ }
+
+ if os.Getenv("OUTPUT_DATABASE_EXECUTE_MIGRATIONS") != "false" {
+ log.Log.Info("Executing database migrations")
+ Db.AutoMigrate(&tables.Instance{}, &tables.ValueType{}, &tables.Value{})
+ }
+}
diff --git a/database/tables/instance.go b/database/tables/instance.go
new file mode 100644
index 0000000000000000000000000000000000000000..1ecfb7f8f909cb621f3da15755fece2a450311f3
--- /dev/null
+++ b/database/tables/instance.go
@@ -0,0 +1,7 @@
+package tables
+
+type Instance struct {
+ ID uint `gorm:"primaryKey"`
+ Name string
+ Description string
+}
diff --git a/database/tables/valueTypes.go b/database/tables/valueTypes.go
new file mode 100644
index 0000000000000000000000000000000000000000..5e3c14f998d271d0051c92a24ca0f0425146c5ae
--- /dev/null
+++ b/database/tables/valueTypes.go
@@ -0,0 +1,7 @@
+package tables
+
+type ValueType struct {
+ ID uint `gorm:"primaryKey"`
+ Name string
+ Description string
+}
diff --git a/database/tables/values.go b/database/tables/values.go
new file mode 100644
index 0000000000000000000000000000000000000000..51edd6edd8932c3e76878dd423ca40ba27113836
--- /dev/null
+++ b/database/tables/values.go
@@ -0,0 +1,13 @@
+package tables
+
+import "time"
+
+type Value struct {
+ ID uint `gorm:"primaryKey"`
+ Date time.Time
+ InstanceID int
+ Instance Instance
+ ValueTypeID int
+ ValueType ValueType
+ Value string
+}
diff --git a/go.mod b/go.mod
index 97a60efbe837b4d6502015f98416ee8f90f056f6..f1f7efa8326aeda20193ef01614ad175bf2918b4 100644
--- a/go.mod
+++ b/go.mod
@@ -4,17 +4,35 @@ go 1.23.4
require (
github.com/eclipse/paho.mqtt.golang v1.5.0
+ github.com/glebarez/sqlite v1.11.0
github.com/joho/godotenv v1.5.1
+ github.com/nekomeowww/gorm-logger-logrus v1.0.8
github.com/sirupsen/logrus v1.9.3
github.com/wind-c/comqtt/v2 v2.6.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1
+ gorm.io/driver/mysql v1.5.7
+ gorm.io/gorm v1.25.12
)
require (
+ filippo.io/edwards25519 v1.1.0 // indirect
+ github.com/dustin/go-humanize v1.0.1 // indirect
+ github.com/glebarez/go-sqlite v1.21.2 // indirect
+ github.com/go-sql-driver/mysql v1.8.1 // indirect
+ github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
+ github.com/jinzhu/inflection v1.0.0 // indirect
+ github.com/jinzhu/now v1.1.5 // indirect
+ github.com/mattn/go-isatty v0.0.20 // indirect
+ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rs/xid v1.5.0 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.22.0 // indirect
+ golang.org/x/text v0.16.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
+ modernc.org/libc v1.22.5 // indirect
+ modernc.org/mathutil v1.5.0 // indirect
+ modernc.org/memory v1.5.0 // indirect
+ modernc.org/sqlite v1.23.1 // indirect
)
diff --git a/go.sum b/go.sum
index 04b0286ab491326e40257bfeb9b34ed69e0ec4bc..2713a237a4659fe702ee56a9d20f7b7dc0f5f425 100644
--- a/go.sum
+++ b/go.sum
@@ -1,16 +1,42 @@
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o=
github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk=
+github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
+github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
+github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
+github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
+github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/nekomeowww/gorm-logger-logrus v1.0.8 h1:/XdgNrSBSrTwiZ7fgf8Y9zW9juW1rVmRoY7OkX/tPnY=
+github.com/nekomeowww/gorm-logger-logrus v1.0.8/go.mod h1:V266BSFFzJF1jnKWVAucFsh4U+ib3pEqJCzQY5zm7iA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
@@ -26,8 +52,11 @@ golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
@@ -35,3 +64,16 @@ gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYs
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
+gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
+gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
+gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
+modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
+modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
+modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
+modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
+modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
+modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
+modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
diff --git a/main.go b/main.go
index 26cb826c56625dd59de61e6af4bec586071a480b..1c83f0939155683b2d5c3cda3abd891478598577 100644
--- a/main.go
+++ b/main.go
@@ -5,20 +5,32 @@ import (
"time"
_ "github.com/joho/godotenv/autoload"
+ "jonasled.dev/jonasled/ems-esp-logger/database"
"jonasled.dev/jonasled/ems-esp-logger/log"
+ "jonasled.dev/jonasled/ems-esp-logger/messageworker"
"jonasled.dev/jonasled/ems-esp-logger/mqttclient"
"jonasled.dev/jonasled/ems-esp-logger/mqttserver"
+ "jonasled.dev/jonasled/ems-esp-logger/queue"
)
func main() {
log.Init()
+
+ queue.Init()
+
+ if os.Getenv("OUTPUT_DATABSE") != "" {
+ database.Init()
+ database.CreateInstance()
+ }
+
if os.Getenv("MQTT_SERVER_ENABLED") == "true" {
log.Log.Info("Starting embedded MQTT server")
mqttserver.Start()
}
mqttclient.Init()
+ go messageworker.Run()
for {
- time.Sleep(time.Second)
+ time.Sleep(time.Second) // reduce CPU usage by adding a short sleep here instead of a empty for loop
}
}
diff --git a/messageworker/main.go b/messageworker/main.go
new file mode 100644
index 0000000000000000000000000000000000000000..99167c82601442d32c2db7c4986b42a5de2b9ff6
--- /dev/null
+++ b/messageworker/main.go
@@ -0,0 +1,56 @@
+package messageworker
+
+import (
+ "encoding/json"
+ "time"
+
+ "jonasled.dev/jonasled/ems-esp-logger/database"
+ "jonasled.dev/jonasled/ems-esp-logger/database/tables"
+ "jonasled.dev/jonasled/ems-esp-logger/helper"
+ "jonasled.dev/jonasled/ems-esp-logger/log"
+ "jonasled.dev/jonasled/ems-esp-logger/queue"
+ "jonasled.dev/jonasled/ems-esp-logger/types"
+)
+
+func Run() {
+ for {
+ taskData, err := queue.MainQueue.Dequeue()
+ if err != nil {
+ time.Sleep(time.Second)
+ continue
+ }
+ log.Log.Debug("Received new task: ", taskData)
+ var task types.Task
+ err = json.Unmarshal([]byte(taskData), &task)
+ if err != nil {
+ log.Log.Error("Failed decoding task as JSON: ", err.Error())
+ continue
+ }
+ var instance tables.Instance
+ err = database.Db.Where("name = ?", task.Instance).First(&instance).Error
+ if err != nil {
+ log.Log.Error("Failed retreiving instance from database, pushing task back to queue: ", err.Error())
+ queue.MainQueue.Enqueue(taskData, 60)
+ continue
+ }
+
+ var jsonData map[string]interface{}
+ err = json.Unmarshal([]byte(task.Data), &jsonData)
+ if err != nil {
+ log.Log.Error("Failed decoding boiler JSON: ", err.Error())
+ queue.MainQueue.Enqueue(taskData, 60)
+ continue
+ }
+ for key, value := range jsonData {
+ valueType := database.GetOrCreateValueType(key)
+ dbValue := tables.Value{
+ Date: task.Date,
+ ValueType: valueType,
+ Value: helper.AnyToString(value),
+ Instance: instance,
+ }
+ database.Db.Create(&dbValue)
+ }
+ log.Log.Info("Stored boiler data in database")
+ }
+}
diff --git a/mqttclient/main.go b/mqttclient/main.go
index f459ffc630cd6d842c82ff9cda190264e1efd6f8..b349ef54911931b3440d377ca8c31acd233b9fcc 100644
--- a/mqttclient/main.go
+++ b/mqttclient/main.go
@@ -10,6 +10,8 @@ import (
"jonasled.dev/jonasled/ems-esp-logger/csv"
"jonasled.dev/jonasled/ems-esp-logger/helper"
"jonasled.dev/jonasled/ems-esp-logger/log"
+ "jonasled.dev/jonasled/ems-esp-logger/queue"
+ "jonasled.dev/jonasled/ems-esp-logger/types"
)
var messagePubHandler mqtt.MessageHandler = func(client mqtt.Client, msg mqtt.Message) {
@@ -20,9 +22,22 @@ var messagePubHandlerBoiler mqtt.MessageHandler = func(client mqtt.Client, msg m
log.Log.Debugf("Received message for boiler: %s from topic: %s\n", msg.Payload(), msg.Topic())
log.Log.Info("Received new boiler data")
csv.JsonToCsv(string(msg.Payload()))
+
if os.Getenv("OUTPUT_FILE_NAME_RAW") != "" {
dumpRawData(string(msg.Payload()), os.Getenv("OUTPUT_FILE_NAME_RAW"))
}
+ if os.Getenv("OUTPUT_DATABSE") != "" {
+ task := types.Task{
+ Date: time.Now().UTC(),
+ Data: string(msg.Payload()),
+ Instance: os.Getenv("INSTANCE_NAME"),
+ }
+ taskData, err := json.Marshal(task)
+ if err != nil {
+ log.Log.Error("Failed encoding new data task as JSON: ", err.Error())
+ }
+ queue.MainQueue.Enqueue(string(taskData), 0)
+ }
}
@@ -53,10 +68,12 @@ func Init() {
}
func dumpRawData(data string, filename string) {
+
var jsonData map[string]interface{}
err := json.Unmarshal([]byte(data), &jsonData)
if err != nil {
- panic(err)
+ log.Log.Error(err)
+ return
}
currentTime := time.Now().Local().Format("2006-01-02 15:04:05")
@@ -64,12 +81,14 @@ func dumpRawData(data string, filename string) {
updatedData, err := json.Marshal(jsonData)
if err != nil {
- panic(err)
+ log.Log.Error(err)
+ return
}
file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
- panic(err)
+ log.Log.Error(err)
+ return
}
defer file.Close()
diff --git a/queue/main.go b/queue/main.go
new file mode 100644
index 0000000000000000000000000000000000000000..c802f1d84b737d12e97c6709bb1676310afe012a
--- /dev/null
+++ b/queue/main.go
@@ -0,0 +1,36 @@
+package queue
+
+import (
+ "fmt"
+ "os"
+ "os/signal"
+ "syscall"
+
+ "github.com/wind-c/comqtt/v2/cluster/log"
+)
+
+var MainQueue Queue
+
+func Init() {
+ queue, err := NewQueue(os.Getenv("QUEUE_STATE_FOLDER") + "/main.json")
+ if err != nil {
+ log.Fatal("Error initializing main queue:", err)
+ }
+
+ // Set up signal handling for clean exit
+ exitChan := make(chan os.Signal, 1)
+ signal.Notify(exitChan, os.Interrupt, syscall.SIGTERM)
+
+ MainQueue = *queue
+ go func() {
+ <-exitChan
+ fmt.Println("\nSaving main queue state before exiting...")
+ fmt.Println("\nCurrent queue size: ", MainQueue.GetCurrentSize())
+ if err := MainQueue.Save(); err != nil {
+ fmt.Println("Error saving main queue state:", err)
+ } else {
+ fmt.Println("Main queue state saved successfully.")
+ }
+ os.Exit(0)
+ }()
+}
diff --git a/queue/queue.go b/queue/queue.go
new file mode 100644
index 0000000000000000000000000000000000000000..8f00ac087da1abc1b1bd64a17e0e4e9e26a15a46
--- /dev/null
+++ b/queue/queue.go
@@ -0,0 +1,82 @@
+package queue
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "time"
+
+ "jonasled.dev/jonasled/ems-esp-logger/log"
+)
+
+type QueueElement struct {
+ Value string `json:"value"`
+ EnqueueTime time.Time `json:"enqueue_time"`
+ MinDuration int `json:"min_duration"` // Minimum duration in seconds
+}
+
+type Queue struct {
+ Elements []QueueElement
+ FilePath string
+}
+
+func NewQueue(filePath string) (*Queue, error) {
+ q := &Queue{
+ Elements: make([]QueueElement, 0),
+ FilePath: filePath,
+ }
+
+ // Load state from file
+ if _, err := os.Stat(filePath); err == nil {
+ data, err := os.ReadFile(filePath)
+ if err != nil {
+ return nil, fmt.Errorf("error reading state file: %w", err)
+ }
+ if err := json.Unmarshal(data, &q.Elements); err != nil {
+ return nil, fmt.Errorf("error parsing state file: %w", err)
+ }
+ log.Log.Info("Initialized queue, current size: ", q.GetCurrentSize())
+ }
+ return q, nil
+}
+
+func (q *Queue) Enqueue(element string, minDuration int) {
+ q.Elements = append(q.Elements, QueueElement{
+ Value: element,
+ EnqueueTime: time.Now(),
+ MinDuration: minDuration,
+ })
+}
+
+func (q *Queue) Dequeue() (string, error) {
+ if len(q.Elements) == 0 {
+ return "", fmt.Errorf("queue is empty")
+ }
+
+ currentTime := time.Now()
+ for i, elem := range q.Elements {
+ // Check if the element has satisfied its minimum duration in the queue
+ if currentTime.Sub(elem.EnqueueTime).Seconds() >= float64(elem.MinDuration) {
+ // Remove the element from the queue
+ q.Elements = append(q.Elements[:i], q.Elements[i+1:]...)
+ return elem.Value, nil
+ }
+ }
+
+ return "", fmt.Errorf("no elements are ready for dequeuing")
+}
+
+func (q *Queue) GetCurrentSize() int {
+ return len(q.Elements)
+}
+
+func (q *Queue) Save() error {
+ data, err := json.Marshal(q.Elements)
+ if err != nil {
+ return fmt.Errorf("error serializing state: %w", err)
+ }
+ if err := os.WriteFile(q.FilePath, data, 0644); err != nil {
+ return fmt.Errorf("error writing state to file: %w", err)
+ }
+ return nil
+}
diff --git a/types/task.go b/types/task.go
new file mode 100644
index 0000000000000000000000000000000000000000..44e1d04775e6e9984e28c3a3f85f9c3d97d86132
--- /dev/null
+++ b/types/task.go
@@ -0,0 +1,9 @@
+package types
+
+import "time"
+
+type Task struct {
+ Date time.Time
+ Instance string
+ Data string
+}