From 74a4019dc57b9db5fa0b111004282b74b0305f3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Z=C3=ADpek?= Date: Mon, 1 Apr 2024 10:33:20 +0200 Subject: [PATCH] Data retention --- Dockerfile | 1 + .../dashboard/contexts/DashboardContext.tsx | 22 ++++++--- client/src/utils/hooks/useQueryString.ts | 7 ++- server/.env.example | 2 + server/app/cleaner.go | 15 +++++- server/config/config.go | 48 ++++++++++++------- server/services/alerts_evaluator_service.go | 17 +++---- server/services/config.go | 1 + server/services/sensor_values_service.go | 14 ++++++ 9 files changed, 93 insertions(+), 34 deletions(-) create mode 100644 server/services/config.go diff --git a/Dockerfile b/Dockerfile index 4b1a6a8..b87e77f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,6 +56,7 @@ ENV AUTH_USERNAME admin ENV AUTH_PASSWORD password ENV AUTH_KEY password ENV AUTH_ENABLED true +ENV DATA_RETENTION_IN_DAYS 0 EXPOSE ${PORT} VOLUME [ "/data" ] diff --git a/client/src/pages/dashboard/contexts/DashboardContext.tsx b/client/src/pages/dashboard/contexts/DashboardContext.tsx index 76b9ddf..e9245b5 100644 --- a/client/src/pages/dashboard/contexts/DashboardContext.tsx +++ b/client/src/pages/dashboard/contexts/DashboardContext.tsx @@ -47,13 +47,19 @@ export const DashboardContextProvider = ({ }) => { const viewport = useViewportSize() - const [dashboardId, setDashboardId] = useState(-1) + const { getValues: getQuery, setValues: setQuery } = useQueryString() + + const [dashboardId, setDashboardId] = useState(() => { + const query = getQuery() + const queryDashboardId = +(query['dashboard'] ?? '') + + return !isNaN(queryDashboardId) ? queryDashboardId : -1 + }) + const isDashboardSelected = !isNaN(dashboardId) && dashboardId >= 0 const dashboards = useQuery(['/dashboards'], getDashboards) - const { getValues: getQuery, setValues: setQuery } = useQueryString() - const dashboard = useQuery( ['/dashboards', dashboardId], () => getDashboard(dashboardId), @@ -105,9 +111,9 @@ export const DashboardContextProvider = ({ const [filter, setFilter] = useState(() => { const query = getQuery() - const queryFilter = query.get('interval') - const queryFilterFrom = query.get('from') - const queryFilterTo = query.get('to') + const queryFilter = query['interval'] + const queryFilterFrom = query['from'] + const queryFilterTo = query['to'] const presetInterval = queryFilter && @@ -129,13 +135,15 @@ export const DashboardContextProvider = ({ useEffect(() => { setQuery({ + ...getQuery(), + dashboard: dashboardId.toString(), interval: filter.interval, ...(filter.interval === 'custom' && { from: filter.customFrom.toISOString(), to: filter.customTo.toISOString(), }), }) - }, [filter]) + }, [filter, dashboardId]) const verticalMode = viewport.width < 800 diff --git a/client/src/utils/hooks/useQueryString.ts b/client/src/utils/hooks/useQueryString.ts index 293bb7b..efd5534 100644 --- a/client/src/utils/hooks/useQueryString.ts +++ b/client/src/utils/hooks/useQueryString.ts @@ -21,8 +21,13 @@ export const useQueryString = () => { const getValues = useCallback(() => { const [, searchParams] = valuesRef.current + const result = {} as Record - return searchParams + searchParams.forEach((value, key) => { + result[key] = value + }) + + return result }, []) const setValues = useCallback((values: Record) => { diff --git a/server/.env.example b/server/.env.example index 633b85e..dc103c6 100644 --- a/server/.env.example +++ b/server/.env.example @@ -6,3 +6,5 @@ AUTH_ENABLED=true AUTH_USERNAME=admin AUTH_PASSWORD=password AUTH_KEY=password +# How long should the data be stored +DATA_RETENTION_IN_DAYS=365 diff --git a/server/app/cleaner.go b/server/app/cleaner.go index 2e48142..96ba1cc 100644 --- a/server/app/cleaner.go +++ b/server/app/cleaner.go @@ -1,6 +1,7 @@ package app import ( + "log" "time" ) @@ -9,7 +10,19 @@ func (s *Server) StartCleaner() { go func() { for { - s.Services.Sessions.Cleanup() + err := s.Services.Sessions.Cleanup() + + if err != nil { + log.Println("Error cleaning up sessions:", err) + } + + if s.Config.DataRetentionInDays > 0 { + err := s.Services.SensorValues.Cleanup(s.Config.DataRetentionInDays) + if err != nil { + log.Println("Error cleaning up sensor values:", err) + } + } + <-ticker.C } }() diff --git a/server/config/config.go b/server/config/config.go index e24c48a..ace8cee 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -1,37 +1,51 @@ package config import ( + "fmt" "os" "strconv" ) type Config struct { - Mode string - DatabaseUrl string - Port int - Ip string - AuthEnabled bool - AuthUsername string - AuthPassword string - AuthKey string + Mode string + DatabaseUrl string + Port int + Ip string + AuthEnabled bool + AuthUsername string + AuthPassword string + AuthKey string + DataRetentionInDays int64 } func LoadConfig() *Config { port, err := strconv.Atoi(os.Getenv("PORT")) if err != nil { - panic(err) + panic(fmt.Errorf("PORT must be an integer: %v", err)) + } + + dataRetentionInDaysStr := os.Getenv("DATA_RETENTION_IN_DAYS") + dataRetentionInDays := 0 + + if dataRetentionInDaysStr != "" { + dataRetentionInDays, err = strconv.Atoi(dataRetentionInDaysStr) + + if err != nil { + panic(fmt.Errorf("DATA_RETENTION_IN_DAYS must be an integer: %v", err)) + } } config := Config{ - Mode: os.Getenv("GIN_MODE"), - DatabaseUrl: os.Getenv("DATABASE_URL"), - Port: port, - Ip: os.Getenv("BIND_IP"), - AuthEnabled: os.Getenv("AUTH_ENABLED") != "false", - AuthUsername: os.Getenv("AUTH_USERNAME"), - AuthPassword: os.Getenv("AUTH_PASSWORD"), - AuthKey: os.Getenv("AUTH_KEY"), + Mode: os.Getenv("GIN_MODE"), + DatabaseUrl: os.Getenv("DATABASE_URL"), + Port: port, + Ip: os.Getenv("BIND_IP"), + AuthEnabled: os.Getenv("AUTH_ENABLED") != "false", + AuthUsername: os.Getenv("AUTH_USERNAME"), + AuthPassword: os.Getenv("AUTH_PASSWORD"), + AuthKey: os.Getenv("AUTH_KEY"), + DataRetentionInDays: int64(dataRetentionInDays), } // TODO: Crash when any auth* param is empty diff --git a/server/services/alerts_evaluator_service.go b/server/services/alerts_evaluator_service.go index 06d9ccb..5b26384 100644 --- a/server/services/alerts_evaluator_service.go +++ b/server/services/alerts_evaluator_service.go @@ -84,10 +84,9 @@ func (s *AlertsEvaluatorService) EvaluateAlert(alert *models.AlertItem) error { Value: condition["value"].(float64), } - value, err := s.ctx.Services.SensorValues.GetLatest(sensorValueCondition.SensorId, time.Now().Unix()) - lastValue = value.Value sensorId = int64(sensorValueCondition.SensorId) + value, err := s.ctx.Services.SensorValues.GetLatest(sensorValueCondition.SensorId, time.Now().Unix()) if err != nil { if err == sql.ErrNoRows { lastValue = float64(0) @@ -96,6 +95,7 @@ func (s *AlertsEvaluatorService) EvaluateAlert(alert *models.AlertItem) error { return fmt.Errorf("error getting sensor value: %v", err) } } else { + lastValue = value.Value conditionMet = evaluateSensorValueCondition(sensorValueCondition, value) } @@ -110,22 +110,23 @@ func (s *AlertsEvaluatorService) EvaluateAlert(alert *models.AlertItem) error { ValueUnit: condition["valueUnit"].(string), } - value, err := s.ctx.Services.SensorValues.GetLatest(sensorLastContactCondition.SensorId, time.Now().Unix()) - lastValue = float64(value.Timestamp) sensorId = sensorLastContactCondition.SensorId + value, err := s.ctx.Services.SensorValues.GetLatest(sensorLastContactCondition.SensorId, time.Now().Unix()) + if err != nil { if err == sql.ErrNoRows { lastValue = float64(0) } else { return fmt.Errorf("error getting sensor last contact value: %v", err) } + } else { + lastValue = float64(value.Timestamp) + conditionInSec := int64(sensorLastContactCondition.Value * unitToSeconds(sensorLastContactCondition.ValueUnit)) + conditionMet = time.Now().Unix()-value.Timestamp > conditionInSec + } - conditionInSec := int64(sensorLastContactCondition.Value * unitToSeconds(sensorLastContactCondition.ValueUnit)) - - conditionMet = time.Now().Unix()-value.Timestamp > conditionInSec - break } diff --git a/server/services/config.go b/server/services/config.go new file mode 100644 index 0000000..5e568ea --- /dev/null +++ b/server/services/config.go @@ -0,0 +1 @@ +package services diff --git a/server/services/sensor_values_service.go b/server/services/sensor_values_service.go index b284a6d..72bd764 100644 --- a/server/services/sensor_values_service.go +++ b/server/services/sensor_values_service.go @@ -2,6 +2,7 @@ package services import ( "database/sql" + "fmt" "time" ) @@ -41,6 +42,19 @@ func (s *SensorValuesService) GetLatest(sensorId int64, to int64) (*sensorValue, return &value, nil } +func (s *SensorValuesService) Cleanup(retentionInDays int64) error { + if retentionInDays <= 0 { + return fmt.Errorf("retentionInDays must be greater than 0") + } + + _, err := s.ctx.DB.Exec("DELETE FROM sensor_values WHERE timestamp < ?", time.Now().Unix()-(retentionInDays*24*60*60)) + if err != nil { + return err + } + + return nil +} + func (s *SensorValuesService) getValueListQuery(sensorId int64, from int64, to int64, divide int64) (*sql.Rows, error) { if divide == 1 { return s.ctx.DB.Query("SELECT timestamp, value FROM sensor_values WHERE sensor_id = ? AND timestamp > ? AND timestamp < ? ORDER BY timestamp ASC", sensorId, from, to)