From 7cc599b5b59ab0f65b78049e7c5b2db729618c9a Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 22 Feb 2024 13:58:15 +0200 Subject: [PATCH] Housekeeper stats; Grafana dashboard --- .../DefaultHousekeeperService.java | 23 +- .../HousekeeperReprocessingService.java | 33 +- .../AlarmsUnassignTaskProcessor.java | 2 +- .../AttributesDeletionTaskProcessor.java | 2 +- .../EventsDeletionTaskProcessor.java | 3 +- .../TelemetryDeletionTaskProcessor.java | 2 +- .../stats/HousekeeperStatsService.java | 117 ++++++ .../src/main/resources/thingsboard.yml | 11 +- .../common/stats/DefaultStatsFactory.java | 20 +- .../server/common/stats/StatsFactory.java | 5 + .../server/common/stats/StatsType.java | 7 +- .../provisioning/dashboards/housekeeper.json | 390 ++++++++++++++++++ 12 files changed, 588 insertions(+), 27 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java create mode 100644 docker/monitoring/grafana/provisioning/dashboards/housekeeper.json diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index 76bf40fc36..a324f279fb 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -33,9 +33,9 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.AfterStartUp; -import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor; +import org.thingsboard.server.service.housekeeper.stats.HousekeeperStatsService; import javax.annotation.PreDestroy; import java.util.List; @@ -59,27 +59,28 @@ public class DefaultHousekeeperService implements HousekeeperService { private final TbQueueConsumer> consumer; private final TbQueueProducer> producer; private final HousekeeperReprocessingService reprocessingService; - private final DataDecodingEncodingService dataDecodingEncodingService; + private final HousekeeperStatsService statsService; private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer")); private final ExecutorService executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-task-processor")); - @Value("${queue.core.housekeeper.poll-interval-ms:10000}") + @Value("${queue.core.housekeeper.task-processing-timeout-ms:120000}") + private int taskProcessingTimeout; + @Value("${queue.core.housekeeper.poll-interval-ms:500}") private int pollInterval; - private int taskProcessingTimeout = 120; private boolean stopped; public DefaultHousekeeperService(HousekeeperReprocessingService reprocessingService, TbCoreQueueFactory queueFactory, TbQueueProducerProvider producerProvider, - DataDecodingEncodingService dataDecodingEncodingService, + HousekeeperStatsService statsService, @Lazy List> taskProcessors) { this.consumer = queueFactory.createHousekeeperMsgConsumer(); this.producer = producerProvider.getHousekeeperMsgProducer(); this.reprocessingService = reprocessingService; + this.statsService = statsService; this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p)); - this.dataDecodingEncodingService = dataDecodingEncodingService; } @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) @@ -129,18 +130,23 @@ public class DefaultHousekeeperService implements HousekeeperService { throw new IllegalArgumentException("Unsupported task type " + task.getTaskType()); } + if (log.isDebugEnabled()) { + log.debug("[{}][{}][{}] {} task {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), + msg.getTask().getErrorsCount() == 0 ? "Processing" : "Reprocessing", task.getTaskType()); + } try { Future future = executor.submit(() -> { taskProcessor.process((T) task); return null; }); - future.get(taskProcessingTimeout, TimeUnit.SECONDS); + future.get(taskProcessingTimeout, TimeUnit.MILLISECONDS); + statsService.reportProcessed(task, msg); } catch (InterruptedException e) { throw e; } catch (Throwable e) { Throwable error = e; if (e instanceof ExecutionException) { - error = error.getCause(); + error = e.getCause(); } else if (e instanceof TimeoutException) { error = new TimeoutException("Timeout after " + taskProcessingTimeout + " seconds"); } @@ -148,6 +154,7 @@ public class DefaultHousekeeperService implements HousekeeperService { task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType(), msg.getTask().getAttempt(), task, error); reprocessingService.submitForReprocessing(queueMsg, error); + statsService.reportFailure(task, msg); } } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index 44b2728089..de88b2dcea 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.StringUtils; @@ -28,20 +27,20 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; +import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @TbCoreComponent @@ -54,16 +53,19 @@ public class HousekeeperReprocessingService { private final TbCoreQueueFactory queueFactory; private final TbQueueProducerProvider producerProvider; - private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer")); - - private static final int startDelay = 15; // fixme - to 5 minutes - private static final int reprocessingDelay = 30; // seconds - private static final int maxReprocessingAttempts = 5; + @Value("${queue.core.housekeeper.reprocessing-start-delay-sec:15}") // fixme: to 5 minutes + private int startDelay; + @Value("${queue.core.housekeeper.task-reprocessing-delay-sec:30}") // fixme: to 30 minutes or 1 hour + private int reprocessingDelay; + @Value("${queue.core.housekeeper.max-reprocessing-attempts:10}") + private int maxReprocessingAttempts; @Value("${queue.core.housekeeper.poll-interval-ms:500}") private int pollInterval; + private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer")); + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-scheduler")); + private boolean stopped; - // todo: stats public HousekeeperReprocessingService(@Lazy DefaultHousekeeperService housekeeperService, PartitionService partitionService, TbCoreQueueFactory queueFactory, @@ -74,7 +76,17 @@ public class HousekeeperReprocessingService { this.producerProvider = producerProvider; } - @Scheduled(initialDelay = startDelay, fixedDelay = reprocessingDelay, timeUnit = TimeUnit.SECONDS) + @PostConstruct + private void init() { + scheduler.scheduleWithFixedDelay(() -> { + try { + startReprocessing(); + } catch (Throwable e) { + log.error("Unexpected error during reprocessing", e); + } + }, startDelay, reprocessingDelay, TimeUnit.SECONDS); + } + public void startReprocessing() { if (!partitionService.isMyPartition(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID)) { return; @@ -146,6 +158,7 @@ public class HousekeeperReprocessingService { @PreDestroy private void stop() { stopped = true; + scheduler.shutdownNow(); consumerExecutor.shutdownNow(); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java index 247466af65..68fd05deec 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java @@ -36,7 +36,7 @@ public class AlarmsUnassignTaskProcessor implements HousekeeperTaskProcessor alarms = alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs()); - log.trace("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), alarms.size()); + log.debug("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), alarms.size()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java index e3bac35e9b..0467e50aeb 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java @@ -32,7 +32,7 @@ public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor @Override public void process(HousekeeperTask task) throws Exception { int deletedCount = attributesService.removeAllByEntityId(task.getTenantId(), task.getEntityId()); - log.trace("[{}][{}][{}] Deleted {} attributes", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); + log.debug("[{}][{}][{}] Deleted {} attributes", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java index b95d7fc1e3..481f9f1bbf 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java @@ -24,16 +24,17 @@ import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; @Component @RequiredArgsConstructor public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor { + private final EventService eventService; @Override public void process(HousekeeperTask task) throws Exception { eventService.removeEvents(task.getTenantId(), task.getEntityId(), null, 0L, System.currentTimeMillis()); - throw new RuntimeException("test error"); } @Override public HousekeeperTaskType getTaskType() { return HousekeeperTaskType.DELETE_EVENTS; } + } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java index 7159c43f46..e7da740261 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -40,7 +40,7 @@ public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor< DeleteTsKvQuery deleteQuery = new BaseDeleteTsKvQuery(key, 0, System.currentTimeMillis(), false, true); timeseriesService.remove(task.getTenantId(), task.getEntityId(), List.of(deleteQuery)).get(); } - log.trace("[{}][{}][{}] Deleted {} telemetry keys", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), keys.size()); + log.debug("[{}][{}][{}] Deleted {} telemetry keys", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), keys.size()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java new file mode 100644 index 0000000000..4fcc13837d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java @@ -0,0 +1,117 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.stats; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.stats.DefaultCounter; +import org.thingsboard.server.common.stats.StatsCounter; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.stats.StatsType; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; + +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class HousekeeperStatsService { + + private final Map stats = new EnumMap<>(HousekeeperTaskType.class); + + public HousekeeperStatsService(StatsFactory statsFactory) { + for (HousekeeperTaskType taskType : HousekeeperTaskType.values()) { + stats.put(taskType, new HousekeeperStats(taskType, statsFactory)); + } + } + + @Scheduled(initialDelay = 60, fixedDelay = 60, timeUnit = TimeUnit.SECONDS) + private void reportStats() { + String statsStr = stats.values().stream().map(stats -> { + String countersStr = stats.getCounters().stream() + .filter(counter -> counter.get() > 0) + .map(counter -> counter.getName() + " = " + counter.get()) + .collect(Collectors.joining(", ")); + if (countersStr.isEmpty()) { + return null; + } else { + return stats.getTaskType() + " {" + countersStr + "}"; + } + }).filter(Objects::nonNull).collect(Collectors.joining("; ")); + + if (!statsStr.isEmpty()) { + stats.values().forEach(HousekeeperStats::reset); + log.info("Housekeeper stats: {}", statsStr); + } + } + + public void reportProcessed(HousekeeperTask task, ToHousekeeperServiceMsg msg) { + HousekeeperStats stats = this.stats.get(task.getTaskType()); + if (msg.getTask().getErrorsCount() == 0) { + stats.getProcessedCounter().increment(); + } else { + stats.getReprocessedCounter().increment(); + } + } + + public void reportFailure(HousekeeperTask task, ToHousekeeperServiceMsg msg) { + HousekeeperStats stats = this.stats.get(task.getTaskType()); + if (msg.getTask().getErrorsCount() == 0) { + stats.getFailedProcessingCounter().increment(); + } else { + stats.getFailedReprocessingCounter().increment(); + } + } + + @Getter + static class HousekeeperStats { + private final HousekeeperTaskType taskType; + private final List counters = new ArrayList<>(); + + private final StatsCounter processedCounter; + private final StatsCounter failedProcessingCounter; + private final StatsCounter reprocessedCounter; + private final StatsCounter failedReprocessingCounter; + + public HousekeeperStats(HousekeeperTaskType taskType, StatsFactory statsFactory) { + this.taskType = taskType; + this.processedCounter = register("processed", statsFactory); + this.failedProcessingCounter = register("failedProcessing", statsFactory); + this.reprocessedCounter = register("reprocessed", statsFactory); + this.failedReprocessingCounter = register("failedReprocessing", statsFactory); + } + + private StatsCounter register(String statsName, StatsFactory statsFactory) { + StatsCounter counter = statsFactory.createStatsCounter(StatsType.HOUSEKEEPER.getName(), statsName, Map.of("taskType", taskType.name())); + counters.add(counter); + return counter; + } + + public void reset() { + counters.forEach(DefaultCounter::clear); + } + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index f59e3c86b7..4ced0655f0 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1585,9 +1585,14 @@ queue: # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" housekeeper: - topic: "tb_housekeeper" - reprocessing-topic: "tb_housekeeper.reprocessing" - poll-interval-ms: "1000" + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" + reprocessing-topic: "${TB_HOUSEKEEPER_REPROCESSING_TOPIC:tb_housekeeper.reprocessing}" + poll-interval-ms: "${TB_HOUSEKEEPER_POLL_INTERVAL_MS:500}" + task-processing-timeout-ms: "${TB_HOUSEKEEPER_TASK_PROCESSING_TIMEOUT_MS:120000}" + reprocessing-start-delay-sec: "${TB_HOUSEKEEPER_REPROCESSING_START_DELAY_SEC:15}" # fixme: to 5 minutes + task-reprocessing-delay-sec: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_SEC:30}" # fixme: to 30 minutes or 1 hour + max-reprocessing-attempts: "${TB_HOUSEKEEPER_MAX_REPROCESSING_ATTEMPTS:30}" + vc: # Default topic name for Kafka, RabbitMQ, etc. topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java index a6ec01d089..459f2e0a68 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java @@ -25,6 +25,10 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.StringUtils; import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @Service @@ -62,10 +66,24 @@ public class DefaultStatsFactory implements StatsFactory { @Override public StatsCounter createStatsCounter(String key, String statsName) { + return createStatsCounter(key, statsName, Collections.emptyMap()); + } + + @Override + public StatsCounter createStatsCounter(String key, String statsName, Map tags) { + String[] tagsArr = new String[]{STATS_NAME_TAG, statsName}; + if (!tags.isEmpty()) { + List tagsList = new ArrayList<>(List.of(tagsArr)); + tags.forEach((name, value) -> { + tagsList.add(name); + tagsList.add(value); + }); + tagsArr = tagsList.toArray(String[]::new); + } return new StatsCounter( new AtomicInteger(0), metricsEnabled ? - meterRegistry.counter(key, STATS_NAME_TAG, statsName) + meterRegistry.counter(key, tagsArr) : STUB_COUNTER, statsName ); diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java index 8e6989016b..90a4191ee2 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java @@ -17,9 +17,14 @@ package org.thingsboard.server.common.stats; import io.micrometer.core.instrument.Timer; +import java.util.Map; + public interface StatsFactory { + StatsCounter createStatsCounter(String key, String statsName); + StatsCounter createStatsCounter(String key, String statsName, Map tags); + DefaultCounter createDefaultCounter(String key, String... tags); T createGauge(String key, T number, String... tags); diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java index 318f0418de..6c854ac8d6 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java @@ -16,7 +16,12 @@ package org.thingsboard.server.common.stats; public enum StatsType { - RULE_ENGINE("ruleEngine"), CORE("core"), TRANSPORT("transport"), JS_INVOKE("jsInvoke"), RATE_EXECUTOR("rateExecutor"); + RULE_ENGINE("ruleEngine"), + CORE("core"), + TRANSPORT("transport"), + JS_INVOKE("jsInvoke"), + RATE_EXECUTOR("rateExecutor"), + HOUSEKEEPER("housekeeper"); private String name; diff --git a/docker/monitoring/grafana/provisioning/dashboards/housekeeper.json b/docker/monitoring/grafana/provisioning/dashboards/housekeeper.json new file mode 100644 index 0000000000..8c68cc9428 --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/housekeeper.json @@ -0,0 +1,390 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "9BonzvTSz" + }, + "exemplar": true, + "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"processed\"}[1m]))", + "interval": "", + "legendFormat": "{{taskType}}", + "refId": "A" + } + ], + "title": "Processed", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "9BonzvTSz" + }, + "exemplar": true, + "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"failedProcessing\"}[1m]))", + "interval": "", + "legendFormat": "{{taskType}}", + "refId": "A" + } + ], + "title": "Processing failures", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "9BonzvTSz" + }, + "exemplar": true, + "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"reprocessed\"}[1m]))", + "interval": "", + "legendFormat": "{{taskType}}", + "refId": "A" + } + ], + "title": "Reprocessed", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "9BonzvTSz" + }, + "exemplar": true, + "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"failedReprocessing\"}[1m]))", + "interval": "", + "legendFormat": "{{taskType}}", + "refId": "A" + } + ], + "title": "Reprocessing failures", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Housekeeper", + "uid": "JFJb7voIz", + "version": 5, + "weekStart": "" +} \ No newline at end of file