dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Housekeeper stats; Grafana dashboard

Browse Source
This commit is contained in:
ViacheslavKlimov 2024-02-22 13:58:15 +02:00
parent b72af4ead9
commit 7cc599b5b5
12 changed files with 588 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java
View File

@ -33,9 +33,9 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.queue.util.AfterStartUp;
import org.thingsboard.server.queue.util.DataDecodingEncodingService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor;
import org.thingsboard.server.service.housekeeper.stats.HousekeeperStatsService;
import javax.annotation.PreDestroy;
import java.util.List;
@ -59,27 +59,28 @@ public class DefaultHousekeeperService implements HousekeeperService {
private final TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> consumer;
private final TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> producer;
private final HousekeeperReprocessingService reprocessingService;
private final DataDecodingEncodingService dataDecodingEncodingService;
private final HousekeeperStatsService statsService;
private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer"));
private final ExecutorService executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-task-processor"));
@Value("${queue.core.housekeeper.poll-interval-ms:10000}")
@Value("${queue.core.housekeeper.task-processing-timeout-ms:120000}")
private int taskProcessingTimeout;
@Value("${queue.core.housekeeper.poll-interval-ms:500}")
private int pollInterval;
private int taskProcessingTimeout = 120;
private boolean stopped;
public DefaultHousekeeperService(HousekeeperReprocessingService reprocessingService,
TbCoreQueueFactory queueFactory,
TbQueueProducerProvider producerProvider,
DataDecodingEncodingService dataDecodingEncodingService,
HousekeeperStatsService statsService,
@Lazy List<HousekeeperTaskProcessor<?>> taskProcessors) {
this.consumer = queueFactory.createHousekeeperMsgConsumer();
this.producer = producerProvider.getHousekeeperMsgProducer();
this.reprocessingService = reprocessingService;
this.statsService = statsService;
this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p));
this.dataDecodingEncodingService = dataDecodingEncodingService;
}
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
@ -129,18 +130,23 @@ public class DefaultHousekeeperService implements HousekeeperService {
throw new IllegalArgumentException("Unsupported task type " + task.getTaskType());
}
if (log.isDebugEnabled()) {
log.debug("[{}][{}][{}] {} task {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(),
msg.getTask().getErrorsCount() == 0 ? "Processing" : "Reprocessing", task.getTaskType());
}
try {
Future<Object> future = executor.submit(() -> {
taskProcessor.process((T) task);
return null;
});
future.get(taskProcessingTimeout, TimeUnit.SECONDS);
future.get(taskProcessingTimeout, TimeUnit.MILLISECONDS);
statsService.reportProcessed(task, msg);
} catch (InterruptedException e) {
throw e;
} catch (Throwable e) {
Throwable error = e;
if (e instanceof ExecutionException) {
error = error.getCause();
error = e.getCause();
} else if (e instanceof TimeoutException) {
error = new TimeoutException("Timeout after " + taskProcessingTimeout + " seconds");
}
@ -148,6 +154,7 @@ public class DefaultHousekeeperService implements HousekeeperService {
task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(),
task.getTaskType(), msg.getTask().getAttempt(), task, error);
reprocessingService.submitForReprocessing(queueMsg, error);
statsService.reportFailure(task, msg);
}
}

33
application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java
View File

@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.StringUtils;
@ -28,20 +27,20 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import org.thingsboard.server.queue.TbQueueCallback;
import org.thingsboard.server.queue.TbQueueMsgMetadata;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.queue.util.TbCoreComponent;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@TbCoreComponent
@ -54,16 +53,19 @@ public class HousekeeperReprocessingService {
private final TbCoreQueueFactory queueFactory;
private final TbQueueProducerProvider producerProvider;
private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer"));
private static final int startDelay = 15; // fixme - to 5 minutes
private static final int reprocessingDelay = 30; // seconds
private static final int maxReprocessingAttempts = 5;
@Value("${queue.core.housekeeper.reprocessing-start-delay-sec:15}") // fixme: to 5 minutes
private int startDelay;
@Value("${queue.core.housekeeper.task-reprocessing-delay-sec:30}") // fixme: to 30 minutes or 1 hour
private int reprocessingDelay;
@Value("${queue.core.housekeeper.max-reprocessing-attempts:10}")
private int maxReprocessingAttempts;
@Value("${queue.core.housekeeper.poll-interval-ms:500}")
private int pollInterval;
private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer"));
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-scheduler"));
private boolean stopped;
// todo: stats
public HousekeeperReprocessingService(@Lazy DefaultHousekeeperService housekeeperService,
PartitionService partitionService, TbCoreQueueFactory queueFactory,
@ -74,7 +76,17 @@ public class HousekeeperReprocessingService {
this.producerProvider = producerProvider;
}
@Scheduled(initialDelay = startDelay, fixedDelay = reprocessingDelay, timeUnit = TimeUnit.SECONDS)
@PostConstruct
private void init() {
scheduler.scheduleWithFixedDelay(() -> {
try {
startReprocessing();
} catch (Throwable e) {
log.error("Unexpected error during reprocessing", e);
}
}, startDelay, reprocessingDelay, TimeUnit.SECONDS);
}
public void startReprocessing() {
if (!partitionService.isMyPartition(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID)) {
return;
@ -146,6 +158,7 @@ public class HousekeeperReprocessingService {
@PreDestroy
private void stop() {
stopped = true;
scheduler.shutdownNow();
consumerExecutor.shutdownNow();
}

2
application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java
View File

@ -36,7 +36,7 @@ public class AlarmsUnassignTaskProcessor implements HousekeeperTaskProcessor<Ala
@Override
public void process(AlarmsUnassignHousekeeperTask task) throws Exception {
List<AlarmId> alarms = alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs());
log.trace("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), alarms.size());
log.debug("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), alarms.size());
}
@Override

2
application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java
View File

@ -32,7 +32,7 @@ public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor
@Override
public void process(HousekeeperTask task) throws Exception {
int deletedCount = attributesService.removeAllByEntityId(task.getTenantId(), task.getEntityId());
log.trace("[{}][{}][{}] Deleted {} attributes", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount);
log.debug("[{}][{}][{}] Deleted {} attributes", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount);
}
@Override

3
application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java
View File

@ -24,16 +24,17 @@ import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType;
@Component
@RequiredArgsConstructor
public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor<HousekeeperTask> {
private final EventService eventService;
@Override
public void process(HousekeeperTask task) throws Exception {
eventService.removeEvents(task.getTenantId(), task.getEntityId(), null, 0L, System.currentTimeMillis());
throw new RuntimeException("test error");
}
@Override
public HousekeeperTaskType getTaskType() {
return HousekeeperTaskType.DELETE_EVENTS;
}
}

2
application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java
View File

@ -40,7 +40,7 @@ public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor<
DeleteTsKvQuery deleteQuery = new BaseDeleteTsKvQuery(key, 0, System.currentTimeMillis(), false, true);
timeseriesService.remove(task.getTenantId(), task.getEntityId(), List.of(deleteQuery)).get();
}
log.trace("[{}][{}][{}] Deleted {} telemetry keys", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), keys.size());
log.debug("[{}][{}][{}] Deleted {} telemetry keys", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), keys.size());
}
@Override

117
application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java Normal file
View File

@ -0,0 +1,117 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.housekeeper.stats;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.stats.DefaultCounter;
import org.thingsboard.server.common.stats.StatsCounter;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.stats.StatsType;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import java.util.ArrayList;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Service
@Slf4j
public class HousekeeperStatsService {
private final Map<HousekeeperTaskType, HousekeeperStats> stats = new EnumMap<>(HousekeeperTaskType.class);
public HousekeeperStatsService(StatsFactory statsFactory) {
for (HousekeeperTaskType taskType : HousekeeperTaskType.values()) {
stats.put(taskType, new HousekeeperStats(taskType, statsFactory));
}
}
@Scheduled(initialDelay = 60, fixedDelay = 60, timeUnit = TimeUnit.SECONDS)
private void reportStats() {
String statsStr = stats.values().stream().map(stats -> {
String countersStr = stats.getCounters().stream()
.filter(counter -> counter.get() > 0)
.map(counter -> counter.getName() + " = " + counter.get())
.collect(Collectors.joining(", "));
if (countersStr.isEmpty()) {
return null;
} else {
return stats.getTaskType() + " {" + countersStr + "}";
}
}).filter(Objects::nonNull).collect(Collectors.joining("; "));
if (!statsStr.isEmpty()) {
stats.values().forEach(HousekeeperStats::reset);
log.info("Housekeeper stats: {}", statsStr);
}
}
public void reportProcessed(HousekeeperTask task, ToHousekeeperServiceMsg msg) {
HousekeeperStats stats = this.stats.get(task.getTaskType());
if (msg.getTask().getErrorsCount() == 0) {
stats.getProcessedCounter().increment();
} else {
stats.getReprocessedCounter().increment();
}
}
public void reportFailure(HousekeeperTask task, ToHousekeeperServiceMsg msg) {
HousekeeperStats stats = this.stats.get(task.getTaskType());
if (msg.getTask().getErrorsCount() == 0) {
stats.getFailedProcessingCounter().increment();
} else {
stats.getFailedReprocessingCounter().increment();
}
}
@Getter
static class HousekeeperStats {
private final HousekeeperTaskType taskType;
private final List<StatsCounter> counters = new ArrayList<>();
private final StatsCounter processedCounter;
private final StatsCounter failedProcessingCounter;
private final StatsCounter reprocessedCounter;
private final StatsCounter failedReprocessingCounter;
public HousekeeperStats(HousekeeperTaskType taskType, StatsFactory statsFactory) {
this.taskType = taskType;
this.processedCounter = register("processed", statsFactory);
this.failedProcessingCounter = register("failedProcessing", statsFactory);
this.reprocessedCounter = register("reprocessed", statsFactory);
this.failedReprocessingCounter = register("failedReprocessing", statsFactory);
}
private StatsCounter register(String statsName, StatsFactory statsFactory) {
StatsCounter counter = statsFactory.createStatsCounter(StatsType.HOUSEKEEPER.getName(), statsName, Map.of("taskType", taskType.name()));
counters.add(counter);
return counter;
}
public void reset() {
counters.forEach(DefaultCounter::clear);
}
}
}

11
application/src/main/resources/thingsboard.yml
View File

@ -1585,9 +1585,14 @@ queue:
# Statistics printing interval for Core microservices
print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}"
housekeeper:
topic: "tb_housekeeper"
reprocessing-topic: "tb_housekeeper.reprocessing"
poll-interval-ms: "1000"
topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}"
reprocessing-topic: "${TB_HOUSEKEEPER_REPROCESSING_TOPIC:tb_housekeeper.reprocessing}"
poll-interval-ms: "${TB_HOUSEKEEPER_POLL_INTERVAL_MS:500}"
task-processing-timeout-ms: "${TB_HOUSEKEEPER_TASK_PROCESSING_TIMEOUT_MS:120000}"
reprocessing-start-delay-sec: "${TB_HOUSEKEEPER_REPROCESSING_START_DELAY_SEC:15}" # fixme: to 5 minutes
task-reprocessing-delay-sec: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_SEC:30}" # fixme: to 30 minutes or 1 hour
max-reprocessing-attempts: "${TB_HOUSEKEEPER_MAX_REPROCESSING_ATTEMPTS:30}"
vc:
# Default topic name for Kafka, RabbitMQ, etc.
topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}"

20
common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java
View File

@ -25,6 +25,10 @@ import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.StringUtils;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
@Service
@ -62,10 +66,24 @@ public class DefaultStatsFactory implements StatsFactory {
@Override
public StatsCounter createStatsCounter(String key, String statsName) {
return createStatsCounter(key, statsName, Collections.emptyMap());
}
@Override
public StatsCounter createStatsCounter(String key, String statsName, Map<String, String> tags) {
String[] tagsArr = new String[]{STATS_NAME_TAG, statsName};
if (!tags.isEmpty()) {
List<String> tagsList = new ArrayList<>(List.of(tagsArr));
tags.forEach((name, value) -> {
tagsList.add(name);
tagsList.add(value);
});
tagsArr = tagsList.toArray(String[]::new);
}
return new StatsCounter(
new AtomicInteger(0),
metricsEnabled ?
meterRegistry.counter(key, STATS_NAME_TAG, statsName)
meterRegistry.counter(key, tagsArr)
: STUB_COUNTER,
statsName
);

5
common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java
View File

@ -17,9 +17,14 @@ package org.thingsboard.server.common.stats;
import io.micrometer.core.instrument.Timer;
import java.util.Map;
public interface StatsFactory {
StatsCounter createStatsCounter(String key, String statsName);
StatsCounter createStatsCounter(String key, String statsName, Map<String, String> tags);
DefaultCounter createDefaultCounter(String key, String... tags);
<T extends Number> T createGauge(String key, T number, String... tags);

7
common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java
View File

@ -16,7 +16,12 @@
package org.thingsboard.server.common.stats;
public enum StatsType {
RULE_ENGINE("ruleEngine"), CORE("core"), TRANSPORT("transport"), JS_INVOKE("jsInvoke"), RATE_EXECUTOR("rateExecutor");
RULE_ENGINE("ruleEngine"),
CORE("core"),
TRANSPORT("transport"),
JS_INVOKE("jsInvoke"),
RATE_EXECUTOR("rateExecutor"),
HOUSEKEEPER("housekeeper");
private String name;

390
docker/monitoring/grafana/provisioning/dashboards/housekeeper.json Normal file
View File

@ -0,0 +1,390 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 2,
"links": [],
"liveNow": false,
"panels": [
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "9BonzvTSz"
},
"exemplar": true,
"expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"processed\"}[1m]))",
"interval": "",
"legendFormat": "{{taskType}}",
"refId": "A"
}
],
"title": "Processed",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 0
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "9BonzvTSz"
},
"exemplar": true,
"expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"failedProcessing\"}[1m]))",
"interval": "",
"legendFormat": "{{taskType}}",
"refId": "A"
}
],
"title": "Processing failures",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 10
},
"id": 5,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "9BonzvTSz"
},
"exemplar": true,
"expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"reprocessed\"}[1m]))",
"interval": "",
"legendFormat": "{{taskType}}",
"refId": "A"
}
],
"title": "Reprocessed",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 10
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "9BonzvTSz"
},
"exemplar": true,
"expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"failedReprocessing\"}[1m]))",
"interval": "",
"legendFormat": "{{taskType}}",
"refId": "A"
}
],
"title": "Reprocessing failures",
"type": "timeseries"
}
],
"refresh": "10s",
"schemaVersion": 35,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-15m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Housekeeper",
"uid": "JFJb7voIz",
"version": 5,
"weekStart": ""
}