Refactoring for latencies tracking
This commit is contained in:
parent
333ab064cc
commit
1b9dfa17c2
@ -25,4 +25,8 @@ public class Latencies {
|
|||||||
return String.format("%sRequest", key);
|
return String.format("%sRequest", key);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static String wsUpdate(String key) {
|
||||||
|
return String.format("%sWsUpdate", key);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -15,53 +15,15 @@
|
|||||||
*/
|
*/
|
||||||
package org.thingsboard.monitoring.data;
|
package org.thingsboard.monitoring.data;
|
||||||
|
|
||||||
import com.google.common.util.concurrent.AtomicDouble;
|
import lombok.Data;
|
||||||
import lombok.RequiredArgsConstructor;
|
|
||||||
|
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
@Data(staticConstructor = "of")
|
||||||
|
|
||||||
@RequiredArgsConstructor
|
|
||||||
public class Latency {
|
public class Latency {
|
||||||
|
|
||||||
private final String key;
|
private final String key;
|
||||||
private final AtomicDouble latencySum = new AtomicDouble();
|
private final double value;
|
||||||
private final AtomicInteger counter = new AtomicInteger();
|
|
||||||
|
|
||||||
public synchronized void report(double latencyInMs) {
|
public String getFormattedValue() {
|
||||||
latencySum.addAndGet(latencyInMs);
|
return String.format("%,.2f ms", value);
|
||||||
counter.incrementAndGet();
|
|
||||||
}
|
|
||||||
|
|
||||||
public synchronized double getAvg() {
|
|
||||||
return latencySum.get() / counter.get();
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isNotEmpty() {
|
|
||||||
return counter.get() > 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
public synchronized void reset() {
|
|
||||||
latencySum.set(0.0);
|
|
||||||
counter.set(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getKey() {
|
|
||||||
return key;
|
|
||||||
}
|
|
||||||
|
|
||||||
public synchronized Latency snapshot() {
|
|
||||||
Latency snapshot = new Latency(key);
|
|
||||||
snapshot.latencySum.set(latencySum.get());
|
|
||||||
snapshot.counter.set(counter.get());
|
|
||||||
return snapshot;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String toString() {
|
|
||||||
return "Latency{" +
|
|
||||||
"key='" + key + '\'' +
|
|
||||||
", avgLatency=" + getAvg() +
|
|
||||||
'}';
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -21,11 +21,11 @@ import java.util.Collection;
|
|||||||
|
|
||||||
public class HighLatencyNotification implements Notification {
|
public class HighLatencyNotification implements Notification {
|
||||||
|
|
||||||
private final Collection<Latency> latencies;
|
private final Collection<Latency> highLatencies;
|
||||||
private final int thresholdMs;
|
private final int thresholdMs;
|
||||||
|
|
||||||
public HighLatencyNotification(Collection<Latency> latencies, int thresholdMs) {
|
public HighLatencyNotification(Collection<Latency> highLatencies, int thresholdMs) {
|
||||||
this.latencies = latencies;
|
this.highLatencies = highLatencies;
|
||||||
this.thresholdMs = thresholdMs;
|
this.thresholdMs = thresholdMs;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -33,8 +33,8 @@ public class HighLatencyNotification implements Notification {
|
|||||||
public String getText() {
|
public String getText() {
|
||||||
StringBuilder text = new StringBuilder();
|
StringBuilder text = new StringBuilder();
|
||||||
text.append("Some of the latencies are higher than ").append(thresholdMs).append(" ms:\n");
|
text.append("Some of the latencies are higher than ").append(thresholdMs).append(" ms:\n");
|
||||||
latencies.forEach(latency -> {
|
highLatencies.forEach(latency -> {
|
||||||
text.append(String.format("[%s] %,.2f ms\n", latency.getKey(), latency.getAvg()));
|
text.append(String.format("[%s] %s\n", latency.getKey(), latency.getFormattedValue()));
|
||||||
});
|
});
|
||||||
return text.toString();
|
return text.toString();
|
||||||
}
|
}
|
||||||
|
|||||||
@ -96,7 +96,7 @@ public abstract class BaseHealthChecker<C extends MonitoringConfig, T extends Mo
|
|||||||
} else if (!update.toString().equals(testValue)) {
|
} else if (!update.toString().equals(testValue)) {
|
||||||
throw new ServiceFailureException("Was expecting value " + testValue + " but got " + update);
|
throw new ServiceFailureException("Was expecting value " + testValue + " but got " + update);
|
||||||
}
|
}
|
||||||
reporter.reportLatency(Latencies.WS_UPDATE, stopWatch.getTime());
|
reporter.reportLatency(Latencies.wsUpdate(getKey()), stopWatch.getTime());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -36,7 +36,7 @@ import java.util.UUID;
|
|||||||
@Slf4j
|
@Slf4j
|
||||||
public abstract class BaseMonitoringService<C extends MonitoringConfig<T>, T extends MonitoringTarget> {
|
public abstract class BaseMonitoringService<C extends MonitoringConfig<T>, T extends MonitoringTarget> {
|
||||||
|
|
||||||
@Autowired
|
@Autowired(required = false)
|
||||||
private List<C> configs;
|
private List<C> configs;
|
||||||
private final List<BaseHealthChecker<C, T>> healthCheckers = new LinkedList<>();
|
private final List<BaseHealthChecker<C, T>> healthCheckers = new LinkedList<>();
|
||||||
private final List<UUID> devices = new LinkedList<>();
|
private final List<UUID> devices = new LinkedList<>();
|
||||||
@ -54,6 +54,9 @@ public abstract class BaseMonitoringService<C extends MonitoringConfig<T>, T ext
|
|||||||
|
|
||||||
@PostConstruct
|
@PostConstruct
|
||||||
private void init() {
|
private void init() {
|
||||||
|
if (configs == null || configs.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
tbClient.logIn();
|
tbClient.logIn();
|
||||||
configs.forEach(config -> {
|
configs.forEach(config -> {
|
||||||
config.getTargets().forEach(target -> {
|
config.getTargets().forEach(target -> {
|
||||||
|
|||||||
@ -63,25 +63,20 @@ public class MonitoringReporter {
|
|||||||
private String reportingAssetId;
|
private String reportingAssetId;
|
||||||
|
|
||||||
public void reportLatencies(TbClient tbClient) {
|
public void reportLatencies(TbClient tbClient) {
|
||||||
List<Latency> latencies = this.latencies.values().stream()
|
|
||||||
.filter(Latency::isNotEmpty)
|
|
||||||
.map(latency -> {
|
|
||||||
Latency snapshot = latency.snapshot();
|
|
||||||
latency.reset();
|
|
||||||
return snapshot;
|
|
||||||
})
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
if (latencies.isEmpty()) {
|
if (latencies.isEmpty()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
log.info("Latencies:\n{}", latencies.stream().map(latency -> latency.getKey() + ": " + latency.getAvg() + " ms")
|
log.debug("Latencies:\n{}", latencies.values().stream().map(latency -> latency.getKey() + ": " + latency.getFormattedValue())
|
||||||
.collect(Collectors.joining("\n")) + "\n");
|
.collect(Collectors.joining("\n")) + "\n");
|
||||||
|
|
||||||
if (!latencyReportingEnabled) return;
|
if (!latencyReportingEnabled) return;
|
||||||
|
|
||||||
if (latencies.stream().anyMatch(latency -> latency.getAvg() >= (double) latencyThresholdMs)) {
|
List<Latency> highLatencies = latencies.values().stream()
|
||||||
HighLatencyNotification highLatencyNotification = new HighLatencyNotification(latencies, latencyThresholdMs);
|
.filter(latency -> latency.getValue() >= (double) latencyThresholdMs)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
if (!highLatencies.isEmpty()) {
|
||||||
|
HighLatencyNotification highLatencyNotification = new HighLatencyNotification(highLatencies, latencyThresholdMs);
|
||||||
notificationService.sendNotification(highLatencyNotification);
|
notificationService.sendNotification(highLatencyNotification);
|
||||||
|
log.warn("{}", highLatencyNotification.getText());
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@ -99,10 +94,11 @@ public class MonitoringReporter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
ObjectNode msg = JacksonUtil.newObjectNode();
|
ObjectNode msg = JacksonUtil.newObjectNode();
|
||||||
latencies.forEach(latency -> {
|
latencies.values().forEach(latency -> {
|
||||||
msg.set(latency.getKey(), new DoubleNode(latency.getAvg()));
|
msg.set(latency.getKey(), new DoubleNode(latency.getValue()));
|
||||||
});
|
});
|
||||||
tbClient.saveEntityTelemetry(new AssetId(UUID.fromString(reportingAssetId)), "time", msg);
|
tbClient.saveEntityTelemetry(new AssetId(UUID.fromString(reportingAssetId)), "time", msg);
|
||||||
|
latencies.clear();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("Failed to report latencies: {}", e.getMessage());
|
log.error("Failed to report latencies: {}", e.getMessage());
|
||||||
}
|
}
|
||||||
@ -112,7 +108,7 @@ public class MonitoringReporter {
|
|||||||
String latencyKey = key + "Latency";
|
String latencyKey = key + "Latency";
|
||||||
double latencyInMs = (double) latencyInNanos / 1000_000;
|
double latencyInMs = (double) latencyInNanos / 1000_000;
|
||||||
log.trace("Reporting latency [{}]: {} ms", key, latencyInMs);
|
log.trace("Reporting latency [{}]: {} ms", key, latencyInMs);
|
||||||
latencies.computeIfAbsent(latencyKey, k -> new Latency(latencyKey)).report(latencyInMs);
|
latencies.put(latencyKey, Latency.of(latencyKey, latencyInMs));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void serviceFailure(Object serviceKey, Throwable error) {
|
public void serviceFailure(Object serviceKey, Throwable error) {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user