From 671c7db5c9347f99b0010ad26b1117c708531d3c Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 21 Mar 2023 17:19:51 +0100 Subject: [PATCH] added new metrics --- .../system/DefaultSystemInfoService.java | 104 ++++++++++++++---- common/cluster-api/src/main/proto/queue.proto | 10 +- .../server/common/data/SystemInfoData.java | 14 ++- .../DefaultTbServiceInfoProvider.java | 20 ++++ common/util/pom.xml | 4 + .../thingsboard/common/util/SystemUtil.java | 54 ++++++++- pom.xml | 6 + 7 files changed, 180 insertions(+), 32 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java index 41d2f6cdbf..8f05f26648 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java @@ -21,22 +21,27 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.SystemInfo; import org.thingsboard.server.common.data.SystemInfoData; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import org.thingsboard.server.queue.discovery.DiscoveryService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; -import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.ArrayList; import java.util.Collections; @@ -45,15 +50,19 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import static org.thingsboard.common.util.SystemUtil.getFreeMemory; import static org.thingsboard.common.util.SystemUtil.getCpuUsage; import static org.thingsboard.common.util.SystemUtil.getFreeDiscSpace; import static org.thingsboard.common.util.SystemUtil.getMemoryUsage; +import static org.thingsboard.common.util.SystemUtil.getTotalCpuUsage; +import static org.thingsboard.common.util.SystemUtil.getTotalDiscSpace; +import static org.thingsboard.common.util.SystemUtil.getTotalMemory; @TbCoreComponent @Service @RequiredArgsConstructor @Slf4j -public class DefaultSystemInfoService implements SystemInfoService { +public class DefaultSystemInfoService extends TbApplicationEventListener implements SystemInfoService { public static final FutureCallback CALLBACK = new FutureCallback<>() { @Override @@ -67,6 +76,7 @@ public class DefaultSystemInfoService implements SystemInfoService { }; private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; private final DiscoveryService discoveryService; private final TelemetrySubscriptionService telemetryService; private ScheduledExecutorService scheduler; @@ -74,18 +84,15 @@ public class DefaultSystemInfoService implements SystemInfoService { @Value("${zk.enabled:false}") private boolean zkEnabled; - @PostConstruct - private void init() { - if (!zkEnabled) { - scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-system-info-scheduler")); - scheduler.scheduleAtFixedRate(this::saveCurrentSystemInfo, 0, 1, TimeUnit.MINUTES); - } - } - - @PreDestroy - private void destroy() { - if (scheduler != null) { - scheduler.shutdownNow(); + @Override + protected void onTbApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { + if (scheduler == null && partitionService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID).isMyPartition()) { + scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-system-info-scheduler")); + scheduler.scheduleAtFixedRate(this::saveCurrentSystemInfo, 0, 1, TimeUnit.MINUTES); + } else { + destroy(); + } } } @@ -96,13 +103,7 @@ public class DefaultSystemInfoService implements SystemInfoService { ServiceInfo serviceInfo = serviceInfoProvider.getServiceInfoWithCurrentSystemInfo(); if (zkEnabled) { - List clusterSystemData = new ArrayList<>(); - clusterSystemData.add(createSystemInfoData(serviceInfo)); - this.discoveryService.getOtherServers() - .stream() - .map(this::createSystemInfoData) - .forEach(clusterSystemData::add); - systemInfo.setSystemData(clusterSystemData); + systemInfo.setSystemData(getSystemData(serviceInfo)); } else { systemInfo.setMonolith(true); systemInfo.setSystemData(Collections.singletonList(createSystemInfoData(serviceInfo))); @@ -112,6 +113,22 @@ public class DefaultSystemInfoService implements SystemInfoService { } protected void saveCurrentSystemInfo() { + if (zkEnabled) { + saveCurrentClusterSystemInfo(); + } else { + saveCurrentMonolithSystemInfo(); + } + } + + private void saveCurrentClusterSystemInfo() { + long ts = System.currentTimeMillis(); + + List clusterSystemData = getSystemData(serviceInfoProvider.getServiceInfoWithCurrentSystemInfo()); + BasicTsKvEntry clusterDataKv = new BasicTsKvEntry(ts, new JsonDataEntry("clusterSystemData", JacksonUtil.toString(clusterSystemData))); + doSave(Collections.singletonList(clusterDataKv)); + } + + private void saveCurrentMonolithSystemInfo() { long ts = System.currentTimeMillis(); List tsList = new ArrayList<>(); @@ -119,16 +136,46 @@ public class DefaultSystemInfoService implements SystemInfoService { if (memoryUsage != null) { tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("memoryUsage", memoryUsage))); } + Long totalMemory = getTotalMemory(); + if (totalMemory != null) { + tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("totalMemory", totalMemory))); + } + Long freeMemory = getFreeMemory(); + if (freeMemory != null) { + tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("freeMemory", freeMemory))); + } Double cpuUsage = getCpuUsage(); if (cpuUsage != null) { tsList.add(new BasicTsKvEntry(ts, new DoubleDataEntry("cpuUsage", cpuUsage))); } + Double totalCpuUsage = getTotalCpuUsage(); + if (totalCpuUsage != null) { + tsList.add(new BasicTsKvEntry(ts, new DoubleDataEntry("totalCpuUsage", totalCpuUsage))); + } Long freeDiscSpace = getFreeDiscSpace(); if (freeDiscSpace != null) { tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("freeDiscSpace", freeDiscSpace))); } + Long totalDiscSpace = getTotalDiscSpace(); + if (totalDiscSpace != null) { + tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("totalDiscSpace", totalDiscSpace))); + } - telemetryService.saveAndNotifyInternal(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, tsList, CALLBACK); + doSave(tsList); + } + + private void doSave(List telemetry) { + telemetryService.saveAndNotifyInternal(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, telemetry, CALLBACK); + } + + private List getSystemData(ServiceInfo serviceInfo) { + List clusterSystemData = new ArrayList<>(); + clusterSystemData.add(createSystemInfoData(serviceInfo)); + this.discoveryService.getOtherServers() + .stream() + .map(this::createSystemInfoData) + .forEach(clusterSystemData::add); + return clusterSystemData; } private SystemInfoData createSystemInfoData(ServiceInfo serviceInfo) { @@ -136,10 +183,21 @@ public class DefaultSystemInfoService implements SystemInfoService { SystemInfoData infoData = new SystemInfoData(); infoData.setServiceId(serviceInfo.getServiceId()); infoData.setServiceType(serviceTypes.size() > 1 ? "MONOLITH" : serviceTypes.get(0)); - infoData.setMemUsage(serviceInfo.getSystemInfo().getMemoryUsage()); + infoData.setMemoryUsage(serviceInfo.getSystemInfo().getMemoryUsage()); + infoData.setTotalMemory(serviceInfo.getSystemInfo().getTotalMemory()); + infoData.setFreeMemory(serviceInfo.getSystemInfo().getFreeMemory()); infoData.setCpuUsage(serviceInfo.getSystemInfo().getCpuUsage()); + infoData.setTotalCpuUsage(serviceInfo.getSystemInfo().getTotalCpuUsage()); infoData.setFreeDiscSpace(serviceInfo.getSystemInfo().getFreeDiscSpace()); + infoData.setTotalDiscSpace(serviceInfo.getSystemInfo().getTotalDiscSpace()); return infoData; } + @PreDestroy + private void destroy() { + if (scheduler != null) { + scheduler.shutdownNow(); + scheduler = null; + } + } } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 24d1f67db3..0af667356d 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -31,9 +31,13 @@ message ServiceInfo { } message SystemInfoProto { - int64 memoryUsage = 1; - double cpuUsage = 2; - int64 freeDiscSpace = 3; + double cpuUsage = 1; + double totalCpuUsage = 2; + int64 memoryUsage = 3; + int64 totalMemory = 4; + int64 freeMemory = 5; + int64 freeDiscSpace = 6; + int64 totalDiscSpace = 7; } /** diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java index 9345fb4feb..e4510e0f48 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java @@ -28,8 +28,16 @@ public class SystemInfoData { private String serviceType; @ApiModelProperty(position = 3, value = "CPU usage.") private Double cpuUsage; - @ApiModelProperty(position = 4, value = "Memory usage.") - private Long memUsage; - @ApiModelProperty(position = 5, value = "Free disc space.") + @ApiModelProperty(position = 4, value = "Total CPU usage.") + private Double totalCpuUsage; + @ApiModelProperty(position = 5, value = "Memory usage in bytes.") + private Long memoryUsage; + @ApiModelProperty(position = 6, value = "Total memory in bytes.") + private Long totalMemory; + @ApiModelProperty(position = 6, value = "Free memory in bytes.") + private Long freeMemory; + @ApiModelProperty(position = 7, value = "Free disc space in bytes.") private Long freeDiscSpace; + @ApiModelProperty(position = 7, value = "Total disc space in bytes.") + private Long totalDiscSpace; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index b9377f3174..db9334d3a0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -37,9 +37,13 @@ import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import static org.thingsboard.common.util.SystemUtil.getFreeMemory; import static org.thingsboard.common.util.SystemUtil.getCpuUsage; import static org.thingsboard.common.util.SystemUtil.getFreeDiscSpace; import static org.thingsboard.common.util.SystemUtil.getMemoryUsage; +import static org.thingsboard.common.util.SystemUtil.getTotalCpuUsage; +import static org.thingsboard.common.util.SystemUtil.getTotalDiscSpace; +import static org.thingsboard.common.util.SystemUtil.getTotalMemory; @Component @Slf4j @@ -118,14 +122,30 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { if (memoryUsage != null) { builder.setMemoryUsage(memoryUsage); } + Long totalMemory = getTotalMemory(); + if (totalMemory != null) { + builder.setTotalMemory(totalMemory); + } + Long freeMemory = getFreeMemory(); + if (freeMemory != null) { + builder.setFreeMemory(freeMemory); + } Double cpuUsage = getCpuUsage(); if (cpuUsage != null) { builder.setCpuUsage(cpuUsage); } + Double totalCpuUsage = getTotalCpuUsage(); + if (totalCpuUsage != null) { + builder.setTotalCpuUsage(totalCpuUsage); + } Long freeDiscSpace = getFreeDiscSpace(); if (freeDiscSpace != null) { builder.setFreeDiscSpace(freeDiscSpace); } + Long totalDiscSpace = getTotalDiscSpace(); + if (totalDiscSpace != null) { + builder.setTotalDiscSpace(totalDiscSpace); + } return builder.build(); } diff --git a/common/util/pom.xml b/common/util/pom.xml index a222724acd..18b08621ec 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -88,6 +88,10 @@ org.thingsboard.common data + + com.github.dblock + oshi-core + diff --git a/common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java b/common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java index 4533bbf65e..18ddf1a0a0 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java @@ -16,10 +16,11 @@ package org.thingsboard.common.util; import lombok.extern.slf4j.Slf4j; +import oshi.SystemInfo; +import oshi.hardware.HardwareAbstractionLayer; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; -import java.lang.management.OperatingSystemMXBean; import java.nio.file.FileStore; import java.nio.file.Files; import java.nio.file.Paths; @@ -27,6 +28,13 @@ import java.nio.file.Paths; @Slf4j public class SystemUtil { + private static final HardwareAbstractionLayer HARDWARE; + + static { + SystemInfo si = new SystemInfo(); + HARDWARE = si.getHardware(); + } + public static Long getMemoryUsage() { try { MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); @@ -37,16 +45,42 @@ public class SystemUtil { return null; } + public static Long getTotalMemory() { + try { + return HARDWARE.getMemory().getTotal(); + } catch (Exception e) { + log.debug("Failed to get total memory!!!", e); + } + return null; + } + + public static Long getFreeMemory() { + try { + return HARDWARE.getMemory().getAvailable(); + } catch (Exception e) { + log.debug("Failed to get free memory!!!", e); + } + return null; + } + public static Double getCpuUsage() { try { - OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); - return osBean.getSystemLoadAverage(); + return prepare(HARDWARE.getProcessor().getSystemLoadAverage()); } catch (Exception e) { log.debug("Failed to get cpu usage!!!", e); } return null; } + public static Double getTotalCpuUsage() { + try { + return prepare(HARDWARE.getProcessor().getSystemCpuLoad() * 100); + } catch (Exception e) { + log.debug("Failed to get total cpu usage!!!", e); + } + return null; + } + public static Long getFreeDiscSpace() { try { FileStore store = Files.getFileStore(Paths.get("/")); @@ -56,4 +90,18 @@ public class SystemUtil { } return null; } + + public static Long getTotalDiscSpace() { + try { + FileStore store = Files.getFileStore(Paths.get("/")); + return store.getTotalSpace(); + } catch (Exception e) { + log.debug("Failed to get total disc space!!!", e); + } + return null; + } + + private static Double prepare(Double d) { + return (int) (d * 100) / 100.0; + } } diff --git a/pom.xml b/pom.xml index 65d9de814e..7b6d565ead 100755 --- a/pom.xml +++ b/pom.xml @@ -149,6 +149,7 @@ 2.21.0 2.12.0 1.12.1 + 3.4.0 @@ -1987,6 +1988,11 @@ exp4j ${exp4j.version} + + com.github.dblock + oshi-core + ${oshi.version} +