From b6cfc32b6be6c29dfc504bed3c264d0e8a3d3fcc Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 16 Jul 2024 11:22:54 +0300 Subject: [PATCH 1/7] TbHttpClient - fixed proxy ssl configuration --- .../java/org/thingsboard/rule/engine/rest/TbHttpClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index 26de3ad5e5..b6a7234cac 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -118,8 +118,8 @@ public class TbHttpClient { o.username(proxyUser).password(u -> proxyPassword); } }); - SslContext sslContext = SslContextBuilder.forClient().build(); - httpClient.secure(t -> t.sslContext(sslContext)); + SslContext sslContext = config.getCredentials().initSslContext(); + httpClient = httpClient.secure(t -> t.sslContext(sslContext)); } } else if (!config.isUseSimpleClientHttpFactory()) { if (CredentialsType.CERT_PEM == config.getCredentials().getType()) { From ed8a20b89d86361137f8288933d7e39cbe1db7c2 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 16 Jul 2024 11:23:23 +0300 Subject: [PATCH 2/7] EdgeEventSourcing - ignore EntityAlarm save events. Alarms pushed to edge using push_to_edge node --- .../server/service/edge/EdgeEventSourcingListener.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 11974d2ebc..3de4c95cc3 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmApiCallResult; import org.thingsboard.server.common.data.alarm.AlarmComment; +import org.thingsboard.server.common.data.alarm.EntityAlarm; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; @@ -194,7 +195,7 @@ public class EdgeEventSourcingListener { } break; case ALARM: - if (entity instanceof AlarmApiCallResult || entity instanceof Alarm) { + if (entity instanceof AlarmApiCallResult || entity instanceof Alarm || entity instanceof EntityAlarm) { return false; } break; From a2e4cb9a7d7b27706a546ec4e5e28d3830133de5 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Thu, 27 Jun 2024 19:06:38 +0300 Subject: [PATCH 3/7] Handle properly DELETE cases for ASSET/DEVICE to push ENTITY_DELETED msg into correct rule chain of the profile --- .../queue/DefaultTbClusterService.java | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index eabec663a3..7538e498eb 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -22,6 +22,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.DataConstants; @@ -35,6 +36,7 @@ import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; @@ -45,6 +47,7 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; @@ -205,7 +208,7 @@ public class DefaultTbClusterService implements TbClusterService { return; } } else { - HasRuleEngineProfile ruleEngineProfile = getRuleEngineProfileForEntityOrElseNull(tenantId, entityId); + HasRuleEngineProfile ruleEngineProfile = getRuleEngineProfileForEntityOrElseNull(tenantId, entityId, tbMsg); tbMsg = transformMsg(tbMsg, ruleEngineProfile); } @@ -213,13 +216,39 @@ public class DefaultTbClusterService implements TbClusterService { toRuleEngineMsgs.incrementAndGet(); } - private HasRuleEngineProfile getRuleEngineProfileForEntityOrElseNull(TenantId tenantId, EntityId entityId) { + private HasRuleEngineProfile getRuleEngineProfileForEntityOrElseNull(TenantId tenantId, EntityId entityId, TbMsg tbMsg) { if (entityId.getEntityType().equals(EntityType.DEVICE)) { - return deviceProfileCache.get(tenantId, new DeviceId(entityId.getId())); + if (TbMsgType.ENTITY_DELETED.equals(tbMsg.getInternalType())) { + try { + Device deletedDevice = JacksonUtil.fromString(tbMsg.getData(), Device.class); + if (deletedDevice == null) { + return null; + } + return deviceProfileCache.get(tenantId, deletedDevice.getDeviceProfileId()); + } catch (Exception e) { + log.warn("[{}][{}] Failed to deserialize device: {}", tenantId, entityId, tbMsg, e); + return null; + } + } else { + return deviceProfileCache.get(tenantId, new DeviceId(entityId.getId())); + } } else if (entityId.getEntityType().equals(EntityType.DEVICE_PROFILE)) { return deviceProfileCache.get(tenantId, new DeviceProfileId(entityId.getId())); } else if (entityId.getEntityType().equals(EntityType.ASSET)) { - return assetProfileCache.get(tenantId, new AssetId(entityId.getId())); + if (TbMsgType.ENTITY_DELETED.equals(tbMsg.getInternalType())) { + try { + Asset deletedAsset = JacksonUtil.fromString(tbMsg.getData(), Asset.class); + if (deletedAsset == null) { + return null; + } + return assetProfileCache.get(tenantId, deletedAsset.getAssetProfileId()); + } catch (Exception e) { + log.warn("[{}][{}] Failed to deserialize asset: {}", tenantId, entityId, tbMsg, e); + return null; + } + } else { + return assetProfileCache.get(tenantId, new AssetId(entityId.getId())); + } } else if (entityId.getEntityType().equals(EntityType.ASSET_PROFILE)) { return assetProfileCache.get(tenantId, new AssetProfileId(entityId.getId())); } From b7c505efcd1911d953e4d0077d7c3245b98c6ee8 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 17 Jul 2024 11:59:20 +0300 Subject: [PATCH 4/7] DefaultTbClusterService.getRuleEngineProfileForEntityOrElseNull - Added unit tests --- .../queue/DefaultTbClusterService.java | 2 +- .../queue/DefaultTbClusterServiceTest.java | 49 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 7538e498eb..49ec72d27a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -216,7 +216,7 @@ public class DefaultTbClusterService implements TbClusterService { toRuleEngineMsgs.incrementAndGet(); } - private HasRuleEngineProfile getRuleEngineProfileForEntityOrElseNull(TenantId tenantId, EntityId entityId, TbMsg tbMsg) { + HasRuleEngineProfile getRuleEngineProfileForEntityOrElseNull(TenantId tenantId, EntityId entityId, TbMsg tbMsg) { if (entityId.getEntityType().equals(EntityType.DEVICE)) { if (TbMsgType.ENTITY_DELETED.equals(tbMsg.getInternalType())) { try { diff --git a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java index 22f3620d8c..78cca2de62 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java @@ -23,11 +23,20 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; @@ -249,4 +258,44 @@ public class DefaultTbClusterServiceTest { queue.setPartitions(10); return queue; } + + @Test + public void testGetRuleEngineProfileForUpdatedAndDeletedDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + TenantId tenantId = new TenantId(UUID.randomUUID()); + DeviceProfileId deviceProfileId = new DeviceProfileId(UUID.randomUUID()); + + Device device = new Device(deviceId); + device.setDeviceProfileId(deviceProfileId); + + // device updated + TbMsg tbMsg = TbMsg.builder().internalType(TbMsgType.ENTITY_UPDATED).build(); + ((DefaultTbClusterService) clusterService).getRuleEngineProfileForEntityOrElseNull(tenantId, deviceId, tbMsg); + verify(deviceProfileCache, times(1)).get(tenantId, deviceId); + + // device deleted + tbMsg = TbMsg.builder().internalType(TbMsgType.ENTITY_DELETED).data(JacksonUtil.toString(device)).build(); + ((DefaultTbClusterService) clusterService).getRuleEngineProfileForEntityOrElseNull(tenantId, deviceId, tbMsg); + verify(deviceProfileCache, times(1)).get(tenantId, deviceProfileId); + } + + @Test + public void testGetRuleEngineProfileForUpdatedAndDeletedAsset() { + AssetId assetId = new AssetId(UUID.randomUUID()); + TenantId tenantId = new TenantId(UUID.randomUUID()); + AssetProfileId assetProfileId = new AssetProfileId(UUID.randomUUID()); + + Asset asset = new Asset(assetId); + asset.setAssetProfileId(assetProfileId); + + // asset updated + TbMsg tbMsg = TbMsg.builder().internalType(TbMsgType.ENTITY_UPDATED).build(); + ((DefaultTbClusterService) clusterService).getRuleEngineProfileForEntityOrElseNull(tenantId, assetId, tbMsg); + verify(assetProfileCache, times(1)).get(tenantId, assetId); + + // asset deleted + tbMsg = TbMsg.builder().internalType(TbMsgType.ENTITY_DELETED).data(JacksonUtil.toString(asset)).build(); + ((DefaultTbClusterService) clusterService).getRuleEngineProfileForEntityOrElseNull(tenantId, assetId, tbMsg); + verify(assetProfileCache, times(1)).get(tenantId, assetProfileId); + } } \ No newline at end of file From 8d34a2003ec0a62c581ce4df855f65fbd5c3d7a7 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 19 Jul 2024 18:26:34 +0300 Subject: [PATCH 5/7] Fix entities deletion task reprocessing --- .../ruleChain/RuleEngineComponentActor.java | 3 +++ .../dao/resource/BaseResourceService.java | 3 +++ .../server/dao/rule/BaseRuleChainService.java | 22 +++++++++++-------- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleEngineComponentActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleEngineComponentActor.java index bae9216446..f05fdb1fb9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleEngineComponentActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleEngineComponentActor.java @@ -50,6 +50,9 @@ public abstract class RuleEngineComponentActor referencingRuleNodes = getReferencingRuleChainNodes(tenantId, ruleChainId); Set referencingRuleChainIds = referencingRuleNodes.stream().map(RuleNode::getRuleChainId).collect(Collectors.toSet()); - if (ruleChain != null) { - if (ruleChain.isRoot()) { - throw new DataValidationException("Deletion of Root Tenant Rule Chain is prohibited!"); - } - if (RuleChainType.EDGE.equals(ruleChain.getType())) { - for (Edge edge : new PageDataIterable<>(link -> edgeService.findEdgesByTenantIdAndEntityId(tenantId, ruleChainId, link), DEFAULT_PAGE_SIZE)) { - if (edge.getRootRuleChainId() != null && edge.getRootRuleChainId().equals(ruleChainId)) { - throw new DataValidationException("Can't delete rule chain that is root for edge [" + edge.getName() + "]. Please assign another root rule chain first to the edge!"); - } + if (ruleChain.isRoot()) { + throw new DataValidationException("Deletion of Root Tenant Rule Chain is prohibited!"); + } + if (RuleChainType.EDGE.equals(ruleChain.getType())) { + for (Edge edge : new PageDataIterable<>(link -> edgeService.findEdgesByTenantIdAndEntityId(tenantId, ruleChainId, link), DEFAULT_PAGE_SIZE)) { + if (edge.getRootRuleChainId() != null && edge.getRootRuleChainId().equals(ruleChainId)) { + throw new DataValidationException("Can't delete rule chain that is root for edge [" + edge.getName() + "]. Please assign another root rule chain first to the edge!"); } } } @@ -457,6 +458,9 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { if (force) { RuleChain ruleChain = findRuleChainById(tenantId, (RuleChainId) id); + if (ruleChain == null) { + return; + } checkRuleNodesAndDelete(tenantId, ruleChain, null); } else { deleteRuleChainById(tenantId, (RuleChainId) id); From f34361e1eac041bf4de5002e5101ed6a50697b2b Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 30 Jul 2024 12:02:01 +0300 Subject: [PATCH 6/7] Fix Nashorn sandbox script compile error (ScriptCPUAbuseException) --- .../script/NashornJsInvokeServiceTest.java | 47 +++++++++++++++++-- pom.xml | 2 +- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java b/application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java index 28834a0ab7..33e7ccea23 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java @@ -39,12 +39,13 @@ import java.util.concurrent.TimeoutException; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.thingsboard.server.common.data.msg.TbMsgType.POST_TELEMETRY_REQUEST; @DaoSqlTest @TestPropertySource(properties = { "js.evaluator=local", - "js.max_script_body_size=50", + "js.max_script_body_size=10000", "js.max_total_args_size=50", "js.max_result_size=50", "js.local.max_errors=2", @@ -87,7 +88,7 @@ class NashornJsInvokeServiceTest extends AbstractControllerTest { @Test void givenSimpleScriptMultiThreadTestPerformance() throws ExecutionException, InterruptedException, TimeoutException { - int iterations = 1000*4; + int iterations = 1000 * 4; List> futures = new ArrayList<>(iterations); UUID scriptId = evalScript("return msg.temperature > 20 ;"); // warmup @@ -125,7 +126,7 @@ class NashornJsInvokeServiceTest extends AbstractControllerTest { @Test void givenTooBigScriptForEval_thenReturnError() { - String hugeScript = "var a = 'qwertyqwertywertyqwabababer'; return {a: a};"; + String hugeScript = "var a = '" + "a".repeat(10000) + "'; return {a: a};"; assertThatThrownBy(() -> { evalScript(hugeScript); @@ -159,6 +160,46 @@ class NashornJsInvokeServiceTest extends AbstractControllerTest { assertThatScriptIsBlocked(scriptId); } + @Test + void givenComplexScript_testCompile() { + String script = """ + function(data) { + if (data.get("propertyA") == "a special value 1" || data.get("propertyA") == "a special value 2") { + return "a special value 1"; + } else if (data.get("propertyB") == "a special value 3" && (data.get("propertyC") == "a special value 1" || data.get("propertyJ") == "a special value 1" || data.get("propertyV") == "a special value 1")) { + return "a special value 1"; + } else if (data.get("propertyB") == "4" && (data.get("propertyD") == "a special value 1" || data.get("propertyV") == "a special value 1" || data.get("propertyW") == "a special value 1")) { + return "a special value 1"; + } else if (data.get("propertyB") == "a special value 2" && (data.get("propertyE") == "a special value 1" || data.get("propertyF") == "a special value 1" || data.get("propertyL") == "a special value 1")) { + return "a special value 1"; + } else if (data.get("propertyB") == "a special value 3" && (data.get("propertyE") == "a special value 1" || data.get("propertyF") == "a special value 1" || data.get("propertyL") == "a special value 1")) { + return "a special value 1"; + } else if (data.get("propertyB") == "a special value 3" && (data.get("propertyM") == "a special value 1" || data.get("propertyY") == "a special value 1" || data.get("propertyH") == "a special value 1")) { + return "a special value 1"; + } else if (data.get("propertyB") == "a special value 3" && (data.get("propertyM") == "a special value 1" || data.get("propertyY") == "a special value 1" || data.get("propertyH") == "a special value 1")) { + return "a special value 1"; + } else if (data.get("propertyB") == "a special value 3" && (data.get("propertyM") == "a special value 1" || data.get("propertyY") == "a special value 1" || data.get("propertyH") == "a special value 1")) { + return "a special value 1"; + } else if (data.get("propertyB") == "a special value 3" && (data.get("propertyM") == "a special value 1" || data.get("propertyY") == "a special value 1" || data.get("propertyH") == "a special value 1")) { + return "a special value 1"; + } else if (data.get("propertyB") == "a special value 3" && (data.get("propertyM") == "a special value 1" || data.get("propertyY") == "a special value 1" || data.get("propertyH") == "a special value 1")) { + return "a special value 1"; + } else if (data.get("propertyB") == "a special value 3" && (data.get("propertyM") == "a special value 1" || data.get("propertyY") == "a special value 1" || data.get("propertyH") == "a special value 1")) { + return "a special value 1"; + } else if (data.get("propertyB") == "a special value 3" && (data.get("propertyM") == "a special value 1" || data.get("propertyY") == "a special value 1" || data.get("propertyH") == "a special value 1")) { + return "a special value 1"; + } else { + return "0" + }; + } + """; + + // with delight-nashorn-sandbox 0.4.2, this would throw delight.nashornsandbox.exceptions.ScriptCPUAbuseException: Regular expression running for too many iterations. The operation could NOT be gracefully interrupted. + assertDoesNotThrow(() -> { + evalScript(script); + }); + } + private void assertThatScriptIsBlocked(UUID scriptId) { assertThatThrownBy(() -> { invokeScript(scriptId, "{}"); diff --git a/pom.xml b/pom.xml index f75e782bef..9b8943cff3 100755 --- a/pom.xml +++ b/pom.xml @@ -105,7 +105,7 @@ org/thingsboard/server/extensions/core/plugin/telemetry/gen/**/* 8.13.2 - 0.4.2 + 0.4.5 15.4