From dfb0549424a4f4d5a7c6d7734fa8282a6c2010b7 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 17 Feb 2025 16:32:59 +0200 Subject: [PATCH 01/26] VC support for calculated fields --- .../DefaultEntitiesExportImportService.java | 3 +- .../impl/CalculatedFieldExportService.java | 42 ++++++++++ .../impl/CalculatedFieldImportService.java | 78 +++++++++++++++++++ .../common/data/cf/CalculatedField.java | 17 ++++ .../server/common/data/sync/JsonTbEntity.java | 4 +- .../server/dao/cf/CalculatedFieldDao.java | 3 +- .../dao/sql/cf/CalculatedFieldRepository.java | 11 ++- .../dao/sql/cf/JpaCalculatedFieldDao.java | 20 +++++ 8 files changed, 174 insertions(+), 4 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/CalculatedFieldExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CalculatedFieldImportService.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index a9b278a19d..de437144ae 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -69,7 +69,8 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS EntityType.DASHBOARD, EntityType.ASSET_PROFILE, EntityType.ASSET, EntityType.DEVICE_PROFILE, EntityType.DEVICE, EntityType.ENTITY_VIEW, EntityType.WIDGET_TYPE, EntityType.WIDGETS_BUNDLE, - EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_RULE + EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_RULE, + EntityType.CALCULATED_FIELD ); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/CalculatedFieldExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/CalculatedFieldExportService.java new file mode 100644 index 0000000000..fc1465b29c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/CalculatedFieldExportService.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +import java.util.Set; + +@Service +@TbCoreComponent +public class CalculatedFieldExportService extends BaseEntityExportService> { + + @Override + protected void setRelatedEntities(EntitiesExportCtx ctx, CalculatedField calculatedField, EntityExportData exportData) { + calculatedField.setEntityId(getExternalIdOrElseInternal(ctx, calculatedField.getEntityId())); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.CALCULATED_FIELD); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CalculatedFieldImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CalculatedFieldImportService.java new file mode 100644 index 0000000000..07a306d511 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CalculatedFieldImportService.java @@ -0,0 +1,78 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.service.ConstraintValidator; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class CalculatedFieldImportService extends BaseEntityImportService> { + + private final CalculatedFieldService calculatedFieldService; + + @Override + protected void setOwner(TenantId tenantId, CalculatedField calculatedField, IdProvider idProvider) { + calculatedField.setTenantId(tenantId); + } + + @Override + protected CalculatedField prepare(EntitiesImportCtx ctx, CalculatedField calculatedField, CalculatedField oldEntity, EntityExportData exportData, IdProvider idProvider) { + calculatedField.setEntityId(idProvider.getInternalId(calculatedField.getEntityId())); + return calculatedField; + } + + @Override + protected CalculatedField saveOrUpdate(EntitiesImportCtx ctx, CalculatedField calculatedField, EntityExportData exportData, IdProvider idProvider) { + ConstraintValidator.validateFields(calculatedField); + return calculatedFieldService.save(calculatedField); + } + + @Override + protected CalculatedField deepCopy(CalculatedField calculatedField) { + return new CalculatedField(calculatedField); + } + + @Override + protected void onEntitySaved(User user, CalculatedField savedEntity, CalculatedField oldEntity) throws ThingsboardException { + entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, null, + oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, null); + } + + @Override + protected void cleanupForComparison(CalculatedField e) { + super.cleanupForComparison(e); + } + + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index f4b92b3802..0c6278be2b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -37,11 +37,14 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; +import java.io.Serial; + @Schema @Data @EqualsAndHashCode(callSuper = true) public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, ExportableEntity, HasDebugSettings { + @Serial private static final long serialVersionUID = 4491966747773381420L; private TenantId tenantId; @@ -78,6 +81,20 @@ public class CalculatedField extends BaseData implements HasN super(id); } + public CalculatedField(CalculatedField other) { + super(other); + this.tenantId = other.tenantId; + this.entityId = other.entityId; + this.type = other.type; + this.name = other.name; + this.configurationVersion = other.configurationVersion; + this.configuration = other.configuration; + this.version = other.version; + this.externalId = other.externalId; + this.debugMode = other.debugMode; + this.debugSettings = other.debugSettings; + } + public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { this.tenantId = tenantId; this.entityId = entityId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java index 1b5459ec1a..6af6f6a034 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.notification.rule.NotificationRule; import org.thingsboard.server.common.data.notification.targets.NotificationTarget; import org.thingsboard.server.common.data.notification.template.NotificationTemplate; @@ -58,7 +59,8 @@ import java.lang.annotation.Target; @Type(name = "NOTIFICATION_TEMPLATE", value = NotificationTemplate.class), @Type(name = "NOTIFICATION_TARGET", value = NotificationTarget.class), @Type(name = "NOTIFICATION_RULE", value = NotificationRule.class), - @Type(name = "TB_RESOURCE", value = TbResource.class) + @Type(name = "TB_RESOURCE", value = TbResource.class), + @Type(name = "CALCULATED_FIELD", value = CalculatedField.class) }) @JsonIgnoreProperties(value = {"tenantId", "createdTime", "version"}, ignoreUnknown = true) public @interface JsonTbEntity { diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 3efb4011ed..99683aaa77 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -22,10 +22,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import java.util.List; -public interface CalculatedFieldDao extends Dao { +public interface CalculatedFieldDao extends Dao, ExportableEntityDao { List findAllByTenantId(TenantId tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index 2aeca659bc..9a5c76df97 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -18,13 +18,16 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import java.util.List; import java.util.UUID; -public interface CalculatedFieldRepository extends JpaRepository { +@Repository +public interface CalculatedFieldRepository extends JpaRepository, ExportableEntityRepository { boolean existsByTenantIdAndEntityId(UUID tenantId, UUID entityId); @@ -34,10 +37,16 @@ public interface CalculatedFieldRepository extends JpaRepository findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId, Pageable pageable); + Page findByTenantId(UUID tenantId, Pageable pageable); + List findAllByTenantId(UUID tenantId); List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); long countByTenantIdAndEntityId(UUID tenantId, UUID entityId); + CalculatedFieldId findExternalIdById(UUID id); + + CalculatedFieldEntity findByTenantIdAndName(UUID tenantId, String name); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 703bdfbf6f..865847e15a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -93,6 +93,26 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findByTenantId(UUID tenantId, PageLink pageLink) { + return DaoUtil.toPageData(calculatedFieldRepository.findByTenantId(tenantId, DaoUtil.toPageable(pageLink))); + } + + @Override + public CalculatedField findByTenantIdAndName(UUID tenantId, String name) { + return DaoUtil.getData(calculatedFieldRepository.findByTenantIdAndName(tenantId, name)); + } + + @Override + public CalculatedField findByTenantIdAndExternalId(UUID tenantId, UUID externalId) { + return DaoUtil.getData(calculatedFieldRepository.findByTenantIdAndExternalId(tenantId, externalId)); + } + + @Override + public CalculatedFieldId getExternalIdByInternal(CalculatedFieldId internalId) { + return calculatedFieldRepository.findExternalIdById(internalId.getId()); + } + @Override protected Class getEntityClass() { return CalculatedFieldEntity.class; From 85f2fcd4d519bae15219c36e3ae345655f2ddcc2 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 17 Feb 2025 17:47:32 +0200 Subject: [PATCH 02/26] Add tests --- .../sync/ie/ExportImportServiceSqlTest.java | 71 ++++++++++++++++++- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java index 0beaca9af9..3e47db1bc0 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java @@ -43,6 +43,15 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.DeviceData; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; @@ -51,6 +60,7 @@ import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceId; @@ -79,6 +89,7 @@ import org.thingsboard.server.common.data.util.ThrowingRunnable; import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; @@ -145,6 +156,8 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { protected TenantService tenantService; @Autowired protected EntityViewService entityViewService; + @Autowired + protected CalculatedFieldService calculatedFieldService; protected TenantId tenantId1; protected User tenantAdmin1; @@ -191,9 +204,10 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { Asset asset = createAsset(tenantId1, null, assetProfile.getId(), "Asset 1"); DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device 1"); + CalculatedField calculatedField = createCalculatedField(tenantId1, device.getId(), device.getId()); Map entitiesExportData = Stream.of(customer.getId(), asset.getId(), device.getId(), - ruleChain.getId(), dashboard.getId(), assetProfile.getId(), deviceProfile.getId()) + ruleChain.getId(), dashboard.getId(), assetProfile.getId(), deviceProfile.getId(), calculatedField.getId()) .map(entityId -> { try { return exportEntity(tenantAdmin1, entityId, EntityExportSettings.builder() @@ -245,7 +259,6 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { verify(entityActionService, Mockito.never()).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), any(), eq(ActionType.UPDATED), isNull()); - EntityExportData updatedAssetEntity = getAndClone(entitiesExportData, EntityType.ASSET); updatedAssetEntity.getEntity().setLabel("t" + updatedAssetEntity.getEntity().getLabel()); Asset updatedAsset = importEntity(tenantAdmin2, updatedAssetEntity).getSavedEntity(); @@ -272,6 +285,19 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { updatedDeviceEntity.getEntity().setLabel("t" + updatedDeviceEntity.getEntity().getLabel()); Device updatedDevice = importEntity(tenantAdmin2, updatedDeviceEntity).getSavedEntity(); verify(tbClusterService).onDeviceUpdated(eq(updatedDevice), eq(importedDevice)); + + CalculatedField importedCalculatedField = (CalculatedField) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.CALCULATED_FIELD)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedCalculatedField.getId()), eq(importedCalculatedField), + any(), eq(ActionType.ADDED), isNull()); + importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.CALCULATED_FIELD)); + verify(entityActionService, Mockito.never()).logEntityAction(any(), eq(importedCalculatedField.getId()), eq(importedCalculatedField), + any(), eq(ActionType.UPDATED), isNull()); + + EntityExportData updatedCalculatedFieldEntity = getAndClone(entitiesExportData, EntityType.CALCULATED_FIELD); + updatedCalculatedFieldEntity.getEntity().setName("t" + updatedCalculatedFieldEntity.getEntity().getName()); + CalculatedField updatedCalculatedField = importEntity(tenantAdmin2, updatedCalculatedFieldEntity).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(updatedCalculatedField.getId()), eq(updatedCalculatedField), + any(), eq(ActionType.UPDATED), isNull()); } @Test @@ -290,9 +316,11 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { Device device = createDevice(tenantId1, customer.getId(), deviceProfile.getId(), "Device 1"); EntityView entityView = createEntityView(tenantId1, customer.getId(), device.getId(), "Entity view 1"); + CalculatedField calculatedField = createCalculatedField(tenantId1, device.getId(), device.getId()); + Map ids = new HashMap<>(); for (EntityId entityId : List.of(customer.getId(), ruleChain.getId(), dashboard.getId(), assetProfile.getId(), asset.getId(), - deviceProfile.getId(), device.getId(), entityView.getId(), ruleChain.getId(), dashboard.getId())) { + deviceProfile.getId(), device.getId(), entityView.getId(), ruleChain.getId(), dashboard.getId(), calculatedField.getId())) { EntityExportData exportData = exportEntity(getSecurityUser(tenantAdmin1), entityId); EntityImportResult importResult = importEntity(getSecurityUser(tenantAdmin2), exportData, EntityImportSettings.builder() .saveCredentials(false) @@ -333,6 +361,10 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { assertThat(exportedEntityView.getCustomerId()).isEqualTo(customer.getId()); assertThat(exportedEntityView.getEntityId()).isEqualTo(device.getId()); + CalculatedField exportedCalculatedField = (CalculatedField) exportEntity(tenantAdmin2, (CalculatedFieldId) ids.get(calculatedField.getId())).getEntity(); + assertThat(exportedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(exportedCalculatedField.getEntityId()).isEqualTo(device.getId()); + deviceProfile.setDefaultDashboardId(null); deviceProfileService.saveDeviceProfile(deviceProfile); DeviceProfile importedDeviceProfile = deviceProfileService.findDeviceProfileById(tenantId2, (DeviceProfileId) ids.get(deviceProfile.getId())); @@ -549,6 +581,39 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { return relation; } + private CalculatedField createCalculatedField(TenantId tenantId, EntityId entityId, EntityId referencedEntityId) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setEntityId(entityId); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); + calculatedField.setVersion(1L); + return calculatedFieldService.save(calculatedField); + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("T - (100 - H) / 5"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + return config; + } + protected , I extends EntityId> EntityExportData exportEntity(User user, I entityId) throws Exception { return exportEntity(user, entityId, EntityExportSettings.builder() .exportCredentials(true) From 44a7bf82c4156ef9bbfc1cd0364fe403f0afcfd9 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 17 Feb 2025 18:00:12 +0200 Subject: [PATCH 03/26] Fix ComponentLifecycleMsg: message and builder --- .../server/common/msg/plugin/ComponentLifecycleMsg.java | 4 ++++ .../org/thingsboard/server/common/util/ProtoUtils.java | 7 +++---- common/proto/src/main/proto/queue.proto | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index 208b6ab2f5..7202a11a6c 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.common.msg.cluster.ToAllNodesMsg; +import java.io.Serial; import java.util.Optional; /** @@ -33,6 +34,8 @@ import java.util.Optional; */ @Data public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { + + @Serial private static final long serialVersionUID = -5303421482781273062L; private final TenantId tenantId; @@ -66,4 +69,5 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { public MsgType getMsgType() { return MsgType.COMPONENT_LIFE_CYCLE_MSG; } + } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 07406977d2..057cf875f1 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -134,7 +134,7 @@ public class ProtoUtils { builder.setName(msg.getName()); } if (msg.getOldName() != null) { - builder.setName(msg.getOldName()); + builder.setOldName(msg.getOldName()); } return builder.build(); } @@ -149,8 +149,8 @@ public class ProtoUtils { .tenantId(TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB()))) .entityId(entityId) .event(ComponentLifecycleEvent.values()[proto.getEventValue()]) - .name(proto.getName()) - .oldName(proto.getOldName()); + .name(proto.hasName() ? proto.getName() : null) + .oldName(proto.hasOldName() ? proto.getOldName() : null); if (proto.getProfileIdMSB() != 0 || proto.getProfileIdLSB() != 0) { var profileType = EntityType.DEVICE.equals(entityId.getEntityType()) ? EntityType.DEVICE_PROFILE : EntityType.ASSET_PROFILE; builder.profileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB()))); @@ -166,7 +166,6 @@ public class ProtoUtils { return entityTypeByProtoNumber[entityType.getNumber()]; } - public static TransportProtos.ToEdgeSyncRequestMsgProto toProto(ToEdgeSyncRequest request) { return TransportProtos.ToEdgeSyncRequestMsgProto.newBuilder() .setTenantIdMSB(request.getTenantId().getId().getMostSignificantBits()) diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index d0050c310d..e782291f02 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1188,8 +1188,8 @@ message ComponentLifecycleMsgProto { int64 entityIdLSB = 5; ComponentLifecycleEvent event = 6; //Since 4.0. To replace the - string oldName = 7; - string name = 8; + optional string oldName = 7; + optional string name = 8; int64 oldProfileIdMSB = 9; int64 oldProfileIdLSB = 10; int64 profileIdMSB = 11; From 4daf4ba9888d0ef98d0eb5f17843873f192fa5c7 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Wed, 19 Feb 2025 17:36:06 +0200 Subject: [PATCH 04/26] Fix refEntity import-export --- .../ie/exporting/impl/CalculatedFieldExportService.java | 7 +++++++ .../ie/importing/impl/CalculatedFieldImportService.java | 7 +++++++ .../server/service/sync/ie/ExportImportServiceSqlTest.java | 3 ++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/CalculatedFieldExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/CalculatedFieldExportService.java index fc1465b29c..3612f9f4e0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/CalculatedFieldExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/CalculatedFieldExportService.java @@ -19,6 +19,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; @@ -32,6 +33,12 @@ public class CalculatedFieldExportService extends BaseEntityExportService ctx, CalculatedField calculatedField, EntityExportData exportData) { calculatedField.setEntityId(getExternalIdOrElseInternal(ctx, calculatedField.getEntityId())); + calculatedField.getConfiguration().getArguments().values().forEach(argument -> { + if (argument.getRefEntityId() != null) { + EntityId internalEntityId = getExternalIdOrElseInternal(ctx, argument.getRefEntityId()); + argument.setRefEntityId(internalEntityId); + } + }); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CalculatedFieldImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CalculatedFieldImportService.java index 07a306d511..978ac758df 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CalculatedFieldImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CalculatedFieldImportService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.dao.cf.CalculatedFieldService; @@ -45,6 +46,12 @@ public class CalculatedFieldImportService extends BaseEntityImportService exportData, IdProvider idProvider) { calculatedField.setEntityId(idProvider.getInternalId(calculatedField.getEntityId())); + calculatedField.getConfiguration().getArguments().values().forEach(argument -> { + if (argument.getRefEntityId() != null) { + EntityId internalEntityId = idProvider.getInternalId(argument.getRefEntityId()); + argument.setRefEntityId(internalEntityId); + } + }); return calculatedField; } diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java index 3e47db1bc0..7d1fee8ac3 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java @@ -204,7 +204,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { Asset asset = createAsset(tenantId1, null, assetProfile.getId(), "Asset 1"); DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device 1"); - CalculatedField calculatedField = createCalculatedField(tenantId1, device.getId(), device.getId()); + CalculatedField calculatedField = createCalculatedField(tenantId1, device.getId(), asset.getId()); Map entitiesExportData = Stream.of(customer.getId(), asset.getId(), device.getId(), ruleChain.getId(), dashboard.getId(), assetProfile.getId(), deviceProfile.getId(), calculatedField.getId()) @@ -364,6 +364,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { CalculatedField exportedCalculatedField = (CalculatedField) exportEntity(tenantAdmin2, (CalculatedFieldId) ids.get(calculatedField.getId())).getEntity(); assertThat(exportedCalculatedField.getName()).isEqualTo(calculatedField.getName()); assertThat(exportedCalculatedField.getEntityId()).isEqualTo(device.getId()); + assertThat(exportedCalculatedField.getConfiguration().getReferencedEntities()).isEqualTo(calculatedField.getConfiguration().getReferencedEntities()); deviceProfile.setDefaultDashboardId(null); deviceProfileService.saveDeviceProfile(deviceProfile); From 5372f22b8a314e84823d39cb4a19a762f206cf59 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 28 Feb 2025 17:08:24 +0200 Subject: [PATCH 05/26] Added latest debug event call in Calculated Field Dialog on Test function --- .../core/http/calculated-fields.service.ts | 5 +++++ .../calculated-field-dialog.component.ts | 22 ++++++++++++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index fe5b0f7b52..66c0cb609e 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -23,6 +23,7 @@ import { CalculatedField, CalculatedFieldTestScriptInputParams } from '@shared/m import { PageLink } from '@shared/models/page/page-link'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityTestScriptResult } from '@shared/models/entity.models'; +import { CalculatedFieldEventBody } from '@shared/models/event.models'; @Injectable({ providedIn: 'root' @@ -53,4 +54,8 @@ export class CalculatedFieldsService { public testScript(inputParams: CalculatedFieldTestScriptInputParams, config?: RequestConfig): Observable { return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); } + + public getLatestCalculatedFieldDebugEvent(id: string, config?: RequestConfig): Observable { + return this.http.get(`/api/calculatedField/${id}/debug`, defaultHttpOptionsFromConfig(config)); + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index c3e3d52a72..65654efd70 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -35,7 +35,7 @@ import { import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; -import { map, startWith } from 'rxjs/operators'; +import { map, startWith, switchMap } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; @@ -136,10 +136,22 @@ export class CalculatedFieldDialogComponent extends DialogComponent { - this.configFormGroup.get('expressionSCRIPT').setValue(expression); - this.configFormGroup.get('expressionSCRIPT').markAsDirty(); - }); + const calculatedFieldId = this.data.value?.id?.id; + + (calculatedFieldId + ? this.calculatedFieldsService.getLatestCalculatedFieldDebugEvent(calculatedFieldId) + .pipe( + switchMap(event => { + const args = event?.arguments ? JSON.parse(event.arguments) : null; + return this.data.getTestScriptDialogFn(this.fromGroupValue, args, false); + }), + takeUntilDestroyed(this.destroyRef) + ) + : this.data.getTestScriptDialogFn(this.fromGroupValue, null, false)) + .subscribe(expression => { + this.configFormGroup.get('expressionSCRIPT').setValue(expression); + this.configFormGroup.get('expressionSCRIPT').markAsDirty(); + }); } private applyDialogData(): void { From 99beae211577c39944c136e411ab15d9e8b6f0a2 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 28 Feb 2025 17:17:55 +0200 Subject: [PATCH 06/26] refactoring --- .../calculated-field-dialog.component.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 65654efd70..68abd646e3 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -39,6 +39,7 @@ import { map, startWith, switchMap } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { Observable } from 'rxjs'; @Component({ selector: 'tb-calculated-field-dialog', @@ -137,9 +138,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent; - (calculatedFieldId - ? this.calculatedFieldsService.getLatestCalculatedFieldDebugEvent(calculatedFieldId) + if (calculatedFieldId) { + testScriptDialogResult$ = this.calculatedFieldsService.getLatestCalculatedFieldDebugEvent(calculatedFieldId) .pipe( switchMap(event => { const args = event?.arguments ? JSON.parse(event.arguments) : null; @@ -147,11 +149,14 @@ export class CalculatedFieldDialogComponent extends DialogComponent { - this.configFormGroup.get('expressionSCRIPT').setValue(expression); - this.configFormGroup.get('expressionSCRIPT').markAsDirty(); - }); + } else { + testScriptDialogResult$ = this.data.getTestScriptDialogFn(this.fromGroupValue, null, false); + } + + testScriptDialogResult$.subscribe(expression => { + this.configFormGroup.get('expressionSCRIPT').setValue(expression); + this.configFormGroup.get('expressionSCRIPT').markAsDirty(); + }); } private applyDialogData(): void { From ac52098207e4330fa44225f07bc08614d019bf9f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 28 Feb 2025 17:46:25 +0200 Subject: [PATCH 07/26] CF: merge function draft --- .../api/tbel/DefaultTbelInvokeService.java | 1 + .../script/api/tbel/TbTimeWindow.java | 2 + .../thingsboard/script/api/tbel/TbUtils.java | 51 ++++++++++++++-- .../api/tbel/TbelCfTsMultiDoubleVal.java | 32 ++++++++++ .../script/api/tbel/TbelCfTsRollingArg.java | 5 -- .../script/api/tbel/TbelCfTsRollingData.java | 61 +++++++++++++++++++ 6 files changed, 143 insertions(+), 9 deletions(-) create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingData.java diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java index f626746a64..962d384a46 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java @@ -136,6 +136,7 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem parserConfig.registerDataType("TbelCfSingleValueArg", TbelCfSingleValueArg.class, TbelCfSingleValueArg::memorySize); parserConfig.registerDataType("TbelCfTsRollingArg", TbelCfTsRollingArg.class, TbelCfTsRollingArg::memorySize); parserConfig.registerDataType("TbelCfTsDoubleVal", TbelCfTsDoubleVal.class, TbelCfTsDoubleVal::memorySize); + parserConfig.registerDataType("TbelCfTsRollingData", TbelCfTsRollingData.class, TbelCfTsRollingData::memorySize); parserConfig.registerDataType("TbTimeWindow", TbTimeWindow.class, TbTimeWindow::memorySize); TbUtils.register(parserConfig); executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "tbel-executor")); diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java index ce61965317..5048611838 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java @@ -17,9 +17,11 @@ package org.thingsboard.script.api.tbel; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; @Data @AllArgsConstructor +@NoArgsConstructor public class TbTimeWindow implements TbelCfObject { public static final long OBJ_SIZE = 32L; diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 9a57240d23..b108bbe33e 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2025 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -44,6 +44,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.regex.Matcher; import static java.lang.Character.MAX_RADIX; @@ -367,6 +368,8 @@ public class TbUtils { byte[].class, int.class))); parserConfig.addImport("parseBinaryArrayToInt", new MethodStub(TbUtils.class.getMethod("parseBinaryArrayToInt", byte[].class, int.class, int.class))); + parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("mergeCfTsRollingArgs", + TbelCfTsRollingArg.class, TbelCfTsRollingArg.class))); } public static String btoa(String input) { @@ -1506,5 +1509,45 @@ public class TbUtils { } return hex; } + + public static TbelCfTsRollingData mergeCfTsRollingArgs(TbelCfTsRollingArg a, TbelCfTsRollingArg b) { + return mergeCfTsRollingArgs(Arrays.asList(a, b), null); + } + + public static TbelCfTsRollingData mergeCfTsRollingArgs(List args, Map settings) { + TreeSet allTimestamps = new TreeSet<>(); + long startTs = Long.MAX_VALUE; + long endTs = Long.MIN_VALUE; + for (TbelCfTsRollingArg arg : args) { + for (TbelCfTsDoubleVal val : arg.getValues()) { + allTimestamps.add(val.getTs()); + } + startTs = Math.min(startTs, arg.getTimeWindow().getStartTs()); + endTs = Math.max(endTs, arg.getTimeWindow().getEndTs()); + } + + List data = new ArrayList<>(); + + int[] lastIndex = new int[args.size()]; + double[] result = new double[args.size()]; + Arrays.fill(result, Double.NaN); + + var tw = new TbTimeWindow(startTs, endTs, allTimestamps.size()); + + for (long ts : allTimestamps) { + for (int i = 0; i < args.size(); i++) { + var arg = args.get(i); + var values = arg.getValues(); + while (lastIndex[i] < values.size() && values.get(lastIndex[i]).getTs() <= ts) { + result[i] = values.get(lastIndex[i]).getValue(); + lastIndex[i]++; + } + } + data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); + } + + return new TbelCfTsRollingData(tw, data); + } + } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java new file mode 100644 index 0000000000..33e4a65bb8 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import lombok.Data; + +@Data +public class TbelCfTsMultiDoubleVal implements TbelCfObject { + + public static final long OBJ_SIZE = 32L; // Approximate calculation; + + private final long ts; + private final double[] values; + + @Override + public long memorySize() { + return OBJ_SIZE + values.length * 8L; + } +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java index 807d498a16..74c9d77454 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java @@ -266,11 +266,6 @@ public class TbelCfTsRollingArg implements TbelCfArg, Iterable action) { - values.forEach(action); - } - @Override public String getType() { return "TS_ROLLING"; diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingData.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingData.java new file mode 100644 index 0000000000..646e826915 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingData.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.function.Consumer; + +import static org.thingsboard.script.api.tbel.TbelCfTsDoubleVal.OBJ_SIZE; + +public class TbelCfTsRollingData implements TbelCfObject, Iterable { + + @Getter + private final TbTimeWindow timeWindow; + @Getter + private final List values; + + public TbelCfTsRollingData(TbTimeWindow timeWindow, List values) { + this.timeWindow = timeWindow; + this.values = Collections.unmodifiableList(values); + } + + @Override + public long memorySize() { + return 12 + values.size() * OBJ_SIZE; + } + + @JsonIgnore + public List getValue() { + return values; + } + + @JsonIgnore + public int getSize() { + return values.size(); + } + + @Override + public Iterator iterator() { + return values.iterator(); + } + +} From 2b51451896f74aad4854bd199b3abd41dc4d0fa3 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 28 Feb 2025 18:17:20 +0200 Subject: [PATCH 08/26] Change cf strategy for vc --- .../DefaultEntitiesExportImportService.java | 13 +- .../sync/ie/EntitiesExportImportService.java | 1 - .../ie/exporting/impl/AssetExportService.java | 17 ++- .../impl/AssetProfileExportService.java | 17 ++- .../BaseCalculatedFieldsExportService.java | 51 ++++++++ .../impl/CalculatedFieldExportService.java | 49 ------- .../exporting/impl/DeviceExportService.java | 11 +- .../impl/DeviceProfileExportService.java | 17 ++- .../ie/importing/impl/AssetImportService.java | 18 ++- .../impl/AssetProfileImportService.java | 18 ++- .../BaseCalculatedFieldsImportService.java | 123 ++++++++++++++++++ .../impl/CalculatedFieldImportService.java | 85 ------------ .../importing/impl/DeviceImportService.java | 48 ++++--- .../impl/DeviceProfileImportService.java | 18 ++- .../DefaultEntitiesVersionControlService.java | 10 +- .../sync/vc/data/EntitiesExportCtx.java | 2 + .../sync/vc/data/EntitiesImportCtx.java | 12 +- .../sync/vc/data/EntityTypeExportCtx.java | 1 + .../sync/ie/ExportImportServiceSqlTest.java | 50 ++++--- .../service/sync/vc/VersionControlTest.java | 11 +- .../common/data/sync/ie/AssetExportData.java | 28 ++++ .../data/sync/ie/AssetProfileExportData.java | 28 ++++ .../sync/ie/CalculatedFieldExportData.java | 56 ++++++++ .../common/data/sync/ie/DeviceExportData.java | 3 +- .../data/sync/ie/DeviceProfileExportData.java | 28 ++++ .../common/data/sync/ie/EntityExportData.java | 8 ++ .../data/sync/ie/EntityExportSettings.java | 3 + .../data/sync/ie/EntityImportSettings.java | 3 + .../request/create/VersionCreateConfig.java | 5 + .../vc/request/load/VersionLoadConfig.java | 1 + .../dao/cf/BaseCalculatedFieldService.java | 1 - .../model/sql/CalculatedFieldLinkEntity.java | 11 +- 32 files changed, 525 insertions(+), 222 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseCalculatedFieldsExportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/CalculatedFieldExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseCalculatedFieldsImportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CalculatedFieldImportService.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AssetExportData.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AssetProfileExportData.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/CalculatedFieldExportData.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceProfileExportData.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 7d0f5ebf48..94b0ddc07e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.util.ThrowingRunnable; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -61,6 +62,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> importServices = new HashMap<>(); private final RelationService relationService; + private final CalculatedFieldService calculatedFieldService; private final RateLimitService rateLimitService; private final TbLogEntityActionService logEntityActionService; @@ -69,11 +71,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS EntityType.DASHBOARD, EntityType.ASSET_PROFILE, EntityType.ASSET, EntityType.DEVICE_PROFILE, EntityType.DEVICE, EntityType.ENTITY_VIEW, EntityType.WIDGET_TYPE, EntityType.WIDGETS_BUNDLE, - EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_RULE, - EntityType.CALCULATED_FIELD + EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_RULE ); - @Override public , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId) throws ThingsboardException { if (!rateLimitService.checkRateLimit(LimitedApi.ENTITY_EXPORT, ctx.getTenantId())) { @@ -128,15 +128,18 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS logEntityActionService.logEntityRelationAction(ctx.getTenantId(), null, relation, ctx.getUser(), ActionType.RELATION_ADD_OR_UPDATE, null, relation); } - } + ctx.getCalculatedFields().forEach((calculatedField, created) -> { + var savedCalculatedField = calculatedFieldService.save(calculatedField); + logEntityActionService.logEntityAction(ctx.getTenantId(), savedCalculatedField.getId(), savedCalculatedField, created ? ActionType.ADDED : ActionType.UPDATED, ctx.getUser()); + }); + } @Override public Comparator getEntityTypeComparatorForImport() { return Comparator.comparing(SUPPORTED_ENTITY_TYPES::indexOf); } - @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { EntityExportService exportService = exportServices.get(entityType); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 2b7e7d3593..b77447df8d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -32,7 +32,6 @@ public interface EntitiesExportImportService { , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData) throws ThingsboardException; - void saveReferencesAndRelations(EntitiesImportCtx ctx) throws ThingsboardException; Comparator getEntityTypeComparatorForImport(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java index 56d5bb0934..6f56fa5145 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java @@ -19,7 +19,8 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.AssetExportData; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; @@ -27,12 +28,22 @@ import java.util.Set; @Service @TbCoreComponent -public class AssetExportService extends BaseEntityExportService> { +public class AssetExportService extends BaseCalculatedFieldsExportService { + + protected AssetExportService(CalculatedFieldService calculatedFieldService) { + super(calculatedFieldService); + } @Override - protected void setRelatedEntities(EntitiesExportCtx ctx, Asset asset, EntityExportData exportData) { + protected void setRelatedEntities(EntitiesExportCtx ctx, Asset asset, AssetExportData exportData) { asset.setCustomerId(getExternalIdOrElseInternal(ctx, asset.getCustomerId())); asset.setAssetProfileId(getExternalIdOrElseInternal(ctx, asset.getAssetProfileId())); + setCalculatedFields(ctx, asset, exportData); + } + + @Override + protected AssetExportData newExportData() { + return new AssetExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetProfileExportService.java index b670daf06c..2314db92f5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetProfileExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetProfileExportService.java @@ -19,7 +19,8 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.AssetProfileExportData; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; @@ -27,13 +28,23 @@ import java.util.Set; @Service @TbCoreComponent -public class AssetProfileExportService extends BaseEntityExportService> { +public class AssetProfileExportService extends BaseCalculatedFieldsExportService { + + protected AssetProfileExportService(CalculatedFieldService calculatedFieldService) { + super(calculatedFieldService); + } @Override - protected void setRelatedEntities(EntitiesExportCtx ctx, AssetProfile assetProfile, EntityExportData exportData) { + protected void setRelatedEntities(EntitiesExportCtx ctx, AssetProfile assetProfile, AssetProfileExportData exportData) { assetProfile.setDefaultDashboardId(getExternalIdOrElseInternal(ctx, assetProfile.getDefaultDashboardId())); assetProfile.setDefaultRuleChainId(getExternalIdOrElseInternal(ctx, assetProfile.getDefaultRuleChainId())); assetProfile.setDefaultEdgeRuleChainId(getExternalIdOrElseInternal(ctx, assetProfile.getDefaultEdgeRuleChainId())); + setCalculatedFields(ctx, assetProfile, exportData); + } + + @Override + protected AssetProfileExportData newExportData() { + return new AssetProfileExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseCalculatedFieldsExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseCalculatedFieldsExportService.java new file mode 100644 index 0000000000..e28f8aaa79 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseCalculatedFieldsExportService.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ie.CalculatedFieldExportData; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +import java.util.List; + +public abstract class BaseCalculatedFieldsExportService & HasTenantId, D extends CalculatedFieldExportData> extends BaseEntityExportService { + + protected final CalculatedFieldService calculatedFieldService; + + protected BaseCalculatedFieldsExportService(CalculatedFieldService calculatedFieldService) { + this.calculatedFieldService = calculatedFieldService; + } + + protected void setCalculatedFields(EntitiesExportCtx ctx, E entity, D exportData) { + if (ctx.getSettings().isExportCalculatedFields()) { + List calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(ctx.getTenantId(), entity.getId()); + calculatedFields.forEach(calculatedField -> { + calculatedField.getConfiguration().getArguments().values().forEach(argument -> { + if (argument.getRefEntityId() != null) { + EntityId externalId = getExternalIdOrElseInternal(ctx, argument.getRefEntityId()); + argument.setRefEntityId(externalId); + } + }); + }); + exportData.setCalculatedFields(calculatedFields); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/CalculatedFieldExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/CalculatedFieldExportService.java deleted file mode 100644 index b42f3c6647..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/CalculatedFieldExportService.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.ie.exporting.impl; - -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; - -import java.util.Set; - -@Service -@TbCoreComponent -public class CalculatedFieldExportService extends BaseEntityExportService> { - - @Override - protected void setRelatedEntities(EntitiesExportCtx ctx, CalculatedField calculatedField, EntityExportData exportData) { - calculatedField.setEntityId(getExternalIdOrElseInternal(ctx, calculatedField.getEntityId())); - calculatedField.getConfiguration().getArguments().values().forEach(argument -> { - if (argument.getRefEntityId() != null) { - EntityId internalEntityId = getExternalIdOrElseInternal(ctx, argument.getRefEntityId()); - argument.setRefEntityId(internalEntityId); - } - }); - } - - @Override - public Set getSupportedEntityTypes() { - return Set.of(EntityType.CALCULATED_FIELD); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java index 7d5f7ee57e..f1b5ade487 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java @@ -15,12 +15,12 @@ */ package org.thingsboard.server.service.sync.ie.exporting.impl; -import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.sync.ie.DeviceExportData; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; @@ -29,11 +29,15 @@ import java.util.Set; @Service @TbCoreComponent -@RequiredArgsConstructor -public class DeviceExportService extends BaseEntityExportService { +public class DeviceExportService extends BaseCalculatedFieldsExportService { private final DeviceCredentialsService deviceCredentialsService; + public DeviceExportService(CalculatedFieldService calculatedFieldService, DeviceCredentialsService deviceCredentialsService) { + super(calculatedFieldService); + this.deviceCredentialsService = deviceCredentialsService; + } + @Override protected void setRelatedEntities(EntitiesExportCtx ctx, Device device, DeviceExportData exportData) { device.setCustomerId(getExternalIdOrElseInternal(ctx, device.getCustomerId())); @@ -44,6 +48,7 @@ public class DeviceExportService extends BaseEntityExportService> { +public class DeviceProfileExportService extends BaseCalculatedFieldsExportService { + + protected DeviceProfileExportService(CalculatedFieldService calculatedFieldService) { + super(calculatedFieldService); + } @Override - protected void setRelatedEntities(EntitiesExportCtx ctx, DeviceProfile deviceProfile, EntityExportData exportData) { + protected void setRelatedEntities(EntitiesExportCtx ctx, DeviceProfile deviceProfile, DeviceProfileExportData exportData) { deviceProfile.setDefaultDashboardId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultDashboardId())); deviceProfile.setDefaultRuleChainId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultRuleChainId())); deviceProfile.setDefaultEdgeRuleChainId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultEdgeRuleChainId())); + setCalculatedFields(ctx, deviceProfile, exportData); + } + + @Override + protected DeviceProfileExportData newExportData() { + return new DeviceProfileExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java index 00ce23b2e8..0b0649072e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java @@ -15,24 +15,28 @@ */ package org.thingsboard.server.service.sync.ie.importing.impl; -import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.AssetExportData; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @Service @TbCoreComponent -@RequiredArgsConstructor -public class AssetImportService extends BaseEntityImportService> { +public class AssetImportService extends BaseCalculatedFieldsImportService { private final AssetService assetService; + public AssetImportService(CalculatedFieldService calculatedFieldService, AssetService assetService) { + super(calculatedFieldService); + this.assetService = assetService; + } + @Override protected void setOwner(TenantId tenantId, Asset asset, IdProvider idProvider) { asset.setTenantId(tenantId); @@ -40,14 +44,14 @@ public class AssetImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected Asset prepare(EntitiesImportCtx ctx, Asset asset, Asset old, AssetExportData exportData, IdProvider idProvider) { asset.setAssetProfileId(idProvider.getInternalId(asset.getAssetProfileId())); return asset; } @Override - protected Asset saveOrUpdate(EntitiesImportCtx ctx, Asset asset, EntityExportData exportData, IdProvider idProvider) { - return assetService.saveAsset(asset); + protected Asset saveOrUpdate(EntitiesImportCtx ctx, Asset asset, AssetExportData exportData, IdProvider idProvider) { + return saveOrUpdateEntity(ctx, asset, exportData, idProvider, assetService::saveAsset); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java index 97c54957bf..6797c92c81 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.sync.ie.importing.impl; -import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; @@ -23,25 +22,30 @@ import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.AssetProfileExportData; import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @Service @TbCoreComponent -@RequiredArgsConstructor -public class AssetProfileImportService extends BaseEntityImportService> { +public class AssetProfileImportService extends BaseCalculatedFieldsImportService { private final AssetProfileService assetProfileService; + public AssetProfileImportService(CalculatedFieldService calculatedFieldService, AssetProfileService assetProfileService) { + super(calculatedFieldService); + this.assetProfileService = assetProfileService; + } + @Override protected void setOwner(TenantId tenantId, AssetProfile assetProfile, IdProvider idProvider) { assetProfile.setTenantId(tenantId); } @Override - protected AssetProfile prepare(EntitiesImportCtx ctx, AssetProfile assetProfile, AssetProfile old, EntityExportData exportData, IdProvider idProvider) { + protected AssetProfile prepare(EntitiesImportCtx ctx, AssetProfile assetProfile, AssetProfile old, AssetProfileExportData exportData, IdProvider idProvider) { assetProfile.setDefaultRuleChainId(idProvider.getInternalId(assetProfile.getDefaultRuleChainId())); assetProfile.setDefaultDashboardId(idProvider.getInternalId(assetProfile.getDefaultDashboardId())); assetProfile.setDefaultEdgeRuleChainId(idProvider.getInternalId(assetProfile.getDefaultEdgeRuleChainId())); @@ -49,8 +53,8 @@ public class AssetProfileImportService extends BaseEntityImportService exportData, IdProvider idProvider) { - return assetProfileService.saveAssetProfile(assetProfile); + protected AssetProfile saveOrUpdate(EntitiesImportCtx ctx, AssetProfile assetProfile, AssetProfileExportData exportData, IdProvider idProvider) { + return saveOrUpdateEntity(ctx, assetProfile, exportData, idProvider, assetProfileService::saveAssetProfile); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseCalculatedFieldsImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseCalculatedFieldsImportService.java new file mode 100644 index 0000000000..ffa51938f0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseCalculatedFieldsImportService.java @@ -0,0 +1,123 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ie.CalculatedFieldExportData; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +public abstract class BaseCalculatedFieldsImportService & HasTenantId, D extends CalculatedFieldExportData> extends BaseEntityImportService { + + private final CalculatedFieldService calculatedFieldService; + + protected BaseCalculatedFieldsImportService(CalculatedFieldService calculatedFieldService) { + this.calculatedFieldService = calculatedFieldService; + } + + protected E saveOrUpdateEntity(EntitiesImportCtx ctx, E entity, D exportData, IdProvider idProvider, Function saveFunction) { + E savedEntity = saveFunction.apply(entity); + + if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) { + saveCalculatedFields(ctx, savedEntity, exportData, idProvider); + } + return savedEntity; + } + + protected void saveCalculatedFields(EntitiesImportCtx ctx, E savedEntity, D exportData, IdProvider idProvider) { + if (exportData.getCalculatedFields() == null || !ctx.isSaveCalculatedFields()) { + return; + } + + exportData.getCalculatedFields().forEach(calculatedField -> { + calculatedField.setTenantId(savedEntity.getTenantId()); + calculatedField.setExternalId(calculatedField.getId()); + calculatedField.setId(idProvider.getInternalId(calculatedField.getId(), false)); + calculatedField.setEntityId(savedEntity.getId()); + + calculatedField.getConfiguration().getArguments().values().forEach(argument -> { + if (argument.getRefEntityId() != null) { + argument.setRefEntityId(idProvider.getInternalId(argument.getRefEntityId(), false)); + } + }); + + calculatedFieldService.save(calculatedField); + }); + } + + @Override + protected boolean updateRelatedEntitiesIfUnmodified(EntitiesImportCtx ctx, E prepared, D exportData, IdProvider idProvider) { + boolean updated = super.updateRelatedEntitiesIfUnmodified(ctx, prepared, exportData, idProvider); + updated |= updateCalculatedFields(ctx, prepared, exportData, idProvider); + return updated; + } + + private boolean updateCalculatedFields(EntitiesImportCtx ctx, E prepared, D exportData, IdProvider idProvider) { + var calculatedFields = exportData.getCalculatedFields(); + if (calculatedFields == null || !ctx.isSaveCalculatedFields()) { + return false; + } + Map calculatedFieldMap = calculatedFields.stream() + .peek(newField -> { + newField.setTenantId(ctx.getTenantId()); + newField.setExternalId(newField.getId()); + newField.setId(idProvider.getInternalId(newField.getId(), false)); + newField.setEntityId(prepared.getId()); + newField.getConfiguration().getArguments().values().forEach(argument -> { + argument.setRefEntityId(idProvider.getInternalId(argument.getRefEntityId(), false)); + }); + }) + .collect(Collectors.toMap(CalculatedField::getId, field -> field)); + + List existingFields = calculatedFieldService.findCalculatedFieldsByEntityId(ctx.getTenantId(), prepared.getId()); + boolean updated = false; + + Map result = new LinkedHashMap<>(); + for (CalculatedField existingField : existingFields) { + if (calculatedFieldMap.containsKey(existingField.getId())) { + CalculatedField newField = calculatedFieldMap.get(existingField.getId()); + if (!newField.equals(existingField)) { + result.put(newField, false); + } + calculatedFieldMap.remove(existingField.getId()); + } else { + updated = true; + calculatedFieldService.deleteCalculatedField(ctx.getTenantId(), existingField.getId()); + } + } + + for (CalculatedField newField : calculatedFieldMap.values()) { + result.put(newField, true); + } + + if (!result.isEmpty()) { + updated = true; + ctx.addCalculatedFields(result); + } + return updated; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CalculatedFieldImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CalculatedFieldImportService.java deleted file mode 100644 index a723f28b1b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CalculatedFieldImportService.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.ie.importing.impl; - -import lombok.RequiredArgsConstructor; -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.dao.service.ConstraintValidator; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; - -@Service -@TbCoreComponent -@RequiredArgsConstructor -public class CalculatedFieldImportService extends BaseEntityImportService> { - - private final CalculatedFieldService calculatedFieldService; - - @Override - protected void setOwner(TenantId tenantId, CalculatedField calculatedField, IdProvider idProvider) { - calculatedField.setTenantId(tenantId); - } - - @Override - protected CalculatedField prepare(EntitiesImportCtx ctx, CalculatedField calculatedField, CalculatedField oldEntity, EntityExportData exportData, IdProvider idProvider) { - calculatedField.setEntityId(idProvider.getInternalId(calculatedField.getEntityId())); - calculatedField.getConfiguration().getArguments().values().forEach(argument -> { - if (argument.getRefEntityId() != null) { - EntityId internalEntityId = idProvider.getInternalId(argument.getRefEntityId()); - argument.setRefEntityId(internalEntityId); - } - }); - return calculatedField; - } - - @Override - protected CalculatedField saveOrUpdate(EntitiesImportCtx ctx, CalculatedField calculatedField, EntityExportData exportData, IdProvider idProvider) { - ConstraintValidator.validateFields(calculatedField); - return calculatedFieldService.save(calculatedField); - } - - @Override - protected CalculatedField deepCopy(CalculatedField calculatedField) { - return new CalculatedField(calculatedField); - } - - @Override - protected void onEntitySaved(User user, CalculatedField savedEntity, CalculatedField oldEntity) throws ThingsboardException { - entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, null, - oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, null); - } - - @Override - protected void cleanupForComparison(CalculatedField e) { - super.cleanupForComparison(e); - } - - @Override - public EntityType getEntityType() { - return EntityType.CALCULATED_FIELD; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java index 61b9839cbd..c8e99cdbc1 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -15,13 +15,13 @@ */ package org.thingsboard.server.service.sync.ie.importing.impl; -import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.DeviceExportData; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -29,12 +29,17 @@ import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @Service @TbCoreComponent -@RequiredArgsConstructor -public class DeviceImportService extends BaseEntityImportService { +public class DeviceImportService extends BaseCalculatedFieldsImportService { private final DeviceService deviceService; private final DeviceCredentialsService credentialsService; + public DeviceImportService(CalculatedFieldService calculatedFieldService, DeviceService deviceService, DeviceCredentialsService credentialsService) { + super(calculatedFieldService); + this.deviceService = deviceService; + this.credentialsService = credentialsService; + } + @Override protected void setOwner(TenantId tenantId, Device device, IdProvider idProvider) { device.setTenantId(tenantId); @@ -64,31 +69,44 @@ public class DeviceImportService extends BaseEntityImportService> { +public class DeviceProfileImportService extends BaseCalculatedFieldsImportService { private final DeviceProfileService deviceProfileService; + public DeviceProfileImportService(CalculatedFieldService calculatedFieldService, DeviceProfileService deviceProfileService) { + super(calculatedFieldService); + this.deviceProfileService = deviceProfileService; + } + @Override protected void setOwner(TenantId tenantId, DeviceProfile deviceProfile, IdProvider idProvider) { deviceProfile.setTenantId(tenantId); } @Override - protected DeviceProfile prepare(EntitiesImportCtx ctx, DeviceProfile deviceProfile, DeviceProfile old, EntityExportData exportData, IdProvider idProvider) { + protected DeviceProfile prepare(EntitiesImportCtx ctx, DeviceProfile deviceProfile, DeviceProfile old, DeviceProfileExportData exportData, IdProvider idProvider) { deviceProfile.setDefaultRuleChainId(idProvider.getInternalId(deviceProfile.getDefaultRuleChainId())); deviceProfile.setDefaultEdgeRuleChainId(idProvider.getInternalId(deviceProfile.getDefaultEdgeRuleChainId())); deviceProfile.setDefaultDashboardId(idProvider.getInternalId(deviceProfile.getDefaultDashboardId())); @@ -51,8 +55,8 @@ public class DeviceProfileImportService extends BaseEntityImportService exportData, IdProvider idProvider) { - return deviceProfileService.saveDeviceProfile(deviceProfile); + protected DeviceProfile saveOrUpdate(EntitiesImportCtx ctx, DeviceProfile deviceProfile, DeviceProfileExportData exportData, IdProvider idProvider) { + return saveOrUpdateEntity(ctx, deviceProfile, exportData, idProvider, deviceProfileService::saveDeviceProfile); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index eed18abe52..89b51ce84a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -94,7 +94,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.function.Function; -import java.util.stream.Collectors; import static com.google.common.util.concurrent.Futures.transform; import static org.thingsboard.server.common.data.sync.vc.VcUtils.checkBranchName; @@ -304,6 +303,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .updateRelations(config.isLoadRelations()) .saveAttributes(config.isLoadAttributes()) .saveCredentials(config.isLoadCredentials()) + .saveCalculatedFields(config.isLoadCalculatedFields()) .findExistingByName(false) .build()); ctx.setFinalImportAttempt(true); @@ -327,7 +327,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont var sw = TbStopWatch.create("before"); List entityTypes = request.getEntityTypes().keySet().stream() - .sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList()); + .sorted(exportImportService.getEntityTypeComparatorForImport()).toList(); for (EntityType entityType : entityTypes) { log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); sw.startNew("Entities " + entityType.name()); @@ -362,6 +362,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .updateRelations(config.isLoadRelations()) .saveAttributes(config.isLoadAttributes()) .saveCredentials(config.isLoadCredentials()) + .saveCalculatedFields(config.isLoadCalculatedFields()) .findExistingByName(config.isFindExistingEntityByName()) .build(); } @@ -471,7 +472,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } @Override - public ListenableFuture compareEntityDataToVersion(User user, EntityId entityId, String versionId) throws Exception { + public ListenableFuture compareEntityDataToVersion(User user, EntityId entityId, String versionId) { HasId entity = exportableEntitiesService.findEntityByTenantIdAndId(user.getTenantId(), entityId); if (!(entity instanceof ExportableEntity)) throw new IllegalArgumentException("Unsupported entity type"); @@ -484,6 +485,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .exportRelations(otherVersion.hasRelations()) .exportAttributes(otherVersion.hasAttributes()) .exportCredentials(otherVersion.hasCredentials()) + .exportCalculatedFields(otherVersion.hasCalculatedFields()) .build()); EntityExportData currentVersion; try { @@ -503,7 +505,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override - public ListenableFuture> listBranches(TenantId tenantId) throws Exception { + public ListenableFuture> listBranches(TenantId tenantId) { return gitServiceQueue.listBranches(tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java index 669dad347a..b3007c6738 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java @@ -69,6 +69,7 @@ public abstract class EntitiesExportCtx { .exportRelations(config.isSaveRelations()) .exportAttributes(config.isSaveAttributes()) .exportCredentials(config.isSaveCredentials()) + .exportCalculatedFields(config.isSaveCalculatedFields()) .build(); } @@ -85,4 +86,5 @@ public abstract class EntitiesExportCtx { log.debug("[{}][{}] Local cache put: {}", internalId.getEntityType(), internalId.getId(), externalId); externalIdMap.put(internalId, externalId != null ? externalId : internalId); } + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index ab836fa13d..c1a8dad161 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -19,6 +19,7 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -31,6 +32,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -54,6 +56,7 @@ public class EntitiesImportCtx { private final Set notFoundIds = new HashSet<>(); private final Set relations = new LinkedHashSet<>(); + private final Map calculatedFields = new LinkedHashMap<>(); private boolean finalImportAttempt = false; private EntityImportSettings settings; @@ -91,6 +94,10 @@ public class EntitiesImportCtx { return getSettings().isSaveCredentials(); } + public boolean isSaveCalculatedFields() { + return getSettings().isSaveCalculatedFields(); + } + public EntityId getInternalId(EntityId externalId) { var result = externalToInternalIdMap.get(externalId); log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss"); @@ -120,6 +127,10 @@ public class EntitiesImportCtx { relations.addAll(values); } + public void addCalculatedFields(Map calculatedFieldMap) { + calculatedFields.putAll(calculatedFieldMap); + } + public void addReferenceCallback(EntityId externalId, ThrowingRunnable tr) { if (tr != null) { referenceCallbacks.put(externalId, tr); @@ -140,5 +151,4 @@ public class EntitiesImportCtx { return notFoundIds.contains(externalId); } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java index e0a5b40281..55c8a763b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java @@ -39,6 +39,7 @@ public class EntityTypeExportCtx extends EntitiesExportCtx .exportRelations(config.isSaveRelations()) .exportAttributes(config.isSaveAttributes()) .exportCredentials(config.isSaveCredentials()) + .exportCalculatedFields(config.isSaveCalculatedFields()) .build(); this.overwrite = ObjectUtils.defaultIfNull(config.getSyncStrategy(), defaultSyncStrategy) == SyncStrategy.OVERWRITE; } diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java index 8870194685..9184a3c034 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java @@ -80,6 +80,7 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; @@ -207,11 +208,12 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { CalculatedField calculatedField = createCalculatedField(tenantId1, device.getId(), asset.getId()); Map entitiesExportData = Stream.of(customer.getId(), asset.getId(), device.getId(), - ruleChain.getId(), dashboard.getId(), assetProfile.getId(), deviceProfile.getId(), calculatedField.getId()) + ruleChain.getId(), dashboard.getId(), assetProfile.getId(), deviceProfile.getId()) .map(entityId -> { try { return exportEntity(tenantAdmin1, entityId, EntityExportSettings.builder() .exportCredentials(false) + .exportCalculatedFields(true) .build()); } catch (Exception e) { throw new RuntimeException(e); @@ -281,23 +283,28 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DEVICE)); verify(tbClusterService, Mockito.never()).onDeviceUpdated(eq(importedDevice), eq(importedDevice)); + // calculated field of imported device: + List calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId2, importedDevice.getId()); + assertThat(calculatedFields.size()).isOne(); + var importedCalculatedField = calculatedFields.get(0); + assertThat(importedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(importedCalculatedField.getExternalId()).isEqualTo(calculatedField.getId()); + EntityExportData updatedDeviceEntity = getAndClone(entitiesExportData, EntityType.DEVICE); updatedDeviceEntity.getEntity().setLabel("t" + updatedDeviceEntity.getEntity().getLabel()); Device updatedDevice = importEntity(tenantAdmin2, updatedDeviceEntity).getSavedEntity(); verify(tbClusterService).onDeviceUpdated(eq(updatedDevice), eq(importedDevice)); - CalculatedField importedCalculatedField = (CalculatedField) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.CALCULATED_FIELD)).getSavedEntity(); - verify(entityActionService).logEntityAction(any(), eq(importedCalculatedField.getId()), eq(importedCalculatedField), - any(), eq(ActionType.ADDED), isNull()); - importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.CALCULATED_FIELD)); - verify(entityActionService, Mockito.never()).logEntityAction(any(), eq(importedCalculatedField.getId()), eq(importedCalculatedField), - any(), eq(ActionType.UPDATED), isNull()); + // update calculated field: + DeviceExportData deviceExportData = (DeviceExportData) getAndClone(entitiesExportData, EntityType.DEVICE); + deviceExportData.setCalculatedFields(deviceExportData.getCalculatedFields().stream().peek(field -> field.setName("t_" + field.getName())).toList()); + importEntity(tenantAdmin2, deviceExportData).getSavedEntity(); - EntityExportData updatedCalculatedFieldEntity = getAndClone(entitiesExportData, EntityType.CALCULATED_FIELD); - updatedCalculatedFieldEntity.getEntity().setName("t" + updatedCalculatedFieldEntity.getEntity().getName()); - CalculatedField updatedCalculatedField = importEntity(tenantAdmin2, updatedCalculatedFieldEntity).getSavedEntity(); - verify(entityActionService).logEntityAction(any(), eq(updatedCalculatedField.getId()), eq(updatedCalculatedField), - any(), eq(ActionType.UPDATED), isNull()); + calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId2, importedDevice.getId()); + assertThat(calculatedFields.size()).isOne(); + importedCalculatedField = calculatedFields.get(0); + assertThat(importedCalculatedField.getExternalId()).isEqualTo(calculatedField.getId()); + assertThat(importedCalculatedField.getName()).startsWith("t_"); } @Test @@ -320,10 +327,11 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { Map ids = new HashMap<>(); for (EntityId entityId : List.of(customer.getId(), ruleChain.getId(), dashboard.getId(), assetProfile.getId(), asset.getId(), - deviceProfile.getId(), device.getId(), entityView.getId(), ruleChain.getId(), dashboard.getId(), calculatedField.getId())) { + deviceProfile.getId(), device.getId(), entityView.getId(), ruleChain.getId(), dashboard.getId())) { EntityExportData exportData = exportEntity(getSecurityUser(tenantAdmin1), entityId); EntityImportResult importResult = importEntity(getSecurityUser(tenantAdmin2), exportData, EntityImportSettings.builder() .saveCredentials(false) + .saveCalculatedFields(true) .build()); ids.put(entityId, (EntityId) importResult.getSavedEntity().getId()); } @@ -353,19 +361,20 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { assertThat(exportedDeviceProfile.getDefaultRuleChainId()).isEqualTo(ruleChain.getId()); assertThat(exportedDeviceProfile.getDefaultDashboardId()).isEqualTo(dashboard.getId()); - Device exportedDevice = (Device) exportEntity(tenantAdmin2, (DeviceId) ids.get(device.getId())).getEntity(); + EntityExportData entityExportData = exportEntity(tenantAdmin2, (DeviceId) ids.get(device.getId())); + Device exportedDevice = entityExportData.getEntity(); assertThat(exportedDevice.getCustomerId()).isEqualTo(customer.getId()); assertThat(exportedDevice.getDeviceProfileId()).isEqualTo(deviceProfile.getId()); + List calculatedFields = ((DeviceExportData) entityExportData).getCalculatedFields(); + assertThat(calculatedFields.size()).isOne(); + CalculatedField field = calculatedFields.get(0); + assertThat(field.getName()).isEqualTo(calculatedField.getName()); + EntityView exportedEntityView = (EntityView) exportEntity(tenantAdmin2, (EntityViewId) ids.get(entityView.getId())).getEntity(); assertThat(exportedEntityView.getCustomerId()).isEqualTo(customer.getId()); assertThat(exportedEntityView.getEntityId()).isEqualTo(device.getId()); - CalculatedField exportedCalculatedField = (CalculatedField) exportEntity(tenantAdmin2, (CalculatedFieldId) ids.get(calculatedField.getId())).getEntity(); - assertThat(exportedCalculatedField.getName()).isEqualTo(calculatedField.getName()); - assertThat(exportedCalculatedField.getEntityId()).isEqualTo(device.getId()); - assertThat(exportedCalculatedField.getConfiguration().getReferencedEntities()).isEqualTo(calculatedField.getConfiguration().getReferencedEntities()); - deviceProfile.setDefaultDashboardId(null); deviceProfileService.saveDeviceProfile(deviceProfile); DeviceProfile importedDeviceProfile = deviceProfileService.findDeviceProfileById(tenantId2, (DeviceProfileId) ids.get(deviceProfile.getId())); @@ -373,7 +382,6 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { deviceProfileService.saveDeviceProfile(importedDeviceProfile); } - protected Device createDevice(TenantId tenantId, CustomerId customerId, DeviceProfileId deviceProfileId, String name) { Device device = new Device(); device.setTenantId(tenantId); @@ -618,6 +626,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { protected , I extends EntityId> EntityExportData exportEntity(User user, I entityId) throws Exception { return exportEntity(user, entityId, EntityExportSettings.builder() .exportCredentials(true) + .exportCalculatedFields(true) .build()); } @@ -628,6 +637,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { protected , I extends EntityId> EntityImportResult importEntity(User user, EntityExportData exportData) throws Exception { return importEntity(user, exportData, EntityImportSettings.builder() .saveCredentials(true) + .saveCalculatedFields(true) .build()); } diff --git a/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java index 28eaf06da7..453fea676f 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java @@ -576,12 +576,12 @@ public class VersionControlTest extends AbstractControllerTest { Asset asset = createAsset(null, null, "Asset 1"); Device device = createDevice(null, null, "Device 1", "test1"); CalculatedField calculatedField = createCalculatedField("CalculatedField1", device.getId(), asset.getId()); - String versionId = createVersion("calculated fields of asset and device", EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE, EntityType.CALCULATED_FIELD); + String versionId = createVersion("calculated fields of asset and device", EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE); loginTenant2(); loadVersion(versionId, config -> { config.setLoadCredentials(false); - }, EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE, EntityType.CALCULATED_FIELD); + }, EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE); Asset importedAsset = findAsset(asset.getName()); Device importedDevice = findDevice(device.getName()); @@ -602,9 +602,9 @@ public class VersionControlTest extends AbstractControllerTest { public void testVcWithCalculatedFields_sameTenant() throws Exception { Asset asset = createAsset(null, null, "Asset 1"); CalculatedField calculatedField = createCalculatedField("CalculatedField", asset.getId(), asset.getId()); - String versionId = createVersion("asset and field", EntityType.ASSET, EntityType.CALCULATED_FIELD); + String versionId = createVersion("asset and field", EntityType.ASSET); - loadVersion(versionId, EntityType.ASSET, EntityType.CALCULATED_FIELD); + loadVersion(versionId, EntityType.ASSET); CalculatedField importedCalculatedField = findCalculatedFieldByEntityId(asset.getId()); checkImportedEntity(tenantId1, calculatedField, tenantId1, importedCalculatedField); assertThat(importedCalculatedField.getName()).isEqualTo(calculatedField.getName()); @@ -676,10 +676,10 @@ public class VersionControlTest extends AbstractControllerTest { request.setEntityTypes(Arrays.stream(entityTypes).collect(Collectors.toMap(t -> t, entityType -> { EntityTypeVersionCreateConfig config = new EntityTypeVersionCreateConfig(); config.setAllEntities(true); - config.setSaveRelations(true); config.setSaveAttributes(true); config.setSaveCredentials(true); + config.setSaveCalculatedFields(true); return config; }))); @@ -745,6 +745,7 @@ public class VersionControlTest extends AbstractControllerTest { config.setLoadAttributes(true); config.setLoadRelations(true); config.setLoadCredentials(true); + config.setLoadCalculatedFields(true); config.setRemoveOtherEntities(false); config.setFindExistingEntityByName(true); configModifier.accept(config); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AssetExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AssetExportData.java new file mode 100644 index 0000000000..4bb836d5d9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AssetExportData.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.thingsboard.server.common.data.asset.Asset; + +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +@Data +public class AssetExportData extends CalculatedFieldExportData { + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AssetProfileExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AssetProfileExportData.java new file mode 100644 index 0000000000..42d9523828 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AssetProfileExportData.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.thingsboard.server.common.data.asset.AssetProfile; + +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +@Data +public class AssetProfileExportData extends CalculatedFieldExportData { + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/CalculatedFieldExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/CalculatedFieldExportData.java new file mode 100644 index 0000000000..7741f88822 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/CalculatedFieldExportData.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.cf.CalculatedField; + +import java.util.Comparator; +import java.util.List; + +@Data +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +public class CalculatedFieldExportData> extends EntityExportData { + + public static final Comparator calculatedFieldsComparator = Comparator.comparing(CalculatedField::getName); + + @JsonProperty(index = 102) + @JsonIgnoreProperties({"entityId", "createdTime", "version"}) + private List calculatedFields; + + @JsonIgnore + @Override + public boolean hasCalculatedFields() { + return calculatedFields != null; + } + + @Override + public CalculatedFieldExportData sort() { + super.sort(); + if (calculatedFields != null && !calculatedFields.isEmpty()) { + calculatedFields.sort(calculatedFieldsComparator); + } + return this; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java index bfa82caa2b..a391067c74 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; @EqualsAndHashCode(callSuper = true) @ToString(callSuper = true) @Data -public class DeviceExportData extends EntityExportData { +public class DeviceExportData extends CalculatedFieldExportData { @JsonProperty(index = 3) @JsonIgnoreProperties({"id", "deviceId", "createdTime", "version"}) @@ -38,4 +38,5 @@ public class DeviceExportData extends EntityExportData { public boolean hasCredentials() { return credentials != null; } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceProfileExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceProfileExportData.java new file mode 100644 index 0000000000..b6b4d9bdcf --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceProfileExportData.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.thingsboard.server.common.data.DeviceProfile; + +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +@Data +public class DeviceProfileExportData extends CalculatedFieldExportData { + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 7bc9a63c29..5b7d95db1a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -37,7 +37,10 @@ import java.util.Map; @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = As.EXISTING_PROPERTY, visible = true, defaultImpl = EntityExportData.class) @JsonSubTypes({ + @Type(name = "DEVICE_PROFILE", value = DeviceProfileExportData.class), + @Type(name = "ASSET_PROFILE", value = AssetProfileExportData.class), @Type(name = "DEVICE", value = DeviceExportData.class), + @Type(name = "ASSET", value = AssetExportData.class), @Type(name = "RULE_CHAIN", value = RuleChainExportData.class), @Type(name = "WIDGET_TYPE", value = WidgetTypeExportData.class), @Type(name = "WIDGETS_BUNDLE", value = WidgetsBundleExportData.class) @@ -96,4 +99,9 @@ public class EntityExportData> { return relations != null; } + @JsonIgnore + public boolean hasCalculatedFields() { + return false; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java index 46dc6860ee..078ca29e28 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java @@ -25,7 +25,10 @@ import lombok.NoArgsConstructor; @NoArgsConstructor @Builder public class EntityExportSettings { + private boolean exportRelations; private boolean exportAttributes; private boolean exportCredentials; + private boolean exportCalculatedFields; + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 2abdb55b65..d49801537c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -25,8 +25,11 @@ import lombok.NoArgsConstructor; @NoArgsConstructor @Builder public class EntityImportSettings { + private boolean findExistingByName; private boolean updateRelations; private boolean saveAttributes; private boolean saveCredentials; + private boolean saveCalculatedFields; + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java index 5a02b18364..6d1ac2283b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -17,13 +17,18 @@ package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; +import java.io.Serial; import java.io.Serializable; @Data public class VersionCreateConfig implements Serializable { + + @Serial private static final long serialVersionUID = 1223723167716612772L; private boolean saveRelations; private boolean saveAttributes; private boolean saveCredentials; + private boolean saveCalculatedFields; + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 0ae263dada..7f3ce89372 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -23,5 +23,6 @@ public class VersionLoadConfig { private boolean loadRelations; private boolean loadAttributes; private boolean loadCredentials; + private boolean loadCalculatedFields; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 416bff6ea3..dceef73aba 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.cf; -import com.google.common.util.concurrent.ListenableFuture; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 2668a1a0d8..3a529b7d93 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.dao.model.sql; -import com.fasterxml.jackson.databind.JsonNode; import jakarta.persistence.Column; -import jakarta.persistence.Convert; import jakarta.persistence.Entity; import jakarta.persistence.Table; import lombok.Data; @@ -60,6 +58,15 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity Date: Mon, 3 Mar 2025 11:36:47 +0200 Subject: [PATCH 09/26] Added UI for calculated fields version control --- .../vc/auto-commit-settings.component.html | 3 +++ .../vc/auto-commit-settings.component.ts | 14 +++++++++++--- .../vc/entity-types-version-create.component.html | 3 +++ .../vc/entity-types-version-create.component.ts | 7 ++++++- .../vc/entity-types-version-load.component.html | 3 +++ .../vc/entity-types-version-load.component.ts | 7 ++++++- .../vc/entity-version-create.component.html | 3 +++ .../vc/entity-version-create.component.ts | 6 +++++- .../vc/entity-version-restore.component.html | 3 +++ .../vc/entity-version-restore.component.ts | 6 ++++-- ui-ngx/src/app/shared/models/vc.models.ts | 7 +++++++ .../src/assets/locale/locale.constant-en_US.json | 2 ++ 12 files changed, 56 insertions(+), 8 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html index 053d0f5006..d80892664d 100644 --- a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html @@ -84,6 +84,9 @@ {{ 'version-control.export-relations' | translate }} + + {{ 'version-control.export-calculated-fields' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts index e0a88a3e09..d55515a959 100644 --- a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts @@ -25,7 +25,11 @@ import { TranslateService } from '@ngx-translate/core'; import { DialogService } from '@core/services/dialog.service'; import { catchError, map, mergeMap } from 'rxjs/operators'; import { Observable, of } from 'rxjs'; -import { EntityTypeVersionCreateConfig, exportableEntityTypes } from '@shared/models/vc.models'; +import { + EntityTypeVersionCreateConfig, + exportableEntityTypes, + typesWithCalculatedFields +} from '@shared/models/vc.models'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; @@ -43,6 +47,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit isReadOnly: Observable; + readonly typesWithCalculatedFields = typesWithCalculatedFields; + constructor(protected store: Store, private adminService: AdminService, private dialogService: DialogService, @@ -104,7 +110,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit branch: null, saveAttributes: true, saveRelations: false, - saveCredentials: true + saveCredentials: true, + saveCalculatedFields: true, }; const allowed = this.allowedEntityTypes(); let entityType: EntityType = null; @@ -206,7 +213,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit branch: [config.branch, []], saveRelations: [config.saveRelations, []], saveAttributes: [config.saveAttributes, []], - saveCredentials: [config.saveCredentials, []] + saveCredentials: [config.saveCredentials, []], + saveCalculatedFields: [config.saveCalculatedFields, []] }) } ); diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html index b9a9a6c01c..8fdf22f6cd 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -72,6 +72,9 @@ {{ 'version-control.export-relations' | translate }} + + {{ 'version-control.export-calculated-fields' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts index 8b16905587..8618e405b0 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts @@ -33,7 +33,8 @@ import { EntityTypeVersionCreateConfig, exportableEntityTypes, SyncStrategy, - syncStrategyTranslationMap + syncStrategyTranslationMap, + typesWithCalculatedFields } from '@shared/models/vc.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; @@ -79,6 +80,8 @@ export class EntityTypesVersionCreateComponent extends PageComponent implements loading = true; + readonly typesWithCalculatedFields = typesWithCalculatedFields; + constructor(protected store: Store, private translate: TranslateService, private fb: UntypedFormBuilder, @@ -150,6 +153,7 @@ export class EntityTypesVersionCreateComponent extends PageComponent implements saveRelations: [config.saveRelations, []], saveAttributes: [config.saveAttributes, []], saveCredentials: [config.saveCredentials, []], + saveCalculatedFields: [config.saveCalculatedFields, []], allEntities: [config.allEntities, []], entityIds: [config.entityIds, [Validators.required]] }) @@ -202,6 +206,7 @@ export class EntityTypesVersionCreateComponent extends PageComponent implements saveAttributes: true, saveRelations: true, saveCredentials: true, + saveCalculatedFields: true, allEntities: true, entityIds: [] }; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html index 4736bb1ee2..b877d19b63 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html @@ -72,6 +72,9 @@ {{ 'version-control.load-relations' | translate }} + + {{ 'version-control.load-calculated-fields' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts index 23f09ae977..f06abaf3a8 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts @@ -31,7 +31,8 @@ import { PageComponent } from '@shared/components/page.component'; import { entityTypesWithoutRelatedData, EntityTypeVersionLoadConfig, - exportableEntityTypes + exportableEntityTypes, + typesWithCalculatedFields } from '@shared/models/vc.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; @@ -75,6 +76,8 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On loading = true; + readonly typesWithCalculatedFields = typesWithCalculatedFields; + constructor(protected store: Store, private translate: TranslateService, private popoverService: TbPopoverService, @@ -145,6 +148,7 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On loadRelations: [config.loadRelations, []], loadAttributes: [config.loadAttributes, []], loadCredentials: [config.loadCredentials, []], + loadCalculatedFields: [config.loadCalculatedFields, []], removeOtherEntities: [config.removeOtherEntities, []], findExistingEntityByName: [config.findExistingEntityByName, []] }) @@ -180,6 +184,7 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On loadAttributes: true, loadRelations: true, loadCredentials: true, + loadCalculatedFields: true, removeOtherEntities: false, findExistingEntityByName: true }; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html index 10c2dc5e40..da4c31a1b8 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -47,6 +47,9 @@ {{ 'version-control.export-relations' | translate }} + + {{ 'version-control.export-calculated-fields' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts index 48b2a2a174..0c4b696ad1 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -20,6 +20,7 @@ import { UntypedFormBuilder, UntypedFormGroup, Validators } from '@angular/forms import { entityTypesWithoutRelatedData, SingleEntityVersionCreateRequest, + typesWithCalculatedFields, VersionCreateRequestType, VersionCreationResult } from '@shared/models/vc.models'; @@ -71,6 +72,8 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni private versionCreateResultSubscription: Subscription; + readonly typesWithCalculatedFields = typesWithCalculatedFields; + constructor(protected store: Store, private entitiesVersionControlService: EntitiesVersionControlService, private cd: ChangeDetectorRef, @@ -115,7 +118,8 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni ? this.createVersionFormGroup.get('saveRelations').value : false, saveAttributes: !entityTypesWithoutRelatedData.has(this.entityId.entityType) ? this.createVersionFormGroup.get('saveAttributes').value : false, - saveCredentials: this.entityId.entityType === EntityType.DEVICE ? this.createVersionFormGroup.get('saveCredentials').value : false + saveCredentials: this.entityId.entityType === EntityType.DEVICE ? this.createVersionFormGroup.get('saveCredentials').value : false, + saveCalculatedFields: typesWithCalculatedFields.has(this.entityId.entityType) ? this.createVersionFormGroup.get('saveCalculatedFields').value : false, }, type: VersionCreateRequestType.SINGLE_ENTITY }; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html index 4bc61757c4..1183223850 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -36,6 +36,9 @@ {{ 'version-control.load-relations' | translate }} + + {{ 'version-control.load-calculated-fields' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts index df2e8c19ff..4456eb7ce7 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -79,7 +79,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn this.restoreFormGroup = this.fb.group({ loadAttributes: [true, []], loadRelations: [true, []], - loadCredentials: [true, []] + loadCredentials: [true, []], + loadCalculatedFields: [true, []] }); this.entitiesVersionControlService.getEntityDataInfo(this.externalEntityId, this.versionId).subscribe((data) => { this.entityDataInfo = data; @@ -110,7 +111,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn config: { loadRelations: this.entityDataInfo.hasRelations ? this.restoreFormGroup.get('loadRelations').value : false, loadAttributes: this.entityDataInfo.hasAttributes ? this.restoreFormGroup.get('loadAttributes').value : false, - loadCredentials: this.entityDataInfo.hasCredentials ? this.restoreFormGroup.get('loadCredentials').value : false + loadCredentials: this.entityDataInfo.hasCredentials ? this.restoreFormGroup.get('loadCredentials').value : false, + loadCalculatedFields: this.entityDataInfo.hasCalculatedFields ? this.restoreFormGroup.get('loadCalculatedFields').value : false }, type: VersionLoadRequestType.SINGLE_ENTITY }; diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 6cc8de02eb..3795518ffc 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -49,6 +49,7 @@ export interface VersionCreateConfig { saveRelations: boolean; saveAttributes: boolean; saveCredentials: boolean; + saveCalculatedFields: boolean; } export enum VersionCreateRequestType { @@ -106,6 +107,7 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: syncStrategy: null, saveAttributes: !entityTypesWithoutRelatedData.has(entityType), saveRelations: !entityTypesWithoutRelatedData.has(entityType), + saveCalculatedFields: typesWithCalculatedFields.has(entityType), saveCredentials: true, allEntities: true, entityIds: [] @@ -118,6 +120,7 @@ export interface VersionLoadConfig { loadRelations: boolean; loadAttributes: boolean; loadCredentials: boolean; + loadCalculatedFields: boolean; } export enum VersionLoadRequestType { @@ -154,6 +157,7 @@ export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: En loadAttributes: !entityTypesWithoutRelatedData.has(entityType), loadRelations: !entityTypesWithoutRelatedData.has(entityType), loadCredentials: true, + loadCalculatedFields: typesWithCalculatedFields.has(entityType), removeOtherEntities: false, findExistingEntityByName: true }; @@ -254,4 +258,7 @@ export interface EntityDataInfo { hasRelations: boolean; hasAttributes: boolean; hasCredentials: boolean; + hasCalculatedFields: boolean; } + +export const typesWithCalculatedFields = new Set([EntityType.DEVICE, EntityType.ASSET, EntityType.ASSET_PROFILE, EntityType.DEVICE_PROFILE]); diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 02c604d8ae..6db2fa9ff1 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -6320,6 +6320,7 @@ "export-relations": "Export relations", "export-attributes": "Export attributes", "export-credentials": "Export credentials", + "export-calculated-fields": "Export calculated fields", "entity-versions": "Entity versions", "versions": "Versions", "created-time": "Created time", @@ -6336,6 +6337,7 @@ "load-relations": "Load relations", "load-attributes": "Load attributes", "load-credentials": "Load credentials", + "load-calculated-fields": "Load calculated fields", "compare-with-current": "Compare with current", "diff-entity-with-version": "Diff with entity version '{{versionName}}'", "previous-difference": "Previous Difference", From f467fc009e2d2e29484799bec0941378188c407c Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 3 Mar 2025 16:58:26 +0200 Subject: [PATCH 10/26] Refactor cf vc --- .../DefaultEntitiesExportImportService.java | 7 +- .../ie/exporting/impl/AssetExportService.java | 18 +-- .../impl/AssetProfileExportService.java | 17 +-- .../BaseCalculatedFieldsExportService.java | 51 -------- .../impl/DefaultEntityExportService.java | 21 +++ .../exporting/impl/DeviceExportService.java | 11 +- .../impl/DeviceProfileExportService.java | 17 +-- .../ie/importing/impl/AssetImportService.java | 22 ++-- .../impl/AssetProfileImportService.java | 22 ++-- .../BaseCalculatedFieldsImportService.java | 123 ------------------ .../impl/BaseEntityImportService.java | 57 +++++++- .../importing/impl/DeviceImportService.java | 40 ++---- .../impl/DeviceProfileImportService.java | 22 ++-- .../sync/vc/data/EntitiesImportCtx.java | 6 +- .../sync/ie/ExportImportServiceSqlTest.java | 2 - .../service/sync/vc/VersionControlTest.java | 6 +- .../common/data/cf/CalculatedField.java | 37 +++--- .../common/data/id/CalculatedFieldId.java | 3 + .../server/common/data/sync/JsonTbEntity.java | 4 +- .../common/data/sync/ie/AssetExportData.java | 28 ---- .../data/sync/ie/AssetProfileExportData.java | 28 ---- .../sync/ie/CalculatedFieldExportData.java | 56 -------- .../common/data/sync/ie/DeviceExportData.java | 2 +- .../data/sync/ie/DeviceProfileExportData.java | 28 ---- .../common/data/sync/ie/EntityExportData.java | 14 +- .../server/dao/cf/CalculatedFieldDao.java | 3 +- .../server/dao/model/ModelConstants.java | 1 - .../dao/model/sql/CalculatedFieldEntity.java | 10 -- .../dao/sql/cf/CalculatedFieldRepository.java | 11 +- ...efaultNativeCalculatedFieldRepository.java | 1 - .../dao/sql/cf/JpaCalculatedFieldDao.java | 20 --- .../main/resources/sql/schema-entities.sql | 4 +- .../service/CalculatedFieldServiceTest.java | 14 -- 33 files changed, 181 insertions(+), 525 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseCalculatedFieldsExportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseCalculatedFieldsImportService.java delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AssetExportData.java delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AssetProfileExportData.java delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/CalculatedFieldExportData.java delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceProfileExportData.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 94b0ddc07e..d45f32c918 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.cache.limits.RateLimitService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; @@ -129,10 +130,10 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS relation, ctx.getUser(), ActionType.RELATION_ADD_OR_UPDATE, null, relation); } - ctx.getCalculatedFields().forEach((calculatedField, created) -> { + for (CalculatedField calculatedField : ctx.getCalculatedFields()) { var savedCalculatedField = calculatedFieldService.save(calculatedField); - logEntityActionService.logEntityAction(ctx.getTenantId(), savedCalculatedField.getId(), savedCalculatedField, created ? ActionType.ADDED : ActionType.UPDATED, ctx.getUser()); - }); + logEntityActionService.logEntityAction(ctx.getTenantId(), savedCalculatedField.getId(), savedCalculatedField, ActionType.ADDED, ctx.getUser()); + } } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java index 6f56fa5145..fbda92b6be 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java @@ -15,12 +15,12 @@ */ package org.thingsboard.server.service.sync.ie.exporting.impl; +import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.sync.ie.AssetExportData; -import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; @@ -28,22 +28,18 @@ import java.util.Set; @Service @TbCoreComponent -public class AssetExportService extends BaseCalculatedFieldsExportService { - - protected AssetExportService(CalculatedFieldService calculatedFieldService) { - super(calculatedFieldService); - } +@RequiredArgsConstructor +public class AssetExportService extends BaseEntityExportService> { @Override - protected void setRelatedEntities(EntitiesExportCtx ctx, Asset asset, AssetExportData exportData) { + protected void setRelatedEntities(EntitiesExportCtx ctx, Asset asset, EntityExportData exportData) { asset.setCustomerId(getExternalIdOrElseInternal(ctx, asset.getCustomerId())); asset.setAssetProfileId(getExternalIdOrElseInternal(ctx, asset.getAssetProfileId())); - setCalculatedFields(ctx, asset, exportData); } @Override - protected AssetExportData newExportData() { - return new AssetExportData(); + protected EntityExportData newExportData() { + return new EntityExportData<>(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetProfileExportService.java index 2314db92f5..b670daf06c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetProfileExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetProfileExportService.java @@ -19,8 +19,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.sync.ie.AssetProfileExportData; -import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; @@ -28,23 +27,13 @@ import java.util.Set; @Service @TbCoreComponent -public class AssetProfileExportService extends BaseCalculatedFieldsExportService { - - protected AssetProfileExportService(CalculatedFieldService calculatedFieldService) { - super(calculatedFieldService); - } +public class AssetProfileExportService extends BaseEntityExportService> { @Override - protected void setRelatedEntities(EntitiesExportCtx ctx, AssetProfile assetProfile, AssetProfileExportData exportData) { + protected void setRelatedEntities(EntitiesExportCtx ctx, AssetProfile assetProfile, EntityExportData exportData) { assetProfile.setDefaultDashboardId(getExternalIdOrElseInternal(ctx, assetProfile.getDefaultDashboardId())); assetProfile.setDefaultRuleChainId(getExternalIdOrElseInternal(ctx, assetProfile.getDefaultRuleChainId())); assetProfile.setDefaultEdgeRuleChainId(getExternalIdOrElseInternal(ctx, assetProfile.getDefaultEdgeRuleChainId())); - setCalculatedFields(ctx, assetProfile, exportData); - } - - @Override - protected AssetProfileExportData newExportData() { - return new AssetProfileExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseCalculatedFieldsExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseCalculatedFieldsExportService.java deleted file mode 100644 index e28f8aaa79..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseCalculatedFieldsExportService.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.ie.exporting.impl; - -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.sync.ie.CalculatedFieldExportData; -import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; - -import java.util.List; - -public abstract class BaseCalculatedFieldsExportService & HasTenantId, D extends CalculatedFieldExportData> extends BaseEntityExportService { - - protected final CalculatedFieldService calculatedFieldService; - - protected BaseCalculatedFieldsExportService(CalculatedFieldService calculatedFieldService) { - this.calculatedFieldService = calculatedFieldService; - } - - protected void setCalculatedFields(EntitiesExportCtx ctx, E entity, D exportData) { - if (ctx.getSettings().isExportCalculatedFields()) { - List calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(ctx.getTenantId(), entity.getId()); - calculatedFields.forEach(calculatedField -> { - calculatedField.getConfiguration().getArguments().values().forEach(argument -> { - if (argument.getRefEntityId() != null) { - EntityId externalId = getExternalIdOrElseInternal(ctx, argument.getRefEntityId()); - argument.setRefEntityId(externalId); - } - }); - }); - exportData.setCalculatedFields(calculatedFields); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index b0bea68290..5c2fcc7dc6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.sync.ie.AttributeExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; @@ -59,6 +61,8 @@ public class DefaultEntityExportService ctx, I entityId) throws ThingsboardException { @@ -98,6 +102,10 @@ public class DefaultEntityExportService> attributes = exportAttributes(ctx, entity); exportData.setAttributes(attributes); } + if (ctx.getSettings().isExportCalculatedFields()) { + List calculatedFields = exportCalculatedFields(ctx, entity.getId()); + exportData.setCalculatedFields(calculatedFields); + } } private List exportRelations(EntitiesExportCtx ctx, E entity) throws ThingsboardException { @@ -141,6 +149,19 @@ public class DefaultEntityExportService exportCalculatedFields(EntitiesExportCtx ctx, EntityId entityId) { + List calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(ctx.getTenantId(), entityId); + calculatedFields.forEach(calculatedField -> { + calculatedField.setEntityId(getExternalIdOrElseInternal(ctx, entityId)); + calculatedField.getConfiguration().getArguments().values().forEach(argument -> { + if (argument.getRefEntityId() != null) { + argument.setRefEntityId(getExternalIdOrElseInternal(ctx, argument.getRefEntityId())); + } + }); + }); + return calculatedFields; + } + protected ID getExternalIdOrElseInternal(EntitiesExportCtx ctx, ID internalId) { if (internalId == null || internalId.isNullUid()) return internalId; var result = ctx.getExternalId(internalId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java index f1b5ade487..7d5f7ee57e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java @@ -15,12 +15,12 @@ */ package org.thingsboard.server.service.sync.ie.exporting.impl; +import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.sync.ie.DeviceExportData; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; @@ -29,15 +29,11 @@ import java.util.Set; @Service @TbCoreComponent -public class DeviceExportService extends BaseCalculatedFieldsExportService { +@RequiredArgsConstructor +public class DeviceExportService extends BaseEntityExportService { private final DeviceCredentialsService deviceCredentialsService; - public DeviceExportService(CalculatedFieldService calculatedFieldService, DeviceCredentialsService deviceCredentialsService) { - super(calculatedFieldService); - this.deviceCredentialsService = deviceCredentialsService; - } - @Override protected void setRelatedEntities(EntitiesExportCtx ctx, Device device, DeviceExportData exportData) { device.setCustomerId(getExternalIdOrElseInternal(ctx, device.getCustomerId())); @@ -48,7 +44,6 @@ public class DeviceExportService extends BaseCalculatedFieldsExportService { - - protected DeviceProfileExportService(CalculatedFieldService calculatedFieldService) { - super(calculatedFieldService); - } +public class DeviceProfileExportService extends BaseEntityExportService> { @Override - protected void setRelatedEntities(EntitiesExportCtx ctx, DeviceProfile deviceProfile, DeviceProfileExportData exportData) { + protected void setRelatedEntities(EntitiesExportCtx ctx, DeviceProfile deviceProfile, EntityExportData exportData) { deviceProfile.setDefaultDashboardId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultDashboardId())); deviceProfile.setDefaultRuleChainId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultRuleChainId())); deviceProfile.setDefaultEdgeRuleChainId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultEdgeRuleChainId())); - setCalculatedFields(ctx, deviceProfile, exportData); - } - - @Override - protected DeviceProfileExportData newExportData() { - return new DeviceProfileExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java index 0b0649072e..7cd4c3aca1 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java @@ -15,28 +15,24 @@ */ package org.thingsboard.server.service.sync.ie.importing.impl; +import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ie.AssetExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.dao.asset.AssetService; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @Service @TbCoreComponent -public class AssetImportService extends BaseCalculatedFieldsImportService { +@RequiredArgsConstructor +public class AssetImportService extends BaseEntityImportService> { private final AssetService assetService; - public AssetImportService(CalculatedFieldService calculatedFieldService, AssetService assetService) { - super(calculatedFieldService); - this.assetService = assetService; - } - @Override protected void setOwner(TenantId tenantId, Asset asset, IdProvider idProvider) { asset.setTenantId(tenantId); @@ -44,14 +40,18 @@ public class AssetImportService extends BaseCalculatedFieldsImportService exportData, IdProvider idProvider) { asset.setAssetProfileId(idProvider.getInternalId(asset.getAssetProfileId())); return asset; } @Override - protected Asset saveOrUpdate(EntitiesImportCtx ctx, Asset asset, AssetExportData exportData, IdProvider idProvider) { - return saveOrUpdateEntity(ctx, asset, exportData, idProvider, assetService::saveAsset); + protected Asset saveOrUpdate(EntitiesImportCtx ctx, Asset asset, EntityExportData exportData, IdProvider idProvider) { + Asset savedAsset = assetService.saveAsset(asset); + if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) { + importCalculatedFields(ctx, savedAsset, exportData, idProvider); + } + return savedAsset; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java index 6797c92c81..32a0090a4a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.sync.ie.importing.impl; +import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; @@ -22,30 +23,25 @@ import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ie.AssetProfileExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.dao.asset.AssetProfileService; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @Service @TbCoreComponent -public class AssetProfileImportService extends BaseCalculatedFieldsImportService { +@RequiredArgsConstructor +public class AssetProfileImportService extends BaseEntityImportService> { private final AssetProfileService assetProfileService; - public AssetProfileImportService(CalculatedFieldService calculatedFieldService, AssetProfileService assetProfileService) { - super(calculatedFieldService); - this.assetProfileService = assetProfileService; - } - @Override protected void setOwner(TenantId tenantId, AssetProfile assetProfile, IdProvider idProvider) { assetProfile.setTenantId(tenantId); } @Override - protected AssetProfile prepare(EntitiesImportCtx ctx, AssetProfile assetProfile, AssetProfile old, AssetProfileExportData exportData, IdProvider idProvider) { + protected AssetProfile prepare(EntitiesImportCtx ctx, AssetProfile assetProfile, AssetProfile old, EntityExportData exportData, IdProvider idProvider) { assetProfile.setDefaultRuleChainId(idProvider.getInternalId(assetProfile.getDefaultRuleChainId())); assetProfile.setDefaultDashboardId(idProvider.getInternalId(assetProfile.getDefaultDashboardId())); assetProfile.setDefaultEdgeRuleChainId(idProvider.getInternalId(assetProfile.getDefaultEdgeRuleChainId())); @@ -53,8 +49,12 @@ public class AssetProfileImportService extends BaseCalculatedFieldsImportService } @Override - protected AssetProfile saveOrUpdate(EntitiesImportCtx ctx, AssetProfile assetProfile, AssetProfileExportData exportData, IdProvider idProvider) { - return saveOrUpdateEntity(ctx, assetProfile, exportData, idProvider, assetProfileService::saveAssetProfile); + protected AssetProfile saveOrUpdate(EntitiesImportCtx ctx, AssetProfile assetProfile, EntityExportData exportData, IdProvider idProvider) { + AssetProfile saved = assetProfileService.saveAssetProfile(assetProfile); + if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) { + importCalculatedFields(ctx, saved, exportData, idProvider); + } + return saved; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseCalculatedFieldsImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseCalculatedFieldsImportService.java deleted file mode 100644 index ffa51938f0..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseCalculatedFieldsImportService.java +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.ie.importing.impl; - -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.sync.ie.CalculatedFieldExportData; -import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - -public abstract class BaseCalculatedFieldsImportService & HasTenantId, D extends CalculatedFieldExportData> extends BaseEntityImportService { - - private final CalculatedFieldService calculatedFieldService; - - protected BaseCalculatedFieldsImportService(CalculatedFieldService calculatedFieldService) { - this.calculatedFieldService = calculatedFieldService; - } - - protected E saveOrUpdateEntity(EntitiesImportCtx ctx, E entity, D exportData, IdProvider idProvider, Function saveFunction) { - E savedEntity = saveFunction.apply(entity); - - if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) { - saveCalculatedFields(ctx, savedEntity, exportData, idProvider); - } - return savedEntity; - } - - protected void saveCalculatedFields(EntitiesImportCtx ctx, E savedEntity, D exportData, IdProvider idProvider) { - if (exportData.getCalculatedFields() == null || !ctx.isSaveCalculatedFields()) { - return; - } - - exportData.getCalculatedFields().forEach(calculatedField -> { - calculatedField.setTenantId(savedEntity.getTenantId()); - calculatedField.setExternalId(calculatedField.getId()); - calculatedField.setId(idProvider.getInternalId(calculatedField.getId(), false)); - calculatedField.setEntityId(savedEntity.getId()); - - calculatedField.getConfiguration().getArguments().values().forEach(argument -> { - if (argument.getRefEntityId() != null) { - argument.setRefEntityId(idProvider.getInternalId(argument.getRefEntityId(), false)); - } - }); - - calculatedFieldService.save(calculatedField); - }); - } - - @Override - protected boolean updateRelatedEntitiesIfUnmodified(EntitiesImportCtx ctx, E prepared, D exportData, IdProvider idProvider) { - boolean updated = super.updateRelatedEntitiesIfUnmodified(ctx, prepared, exportData, idProvider); - updated |= updateCalculatedFields(ctx, prepared, exportData, idProvider); - return updated; - } - - private boolean updateCalculatedFields(EntitiesImportCtx ctx, E prepared, D exportData, IdProvider idProvider) { - var calculatedFields = exportData.getCalculatedFields(); - if (calculatedFields == null || !ctx.isSaveCalculatedFields()) { - return false; - } - Map calculatedFieldMap = calculatedFields.stream() - .peek(newField -> { - newField.setTenantId(ctx.getTenantId()); - newField.setExternalId(newField.getId()); - newField.setId(idProvider.getInternalId(newField.getId(), false)); - newField.setEntityId(prepared.getId()); - newField.getConfiguration().getArguments().values().forEach(argument -> { - argument.setRefEntityId(idProvider.getInternalId(argument.getRefEntityId(), false)); - }); - }) - .collect(Collectors.toMap(CalculatedField::getId, field -> field)); - - List existingFields = calculatedFieldService.findCalculatedFieldsByEntityId(ctx.getTenantId(), prepared.getId()); - boolean updated = false; - - Map result = new LinkedHashMap<>(); - for (CalculatedField existingField : existingFields) { - if (calculatedFieldMap.containsKey(existingField.getId())) { - CalculatedField newField = calculatedFieldMap.get(existingField.getId()); - if (!newField.equals(existingField)) { - result.put(newField, false); - } - calculatedFieldMap.remove(existingField.getId()); - } else { - updated = true; - calculatedFieldService.deleteCalculatedField(ctx.getTenantId(), existingField.getId()); - } - } - - for (CalculatedField newField : calculatedFieldMap.values()) { - result.put(newField, true); - } - - if (!result.isEmpty()) { - updated = true; - ctx.addCalculatedFields(result); - } - return updated; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 7044b235a0..65f2b4f032 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.HasDefaultOption; import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -50,6 +51,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.sync.ie.AttributeExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; @@ -78,6 +80,8 @@ public abstract class BaseEntityImportService existing = calculatedFieldService.findCalculatedFieldsByEntityId(ctx.getTenantId(), savedEntity.getId()); + List fieldsToSave = exportData.getCalculatedFields().stream() + .peek(calculatedField -> { + calculatedField.setTenantId(ctx.getTenantId()); + calculatedField.setEntityId(savedEntity.getId()); + calculatedField.getConfiguration().getArguments().values().forEach(argument -> { + if (argument.getRefEntityId() != null) { + argument.setRefEntityId(idProvider.getInternalId(argument.getRefEntityId(), false)); + } + }); + }).toList(); + + for (CalculatedField existingField : existing) { + boolean found = fieldsToSave.stream().anyMatch(importedField -> compareCalculatedFields(existingField, importedField)); + if (!found) { + calculatedFieldService.deleteCalculatedField(ctx.getTenantId(), existingField.getId()); + updated = true; + } + } + + for (CalculatedField calculatedField : fieldsToSave) { + boolean found = existing.stream().anyMatch(existingField -> compareCalculatedFields(existingField, calculatedField)); + if (!found) { + calculatedFieldService.save(calculatedField); + updated = true; + } + } + return updated; + } + + private boolean compareCalculatedFields(CalculatedField existingField, CalculatedField newField) { + CalculatedField oldCopy = new CalculatedField(existingField); + CalculatedField newCopy = new CalculatedField(newField); + oldCopy.setId(null); + newCopy.setId(null); + oldCopy.setVersion(null); + newCopy.setVersion(null); + oldCopy.setCreatedTime(0); + newCopy.setCreatedTime(0); + return oldCopy.equals(newCopy); + } + protected void onEntitySaved(User user, E savedEntity, E oldEntity) throws ThingsboardException { logEntityActionService.logEntityAction(user.getTenantId(), savedEntity.getId(), savedEntity, null, oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, user); } - @SuppressWarnings("unchecked") protected E findExistingEntity(EntitiesImportCtx ctx, E entity, IdProvider idProvider) { return (E) Optional.ofNullable(entitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId())) @@ -313,10 +364,10 @@ public abstract class BaseEntityImportService new MissingEntityException(externalId)); } - @SuppressWarnings("unchecked") @RequiredArgsConstructor protected class IdProvider { + private final EntitiesImportCtx ctx; private final EntityImportResult importResult; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java index c8e99cdbc1..92bdb1e294 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -15,13 +15,13 @@ */ package org.thingsboard.server.service.sync.ie.importing.impl; +import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.DeviceExportData; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -29,17 +29,12 @@ import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @Service @TbCoreComponent -public class DeviceImportService extends BaseCalculatedFieldsImportService { +@RequiredArgsConstructor +public class DeviceImportService extends BaseEntityImportService { private final DeviceService deviceService; private final DeviceCredentialsService credentialsService; - public DeviceImportService(CalculatedFieldService calculatedFieldService, DeviceService deviceService, DeviceCredentialsService credentialsService) { - super(calculatedFieldService); - this.deviceService = deviceService; - this.credentialsService = credentialsService; - } - @Override protected void setOwner(TenantId tenantId, Device device, IdProvider idProvider) { device.setTenantId(tenantId); @@ -78,7 +73,7 @@ public class DeviceImportService extends BaseCalculatedFieldsImportService { +@RequiredArgsConstructor +public class DeviceProfileImportService extends BaseEntityImportService> { private final DeviceProfileService deviceProfileService; - public DeviceProfileImportService(CalculatedFieldService calculatedFieldService, DeviceProfileService deviceProfileService) { - super(calculatedFieldService); - this.deviceProfileService = deviceProfileService; - } - @Override protected void setOwner(TenantId tenantId, DeviceProfile deviceProfile, IdProvider idProvider) { deviceProfile.setTenantId(tenantId); } @Override - protected DeviceProfile prepare(EntitiesImportCtx ctx, DeviceProfile deviceProfile, DeviceProfile old, DeviceProfileExportData exportData, IdProvider idProvider) { + protected DeviceProfile prepare(EntitiesImportCtx ctx, DeviceProfile deviceProfile, DeviceProfile old, EntityExportData exportData, IdProvider idProvider) { deviceProfile.setDefaultRuleChainId(idProvider.getInternalId(deviceProfile.getDefaultRuleChainId())); deviceProfile.setDefaultEdgeRuleChainId(idProvider.getInternalId(deviceProfile.getDefaultEdgeRuleChainId())); deviceProfile.setDefaultDashboardId(idProvider.getInternalId(deviceProfile.getDefaultDashboardId())); @@ -55,8 +51,12 @@ public class DeviceProfileImportService extends BaseCalculatedFieldsImportServic } @Override - protected DeviceProfile saveOrUpdate(EntitiesImportCtx ctx, DeviceProfile deviceProfile, DeviceProfileExportData exportData, IdProvider idProvider) { - return saveOrUpdateEntity(ctx, deviceProfile, exportData, idProvider, deviceProfileService::saveDeviceProfile); + protected DeviceProfile saveOrUpdate(EntitiesImportCtx ctx, DeviceProfile deviceProfile, EntityExportData exportData, IdProvider idProvider) { + DeviceProfile saved = deviceProfileService.saveDeviceProfile(deviceProfile); + if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) { + importCalculatedFields(ctx, saved, exportData, idProvider); + } + return saved; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index c1a8dad161..726dd12102 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -56,7 +56,7 @@ public class EntitiesImportCtx { private final Set notFoundIds = new HashSet<>(); private final Set relations = new LinkedHashSet<>(); - private final Map calculatedFields = new LinkedHashMap<>(); + private final Set calculatedFields = new LinkedHashSet<>(); private boolean finalImportAttempt = false; private EntityImportSettings settings; @@ -127,8 +127,8 @@ public class EntitiesImportCtx { relations.addAll(values); } - public void addCalculatedFields(Map calculatedFieldMap) { - calculatedFields.putAll(calculatedFieldMap); + public void addCalculatedFields(Collection values) { + calculatedFields.addAll(values); } public void addReferenceCallback(EntityId externalId, ThrowingRunnable tr) { diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java index 9184a3c034..a75fbba095 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java @@ -288,7 +288,6 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { assertThat(calculatedFields.size()).isOne(); var importedCalculatedField = calculatedFields.get(0); assertThat(importedCalculatedField.getName()).isEqualTo(calculatedField.getName()); - assertThat(importedCalculatedField.getExternalId()).isEqualTo(calculatedField.getId()); EntityExportData updatedDeviceEntity = getAndClone(entitiesExportData, EntityType.DEVICE); updatedDeviceEntity.getEntity().setLabel("t" + updatedDeviceEntity.getEntity().getLabel()); @@ -303,7 +302,6 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId2, importedDevice.getId()); assertThat(calculatedFields.size()).isOne(); importedCalculatedField = calculatedFields.get(0); - assertThat(importedCalculatedField.getExternalId()).isEqualTo(calculatedField.getId()); assertThat(importedCalculatedField.getName()).startsWith("t_"); } diff --git a/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java index 453fea676f..d90ca60333 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java @@ -575,7 +575,7 @@ public class VersionControlTest extends AbstractControllerTest { public void testVcWithCalculatedFields_betweenTenants() throws Exception { Asset asset = createAsset(null, null, "Asset 1"); Device device = createDevice(null, null, "Device 1", "test1"); - CalculatedField calculatedField = createCalculatedField("CalculatedField1", device.getId(), asset.getId()); + createCalculatedField("CalculatedField1", device.getId(), asset.getId()); String versionId = createVersion("calculated fields of asset and device", EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE); loginTenant2(); @@ -588,13 +588,13 @@ public class VersionControlTest extends AbstractControllerTest { CalculatedField importedCalculatedField = findCalculatedFieldByEntityId(importedDevice.getId()); checkImportedEntity(tenantId1, device, tenantId2, importedDevice); checkImportedEntity(tenantId1, asset, tenantId2, importedAsset); - checkImportedEntity(tenantId1, calculatedField, tenantId2, importedCalculatedField); List importedCalculatedFields = findCalculatedFieldsByEntityId(importedDevice.getId()); assertThat(importedCalculatedFields).size().isOne(); assertThat(importedCalculatedFields.get(0)).satisfies(importedField -> { assertThat(importedField.getName()).isEqualTo(importedCalculatedField.getName()); assertThat(importedField.getType()).isEqualTo(importedCalculatedField.getType()); + assertThat(importedField.getId()).isNotEqualTo(importedCalculatedField.getId()); }); } @@ -606,7 +606,7 @@ public class VersionControlTest extends AbstractControllerTest { loadVersion(versionId, EntityType.ASSET); CalculatedField importedCalculatedField = findCalculatedFieldByEntityId(asset.getId()); - checkImportedEntity(tenantId1, calculatedField, tenantId1, importedCalculatedField); + assertThat(importedCalculatedField.getId()).isEqualTo(calculatedField.getId()); assertThat(importedCalculatedField.getName()).isEqualTo(calculatedField.getName()); assertThat(importedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration()); assertThat(importedCalculatedField.getType()).isEqualTo(calculatedField.getType()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index b45d7c6921..b86f30ca78 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -23,7 +23,6 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import org.thingsboard.server.common.data.BaseData; -import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasDebugSettings; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; @@ -42,7 +41,7 @@ import java.io.Serial; @Schema @Data @EqualsAndHashCode(callSuper = true) -public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, ExportableEntity, HasDebugSettings { +public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, HasDebugSettings { @Serial private static final long serialVersionUID = 4491966747773381420L; @@ -69,9 +68,6 @@ public class CalculatedField extends BaseData implements HasN @Getter @Setter private Long version; - @Getter - @Setter - private CalculatedFieldId externalId; public CalculatedField() { super(); @@ -81,21 +77,7 @@ public class CalculatedField extends BaseData implements HasN super(id); } - public CalculatedField(CalculatedField other) { - super(other); - this.tenantId = other.tenantId; - this.entityId = other.entityId; - this.type = other.type; - this.name = other.name; - this.configurationVersion = other.configurationVersion; - this.configuration = other.configuration; - this.version = other.version; - this.externalId = other.externalId; - this.debugMode = other.debugMode; - this.debugSettings = other.debugSettings; - } - - public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { + public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version) { this.tenantId = tenantId; this.entityId = entityId; this.type = type; @@ -103,7 +85,19 @@ public class CalculatedField extends BaseData implements HasN this.configurationVersion = configurationVersion; this.configuration = configuration; this.version = version; - this.externalId = externalId; + } + + public CalculatedField(CalculatedField calculatedField) { + super(calculatedField); + this.tenantId = calculatedField.tenantId; + this.entityId = calculatedField.entityId; + this.type = calculatedField.type; + this.name = calculatedField.name; + this.debugMode = calculatedField.debugMode; + this.debugSettings = calculatedField.debugSettings; + this.configurationVersion = calculatedField.configurationVersion; + this.configuration = calculatedField.configuration; + this.version = calculatedField.version; } @Schema(description = "JSON object with the Calculated Field Id. Referencing non-existing Calculated Field Id will cause error.") @@ -129,7 +123,6 @@ public class CalculatedField extends BaseData implements HasN .append(", configurationVersion=").append(configurationVersion) .append(", configuration=").append(configuration) .append(", version=").append(version) - .append(", externalId=").append(externalId) .append(", createdTime=").append(createdTime) .append(", id=").append(id).append(']') .toString(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java index 0a83f1a19f..e17a066d88 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java @@ -20,11 +20,13 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.v3.oas.annotations.media.Schema; import org.thingsboard.server.common.data.EntityType; +import java.io.Serial; import java.util.UUID; @Schema public class CalculatedFieldId extends UUIDBased implements EntityId { + @Serial private static final long serialVersionUID = 1L; @JsonCreator @@ -41,4 +43,5 @@ public class CalculatedFieldId extends UUIDBased implements EntityId { public EntityType getEntityType() { return EntityType.CALCULATED_FIELD; } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java index 5dfe6ac8b2..c763daeb7f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java @@ -28,7 +28,6 @@ import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.notification.rule.NotificationRule; import org.thingsboard.server.common.data.notification.targets.NotificationTarget; import org.thingsboard.server.common.data.notification.template.NotificationTemplate; @@ -59,8 +58,7 @@ import java.lang.annotation.Target; @Type(name = "NOTIFICATION_TEMPLATE", value = NotificationTemplate.class), @Type(name = "NOTIFICATION_TARGET", value = NotificationTarget.class), @Type(name = "NOTIFICATION_RULE", value = NotificationRule.class), - @Type(name = "TB_RESOURCE", value = TbResource.class), - @Type(name = "CALCULATED_FIELD", value = CalculatedField.class) + @Type(name = "TB_RESOURCE", value = TbResource.class) }) @JsonIgnoreProperties(value = {"tenantId", "createdTime", "version"}, ignoreUnknown = true) public @interface JsonTbEntity { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AssetExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AssetExportData.java deleted file mode 100644 index 4bb836d5d9..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AssetExportData.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.sync.ie; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; -import org.thingsboard.server.common.data.asset.Asset; - -@EqualsAndHashCode(callSuper = true) -@ToString(callSuper = true) -@Data -public class AssetExportData extends CalculatedFieldExportData { - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AssetProfileExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AssetProfileExportData.java deleted file mode 100644 index 42d9523828..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AssetProfileExportData.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.sync.ie; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; -import org.thingsboard.server.common.data.asset.AssetProfile; - -@EqualsAndHashCode(callSuper = true) -@ToString(callSuper = true) -@Data -public class AssetProfileExportData extends CalculatedFieldExportData { - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/CalculatedFieldExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/CalculatedFieldExportData.java deleted file mode 100644 index 7741f88822..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/CalculatedFieldExportData.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.sync.ie; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.cf.CalculatedField; - -import java.util.Comparator; -import java.util.List; - -@Data -@EqualsAndHashCode(callSuper = true) -@ToString(callSuper = true) -public class CalculatedFieldExportData> extends EntityExportData { - - public static final Comparator calculatedFieldsComparator = Comparator.comparing(CalculatedField::getName); - - @JsonProperty(index = 102) - @JsonIgnoreProperties({"entityId", "createdTime", "version"}) - private List calculatedFields; - - @JsonIgnore - @Override - public boolean hasCalculatedFields() { - return calculatedFields != null; - } - - @Override - public CalculatedFieldExportData sort() { - super.sort(); - if (calculatedFields != null && !calculatedFields.isEmpty()) { - calculatedFields.sort(calculatedFieldsComparator); - } - return this; - } - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java index a391067c74..98948ca642 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; @EqualsAndHashCode(callSuper = true) @ToString(callSuper = true) @Data -public class DeviceExportData extends CalculatedFieldExportData { +public class DeviceExportData extends EntityExportData { @JsonProperty(index = 3) @JsonIgnoreProperties({"id", "deviceId", "createdTime", "version"}) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceProfileExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceProfileExportData.java deleted file mode 100644 index b6b4d9bdcf..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceProfileExportData.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.sync.ie; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; -import org.thingsboard.server.common.data.DeviceProfile; - -@EqualsAndHashCode(callSuper = true) -@ToString(callSuper = true) -@Data -public class DeviceProfileExportData extends CalculatedFieldExportData { - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 5b7d95db1a..072be6acf2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo.As; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.JsonTbEntity; @@ -37,10 +38,7 @@ import java.util.Map; @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = As.EXISTING_PROPERTY, visible = true, defaultImpl = EntityExportData.class) @JsonSubTypes({ - @Type(name = "DEVICE_PROFILE", value = DeviceProfileExportData.class), - @Type(name = "ASSET_PROFILE", value = AssetProfileExportData.class), @Type(name = "DEVICE", value = DeviceExportData.class), - @Type(name = "ASSET", value = AssetExportData.class), @Type(name = "RULE_CHAIN", value = RuleChainExportData.class), @Type(name = "WIDGET_TYPE", value = WidgetTypeExportData.class), @Type(name = "WIDGETS_BUNDLE", value = WidgetsBundleExportData.class) @@ -58,6 +56,8 @@ public class EntityExportData> { public static final Comparator attrComparator = Comparator .comparing(AttributeExportData::getKey).thenComparing(AttributeExportData::getLastUpdateTs); + public static final Comparator calculatedFieldsComparator = Comparator.comparing(CalculatedField::getName); + @JsonProperty(index = 2) @JsonTbEntity private E entity; @@ -68,6 +68,9 @@ public class EntityExportData> { private List relations; @JsonProperty(index = 101) private Map> attributes; + @JsonProperty(index = 102) + @JsonIgnoreProperties({"id", "entityId", "createdTime", "version"}) + private List calculatedFields; public EntityExportData sort() { if (relations != null && !relations.isEmpty()) { @@ -76,6 +79,9 @@ public class EntityExportData> { if (attributes != null && !attributes.isEmpty()) { attributes.values().forEach(list -> list.sort(attrComparator)); } + if (calculatedFields != null && !calculatedFields.isEmpty()) { + calculatedFields.sort(calculatedFieldsComparator); + } return this; } @@ -101,7 +107,7 @@ public class EntityExportData> { @JsonIgnore public boolean hasCalculatedFields() { - return false; + return calculatedFields != null; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 3f04ab6c67..a966977968 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -22,11 +22,10 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; -import org.thingsboard.server.dao.ExportableEntityDao; import java.util.List; -public interface CalculatedFieldDao extends Dao, ExportableEntityDao { +public interface CalculatedFieldDao extends Dao { List findAllByTenantId(TenantId tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index e2b21ed59b..148908d063 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -729,7 +729,6 @@ public class ModelConstants { public static final String CALCULATED_FIELD_CONFIGURATION_VERSION = "configuration_version"; public static final String CALCULATED_FIELD_CONFIGURATION = "configuration"; public static final String CALCULATED_FIELD_VERSION = "version"; - public static final String CALCULATED_FIELD_EXTERNAL_ID = "external_id"; /** * Calculated field links constants. diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 349091a1ae..de6a1365b1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -40,7 +40,6 @@ import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_C import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION_VERSION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_ID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_TYPE; -import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_EXTERNAL_ID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_NAME; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TABLE_NAME; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TENANT_ID_COLUMN; @@ -82,9 +81,6 @@ public class CalculatedFieldEntity extends BaseVersionedEntity @Column(name = DEBUG_SETTINGS) private String debugSettings; - @Column(name = CALCULATED_FIELD_EXTERNAL_ID) - private UUID externalId; - public CalculatedFieldEntity() { super(); } @@ -101,9 +97,6 @@ public class CalculatedFieldEntity extends BaseVersionedEntity this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); this.version = calculatedField.getVersion(); this.debugSettings = JacksonUtil.toString(calculatedField.getDebugSettings()); - if (calculatedField.getExternalId() != null) { - this.externalId = calculatedField.getExternalId().getId(); - } } @Override @@ -118,9 +111,6 @@ public class CalculatedFieldEntity extends BaseVersionedEntity calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); - if (externalId != null) { - calculatedField.setExternalId(new CalculatedFieldId(externalId)); - } return calculatedField; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index e7e41ed2d0..0f48f3b00d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -18,16 +18,13 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.stereotype.Repository; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import java.util.List; import java.util.UUID; -@Repository -public interface CalculatedFieldRepository extends JpaRepository, ExportableEntityRepository { +public interface CalculatedFieldRepository extends JpaRepository { boolean existsByTenantIdAndEntityId(UUID tenantId, UUID entityId); @@ -37,16 +34,10 @@ public interface CalculatedFieldRepository extends JpaRepository findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId, Pageable pageable); - Page findByTenantId(UUID tenantId, Pageable pageable); - List findAllByTenantId(UUID tenantId); List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); long countByTenantIdAndEntityId(UUID tenantId, UUID entityId); - CalculatedFieldId findExternalIdById(UUID id); - - CalculatedFieldEntity findByTenantIdAndName(UUID tenantId, String name); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index fae3468f1e..e59ff3f4e6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -92,7 +92,6 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); - calculatedField.setExternalId(externalIdObj != null ? new CalculatedFieldId(UUID.fromString((String) externalIdObj)) : null); return calculatedField; }).collect(Collectors.toList()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 01e7888fbf..8922eaca4e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -88,26 +88,6 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findByTenantId(UUID tenantId, PageLink pageLink) { - return DaoUtil.toPageData(calculatedFieldRepository.findByTenantId(tenantId, DaoUtil.toPageable(pageLink))); - } - - @Override - public CalculatedField findByTenantIdAndName(UUID tenantId, String name) { - return DaoUtil.getData(calculatedFieldRepository.findByTenantIdAndName(tenantId, name)); - } - - @Override - public CalculatedField findByTenantIdAndExternalId(UUID tenantId, UUID externalId) { - return DaoUtil.getData(calculatedFieldRepository.findByTenantIdAndExternalId(tenantId, externalId)); - } - - @Override - public CalculatedFieldId getExternalIdByInternal(CalculatedFieldId internalId) { - return calculatedFieldRepository.findExternalIdById(internalId.getId()); - } - @Override protected Class getEntityClass() { return CalculatedFieldEntity.class; diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 5bd34c4f1d..d158912690 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -936,9 +936,7 @@ CREATE TABLE IF NOT EXISTS calculated_field ( configuration varchar(1000000), version BIGINT DEFAULT 1, debug_settings varchar(1024), - external_id UUID, - CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, name), - CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id) + CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, name) ); CREATE TABLE IF NOT EXISTS calculated_field_link ( diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 5a783355d7..2985aa7620 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -101,20 +101,6 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { .hasMessage("Calculated Field with such name is already in exists!"); } - @Test - public void testSaveCalculatedFieldWithExistingExternalId() { - Device device = createTestDevice(); - CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); - calculatedField.setExternalId(new CalculatedFieldId(UUID.fromString("2ef69d0a-89cf-4868-86f8-c50551d87ebe"))); - - calculatedFieldService.save(calculatedField); - - calculatedField.setName("Test 2"); - assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) - .isInstanceOf(DataValidationException.class) - .hasMessage("Calculated Field with such external id already exists!"); - } - @Test public void testFindCalculatedFieldById() { CalculatedField savedCalculatedField = saveValidCalculatedField(); From 9f2d399f4ca35fe125cd60b3abcdf2e99b1c6524 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 3 Mar 2025 17:00:49 +0200 Subject: [PATCH 11/26] Revert AssetExportService --- .../service/sync/ie/exporting/impl/AssetExportService.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java index fbda92b6be..56d5bb0934 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.sync.ie.exporting.impl; -import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; @@ -28,7 +27,6 @@ import java.util.Set; @Service @TbCoreComponent -@RequiredArgsConstructor public class AssetExportService extends BaseEntityExportService> { @Override @@ -37,11 +35,6 @@ public class AssetExportService extends BaseEntityExportService newExportData() { - return new EntityExportData<>(); - } - @Override public Set getSupportedEntityTypes() { return Set.of(EntityType.ASSET); From aecefc84bfdc47afe60b343359d761e5a04f5401 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 3 Mar 2025 19:23:02 +0200 Subject: [PATCH 12/26] fixed NPE, fixed flaky test --- .../subscription/DefaultTbEntityDataSubscriptionService.java | 4 +++- .../org/thingsboard/server/controller/WebsocketApiTest.java | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index 78d7f01404..9e9ca42e83 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -436,7 +436,9 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc ctx.sendWsMsg(update); } else { ctx.doFetchAlarmCount(); - ctx.createAlarmSubscriptions(); + if (entitiesIds != null) { + ctx.createAlarmSubscriptions(); + } TbAlarmCountSubCtx finalCtx = ctx; ScheduledFuture task = scheduler.scheduleWithFixedDelay( () -> refreshDynamicQuery(finalCtx), diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java index 69d50c61d1..9801907d3b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java @@ -83,7 +83,8 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @Slf4j @DaoSqlTest @TestPropertySource(properties = { - "server.ws.alarms_per_alarm_status_subscription_cache_size=5" + "server.ws.alarms_per_alarm_status_subscription_cache_size=5", + "server.ws.dynamic_page_link.refresh_interval=15" }) public class WebsocketApiTest extends AbstractControllerTest { @Autowired From eadc70581725b9f234e89dedda5d3d06e0020233 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 4 Mar 2025 09:16:24 +0200 Subject: [PATCH 13/26] added upgrade script for tenant profile --- .../main/data/upgrade/basic/schema_update.sql | 32 ++++++++++++++++++- .../DefaultTenantProfileConfiguration.java | 10 +++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index ddfac0d768..d694a2bcf6 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -63,4 +63,34 @@ $$; -- UPDATE SAVE TIME SERIES NODES END -ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; \ No newline at end of file +ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; + +-- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS START + +DO $$ + BEGIN + -- Check if the tenant_profile table exists + IF EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_name = 'tenant_profile' + ) THEN + + UPDATE tenant_profile + SET profile_data = profile_data + || jsonb_build_object( + 'configuration', profile_data->'configuration' || jsonb_build_object( + 'maxCalculatedFieldsPerEntity', COALESCE(profile_data->'configuration'->>'maxCalculatedFieldsPerEntity', '5')::bigint, + 'maxArgumentsPerCF', COALESCE(profile_data->'configuration'->>'maxArgumentsPerCF', '10')::bigint, + 'maxDataPointsPerRollingArg', COALESCE(profile_data->'configuration'->>'maxDataPointsPerRollingArg', '1000')::bigint, + 'maxStateSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxStateSizeInKBytes', '32')::bigint, + 'maxSingleValueArgumentSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxSingleValueArgumentSizeInKBytes', '2')::bigint + ) + ) + WHERE profile_data->'configuration' IS NOT NULL; + +END IF; + END; +$$; + +-- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS END diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 9be8a126b2..9c86785a5e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -135,11 +135,11 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private double warnThreshold; - private long maxCalculatedFieldsPerEntity; - private long maxArgumentsPerCF; - private long maxDataPointsPerRollingArg; - private long maxStateSizeInKBytes; - private long maxSingleValueArgumentSizeInKBytes; + private long maxCalculatedFieldsPerEntity = 5; + private long maxArgumentsPerCF = 10; + private long maxDataPointsPerRollingArg = 1000; + private long maxStateSizeInKBytes = 32; + private long maxSingleValueArgumentSizeInKBytes = 2; @Override public long getProfileThreshold(ApiUsageRecordKey key) { From 6a759ae061ca025cca9b5dd62b78559857eae8f1 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Tue, 4 Mar 2025 09:17:16 +0200 Subject: [PATCH 14/26] Fix testVcWithCalculatedFields_betweenTenants --- .../server/service/sync/vc/VersionControlTest.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java index d90ca60333..bb0d490ce0 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java @@ -575,7 +575,7 @@ public class VersionControlTest extends AbstractControllerTest { public void testVcWithCalculatedFields_betweenTenants() throws Exception { Asset asset = createAsset(null, null, "Asset 1"); Device device = createDevice(null, null, "Device 1", "test1"); - createCalculatedField("CalculatedField1", device.getId(), asset.getId()); + CalculatedField calculatedField = createCalculatedField("CalculatedField1", device.getId(), asset.getId()); String versionId = createVersion("calculated fields of asset and device", EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE); loginTenant2(); @@ -585,16 +585,15 @@ public class VersionControlTest extends AbstractControllerTest { Asset importedAsset = findAsset(asset.getName()); Device importedDevice = findDevice(device.getName()); - CalculatedField importedCalculatedField = findCalculatedFieldByEntityId(importedDevice.getId()); checkImportedEntity(tenantId1, device, tenantId2, importedDevice); checkImportedEntity(tenantId1, asset, tenantId2, importedAsset); List importedCalculatedFields = findCalculatedFieldsByEntityId(importedDevice.getId()); assertThat(importedCalculatedFields).size().isOne(); assertThat(importedCalculatedFields.get(0)).satisfies(importedField -> { - assertThat(importedField.getName()).isEqualTo(importedCalculatedField.getName()); - assertThat(importedField.getType()).isEqualTo(importedCalculatedField.getType()); - assertThat(importedField.getId()).isNotEqualTo(importedCalculatedField.getId()); + assertThat(importedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(importedField.getType()).isEqualTo(calculatedField.getType()); + assertThat(importedField.getId()).isNotEqualTo(calculatedField.getId()); }); } From 7994802e4bc3390c6c6701ff726289072ce2890c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 4 Mar 2025 09:39:31 +0200 Subject: [PATCH 15/26] added default values on the UI --- ui-ngx/src/app/shared/models/tenant.model.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 38ff63e3c6..23896141db 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -156,11 +156,11 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan rpcTtlDays: 0, queueStatsTtlDays: 0, ruleEngineExceptionsTtlDays: 0, - maxCalculatedFieldsPerEntity: 0, - maxArgumentsPerCF: 0, - maxDataPointsPerRollingArg: 0, - maxStateSizeInKBytes: 0, - maxSingleValueArgumentSizeInKBytes: 0, + maxCalculatedFieldsPerEntity: 5, + maxArgumentsPerCF: 10, + maxDataPointsPerRollingArg: 1000, + maxStateSizeInKBytes: 32, + maxSingleValueArgumentSizeInKBytes: 2, calculatedFieldDebugEventsRateLimit: '' }; configuration = {...defaultConfiguration, type: TenantProfileType.DEFAULT}; From afd727b097ae4253c7bd8fe445a0c9197eb2d4ad Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 4 Mar 2025 11:22:51 +0200 Subject: [PATCH 16/26] removed check for changing value when update state and moved timeout to config param --- .../server/actors/ActorSystemContext.java | 4 ++++ ...CalculatedFieldEntityMessageProcessor.java | 21 ++++++++----------- .../ctx/state/SingleValueArgumentEntry.java | 4 ---- .../src/main/resources/thingsboard.yml | 2 ++ 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 50a777840c..9ab7adfcc6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -644,6 +644,10 @@ public class ActorSystemContext { @Getter private String deviceStateNodeRateLimitConfig; + @Value("${actors.calculated_fields.calculation_result_timeout:5}") + @Getter + private long cfCalculationResultTimeout; + @Getter @Setter private TbActorSystem actorSystem; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index f7fc204c0f..279628c566 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -275,9 +275,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); boolean stateSizeOk; - if (ctx.isInitialized() && state.isReady()) { - try { - CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); + try { + if (ctx.isInitialized() && state.isReady()) { + CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(systemContext.getCfCalculationResultTimeout(), TimeUnit.SECONDS); state.checkStateSize(ctxId, ctx.getMaxStateSize()); stateSizeOk = state.isSizeOk(); if (stateSizeOk) { @@ -286,21 +286,18 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); } } - } catch (Exception e) { - throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build(); } - } else { + } catch (Exception e) { + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build(); + } finally { state.checkStateSize(ctxId, ctx.getMaxStateSize()); stateSizeOk = state.isSizeOk(); if (stateSizeOk) { - callback.onSuccess(); // State was updated but no calculation performed; + cfStateService.persistState(ctxId, state, callback); + } else { + removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback); } } - if (stateSizeOk) { - cfStateService.persistState(ctxId, state, callback); - } else { - removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback); - } } private void removeStateAndRaiseSizeException(CalculatedFieldEntityCtxId ctxId, CalculatedFieldException ex, TbCallback callback) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index a064a99935..a237f3d022 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -100,10 +100,6 @@ public class SingleValueArgumentEntry implements ArgumentEntry { if (newVersion == null || this.version == null || newVersion > this.version) { this.ts = singleValueEntry.getTs(); this.version = newVersion; - BasicKvEntry newValue = singleValueEntry.getKvEntryValue(); - if (this.kvEntryValue != null && this.kvEntryValue.getValue().equals(newValue.getValue())) { - return false; - } this.kvEntryValue = singleValueEntry.getKvEntryValue(); return true; } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index ec9925bf06..d0ccef8ba9 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -512,6 +512,8 @@ actors: enabled: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}" # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" + # Time in seconds to receive calculation result. + calculation_result_timeout: "${ACTORS_CALCULATION_RESULT_TIMEOUT_SEC:5}" debug: settings: From 8b30d1d93be0b4b5743ff9a77b05395cafc2e661 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 4 Mar 2025 11:31:13 +0200 Subject: [PATCH 17/26] fixed test for new logic --- .../service/cf/ctx/state/SingleValueArgumentEntryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index e83e30663d..2c48ed9167 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -71,6 +71,6 @@ public class SingleValueArgumentEntryTest { @Test void testUpdateEntryWhenValueWasNotChanged() { - assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 237L))).isFalse(); + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 364L))).isTrue(); } } \ No newline at end of file From 6265ff45a8adca7c139d226d81054d287ca406bf Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 4 Mar 2025 11:54:28 +0200 Subject: [PATCH 18/26] merge function draft --- .../api/tbel/DefaultTbelInvokeService.java | 2 + .../script/api/tbel/TbTimeWindow.java | 3 + .../thingsboard/script/api/tbel/TbUtils.java | 60 +++++++++++++--- .../api/tbel/TbelCfTsMultiDoubleVal.java | 34 +++++++++ .../script/api/tbel/TbUtilsTest.java | 72 ++++++++++++++++++- 5 files changed, 160 insertions(+), 11 deletions(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java index 962d384a46..38fed7a2aa 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java @@ -138,6 +138,8 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem parserConfig.registerDataType("TbelCfTsDoubleVal", TbelCfTsDoubleVal.class, TbelCfTsDoubleVal::memorySize); parserConfig.registerDataType("TbelCfTsRollingData", TbelCfTsRollingData.class, TbelCfTsRollingData::memorySize); parserConfig.registerDataType("TbTimeWindow", TbTimeWindow.class, TbTimeWindow::memorySize); + parserConfig.registerDataType("TbelCfTsDoubleVal", TbelCfTsMultiDoubleVal.class, TbelCfTsMultiDoubleVal::memorySize); + TbUtils.register(parserConfig); executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "tbel-executor")); try { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java index 5048611838..b430b23757 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java @@ -35,4 +35,7 @@ public class TbTimeWindow implements TbelCfObject { return OBJ_SIZE; } + public boolean matches(long ts) { + return ts >= startTs && ts < endTs; + } } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 4796cf12bb..e6a0b8d043 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2025 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -368,8 +368,16 @@ public class TbUtils { byte[].class, int.class))); parserConfig.addImport("parseBinaryArrayToInt", new MethodStub(TbUtils.class.getMethod("parseBinaryArrayToInt", byte[].class, int.class, int.class))); - parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("mergeCfTsRollingArgs", + parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", TbelCfTsRollingArg.class, TbelCfTsRollingArg.class))); + parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", + TbelCfTsRollingArg.class, TbelCfTsRollingArg.class, TbTimeWindow.class))); + parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", + TbelCfTsRollingArg.class, TbelCfTsRollingArg.class, TbTimeWindow.class, Map.class))); + parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", + List.class, TbTimeWindow.class))); + parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", + List.class, TbTimeWindow.class, Map.class))); } public static String btoa(String input) { @@ -1510,11 +1518,28 @@ public class TbUtils { return hex; } - public static TbelCfTsRollingData mergeCfTsRollingArgs(TbelCfTsRollingArg a, TbelCfTsRollingArg b) { - return mergeCfTsRollingArgs(Arrays.asList(a, b), null); + public static TbelCfTsRollingData merge(TbelCfTsRollingArg a, TbelCfTsRollingArg b) { + return merge(Arrays.asList(a, b), null, null); } - public static TbelCfTsRollingData mergeCfTsRollingArgs(List args, Map settings) { + public static TbelCfTsRollingData merge(TbelCfTsRollingArg a, TbelCfTsRollingArg b, TbTimeWindow timeWindow) { + return merge(Arrays.asList(a, b), timeWindow, null); + } + + public static TbelCfTsRollingData merge(TbelCfTsRollingArg a, TbelCfTsRollingArg b, TbTimeWindow timeWindow, Map settings) { + return merge(Arrays.asList(a, b), timeWindow, settings); + } + + public static TbelCfTsRollingData merge(List args, TbTimeWindow timeWindow) { + return merge(args, timeWindow, null); + } + + public static TbelCfTsRollingData merge(List args, TbTimeWindow timeWindow, Map settings) { + boolean ignoreNaN = true; + if (settings != null && settings.containsKey("ignoreNaN")) { + ignoreNaN = Boolean.parseBoolean(settings.get("ignoreNaN").toString()); + } + TreeSet allTimestamps = new TreeSet<>(); long startTs = Long.MAX_VALUE; long endTs = Long.MIN_VALUE; @@ -1532,7 +1557,7 @@ public class TbUtils { double[] result = new double[args.size()]; Arrays.fill(result, Double.NaN); - var tw = new TbTimeWindow(startTs, endTs, allTimestamps.size()); + var tw = timeWindow != null ? timeWindow : new TbTimeWindow(startTs, endTs, allTimestamps.size()); for (long ts : allTimestamps) { for (int i = 0; i < args.size(); i++) { @@ -1543,7 +1568,22 @@ public class TbUtils { lastIndex[i]++; } } - data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); + if (tw.matches(ts)) { + if (ignoreNaN) { + boolean skip = false; + for (int i = 0; i < args.size(); i++) { + if (Double.isNaN(result[i])) { + skip = true; + break; + } + } + if (!skip) { + data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); + } + } else { + data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); + } + } } return new TbelCfTsRollingData(tw, data); diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java index 33e4a65bb8..2743bbd0de 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java @@ -15,6 +15,7 @@ */ package org.thingsboard.script.api.tbel; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; @Data @@ -25,6 +26,39 @@ public class TbelCfTsMultiDoubleVal implements TbelCfObject { private final long ts; private final double[] values; + @JsonIgnore + public double getV1() { + return getV(0); + } + + @JsonIgnore + public double getV2() { + return getV(1); + } + + @JsonIgnore + public double getV3() { + return getV(2); + } + + @JsonIgnore + public double getV4() { + return getV(3); + } + + @JsonIgnore + public double getV5() { + return getV(4); + } + + private double getV(int idx) { + if (values.length < idx + 1) { + throw new IllegalArgumentException("Can't get value at index " + idx + ". There are " + values.length + " values present."); + } else { + return values[idx]; + } + } + @Override public long memorySize() { return OBJ_SIZE + values.length * 8L; diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index fbd81948d6..f19a323068 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -1109,7 +1109,7 @@ public class TbUtilsTest { String validInput = Base64.getEncoder().encodeToString(new byte[]{1, 2, 3, 4, 5}); ExecutionArrayList actual = TbUtils.base64ToBytesList(ctx, validInput); ExecutionArrayList expected = new ExecutionArrayList<>(ctx); - expected.addAll(List.of((byte) 1, (byte)2, (byte)3, (byte)4, (byte)5)); + expected.addAll(List.of((byte) 1, (byte) 2, (byte) 3, (byte) 4, (byte) 5)); Assertions.assertEquals(expected, actual); String emptyInput = Base64.getEncoder().encodeToString(new byte[]{}); @@ -1123,6 +1123,7 @@ public class TbUtilsTest { TbUtils.base64ToBytesList(ctx, null); }); } + @Test public void bytesToHex_Test() { byte[] bb = {(byte) 0xBB, (byte) 0xAA}; @@ -1136,6 +1137,75 @@ public class TbUtilsTest { Assertions.assertEquals(expected, actual); } + @Test + public void merge_two_rolling_args_ts_match_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000, 1000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = TbUtils.merge(arg1, arg2); + Assertions.assertEquals(3, result.getSize()); + Assertions.assertNotNull(result.getValues()); + Assertions.assertNotNull(result.getValues().get(0)); + Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); + Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); + Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); + } + + @Test + public void merge_two_rolling_args_with_timewindow_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000, 1000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = TbUtils.merge(arg1, arg2, new TbTimeWindow(0, 10000, 1000)); + Assertions.assertEquals(2, result.getSize()); + Assertions.assertNotNull(result.getValues()); + Assertions.assertNotNull(result.getValues().get(0)); + Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); + Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); + Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); + } + + @Test + public void merge_two_rolling_args_ts_mismatch_default_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000, 1000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(100, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(200, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = TbUtils.merge(arg1, arg2); + Assertions.assertEquals(3, result.getSize()); + Assertions.assertNotNull(result.getValues()); + + TbelCfTsMultiDoubleVal item0 = result.getValues().get(0); + Assertions.assertNotNull(item0); + Assertions.assertEquals(200L, item0.getTs()); + Assertions.assertEquals(1, item0.getValues()[0]); + Assertions.assertEquals(11, item0.getValues()[1]); + } + + @Test + public void merge_two_rolling_args_ts_mismatch_ignore_nan_disabled_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000, 1000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(100, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(200, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = TbUtils.merge(Arrays.asList(arg1, arg2), new TbTimeWindow(0, 60000, 1000), Collections.singletonMap("ignoreNaN", false)); + Assertions.assertEquals(4, result.getSize()); + Assertions.assertNotNull(result.getValues()); + + TbelCfTsMultiDoubleVal item0 = result.getValues().get(0); + Assertions.assertNotNull(item0); + Assertions.assertEquals(100L, item0.getTs()); + Assertions.assertEquals(1, item0.getValues()[0]); + Assertions.assertEquals(Double.NaN, item0.getValues()[1]); + + TbelCfTsMultiDoubleVal item1 = result.getValues().get(1); + Assertions.assertEquals(200L, item1.getTs()); + Assertions.assertEquals(1, item1.getValues()[0]); + Assertions.assertEquals(11, item1.getValues()[1]); + } + private static List toList(byte[] data) { List result = new ArrayList<>(data.length); for (Byte b : data) { From 759bc94d1d797f521c3326ad777c4b45834197ff Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 4 Mar 2025 11:57:57 +0200 Subject: [PATCH 19/26] Review comments --- .../server/actors/ActorSystemContext.java | 2 +- .../CalculatedFieldEntityMessageProcessor.java | 13 +++++++------ application/src/main/resources/thingsboard.yml | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 9ab7adfcc6..de3864df9f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -644,7 +644,7 @@ public class ActorSystemContext { @Getter private String deviceStateNodeRateLimitConfig; - @Value("${actors.calculated_fields.calculation_result_timeout:5}") + @Value("${actors.calculated_fields.calculation_timeout:5}") @Getter private long cfCalculationResultTimeout; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 279628c566..e42ebb593e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -274,13 +274,13 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); - boolean stateSizeOk; + boolean stateSizeChecked = false; try { if (ctx.isInitialized() && state.isReady()) { CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(systemContext.getCfCalculationResultTimeout(), TimeUnit.SECONDS); state.checkStateSize(ctxId, ctx.getMaxStateSize()); - stateSizeOk = state.isSizeOk(); - if (stateSizeOk) { + stateSizeChecked = true; + if (state.isSizeOk()) { cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); @@ -290,9 +290,10 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } catch (Exception e) { throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build(); } finally { - state.checkStateSize(ctxId, ctx.getMaxStateSize()); - stateSizeOk = state.isSizeOk(); - if (stateSizeOk) { + if (!stateSizeChecked) { + state.checkStateSize(ctxId, ctx.getMaxStateSize()); + } + if (state.isSizeOk()) { cfStateService.persistState(ctxId, state, callback); } else { removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index d0ccef8ba9..53258a4296 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -513,7 +513,7 @@ actors: # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" # Time in seconds to receive calculation result. - calculation_result_timeout: "${ACTORS_CALCULATION_RESULT_TIMEOUT_SEC:5}" + calculation_timeout: "${ACTORS_CALCULATION_TIMEOUT_SEC:5}" debug: settings: From 933d77427ce0eb12967d24f94be279e26fbd5eed Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 4 Mar 2025 13:13:09 +0200 Subject: [PATCH 20/26] Merge function --- .../cf/ctx/state/TsRollingArgumentEntry.java | 2 +- .../script/api/tbel/TbTimeWindow.java | 1 - .../thingsboard/script/api/tbel/TbUtils.java | 81 --------------- .../script/api/tbel/TbelCfTsRollingArg.java | 98 ++++++++++++++++++- .../script/api/tbel/TbUtilsTest.java | 69 ------------- .../api/tbel/TbelCfTsRollingArgTest.java | 88 ++++++++++++++++- .../shared/models/calculated-field.models.ts | 89 +++++++++++++---- 7 files changed, 254 insertions(+), 174 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 1114d993a3..b5a680a072 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -89,7 +89,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { for (var e : tsRecords.entrySet()) { values.add(new TbelCfTsDoubleVal(e.getKey(), e.getValue())); } - return new TbelCfTsRollingArg(limit, timeWindow, values); + return new TbelCfTsRollingArg(timeWindow, values); } @Override diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java index b430b23757..1761ef6d0a 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java @@ -28,7 +28,6 @@ public class TbTimeWindow implements TbelCfObject { private long startTs; private long endTs; - private int limit; @Override public long memorySize() { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index e6a0b8d043..a43cc30f23 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -368,16 +368,6 @@ public class TbUtils { byte[].class, int.class))); parserConfig.addImport("parseBinaryArrayToInt", new MethodStub(TbUtils.class.getMethod("parseBinaryArrayToInt", byte[].class, int.class, int.class))); - parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", - TbelCfTsRollingArg.class, TbelCfTsRollingArg.class))); - parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", - TbelCfTsRollingArg.class, TbelCfTsRollingArg.class, TbTimeWindow.class))); - parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", - TbelCfTsRollingArg.class, TbelCfTsRollingArg.class, TbTimeWindow.class, Map.class))); - parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", - List.class, TbTimeWindow.class))); - parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", - List.class, TbTimeWindow.class, Map.class))); } public static String btoa(String input) { @@ -1518,76 +1508,5 @@ public class TbUtils { return hex; } - public static TbelCfTsRollingData merge(TbelCfTsRollingArg a, TbelCfTsRollingArg b) { - return merge(Arrays.asList(a, b), null, null); - } - - public static TbelCfTsRollingData merge(TbelCfTsRollingArg a, TbelCfTsRollingArg b, TbTimeWindow timeWindow) { - return merge(Arrays.asList(a, b), timeWindow, null); - } - - public static TbelCfTsRollingData merge(TbelCfTsRollingArg a, TbelCfTsRollingArg b, TbTimeWindow timeWindow, Map settings) { - return merge(Arrays.asList(a, b), timeWindow, settings); - } - - public static TbelCfTsRollingData merge(List args, TbTimeWindow timeWindow) { - return merge(args, timeWindow, null); - } - - public static TbelCfTsRollingData merge(List args, TbTimeWindow timeWindow, Map settings) { - boolean ignoreNaN = true; - if (settings != null && settings.containsKey("ignoreNaN")) { - ignoreNaN = Boolean.parseBoolean(settings.get("ignoreNaN").toString()); - } - - TreeSet allTimestamps = new TreeSet<>(); - long startTs = Long.MAX_VALUE; - long endTs = Long.MIN_VALUE; - for (TbelCfTsRollingArg arg : args) { - for (TbelCfTsDoubleVal val : arg.getValues()) { - allTimestamps.add(val.getTs()); - } - startTs = Math.min(startTs, arg.getTimeWindow().getStartTs()); - endTs = Math.max(endTs, arg.getTimeWindow().getEndTs()); - } - - List data = new ArrayList<>(); - - int[] lastIndex = new int[args.size()]; - double[] result = new double[args.size()]; - Arrays.fill(result, Double.NaN); - - var tw = timeWindow != null ? timeWindow : new TbTimeWindow(startTs, endTs, allTimestamps.size()); - - for (long ts : allTimestamps) { - for (int i = 0; i < args.size(); i++) { - var arg = args.get(i); - var values = arg.getValues(); - while (lastIndex[i] < values.size() && values.get(lastIndex[i]).getTs() <= ts) { - result[i] = values.get(lastIndex[i]).getValue(); - lastIndex[i]++; - } - } - if (tw.matches(ts)) { - if (ignoreNaN) { - boolean skip = false; - for (int i = 0; i < args.size(); i++) { - if (Double.isNaN(result[i])) { - skip = true; - break; - } - } - if (!skip) { - data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); - } - } else { - data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); - } - } - } - - return new TbelCfTsRollingData(tw, data); - } - } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java index 74c9d77454..3b5a7ac9bd 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java @@ -19,11 +19,15 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Getter; +import org.thingsboard.common.util.JacksonUtil; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.TreeSet; import java.util.function.Consumer; import static org.thingsboard.script.api.tbel.TbelCfTsDoubleVal.OBJ_SIZE; @@ -44,9 +48,9 @@ public class TbelCfTsRollingArg implements TbelCfArg, Iterable values) { + public TbelCfTsRollingArg(long timeWindow, List values) { long ts = System.currentTimeMillis(); - this.timeWindow = new TbTimeWindow(ts - timeWindow, ts, limit); + this.timeWindow = new TbTimeWindow(ts - timeWindow, ts); this.values = Collections.unmodifiableList(values); } @@ -104,6 +108,14 @@ public class TbelCfTsRollingArg implements TbelCfArg, Iterable settings) { + return mergeAll(Collections.singletonList(other), settings); + } + + public TbelCfTsRollingData mergeAll(List others) { + return mergeAll(others, null); + } + + public TbelCfTsRollingData mergeAll(List others, Map settings) { + List args = new ArrayList<>(others.size() + 1); + args.add(this); + args.addAll(others); + + boolean ignoreNaN = true; + if (settings != null && settings.containsKey("ignoreNaN")) { + ignoreNaN = Boolean.parseBoolean(settings.get("ignoreNaN").toString()); + } + + TbTimeWindow timeWindow = null; + if (settings != null && settings.containsKey("timeWindow")) { + var twVar = settings.get("timeWindow"); + if (twVar instanceof TbTimeWindow) { + timeWindow = (TbTimeWindow) settings.get("timeWindow"); + } else if (twVar instanceof Map twMap) { + timeWindow = new TbTimeWindow(Long.valueOf(twMap.get("startTs").toString()), Long.valueOf(twMap.get("endTs").toString())); + } else { + timeWindow = JacksonUtil.fromString(settings.get("timeWindow").toString(), TbTimeWindow.class); + } + } + + TreeSet allTimestamps = new TreeSet<>(); + long startTs = Long.MAX_VALUE; + long endTs = Long.MIN_VALUE; + for (TbelCfTsRollingArg arg : args) { + for (TbelCfTsDoubleVal val : arg.getValues()) { + allTimestamps.add(val.getTs()); + } + startTs = Math.min(startTs, arg.getTimeWindow().getStartTs()); + endTs = Math.max(endTs, arg.getTimeWindow().getEndTs()); + } + + List data = new ArrayList<>(); + + int[] lastIndex = new int[args.size()]; + double[] result = new double[args.size()]; + Arrays.fill(result, Double.NaN); + + for (long ts : allTimestamps) { + for (int i = 0; i < args.size(); i++) { + var arg = args.get(i); + var values = arg.getValues(); + while (lastIndex[i] < values.size() && values.get(lastIndex[i]).getTs() <= ts) { + result[i] = values.get(lastIndex[i]).getValue(); + lastIndex[i]++; + } + } + if (timeWindow == null || timeWindow.matches(ts)) { + if (ignoreNaN) { + boolean skip = false; + for (int i = 0; i < args.size(); i++) { + if (Double.isNaN(result[i])) { + skip = true; + break; + } + } + if (!skip) { + data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); + } + } else { + data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); + } + } + } + + return new TbelCfTsRollingData(timeWindow != null ? timeWindow : new TbTimeWindow(startTs, endTs), data); + } + + @JsonIgnore public int getSize() { return values.size(); diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index f19a323068..71c7a8b78a 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -1137,75 +1137,6 @@ public class TbUtilsTest { Assertions.assertEquals(expected, actual); } - @Test - public void merge_two_rolling_args_ts_match_test() { - TbTimeWindow tw = new TbTimeWindow(0, 60000, 1000); - TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); - TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); - - var result = TbUtils.merge(arg1, arg2); - Assertions.assertEquals(3, result.getSize()); - Assertions.assertNotNull(result.getValues()); - Assertions.assertNotNull(result.getValues().get(0)); - Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); - Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); - Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); - } - - @Test - public void merge_two_rolling_args_with_timewindow_test() { - TbTimeWindow tw = new TbTimeWindow(0, 60000, 1000); - TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); - TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); - - var result = TbUtils.merge(arg1, arg2, new TbTimeWindow(0, 10000, 1000)); - Assertions.assertEquals(2, result.getSize()); - Assertions.assertNotNull(result.getValues()); - Assertions.assertNotNull(result.getValues().get(0)); - Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); - Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); - Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); - } - - @Test - public void merge_two_rolling_args_ts_mismatch_default_test() { - TbTimeWindow tw = new TbTimeWindow(0, 60000, 1000); - TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(100, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); - TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(200, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); - - var result = TbUtils.merge(arg1, arg2); - Assertions.assertEquals(3, result.getSize()); - Assertions.assertNotNull(result.getValues()); - - TbelCfTsMultiDoubleVal item0 = result.getValues().get(0); - Assertions.assertNotNull(item0); - Assertions.assertEquals(200L, item0.getTs()); - Assertions.assertEquals(1, item0.getValues()[0]); - Assertions.assertEquals(11, item0.getValues()[1]); - } - - @Test - public void merge_two_rolling_args_ts_mismatch_ignore_nan_disabled_test() { - TbTimeWindow tw = new TbTimeWindow(0, 60000, 1000); - TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(100, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); - TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(200, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); - - var result = TbUtils.merge(Arrays.asList(arg1, arg2), new TbTimeWindow(0, 60000, 1000), Collections.singletonMap("ignoreNaN", false)); - Assertions.assertEquals(4, result.getSize()); - Assertions.assertNotNull(result.getValues()); - - TbelCfTsMultiDoubleVal item0 = result.getValues().get(0); - Assertions.assertNotNull(item0); - Assertions.assertEquals(100L, item0.getTs()); - Assertions.assertEquals(1, item0.getValues()[0]); - Assertions.assertEquals(Double.NaN, item0.getValues()[1]); - - TbelCfTsMultiDoubleVal item1 = result.getValues().get(1); - Assertions.assertEquals(200L, item1.getTs()); - Assertions.assertEquals(1, item1.getValues()[0]); - Assertions.assertEquals(11, item1.getValues()[1]); - } - private static List toList(byte[] data) { List result = new ArrayList<>(data.length); for (Byte b : data) { diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java index 327b477b6f..69eba2fab2 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java @@ -15,10 +15,15 @@ */ package org.thingsboard.script.api.tbel; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.thingsboard.common.util.JacksonUtil; +import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -33,7 +38,7 @@ public class TbelCfTsRollingArgTest { @BeforeEach void setUp() { rollingArg = new TbelCfTsRollingArg( - new TbTimeWindow(ts - 30000, ts - 10, 10), + new TbTimeWindow(ts - 30000, ts - 10), List.of( new TbelCfTsDoubleVal(ts - 10, Double.NaN), new TbelCfTsDoubleVal(ts - 20, 2.0), @@ -98,7 +103,7 @@ public class TbelCfTsRollingArgTest { void testFirstAndLastWhenOnlyNaNAndIgnoreNaNIsFalse() { assertThat(rollingArg.first()).isEqualTo(2.0); rollingArg = new TbelCfTsRollingArg( - new TbTimeWindow(ts - 30000, ts - 10, 10), + new TbTimeWindow(ts - 30000, ts - 10), List.of( new TbelCfTsDoubleVal(ts - 10, Double.NaN), new TbelCfTsDoubleVal(ts - 40, Double.NaN), @@ -117,7 +122,7 @@ public class TbelCfTsRollingArgTest { @Test void testEmptyValues() { - rollingArg = new TbelCfTsRollingArg(new TbTimeWindow(0, 10, 10), List.of()); + rollingArg = new TbelCfTsRollingArg(new TbTimeWindow(0, 10), List.of()); assertThatThrownBy(rollingArg::sum).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); assertThatThrownBy(rollingArg::max).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); assertThatThrownBy(rollingArg::min).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); @@ -128,4 +133,81 @@ public class TbelCfTsRollingArgTest { assertThatThrownBy(rollingArg::last).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); } + @Test + public void merge_two_rolling_args_ts_match_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = arg1.merge(arg2); + Assertions.assertEquals(3, result.getSize()); + Assertions.assertNotNull(result.getValues()); + Assertions.assertNotNull(result.getValues().get(0)); + Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); + Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); + Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); + } + + @Test + public void merge_two_rolling_args_with_timewindow_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = arg1.merge(arg2, Collections.singletonMap("timeWindow", new TbTimeWindow(0, 10000))); + Assertions.assertEquals(2, result.getSize()); + Assertions.assertNotNull(result.getValues()); + Assertions.assertNotNull(result.getValues().get(0)); + Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); + Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); + Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); + + result = arg1.merge(arg2, Collections.singletonMap("timeWindow", Map.of("startTs", 0L, "endTs", 10000))); + Assertions.assertEquals(2, result.getSize()); + Assertions.assertNotNull(result.getValues()); + Assertions.assertNotNull(result.getValues().get(0)); + Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); + Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); + Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); + } + + @Test + public void merge_two_rolling_args_ts_mismatch_default_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(100, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(200, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = arg1.merge(arg2); + Assertions.assertEquals(3, result.getSize()); + Assertions.assertNotNull(result.getValues()); + + TbelCfTsMultiDoubleVal item0 = result.getValues().get(0); + Assertions.assertNotNull(item0); + Assertions.assertEquals(200L, item0.getTs()); + Assertions.assertEquals(1, item0.getValues()[0]); + Assertions.assertEquals(11, item0.getValues()[1]); + } + + @Test + public void merge_two_rolling_args_ts_mismatch_ignore_nan_disabled_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(100, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(200, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = arg1.merge(arg2, Collections.singletonMap("ignoreNaN", false)); + Assertions.assertEquals(4, result.getSize()); + Assertions.assertNotNull(result.getValues()); + + TbelCfTsMultiDoubleVal item0 = result.getValues().get(0); + Assertions.assertNotNull(item0); + Assertions.assertEquals(100L, item0.getTs()); + Assertions.assertEquals(1, item0.getValues()[0]); + Assertions.assertEquals(Double.NaN, item0.getValues()[1]); + + TbelCfTsMultiDoubleVal item1 = result.getValues().get(1); + Assertions.assertEquals(200L, item1.getTs()); + Assertions.assertEquals(1, item1.getValues()[0]); + Assertions.assertEquals(11, item1.getValues()[1]); + } + } \ No newline at end of file diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index f6d7aaaa1e..4467fecc79 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -272,7 +272,7 @@ export const CalculatedFieldAttributeValueArgumentAutocomplete = { export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { max: { meta: 'function', - description: 'Computes the maximum value in the list of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Returns the maximum value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -288,7 +288,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, min: { meta: 'function', - description: 'Computes the minimum value in the list of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Returns the minimum value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -304,7 +304,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, mean: { meta: 'function', - description: 'Computes the mean value of the rolling argument values list. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Computes the mean value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -318,9 +318,25 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { type: 'number' } }, + avg: { + meta: 'function', + description: 'Computes the average value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The average value, or NaN if applicable', + type: 'number' + } + }, std: { meta: 'function', - description: 'Computes the standard deviation in the list of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Computes the standard deviation of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -336,7 +352,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, median: { meta: 'function', - description: 'Computes the median value of the rolling argument values list. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Computes the median value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -352,7 +368,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, count: { meta: 'function', - description: 'Counts values in the list of rolling argument values. Counts non-NaN values if ignoreNaN is true, otherwise - total size.', + description: 'Counts values of the rolling argument. Counts non-NaN values if ignoreNaN is true, otherwise - total size.', args: [ { name: 'ignoreNaN', @@ -368,7 +384,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, last: { meta: 'function', - description: 'Returns the last non-NaN value in the list of rolling argument values if ignoreNaN is true, otherwise - the last value.', + description: 'Returns the last non-NaN value of the rolling argument values if ignoreNaN is true, otherwise - the last value.', args: [ { name: 'ignoreNaN', @@ -384,7 +400,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, first: { meta: 'function', - description: 'Returns the first non-NaN value in the list of rolling argument values if ignoreNaN is true, otherwise - the first value.', + description: 'Returns the first non-NaN value of the rolling argument values if ignoreNaN is true, otherwise - the first value.', args: [ { name: 'ignoreNaN', @@ -400,7 +416,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, sum: { meta: 'function', - description: 'Computes the sum of values in the list of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Computes the sum of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -413,12 +429,56 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { description: 'The sum of values, or NaN if applicable', type: 'number' } + }, + merge: { + meta: 'function', + description: 'Merges current object with other time series rolling argument into a single object by aligning their timestamped values. Supports optional configurable settings.', + args: [ + { + name: 'other', + description: "A time series rolling argument to be merged with the current object.", + type: "object", + optional: true + }, + { + name: "settings", + description: "Optional settings controlling the merging process. Supported keys: 'ignoreNaN' (boolean, equals true by default) to determine whether NaN values should be ignored; 'timeWindow' (object, empty by default) to apply time window filtering.", + type: "object", + optional: true + } + ], + return: { + description: 'A new object containing merged timestamped values from all provided arguments, aligned based on timestamps and filtered according to settings.', + type: '{ values: { ts: number; values: number[]; }[]; timeWindow: { startTs: number; endTs: number } }; }', + } + }, + mergeAll: { + meta: 'function', + description: 'Merges current object with other time series rolling arguments into a single object by aligning their timestamped values. Supports optional configurable settings.', + args: [ + { + name: 'others', + description: "A list of time series rolling arguments to be merged with the current object.", + type: "object[]", + optional: true + }, + { + name: "settings", + description: "Optional settings controlling the merging process. Supported keys: 'ignoreNaN' (boolean, equals true by default) to determine whether NaN values should be ignored; 'timeWindow' (object, empty by default) to apply time window filtering.", + type: "object", + optional: true + } + ], + return: { + description: 'A new object containing merged timestamped values from all provided arguments, aligned based on timestamps and filtered according to settings.', + type: '{ values: { ts: number; values: number[]; }[]; timeWindow: { startTs: number; endTs: number } }; }', + } } }; export const CalculatedFieldRollingValueArgumentAutocomplete = { meta: 'object', - type: '{ values: { ts: number; value: any; }[]; timeWindow: { startTs: number; endTs: number; limit: number } }; }', + type: '{ values: { ts: number; value: number; }[]; timeWindow: { startTs: number; endTs: number } }; }', description: 'Calculated field rolling value argument.', children: { ...CalculatedFieldRollingValueArgumentFunctionsAutocomplete, @@ -429,7 +489,7 @@ export const CalculatedFieldRollingValueArgumentAutocomplete = { }, timeWindow: { meta: 'object', - type: '{ startTs: number; endTs: number; limit: number }', + type: '{ startTs: number; endTs: number }', description: 'Time window configuration', children: { startTs: { @@ -441,11 +501,6 @@ export const CalculatedFieldRollingValueArgumentAutocomplete = { meta: 'number', type: 'number', description: 'End time stamp', - }, - limit: { - meta: 'number', - type: 'number', - description: 'Limit', } } } @@ -504,7 +559,7 @@ const calculatedFieldSingleArgumentValueHighlightRules: AceHighlightRules = { } const calculatedFieldRollingArgumentValueFunctionsHighlightRules: Array = - ['max', 'min', 'mean', 'std', 'median', 'count', 'last', 'first', 'sum'].map(funcName => ({ + ['max', 'min', 'avg', 'mean', 'std', 'median', 'count', 'last', 'first', 'sum', 'merge', 'mergeAll'].map(funcName => ({ token: 'tb.calculated-field-func', regex: `\\b${funcName}\\b`, next: 'no_regex' From 79d0e706d2188a6740f3b0fc83a0fd3df411960d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 4 Mar 2025 14:02:53 +0200 Subject: [PATCH 21/26] updated upgrade script --- .../main/data/upgrade/basic/schema_update.sql | 37 ++++++------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index d694a2bcf6..e9cbf8ef8e 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -67,30 +67,17 @@ ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; -- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS START -DO $$ - BEGIN - -- Check if the tenant_profile table exists - IF EXISTS ( - SELECT 1 - FROM information_schema.tables - WHERE table_name = 'tenant_profile' - ) THEN - - UPDATE tenant_profile - SET profile_data = profile_data - || jsonb_build_object( - 'configuration', profile_data->'configuration' || jsonb_build_object( - 'maxCalculatedFieldsPerEntity', COALESCE(profile_data->'configuration'->>'maxCalculatedFieldsPerEntity', '5')::bigint, - 'maxArgumentsPerCF', COALESCE(profile_data->'configuration'->>'maxArgumentsPerCF', '10')::bigint, - 'maxDataPointsPerRollingArg', COALESCE(profile_data->'configuration'->>'maxDataPointsPerRollingArg', '1000')::bigint, - 'maxStateSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxStateSizeInKBytes', '32')::bigint, - 'maxSingleValueArgumentSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxSingleValueArgumentSizeInKBytes', '2')::bigint - ) - ) - WHERE profile_data->'configuration' IS NOT NULL; - -END IF; - END; -$$; +UPDATE tenant_profile +SET profile_data = profile_data + || jsonb_build_object( + 'configuration', profile_data->'configuration' || jsonb_build_object( + 'maxCalculatedFieldsPerEntity', COALESCE(profile_data->'configuration'->>'maxCalculatedFieldsPerEntity', '5')::bigint, + 'maxArgumentsPerCF', COALESCE(profile_data->'configuration'->>'maxArgumentsPerCF', '10')::bigint, + 'maxDataPointsPerRollingArg', COALESCE(profile_data->'configuration'->>'maxDataPointsPerRollingArg', '1000')::bigint, + 'maxStateSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxStateSizeInKBytes', '32')::bigint, + 'maxSingleValueArgumentSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxSingleValueArgumentSizeInKBytes', '2')::bigint + ) + ) +WHERE profile_data->'configuration'->>'maxCalculatedFieldsPerEntity' IS NULL; -- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS END From 3365d7f34e44515390a0e823181788ace3241ac7 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 5 Mar 2025 10:03:46 +0200 Subject: [PATCH 22/26] added option to round result for simple type --- .../ctx/state/SimpleCalculatedFieldState.java | 15 +++++++++++- .../state/SimpleCalculatedFieldStateTest.java | 24 ++++++++++++++++++- .../common/data/cf/configuration/Output.java | 1 + .../thingsboard/script/api/tbel/TbUtils.java | 7 +++++- .../script/api/tbel/TbUtilsTest.java | 7 ++++++ 5 files changed, 51 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 25008a60bd..480b334ac3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.kv.BasicKvEntry; @@ -64,7 +65,19 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { double expressionResult = expr.evaluate(); Output output = ctx.getOutput(); - return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), expressionResult)))); + Object result; + Integer decimals = output.getDecimalsByDefault(); + if (decimals != null) { + if (decimals.equals(0)) { + result = TbUtils.toInt(expressionResult); + } else { + result = TbUtils.toFixed(expressionResult, decimals); + } + } else { + result = expressionResult; + } + + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), result)))); } } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index c06e835937..e179e8e706 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.dao.usagerecord.ApiLimitService; @@ -138,7 +139,7 @@ public class SimpleCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49.0))); + assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49))); } @Test @@ -154,6 +155,26 @@ public class SimpleCalculatedFieldStateTest { .hasMessage("Argument 'key2' is not a number."); } + @Test + void testPerformCalculationWhenDecimalsByDefault() throws ExecutionException, InterruptedException { + state.arguments = new HashMap<>(Map.of( + "key1", new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new DoubleDataEntry("key1", 11.3456), 145L), + "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 6, new DoubleDataEntry("key2", 15.1), 165L), + "key3", new SingleValueArgumentEntry(System.currentTimeMillis() - 3, new DoubleDataEntry("key3", 23.1), 184L) + )); + + Output output = getCalculatedFieldConfig().getOutput(); + output.setDecimalsByDefault(3); + ctx.setOutput(output); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49.546))); + } + @Test void testIsReadyWhenNotAllArgPresent() { assertThat(state.isReady()).isFalse(); @@ -219,6 +240,7 @@ public class SimpleCalculatedFieldStateTest { output.setName("output"); output.setType(OutputType.ATTRIBUTES); output.setScope(AttributeScope.SERVER_SCOPE); + output.setDecimalsByDefault(0); config.setOutput(output); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java index 49e393b19f..f2b4948837 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -26,5 +26,6 @@ public class Output { private String name; private OutputType type; private AttributeScope scope; + private Integer decimalsByDefault; } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index a43cc30f23..20a753da14 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -44,7 +44,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeSet; import java.util.regex.Matcher; import static java.lang.Character.MAX_RADIX; @@ -256,6 +255,8 @@ public class TbUtils { double.class, int.class))); parserConfig.addImport("toFixed", new MethodStub(TbUtils.class.getMethod("toFixed", float.class, int.class))); + parserConfig.addImport("toInt", new MethodStub(TbUtils.class.getMethod("toInt", + double.class))); parserConfig.addImport("hexToBytes", new MethodStub(TbUtils.class.getMethod("hexToBytes", ExecutionContext.class, String.class))); parserConfig.addImport("hexToBytesArray", new MethodStub(TbUtils.class.getMethod("hexToBytesArray", @@ -1156,6 +1157,10 @@ public class TbUtils { return BigDecimal.valueOf(value).setScale(precision, RoundingMode.HALF_UP).floatValue(); } + public static int toInt(double value) { + return BigDecimal.valueOf(value).setScale(0, RoundingMode.HALF_UP).intValue(); + } + public static ExecutionHashMap toFlatMap(ExecutionContext ctx, Map json) { return toFlatMap(ctx, json, new ArrayList<>(), true); } diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index 71c7a8b78a..69987d3132 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -1137,6 +1137,13 @@ public class TbUtilsTest { Assertions.assertEquals(expected, actual); } + @Test + void toInt() { + Assertions.assertEquals(1729, TbUtils.toInt(doubleVal)); + Assertions.assertEquals(13, TbUtils.toInt(12.8)); + Assertions.assertEquals(28, TbUtils.toInt(28.0)); + } + private static List toList(byte[] data) { List result = new ArrayList<>(data.length); for (Byte b : data) { From 622580c97cbdec03d47d0e6e0431d20d3e152cdd Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 5 Mar 2025 12:48:59 +0200 Subject: [PATCH 23/26] Update GitHub workflow Ubuntu version to 22.04 --- .github/workflows/check-configuration-files.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-configuration-files.yml b/.github/workflows/check-configuration-files.yml index f280e326c2..21357a8a3c 100644 --- a/.github/workflows/check-configuration-files.yml +++ b/.github/workflows/check-configuration-files.yml @@ -32,7 +32,7 @@ on: jobs: build: name: Check thingsboard.yml file - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout code uses: actions/checkout@v2 From c990169223b52f4c90b66d81fcc65ab93c098e20 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 5 Mar 2025 13:07:38 +0200 Subject: [PATCH 24/26] Update GitHub workflow Python version to 3.13.2 --- .github/workflows/check-configuration-files.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-configuration-files.yml b/.github/workflows/check-configuration-files.yml index 21357a8a3c..561b7d0019 100644 --- a/.github/workflows/check-configuration-files.yml +++ b/.github/workflows/check-configuration-files.yml @@ -36,10 +36,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Set up Python 3.10 + - name: Set up Python 3.13 uses: actions/setup-python@v3 with: - python-version: "3.10.2" + python-version: "3.13.2" architecture: "x64" env: AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache From 59d26eb2e861168591cbf8e984479d79fb34ac5c Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Thu, 6 Mar 2025 11:52:22 +0200 Subject: [PATCH 25/26] Improvements after review --- .../DefaultEntitiesExportImportService.java | 6 ---- .../impl/BaseEntityImportService.java | 2 +- .../sync/vc/data/EntitiesImportCtx.java | 7 ---- .../sync/ie/ExportImportServiceSqlTest.java | 2 +- .../service/sync/vc/VersionControlTest.java | 35 +++++++++++++++++++ 5 files changed, 37 insertions(+), 15 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index d45f32c918..06fe7f4036 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -23,7 +23,6 @@ import org.thingsboard.server.cache.limits.RateLimitService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; @@ -129,11 +128,6 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS logEntityActionService.logEntityRelationAction(ctx.getTenantId(), null, relation, ctx.getUser(), ActionType.RELATION_ADD_OR_UPDATE, null, relation); } - - for (CalculatedField calculatedField : ctx.getCalculatedFields()) { - var savedCalculatedField = calculatedFieldService.save(calculatedField); - logEntityActionService.logEntityAction(ctx.getTenantId(), savedCalculatedField.getId(), savedCalculatedField, ActionType.ADDED, ctx.getUser()); - } } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 65f2b4f032..bfa95af83c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -295,7 +295,7 @@ public abstract class BaseEntityImportService { if (argument.getRefEntityId() != null) { - argument.setRefEntityId(idProvider.getInternalId(argument.getRefEntityId(), false)); + argument.setRefEntityId(idProvider.getInternalId(argument.getRefEntityId(), ctx.isFinalImportAttempt())); } }); }).toList(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index 726dd12102..e24683cd5a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -19,7 +19,6 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -32,7 +31,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -56,7 +54,6 @@ public class EntitiesImportCtx { private final Set notFoundIds = new HashSet<>(); private final Set relations = new LinkedHashSet<>(); - private final Set calculatedFields = new LinkedHashSet<>(); private boolean finalImportAttempt = false; private EntityImportSettings settings; @@ -127,10 +124,6 @@ public class EntitiesImportCtx { relations.addAll(values); } - public void addCalculatedFields(Collection values) { - calculatedFields.addAll(values); - } - public void addReferenceCallback(EntityId externalId, ThrowingRunnable tr) { if (tr != null) { referenceCallbacks.put(externalId, tr); diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java index a75fbba095..da3b214afb 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java @@ -60,7 +60,6 @@ import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceId; @@ -288,6 +287,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { assertThat(calculatedFields.size()).isOne(); var importedCalculatedField = calculatedFields.get(0); assertThat(importedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + verify(tbClusterService).onCalculatedFieldUpdated(eq(importedCalculatedField), isNull(), any()); EntityExportData updatedDeviceEntity = getAndClone(entitiesExportData, EntityType.DEVICE); updatedDeviceEntity.getEntity().setLabel("t" + updatedDeviceEntity.getEntity().getLabel()); diff --git a/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java index bb0d490ce0..06f61ca4e5 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java @@ -597,6 +597,41 @@ public class VersionControlTest extends AbstractControllerTest { }); } + @Test + public void testVcWithReferencedCalculatedFields_betweenTenants() throws Exception { + Asset asset = createAsset(null, null, "Asset 1"); + Device device = createDevice(null, null, "Device 1", "test1"); + CalculatedField deviceCalculatedField = createCalculatedField("CalculatedField1", device.getId(), asset.getId()); + CalculatedField assetCalculatedField = createCalculatedField("CalculatedField2", asset.getId(), device.getId()); + String versionId = createVersion("calculated fields of asset and device", EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE); + + loginTenant2(); + loadVersion(versionId, config -> { + config.setLoadCredentials(false); + }, EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE); + + Asset importedAsset = findAsset(asset.getName()); + Device importedDevice = findDevice(device.getName()); + checkImportedEntity(tenantId1, device, tenantId2, importedDevice); + checkImportedEntity(tenantId1, asset, tenantId2, importedAsset); + + List importedDeviceCalculatedFields = findCalculatedFieldsByEntityId(importedDevice.getId()); + assertThat(importedDeviceCalculatedFields).size().isOne(); + assertThat(importedDeviceCalculatedFields.get(0)).satisfies(importedField -> { + assertThat(importedField.getName()).isEqualTo(deviceCalculatedField.getName()); + assertThat(importedField.getType()).isEqualTo(deviceCalculatedField.getType()); + assertThat(importedField.getId()).isNotEqualTo(deviceCalculatedField.getId()); + }); + + List importedAssetCalculatedFields = findCalculatedFieldsByEntityId(importedAsset.getId()); + assertThat(importedAssetCalculatedFields).size().isOne(); + assertThat(importedAssetCalculatedFields.get(0)).satisfies(importedField -> { + assertThat(importedField.getName()).isEqualTo(assetCalculatedField.getName()); + assertThat(importedField.getType()).isEqualTo(assetCalculatedField.getType()); + assertThat(importedField.getId()).isNotEqualTo(assetCalculatedField.getId()); + }); + } + @Test public void testVcWithCalculatedFields_sameTenant() throws Exception { Asset asset = createAsset(null, null, "Asset 1"); From dca80e71735fe39be844e59b170bb13a535532e3 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Thu, 6 Mar 2025 12:14:09 +0200 Subject: [PATCH 26/26] Remove duplicated call --- .../service/sync/ie/importing/impl/DeviceImportService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java index 92bdb1e294..84e264efdd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -91,7 +91,6 @@ public class DeviceImportService extends BaseEntityImportService