diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 822bd82e00..8dceead8d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -65,8 +65,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final TbNotificationEntityService entityNotificationService; protected static final List SUPPORTED_ENTITY_TYPES = List.of( - EntityType.CUSTOMER, EntityType.ASSET_PROFILE, EntityType.ASSET, EntityType.RULE_CHAIN, - EntityType.DASHBOARD, EntityType.DEVICE_PROFILE, EntityType.DEVICE, + EntityType.CUSTOMER, EntityType.RULE_CHAIN, EntityType.DASHBOARD, + EntityType.ASSET_PROFILE, EntityType.ASSET, + EntityType.DEVICE_PROFILE, EntityType.DEVICE, EntityType.ENTITY_VIEW, EntityType.WIDGETS_BUNDLE ); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index 891a6411a3..79c158cde7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import java.util.Set; +import java.util.regex.Pattern; public abstract class BaseEntityExportService, D extends EntityExportData> extends DefaultEntityExportService { @@ -43,8 +44,8 @@ public abstract class BaseEntityExportService getSupportedEntityTypes(); - protected void replaceUuidsRecursively(EntitiesExportCtx ctx, JsonNode node, Set skipFieldsSet) { - JacksonUtil.replaceUuidsRecursively(node, skipFieldsSet, uuid -> getExternalIdOrElseInternalByUuid(ctx, uuid)); + protected void replaceUuidsRecursively(EntitiesExportCtx ctx, JsonNode node, Set skipFieldsSet, Pattern includedFieldsPattern) { + JacksonUtil.replaceUuidsRecursively(node, skipFieldsSet, includedFieldsPattern, uuid -> getExternalIdOrElseInternalByUuid(ctx, uuid)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java index b408aadd68..0df1e83c7f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java @@ -26,7 +26,6 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; -import java.util.Collections; import java.util.Set; @Service @@ -41,10 +40,10 @@ public class DashboardExportService extends BaseEntityExportService { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 610f1ee0b5..84b39100a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -65,6 +65,7 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Collectors; @Slf4j @@ -386,9 +387,12 @@ public abstract class BaseEntityImportService skipFieldsSet, LinkedHashSet hints) { - JacksonUtil.replaceUuidsRecursively(entityAlias, skipFieldsSet, - uuid -> idProvider.getInternalIdByUuid(uuid, ctx.isFinalImportAttempt(), hints).map(EntityId::getId).orElse(uuid)); + protected void replaceIdsRecursively(EntitiesImportCtx ctx, IdProvider idProvider, JsonNode json, + Set skipFieldsSet, Pattern includedFieldsPattern, + LinkedHashSet hints) { + JacksonUtil.replaceUuidsRecursively(json, skipFieldsSet, includedFieldsPattern, + uuid -> idProvider.getInternalIdByUuid(uuid, ctx.isFinalImportAttempt(), hints) + .map(EntityId::getId).orElse(uuid)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index 250026c71c..eac229f774 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -65,10 +65,10 @@ public class DashboardImportService extends BaseEntityImportService exportData, IdProvider idProvider) { for (JsonNode entityAlias : dashboard.getEntityAliasesConfig()) { - replaceIdsRecursively(ctx, idProvider, entityAlias, Set.of("id"), HINTS); + replaceIdsRecursively(ctx, idProvider, entityAlias, Set.of("id"), null, HINTS); } for (JsonNode widgetConfig : dashboard.getWidgetsConfig()) { - replaceIdsRecursively(ctx, idProvider, JacksonUtil.getSafely(widgetConfig, "config", "actions"), Set.of("id"), HINTS); + replaceIdsRecursively(ctx, idProvider, JacksonUtil.getSafely(widgetConfig, "config", "actions"), Set.of("id"), null, HINTS); } return dashboard; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index eb54fce21e..5c694cdd73 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -42,6 +42,7 @@ import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; +import java.util.regex.Pattern; import java.util.stream.Collectors; @Slf4j @@ -50,11 +51,12 @@ import java.util.stream.Collectors; @RequiredArgsConstructor public class RuleChainImportService extends BaseEntityImportService { - private static final LinkedHashSet HINTS = new LinkedHashSet<>(Arrays.asList(EntityType.RULE_CHAIN, EntityType.DEVICE, EntityType.ASSET)); - private final RuleChainService ruleChainService; private final RuleNodeDao ruleNodeDao; + private static final LinkedHashSet HINTS = new LinkedHashSet<>(Arrays.asList(EntityType.RULE_CHAIN, EntityType.DEVICE, EntityType.ASSET)); + public static final Pattern PROCESSED_CONFIG_FIELDS_PATTERN = Pattern.compile(".*[iI]d.*"); + @Override protected void setOwner(TenantId tenantId, RuleChain ruleChain, IdProvider idProvider) { ruleChain.setTenantId(tenantId); @@ -90,7 +92,8 @@ public class RuleChainImportService extends BaseEntityImportService replaceIdsRecursively(ctx, idProvider, ruleNode.getConfiguration(), Collections.emptySet(), HINTS)); + ruleNodes.forEach(ruleNode -> replaceIdsRecursively(ctx, idProvider, ruleNode.getConfiguration(), + Collections.emptySet(), PROCESSED_CONFIG_FIELDS_PATTERN, HINTS)); Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) .forEach(ruleChainConnectionInfo -> { ruleChainConnectionInfo.setTargetRuleChainId(idProvider.getInternalId(ruleChainConnectionInfo.getTargetRuleChainId(), false)); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index feff708983..0cc73cd945 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -103,7 +103,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu private final Map> pendingRequestMap = new HashMap<>(); private final Map> chunkedMsgs = new ConcurrentHashMap<>(); - @Value("${queue.vc.request-timeout:60000}") + @Value("${queue.vc.request-timeout:180000}") private int requestTimeout; @Value("${queue.vc.msg-chunk-size:500000}") private int msgChunkSize; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 2ce5c3ba21..1ca86150f3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -487,7 +487,7 @@ cache: timeToLiveInMinutes: "${CACHE_SPECS_TWO_FA_VERIFICATION_CODES_TTL:60}" maxSize: "${CACHE_SPECS_TWO_FA_VERIFICATION_CODES_MAX_SIZE:100000}" versionControlTask: - timeToLiveInMinutes: "${CACHE_SPECS_VERSION_CONTROL_TASK_TTL:5}" + timeToLiveInMinutes: "${CACHE_SPECS_VERSION_CONTROL_TASK_TTL:20}" maxSize: "${CACHE_SPECS_VERSION_CONTROL_TASK_MAX_SIZE:100000}" userSettings: timeToLiveInMinutes: "${CACHE_SPECS_USER_SETTINGS_TTL:1440}" @@ -1161,8 +1161,8 @@ queue: topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" partitions: "${TB_QUEUE_VC_PARTITIONS:10}" poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" - pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" - request-timeout: "${TB_QUEUE_VC_REQUEST_TIMEOUT:60000}" + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:180000}" + request-timeout: "${TB_QUEUE_VC_REQUEST_TIMEOUT:180000}" msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:250000}" js: # JS Eval request topic diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java index e493916b8a..42547f0dbf 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java @@ -53,6 +53,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.script.ScriptLanguage; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.sync.ie.DeviceExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; @@ -65,9 +66,11 @@ import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.ota.OtaPackageStateService; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -338,6 +341,47 @@ public class ExportImportServiceSqlTest extends BaseExportImportServiceTest { checkImportedRuleChainData(ruleChain, metaData, importedRuleChain, importedMetaData); } + @Test + public void testImportRuleChain_ruleNodesConfigs() throws Exception { + Customer customer = createCustomer(tenantId1, "Customer 1"); + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain 1"); + RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); + + List nodes = new ArrayList<>(metaData.getNodes()); + RuleNode generatorNode = new RuleNode(); + generatorNode.setName("Generator"); + generatorNode.setType(TbMsgGeneratorNode.class.getName()); + TbMsgGeneratorNodeConfiguration generatorNodeConfig = new TbMsgGeneratorNodeConfiguration(); + generatorNodeConfig.setOriginatorType(EntityType.ASSET_PROFILE); + generatorNodeConfig.setOriginatorId(customer.getId().toString()); + generatorNodeConfig.setPeriodInSeconds(5); + generatorNodeConfig.setMsgCount(1); + generatorNodeConfig.setScriptLang(ScriptLanguage.JS); + UUID someUuid = UUID.randomUUID(); + generatorNodeConfig.setJsScript("var msg = { temp: 42, humidity: 77 };\n" + + "var metadata = { data: 40 };\n" + + "var msgType = \"POST_TELEMETRY_REQUEST\";\n" + + "var someUuid = \"" + someUuid + "\";\n" + + "return { msg: msg, metadata: metadata, msgType: msgType };"); + generatorNode.setConfiguration(mapper.valueToTree(generatorNodeConfig)); + nodes.add(generatorNode); + metaData.setNodes(nodes); + ruleChainService.saveRuleChainMetaData(tenantId1, metaData); + + EntityExportData ruleChainExportData = exportEntity(tenantAdmin1, ruleChain.getId()); + EntityExportData customerExportData = exportEntity(tenantAdmin1, customer.getId()); + + Customer importedCustomer = importEntity(tenantAdmin2, customerExportData).getSavedEntity(); + RuleChain importedRuleChain = importEntity(tenantAdmin2, ruleChainExportData).getSavedEntity(); + RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId2, importedRuleChain.getId()); + + TbMsgGeneratorNodeConfiguration importedGeneratorNodeConfig = JacksonUtil.treeToValue(importedMetaData.getNodes().stream() + .filter(node -> node.getName().equals(generatorNode.getName())) + .findFirst().get().getConfiguration(), TbMsgGeneratorNodeConfiguration.class); + assertThat(importedGeneratorNodeConfig.getOriginatorId()).isEqualTo(importedCustomer.getId().toString()); + assertThat(importedGeneratorNodeConfig.getJsScript()).contains("var someUuid = \"" + someUuid + "\";"); + } + @Test public void testExportImportWithInboundRelations_betweenTenants() throws Exception { diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index e2345c9e6d..6fc119e3ff 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -39,6 +39,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.function.UnaryOperator; +import java.util.regex.Pattern; /** * Created by Valerii Sosliuk on 5/12/2017. @@ -200,7 +201,7 @@ public class JacksonUtil { return node; } - public static void replaceUuidsRecursively(JsonNode node, Set skipFieldsSet, UnaryOperator replacer) { + public static void replaceUuidsRecursively(JsonNode node, Set skipFieldsSet, Pattern includedFieldsPattern, UnaryOperator replacer) { if (node == null) { return; } @@ -212,9 +213,14 @@ public class JacksonUtil { if (skipFieldsSet.contains(fieldName)) { continue; } + if (includedFieldsPattern != null) { + if (!RegexUtils.matches(fieldName, includedFieldsPattern)) { + continue; + } + } var child = objectNode.get(fieldName); if (child.isObject() || child.isArray()) { - replaceUuidsRecursively(child, skipFieldsSet, replacer); + replaceUuidsRecursively(child, skipFieldsSet, includedFieldsPattern, replacer); } else if (child.isTextual()) { String text = child.asText(); String newText = RegexUtils.replace(text, RegexUtils.UUID_PATTERN, uuid -> replacer.apply(UUID.fromString(uuid)).toString()); @@ -228,7 +234,7 @@ public class JacksonUtil { for (int i = 0; i < array.size(); i++) { JsonNode arrayElement = array.get(i); if (arrayElement.isObject() || arrayElement.isArray()) { - replaceUuidsRecursively(arrayElement, skipFieldsSet, replacer); + replaceUuidsRecursively(arrayElement, skipFieldsSet, includedFieldsPattern, replacer); } else if (arrayElement.isTextual()) { String text = arrayElement.asText(); String newText = RegexUtils.replace(text, RegexUtils.UUID_PATTERN, uuid -> replacer.apply(UUID.fromString(uuid)).toString()); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index 618eb11d98..ab2dc417ca 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -122,7 +122,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe @Value("${queue.vc.poll-interval:25}") private long pollDuration; - @Value("${queue.vc.pack-processing-timeout:60000}") + @Value("${queue.vc.pack-processing-timeout:180000}") private long packProcessingTimeout; @Value("${vc.git.io_pool_size:3}") private int ioPoolSize; diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 61ce004c04..9a0040e073 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -168,7 +168,7 @@ queue: topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" partitions: "${TB_QUEUE_VC_PARTITIONS:10}" poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" - pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:180000}" msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:250000}" vc: