Perofrmance improvements

application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java

@@ -96,6 +96,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS
             importResult.getSendEventsCallback().run();
         }
 
+        ctx.putInternalId(exportData.getExternalId(), importResult.getSavedEntity().getId());
         return importResult;
     }
 

application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java

@@ -45,12 +45,10 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup;
 import org.thingsboard.server.common.data.sync.ie.AttributeExportData;
 import org.thingsboard.server.common.data.sync.ie.EntityExportData;
 import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
-import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
 import org.thingsboard.server.dao.relation.RelationService;
 import org.thingsboard.server.service.action.EntityActionService;
 import org.thingsboard.server.service.entitiy.TbNotificationEntityService;
 import org.thingsboard.server.service.security.model.SecurityUser;
-import org.thingsboard.server.service.security.permission.Operation;
 import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
 import org.thingsboard.server.service.sync.ie.importing.EntityImportService;
 import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
@@ -83,13 +81,14 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
     @Transactional(rollbackFor = Exception.class)
     @Override
     public EntityImportResult<E> importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException {
+        EntityImportResult<E> importResult = new EntityImportResult<>();
+        IdProvider idProvider = new IdProvider(ctx, importResult);
+
         E entity = exportData.getEntity();
-        E existingEntity = findExistingEntity(ctx, entity);
+        E existingEntity = findExistingEntity(ctx, entity, idProvider);
 
         entity.setExternalId(entity.getId());
 
-        EntityImportResult<E> importResult = new EntityImportResult<>();
-        IdProvider idProvider = new IdProvider(ctx, importResult);
         setOwner(ctx.getTenantId(), entity, idProvider);
         if (existingEntity == null) {
             entity.setId(null);
@@ -125,39 +124,38 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
         });
 
         if (ctx.isUpdateRelations() && exportData.getRelations() != null) {
-            importRelations(ctx.getUser(), exportData.getRelations(), importResult);
+            importRelations(ctx, exportData.getRelations(), importResult, idProvider);
         }
         if (ctx.isSaveAttributes() && exportData.getAttributes() != null) {
             importAttributes(ctx.getUser(), exportData.getAttributes(), importResult);
         }
     }
 
-    private void importRelations(SecurityUser user, List<EntityRelation> relations, EntityImportResult<E> importResult) {
+    private void importRelations(EntitiesImportCtx ctx, List<EntityRelation> relations, EntityImportResult<E> importResult, IdProvider idProvider) {
+        var tenantId = ctx.getTenantId();
         E entity = importResult.getSavedEntity();
         importResult.addSaveReferencesCallback(() -> {
             for (EntityRelation relation : relations) {
                 if (!relation.getTo().equals(entity.getId())) {
-                    HasId<EntityId> to = findInternalEntity(user.getTenantId(), relation.getTo());
+                    relation.setTo(idProvider.getInternalId(relation.getTo()));
-                    relation.setTo(to.getId());
                 }
                 if (!relation.getFrom().equals(entity.getId())) {
-                    HasId<EntityId> from = findInternalEntity(user.getTenantId(), relation.getFrom());
+                    relation.setFrom(idProvider.getInternalId(relation.getFrom()));
-                    relation.setFrom(from.getId());
                 }
             }
 
             if (importResult.getOldEntity() != null) {
                 List<EntityRelation> existingRelations = new ArrayList<>();
-                existingRelations.addAll(relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON));
+                existingRelations.addAll(relationService.findByTo(tenantId, entity.getId(), RelationTypeGroup.COMMON));
-                existingRelations.addAll(relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON));
+                existingRelations.addAll(relationService.findByFrom(tenantId, entity.getId(), RelationTypeGroup.COMMON));
 
                 for (EntityRelation existingRelation : existingRelations) {
                     if (!relations.contains(existingRelation)) {
-                        relationService.deleteRelation(user.getTenantId(), existingRelation);
+                        relationService.deleteRelation(tenantId, existingRelation);
                         importResult.addSendEventsCallback(() -> {
-                            entityActionService.logEntityAction(user, existingRelation.getFrom(), null, null,
+                            entityActionService.logEntityAction(ctx.getUser(), existingRelation.getFrom(), null, null,
                                     ActionType.RELATION_DELETED, null, existingRelation);
-                            entityActionService.logEntityAction(user, existingRelation.getTo(), null, null,
+                            entityActionService.logEntityAction(ctx.getUser(), existingRelation.getTo(), null, null,
                                     ActionType.RELATION_DELETED, null, existingRelation);
                         });
                     }
@@ -165,11 +163,11 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
             }
 
             for (EntityRelation relation : relations) {
-                relationService.saveRelation(user.getTenantId(), relation);
+                relationService.saveRelation(tenantId, relation);
                 importResult.addSendEventsCallback(() -> {
-                    entityActionService.logEntityAction(user, relation.getFrom(), null, null,
+                    entityActionService.logEntityAction(ctx.getUser(), relation.getFrom(), null, null,
                             ActionType.RELATION_ADD_OR_UPDATE, null, relation);
-                    entityActionService.logEntityAction(user, relation.getTo(), null, null,
+                    entityActionService.logEntityAction(ctx.getUser(), relation.getTo(), null, null,
                             ActionType.RELATION_ADD_OR_UPDATE, null, relation);
                 });
             }
@@ -223,7 +221,7 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
 
 
     @SuppressWarnings("unchecked")
-    protected E findExistingEntity(EntitiesImportCtx ctx, E entity) {
+    protected E findExistingEntity(EntitiesImportCtx ctx, E entity, IdProvider idProvider) {
         return (E) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId()))
                 .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entity.getId())))
                 .or(() -> {

application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java

@@ -30,7 +30,6 @@ import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.DashboardId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.sync.ie.EntityExportData;
-import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
 import org.thingsboard.server.dao.dashboard.DashboardService;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 import org.thingsboard.server.service.security.model.SecurityUser;
@@ -59,8 +58,8 @@ public class DashboardImportService extends BaseEntityImportService<DashboardId,
     }
 
     @Override
-    protected Dashboard findExistingEntity(EntitiesImportCtx ctx, Dashboard dashboard) {
+    protected Dashboard findExistingEntity(EntitiesImportCtx ctx, Dashboard dashboard, IdProvider idProvider) {
-        Dashboard existingDashboard = super.findExistingEntity(ctx, dashboard);
+        Dashboard existingDashboard = super.findExistingEntity(ctx, dashboard, idProvider);
         if (existingDashboard == null && ctx.isFindExistingByName()) {
             existingDashboard = dashboardService.findTenantDashboardsByTitle(ctx.getTenantId(), dashboard.getName()).stream().findFirst().orElse(null);
         }

application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java

@@ -28,7 +28,6 @@ import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
 import org.thingsboard.server.common.data.rule.RuleChain;
 import org.thingsboard.server.common.data.rule.RuleChainMetaData;
 import org.thingsboard.server.common.data.rule.RuleChainType;
-import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
 import org.thingsboard.server.common.data.sync.ie.RuleChainExportData;
 import org.thingsboard.server.dao.rule.RuleChainService;
 import org.thingsboard.server.queue.util.TbCoreComponent;
@@ -53,8 +52,8 @@ public class RuleChainImportService extends BaseEntityImportService<RuleChainId,
     }
 
     @Override
-    protected RuleChain findExistingEntity(EntitiesImportCtx ctx, RuleChain ruleChain) {
+    protected RuleChain findExistingEntity(EntitiesImportCtx ctx, RuleChain ruleChain, IdProvider idProvider) {
-        RuleChain existingRuleChain = super.findExistingEntity(ctx, ruleChain);
+        RuleChain existingRuleChain = super.findExistingEntity(ctx, ruleChain, idProvider);
         if (existingRuleChain == null && ctx.isFindExistingByName()) {
             existingRuleChain = ruleChainService.findTenantRuleChainsByTypeAndName(ctx.getTenantId(), ruleChain.getType(), ruleChain.getName()).stream().findFirst().orElse(null);
         }

application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java

@@ -27,6 +27,7 @@ import org.springframework.stereotype.Service;
 import org.springframework.transaction.support.TransactionCallback;
 import org.springframework.transaction.support.TransactionTemplate;
 import org.thingsboard.common.util.JacksonUtil;
+import org.thingsboard.common.util.TbStopWatch;
 import org.thingsboard.common.util.ThingsBoardExecutors;
 import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.ExportableEntity;
@@ -246,11 +247,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
     private VersionLoadResult loadSingleEntity(SecurityUser user, VersionLoadConfig config, EntityExportData entityData) {
         try {
             var ctx = new EntitiesImportCtx(user, EntityImportSettings.builder()
-                            .updateRelations(config.isLoadRelations())
+                    .updateRelations(config.isLoadRelations())
-                            .saveAttributes(config.isLoadAttributes())
+                    .saveAttributes(config.isLoadAttributes())
-                            .saveCredentials(config.isLoadCredentials())
+                    .saveCredentials(config.isLoadCredentials())
-                            .findExistingByName(false)
+                    .findExistingByName(false)
-                            .build());
+                    .build());
             EntityImportResult<?> importResult = exportImportService.importEntity(ctx, entityData, true, true);
             return VersionLoadResult.success(EntityTypeLoadResult.builder()
                     .entityType(importResult.getEntityType())
@@ -271,6 +272,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
         List<ThrowingRunnable> sendEventsCallbacks = new ArrayList<>();
 
         EntitiesImportCtx ctx = new EntitiesImportCtx(user);
+        var sw = TbStopWatch.create("before");
 
         List<EntityType> entityTypes = request.getEntityTypes().keySet().stream()
                 .sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList());
@@ -294,6 +296,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                         .findExistingByName(config.isFindExistingEntityByName())
                         .build());
                 for (EntityExportData entityData : entityDataList) {
+                    sw.startNew("Entities " + entityType.name());
+                    log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType);
                     EntityImportResult<?> importResult;
                     try {
                         importResult = exportImportService.importEntity(ctx, entityData, false, false);
@@ -322,6 +326,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                     .build());
         }
 
+        sw.startNew("Reimport");
         toReimport.forEach((externalId, importSettings) -> {
             try {
                 EntityExportData entityData = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), externalId).get();
@@ -341,6 +346,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
             }
         });
 
+        sw.startNew("Remove Others");
         request.getEntityTypes().keySet().stream()
                 .filter(entityType -> request.getEntityTypes().get(entityType).isRemoveOtherEntities())
                 .sorted(exportImportService.getEntityTypeComparatorForImport().reversed())
@@ -361,6 +367,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                     });
                 });
 
+        sw.startNew("Callbacks");
+
         for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) {
             try {
                 saveReferencesCallback.run();
@@ -375,6 +383,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                 log.error("Failed to send events for entity", e);
             }
         }
+
+        sw.stop();
+        for (var task : sw.getTaskInfo()) {
+            log.debug("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis());
+        }
+        log.info("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis());
         return VersionLoadResult.success(new ArrayList<>(results.values()));
     }
 
@@ -412,8 +426,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                             .exportCredentials(otherVersion.hasCredentials())
                             .build());
                     return transform(gitServiceQueue.getContentsDiff(user.getTenantId(),
-                                    JacksonUtil.toPrettyString(currentVersion.sort()),
+                            JacksonUtil.toPrettyString(currentVersion.sort()),
-                                    JacksonUtil.toPrettyString(otherVersion.sort())),
+                            JacksonUtil.toPrettyString(otherVersion.sort())),
                             rawDiff -> new EntityDataDiff(currentVersion, otherVersion, rawDiff), MoreExecutors.directExecutor());
                 }, MoreExecutors.directExecutor());
     }

application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java

@@ -16,6 +16,7 @@
 package org.thingsboard.server.service.sync.vc.data;
 
 import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
@@ -24,6 +25,7 @@ import org.thingsboard.server.service.security.model.SecurityUser;
 import java.util.HashMap;
 import java.util.Map;
 
+@Slf4j
 @Data
 public class EntitiesImportCtx {
 
@@ -66,10 +68,13 @@ public class EntitiesImportCtx {
     }
 
     public EntityId getInternalId(EntityId externalId) {
-        return externalToInternalIdMap.get(externalId);
+        var result = externalToInternalIdMap.get(externalId);
+        log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss");
+        return result;
     }
 
     public void putInternalId(EntityId externalId, EntityId internalId) {
+        log.debug("[{}][{}] Local cache put: {}", externalId.getEntityType(), externalId.getId(), internalId);
         externalToInternalIdMap.put(externalId, internalId);
     }
 }

common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java

@@ -96,7 +96,7 @@ public class DefaultTbQueueRequestTemplate<Request extends TbQueueMsg, Response
 
     void mainLoop() {
         while (!stopped) {
-            TbStopWatch sw = TbStopWatch.startNew();
+            TbStopWatch sw = TbStopWatch.create();
             try {
                 fetchAndProcessResponses();
             } catch (Throwable e) {

common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java

@@ -28,12 +28,23 @@ import org.springframework.util.StopWatch;
  * */
 public class TbStopWatch extends StopWatch {
 
-    public static TbStopWatch startNew(){
+    public static TbStopWatch create(){
         TbStopWatch stopWatch = new TbStopWatch();
         stopWatch.start();
         return stopWatch;
     }
 
+    public static TbStopWatch create(String taskName){
+        TbStopWatch stopWatch = new TbStopWatch();
+        stopWatch.start(taskName);
+        return stopWatch;
+    }
+
+    public void startNew(String taskName){
+        stop();
+        start(taskName);
+    }
+
     public long stopAndGetTotalTimeMillis(){
         stop();
         return getTotalTimeMillis();

rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java

@@ -105,7 +105,7 @@ public class TbMsgGeneratorNode implements TbNode {
     public void onMsg(TbContext ctx, TbMsg msg) {
         log.trace("onMsg, config {}, msg {}", config, msg);
         if (initialized.get() && msg.getType().equals(TB_MSG_GENERATOR_NODE_MSG) && msg.getId().equals(nextTickId)) {
-            TbStopWatch sw = TbStopWatch.startNew();
+            TbStopWatch sw = TbStopWatch.create();
             withCallback(generate(ctx, msg),
                     m -> {
                         log.trace("onMsg onSuccess callback, took {}ms, config {}, msg {}", sw.stopAndGetTotalTimeMillis(), config, msg);