dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Perofrmance improvements

Browse Source
This commit is contained in:
Andrii Shvaika 2022-06-13 16:21:05 +03:00
parent 7983f0ee07
commit 82763495e0
9 changed files with 64 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java
View File

@ -96,6 +96,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS
importResult.getSendEventsCallback().run();
}
ctx.putInternalId(exportData.getExternalId(), importResult.getSavedEntity().getId());
return importResult;
}

38
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java
View File

@ -45,12 +45,10 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.sync.ie.AttributeExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.service.action.EntityActionService;
import org.thingsboard.server.service.entitiy.TbNotificationEntityService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
import org.thingsboard.server.service.sync.ie.importing.EntityImportService;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
@ -83,13 +81,14 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
@Transactional(rollbackFor = Exception.class)
@Override
public EntityImportResult<E> importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException {
EntityImportResult<E> importResult = new EntityImportResult<>();
IdProvider idProvider = new IdProvider(ctx, importResult);
E entity = exportData.getEntity();
E existingEntity = findExistingEntity(ctx, entity);
E existingEntity = findExistingEntity(ctx, entity, idProvider);
entity.setExternalId(entity.getId());
EntityImportResult<E> importResult = new EntityImportResult<>();
IdProvider idProvider = new IdProvider(ctx, importResult);
setOwner(ctx.getTenantId(), entity, idProvider);
if (existingEntity == null) {
entity.setId(null);
@ -125,39 +124,38 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
});
if (ctx.isUpdateRelations() && exportData.getRelations() != null) {
importRelations(ctx.getUser(), exportData.getRelations(), importResult);
importRelations(ctx, exportData.getRelations(), importResult, idProvider);
}
if (ctx.isSaveAttributes() && exportData.getAttributes() != null) {
importAttributes(ctx.getUser(), exportData.getAttributes(), importResult);
}
}
private void importRelations(SecurityUser user, List<EntityRelation> relations, EntityImportResult<E> importResult) {
private void importRelations(EntitiesImportCtx ctx, List<EntityRelation> relations, EntityImportResult<E> importResult, IdProvider idProvider) {
var tenantId = ctx.getTenantId();
E entity = importResult.getSavedEntity();
importResult.addSaveReferencesCallback(() -> {
for (EntityRelation relation : relations) {
if (!relation.getTo().equals(entity.getId())) {
HasId<EntityId> to = findInternalEntity(user.getTenantId(), relation.getTo());
relation.setTo(to.getId());
relation.setTo(idProvider.getInternalId(relation.getTo()));
}
if (!relation.getFrom().equals(entity.getId())) {
HasId<EntityId> from = findInternalEntity(user.getTenantId(), relation.getFrom());
relation.setFrom(from.getId());
relation.setFrom(idProvider.getInternalId(relation.getFrom()));
}
}
if (importResult.getOldEntity() != null) {
List<EntityRelation> existingRelations = new ArrayList<>();
existingRelations.addAll(relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON));
existingRelations.addAll(relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON));
existingRelations.addAll(relationService.findByTo(tenantId, entity.getId(), RelationTypeGroup.COMMON));
existingRelations.addAll(relationService.findByFrom(tenantId, entity.getId(), RelationTypeGroup.COMMON));
for (EntityRelation existingRelation : existingRelations) {
if (!relations.contains(existingRelation)) {
relationService.deleteRelation(user.getTenantId(), existingRelation);
relationService.deleteRelation(tenantId, existingRelation);
importResult.addSendEventsCallback(() -> {
entityActionService.logEntityAction(user, existingRelation.getFrom(), null, null,
entityActionService.logEntityAction(ctx.getUser(), existingRelation.getFrom(), null, null,
ActionType.RELATION_DELETED, null, existingRelation);
entityActionService.logEntityAction(user, existingRelation.getTo(), null, null,
entityActionService.logEntityAction(ctx.getUser(), existingRelation.getTo(), null, null,
ActionType.RELATION_DELETED, null, existingRelation);
});
}
@ -165,11 +163,11 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
}
for (EntityRelation relation : relations) {
relationService.saveRelation(user.getTenantId(), relation);
relationService.saveRelation(tenantId, relation);
importResult.addSendEventsCallback(() -> {
entityActionService.logEntityAction(user, relation.getFrom(), null, null,
entityActionService.logEntityAction(ctx.getUser(), relation.getFrom(), null, null,
ActionType.RELATION_ADD_OR_UPDATE, null, relation);
entityActionService.logEntityAction(user, relation.getTo(), null, null,
entityActionService.logEntityAction(ctx.getUser(), relation.getTo(), null, null,
ActionType.RELATION_ADD_OR_UPDATE, null, relation);
});
}
@ -223,7 +221,7 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
@SuppressWarnings("unchecked")
protected E findExistingEntity(EntitiesImportCtx ctx, E entity) {
protected E findExistingEntity(EntitiesImportCtx ctx, E entity, IdProvider idProvider) {
return (E) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId()))
.or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entity.getId())))
.or(() -> {

5
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java
View File

@ -30,7 +30,6 @@ import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DashboardId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
@ -59,8 +58,8 @@ public class DashboardImportService extends BaseEntityImportService<DashboardId,
}
@Override
protected Dashboard findExistingEntity(EntitiesImportCtx ctx, Dashboard dashboard) {
Dashboard existingDashboard = super.findExistingEntity(ctx, dashboard);
protected Dashboard findExistingEntity(EntitiesImportCtx ctx, Dashboard dashboard, IdProvider idProvider) {
Dashboard existingDashboard = super.findExistingEntity(ctx, dashboard, idProvider);
if (existingDashboard == null && ctx.isFindExistingByName()) {
existingDashboard = dashboardService.findTenantDashboardsByTitle(ctx.getTenantId(), dashboard.getName()).stream().findFirst().orElse(null);
}

5
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java
View File

@ -28,7 +28,6 @@ import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
import org.thingsboard.server.common.data.sync.ie.RuleChainExportData;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.queue.util.TbCoreComponent;
@ -53,8 +52,8 @@ public class RuleChainImportService extends BaseEntityImportService<RuleChainId,
}
@Override
protected RuleChain findExistingEntity(EntitiesImportCtx ctx, RuleChain ruleChain) {
RuleChain existingRuleChain = super.findExistingEntity(ctx, ruleChain);
protected RuleChain findExistingEntity(EntitiesImportCtx ctx, RuleChain ruleChain, IdProvider idProvider) {
RuleChain existingRuleChain = super.findExistingEntity(ctx, ruleChain, idProvider);
if (existingRuleChain == null && ctx.isFindExistingByName()) {
existingRuleChain = ruleChainService.findTenantRuleChainsByTypeAndName(ctx.getTenantId(), ruleChain.getType(), ruleChain.getName()).stream().findFirst().orElse(null);
}

28
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
View File

@ -27,6 +27,7 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionCallback;
import org.springframework.transaction.support.TransactionTemplate;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.TbStopWatch;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
@ -246,11 +247,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private VersionLoadResult loadSingleEntity(SecurityUser user, VersionLoadConfig config, EntityExportData entityData) {
try {
var ctx = new EntitiesImportCtx(user, EntityImportSettings.builder()
.updateRelations(config.isLoadRelations())
.saveAttributes(config.isLoadAttributes())
.saveCredentials(config.isLoadCredentials())
.findExistingByName(false)
.build());
.updateRelations(config.isLoadRelations())
.saveAttributes(config.isLoadAttributes())
.saveCredentials(config.isLoadCredentials())
.findExistingByName(false)
.build());
EntityImportResult<?> importResult = exportImportService.importEntity(ctx, entityData, true, true);
return VersionLoadResult.success(EntityTypeLoadResult.builder()
.entityType(importResult.getEntityType())
@ -271,6 +272,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
List<ThrowingRunnable> sendEventsCallbacks = new ArrayList<>();
EntitiesImportCtx ctx = new EntitiesImportCtx(user);
var sw = TbStopWatch.create("before");
List<EntityType> entityTypes = request.getEntityTypes().keySet().stream()
.sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList());
@ -294,6 +296,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
.findExistingByName(config.isFindExistingEntityByName())
.build());
for (EntityExportData entityData : entityDataList) {
sw.startNew("Entities " + entityType.name());
log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType);
EntityImportResult<?> importResult;
try {
importResult = exportImportService.importEntity(ctx, entityData, false, false);
@ -322,6 +326,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
.build());
}
sw.startNew("Reimport");
toReimport.forEach((externalId, importSettings) -> {
try {
EntityExportData entityData = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), externalId).get();
@ -341,6 +346,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}
});
sw.startNew("Remove Others");
request.getEntityTypes().keySet().stream()
.filter(entityType -> request.getEntityTypes().get(entityType).isRemoveOtherEntities())
.sorted(exportImportService.getEntityTypeComparatorForImport().reversed())
@ -361,6 +367,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
});
});
sw.startNew("Callbacks");
for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) {
try {
saveReferencesCallback.run();
@ -375,6 +383,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
log.error("Failed to send events for entity", e);
}
}
sw.stop();
for (var task : sw.getTaskInfo()) {
log.debug("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis());
}
log.info("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis());
return VersionLoadResult.success(new ArrayList<>(results.values()));
}
@ -412,8 +426,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
.exportCredentials(otherVersion.hasCredentials())
.build());
return transform(gitServiceQueue.getContentsDiff(user.getTenantId(),
JacksonUtil.toPrettyString(currentVersion.sort()),
JacksonUtil.toPrettyString(otherVersion.sort())),
JacksonUtil.toPrettyString(currentVersion.sort()),
JacksonUtil.toPrettyString(otherVersion.sort())),
rawDiff -> new EntityDataDiff(currentVersion, otherVersion, rawDiff), MoreExecutors.directExecutor());
}, MoreExecutors.directExecutor());
}

7
application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java
View File

@ -16,6 +16,7 @@
package org.thingsboard.server.service.sync.vc.data;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
@ -24,6 +25,7 @@ import org.thingsboard.server.service.security.model.SecurityUser;
import java.util.HashMap;
import java.util.Map;
@Slf4j
@Data
public class EntitiesImportCtx {
@ -66,10 +68,13 @@ public class EntitiesImportCtx {
}
public EntityId getInternalId(EntityId externalId) {
return externalToInternalIdMap.get(externalId);
var result = externalToInternalIdMap.get(externalId);
log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss");
return result;
}
public void putInternalId(EntityId externalId, EntityId internalId) {
log.debug("[{}][{}] Local cache put: {}", externalId.getEntityType(), externalId.getId(), internalId);
externalToInternalIdMap.put(externalId, internalId);
}
}

2
common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java
View File

@ -96,7 +96,7 @@ public class DefaultTbQueueRequestTemplate<Request extends TbQueueMsg, Response
void mainLoop() {
while (!stopped) {
TbStopWatch sw = TbStopWatch.startNew();
TbStopWatch sw = TbStopWatch.create();
try {
fetchAndProcessResponses();
} catch (Throwable e) {

13
common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java
View File

@ -28,12 +28,23 @@ import org.springframework.util.StopWatch;
* */
public class TbStopWatch extends StopWatch {
public static TbStopWatch startNew(){
public static TbStopWatch create(){
TbStopWatch stopWatch = new TbStopWatch();
stopWatch.start();
return stopWatch;
}
public static TbStopWatch create(String taskName){
TbStopWatch stopWatch = new TbStopWatch();
stopWatch.start(taskName);
return stopWatch;
}
public void startNew(String taskName){
stop();
start(taskName);
}
public long stopAndGetTotalTimeMillis(){
stop();
return getTotalTimeMillis();

2
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java
View File

@ -105,7 +105,7 @@ public class TbMsgGeneratorNode implements TbNode {
public void onMsg(TbContext ctx, TbMsg msg) {
log.trace("onMsg, config {}, msg {}", config, msg);
if (initialized.get() && msg.getType().equals(TB_MSG_GENERATOR_NODE_MSG) && msg.getId().equals(nextTickId)) {
TbStopWatch sw = TbStopWatch.startNew();
TbStopWatch sw = TbStopWatch.create();
withCallback(generate(ctx, msg),
m -> {
log.trace("onMsg onSuccess callback, took {}ms, config {}, msg {}", sw.stopAndGetTotalTimeMillis(), config, msg);