dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Performance improvement for entities export

Browse Source
This commit is contained in:
Andrii Shvaika 2022-06-15 11:09:08 +03:00
parent c63805bcf1
commit aba4fa8215
7 changed files with 40 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java
View File

@ -34,7 +34,7 @@ public class AssetExportService extends BaseEntityExportService<AssetId, Asset,
@Override @Override
protected void setRelatedEntities(EntitiesExportCtx<?> ctx, Asset asset, EntityExportData<Asset> exportData) { protected void setRelatedEntities(EntitiesExportCtx<?> ctx, Asset asset, EntityExportData<Asset> exportData) {
asset.setCustomerId(getExternalIdOrElseInternal(asset.getCustomerId())); asset.setCustomerId(getExternalIdOrElseInternal(ctx, asset.getCustomerId()));
} }
@Override @Override

2
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java
View File

@ -36,7 +36,7 @@ public class DashboardExportService extends BaseEntityExportService<DashboardId,
protected void setRelatedEntities(EntitiesExportCtx<?> ctx, Dashboard dashboard, EntityExportData<Dashboard> exportData) { protected void setRelatedEntities(EntitiesExportCtx<?> ctx, Dashboard dashboard, EntityExportData<Dashboard> exportData) {
if (CollectionUtils.isNotEmpty(dashboard.getAssignedCustomers())) { if (CollectionUtils.isNotEmpty(dashboard.getAssignedCustomers())) {
dashboard.getAssignedCustomers().forEach(customerInfo -> { dashboard.getAssignedCustomers().forEach(customerInfo -> {
customerInfo.setCustomerId(getExternalIdOrElseInternal(customerInfo.getCustomerId())); customerInfo.setCustomerId(getExternalIdOrElseInternal(ctx, customerInfo.getCustomerId()));
}); });
} }
} }

18
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java
View File

@ -53,7 +53,8 @@ import java.util.stream.Collectors;
@Primary @Primary
public class DefaultEntityExportService<I extends EntityId, E extends ExportableEntity<I>, D extends EntityExportData<E>> implements EntityExportService<I, E, D> { public class DefaultEntityExportService<I extends EntityId, E extends ExportableEntity<I>, D extends EntityExportData<E>> implements EntityExportService<I, E, D> {
@Autowired @Lazy @Autowired
@Lazy
protected ExportableEntitiesService exportableEntitiesService; protected ExportableEntitiesService exportableEntitiesService;
@Autowired @Autowired
private RelationService relationService; private RelationService relationService;
@ -82,8 +83,8 @@ public class DefaultEntityExportService<I extends EntityId, E extends Exportable
if (exportSettings.isExportRelations()) { if (exportSettings.isExportRelations()) {
List<EntityRelation> relations = exportRelations(ctx, entity); List<EntityRelation> relations = exportRelations(ctx, entity);
relations.forEach(relation -> { relations.forEach(relation -> {
relation.setFrom(getExternalIdOrElseInternal(relation.getFrom())); relation.setFrom(getExternalIdOrElseInternal(ctx, relation.getFrom()));
relation.setTo(getExternalIdOrElseInternal(relation.getTo())); relation.setTo(getExternalIdOrElseInternal(ctx, relation.getTo()));
}); });
exportData.setRelations(relations); exportData.setRelations(relations);
} }
@ -134,10 +135,15 @@ public class DefaultEntityExportService<I extends EntityId, E extends Exportable
return attributes; return attributes;
} }
protected <ID extends EntityId> ID getExternalIdOrElseInternal(ID internalId) { protected <ID extends EntityId> ID getExternalIdOrElseInternal(EntitiesExportCtx<?> ctx, ID internalId) {
if (internalId == null || internalId.isNullUid()) return internalId; if (internalId == null || internalId.isNullUid()) return internalId;
return Optional.ofNullable(exportableEntitiesService.getExternalIdByInternal(internalId)) var result = ctx.getExternalId(internalId);
.orElse(internalId); if (result == null) {
result = Optional.ofNullable(exportableEntitiesService.getExternalIdByInternal(internalId))
.orElse(internalId);
ctx.putExternalId(internalId, result);
}
return result;
} }
protected D newExportData() { protected D newExportData() {

4
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java
View File

@ -38,8 +38,8 @@ public class DeviceExportService extends BaseEntityExportService<DeviceId, Devic
@Override @Override
protected void setRelatedEntities(EntitiesExportCtx<?> ctx, Device device, DeviceExportData exportData) { protected void setRelatedEntities(EntitiesExportCtx<?> ctx, Device device, DeviceExportData exportData) {
device.setCustomerId(getExternalIdOrElseInternal(device.getCustomerId())); device.setCustomerId(getExternalIdOrElseInternal(ctx, device.getCustomerId()));
device.setDeviceProfileId(getExternalIdOrElseInternal(device.getDeviceProfileId())); device.setDeviceProfileId(getExternalIdOrElseInternal(ctx, device.getDeviceProfileId()));
if (ctx.getSettings().isExportCredentials()) { if (ctx.getSettings().isExportCredentials()) {
exportData.setCredentials(deviceCredentialsService.findDeviceCredentialsByDeviceId(ctx.getTenantId(), device.getId())); exportData.setCredentials(deviceCredentialsService.findDeviceCredentialsByDeviceId(ctx.getTenantId(), device.getId()));
} }

4
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java
View File

@ -33,8 +33,8 @@ public class DeviceProfileExportService extends BaseEntityExportService<DevicePr
@Override @Override
protected void setRelatedEntities(EntitiesExportCtx<?> ctx, DeviceProfile deviceProfile, EntityExportData<DeviceProfile> exportData) { protected void setRelatedEntities(EntitiesExportCtx<?> ctx, DeviceProfile deviceProfile, EntityExportData<DeviceProfile> exportData) {
deviceProfile.setDefaultDashboardId(getExternalIdOrElseInternal(deviceProfile.getDefaultDashboardId())); deviceProfile.setDefaultDashboardId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultDashboardId()));
deviceProfile.setDefaultRuleChainId(getExternalIdOrElseInternal(deviceProfile.getDefaultRuleChainId())); deviceProfile.setDefaultRuleChainId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultRuleChainId()));
} }
@Override @Override

4
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java
View File

@ -33,8 +33,8 @@ public class EntityViewExportService extends BaseEntityExportService<EntityViewI
@Override @Override
protected void setRelatedEntities(EntitiesExportCtx<?> ctx, EntityView entityView, EntityExportData<EntityView> exportData) { protected void setRelatedEntities(EntitiesExportCtx<?> ctx, EntityView entityView, EntityExportData<EntityView> exportData) {
entityView.setEntityId(getExternalIdOrElseInternal(entityView.getEntityId())); entityView.setEntityId(getExternalIdOrElseInternal(ctx, entityView.getEntityId()));
entityView.setCustomerId(getExternalIdOrElseInternal(entityView.getCustomerId())); entityView.setCustomerId(getExternalIdOrElseInternal(ctx, entityView.getCustomerId()));
} }
@Override @Override

20
application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java
View File

@ -17,6 +17,8 @@ package org.thingsboard.server.service.sync.vc.data;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import lombok.Data; import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig;
@ -24,8 +26,11 @@ import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRe
import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.SecurityUser;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
@Slf4j
@Data @Data
public abstract class EntitiesExportCtx<R extends VersionCreateRequest> { public abstract class EntitiesExportCtx<R extends VersionCreateRequest> {
@ -33,12 +38,14 @@ public abstract class EntitiesExportCtx<R extends VersionCreateRequest> {
protected final CommitGitRequest commit; protected final CommitGitRequest commit;
protected final R request; protected final R request;
private final List<ListenableFuture<Void>> futures; private final List<ListenableFuture<Void>> futures;
private final Map<EntityId, EntityId> externalIdMap;
public EntitiesExportCtx(SecurityUser user, CommitGitRequest commit, R request) { public EntitiesExportCtx(SecurityUser user, CommitGitRequest commit, R request) {
this.user = user; this.user = user;
this.commit = commit; this.commit = commit;
this.request = request; this.request = request;
this.futures = new ArrayList<>(); this.futures = new ArrayList<>();
this.externalIdMap = new HashMap<>();
} }
protected <T extends R> EntitiesExportCtx(EntitiesExportCtx<T> other) { protected <T extends R> EntitiesExportCtx(EntitiesExportCtx<T> other) {
@ -46,6 +53,7 @@ public abstract class EntitiesExportCtx<R extends VersionCreateRequest> {
this.commit = other.getCommit(); this.commit = other.getCommit();
this.request = other.getRequest(); this.request = other.getRequest();
this.futures = other.getFutures(); this.futures = other.getFutures();
this.externalIdMap = other.getExternalIdMap();
} }
public void add(ListenableFuture<Void> future) { public void add(ListenableFuture<Void> future) {
@ -65,4 +73,16 @@ public abstract class EntitiesExportCtx<R extends VersionCreateRequest> {
} }
public abstract EntityExportSettings getSettings(); public abstract EntityExportSettings getSettings();
@SuppressWarnings("unchecked")
public <ID extends EntityId> ID getExternalId(ID internalId) {
var result = externalIdMap.get(internalId);
log.debug("[{}][{}] Local cache {} for id", internalId.getEntityType(), internalId.getId(), result != null ? "hit" : "miss");
return (ID) result;
}
public void putExternalId(EntityId internalId, EntityId externalId) {
log.debug("[{}][{}] Local cache put: {}", internalId.getEntityType(), internalId.getId(), externalId);
externalIdMap.put(internalId, externalId);
}
} }