dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/entities-version-control' of github.com:thingsboard/thingsboard into feature/entities-version-control

Browse Source
This commit is contained in:
Igor Kulikov 2022-06-02 10:22:34 +03:00
commit 50c6178c89
26 changed files with 371 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java
View File

@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.sync.vc.EntityDataDiff;
import org.thingsboard.server.common.data.sync.vc.EntityDataInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
@ -296,7 +297,7 @@ public class EntitiesVersionControlController extends BaseController {
" }\n" +
"}\n```")
@PostMapping("/entity")
public DeferredResult<List<VersionLoadResult>> loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException {
public DeferredResult<VersionLoadResult> loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException {
SecurityUser user = getCurrentUser();
try {
return wrapFuture(versionControlService.loadEntitiesVersion(user, request));

4
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java
View File

@ -30,11 +30,11 @@ public abstract class BaseEntityExportService<I extends EntityId, E extends Expo
@Override
protected void setAdditionalExportData(SecurityUser user, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException {
setRelatedEntities(user.getTenantId(), entity, (D) exportData);
setRelatedEntities(user.getTenantId(), entity, (D) exportData, exportSettings);
super.setAdditionalExportData(user, entity, exportData, exportSettings);
}
protected void setRelatedEntities(TenantId tenantId, E mainEntity, D exportData) {}
protected void setRelatedEntities(TenantId tenantId, E mainEntity, D exportData, EntityExportSettings settings) {}
protected abstract D newExportData();

5
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java
View File

@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.dao.device.DeviceCredentialsService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.common.data.sync.ie.DeviceExportData;
@ -35,9 +36,11 @@ public class DeviceExportService extends BaseEntityExportService<DeviceId, Devic
private final DeviceCredentialsService deviceCredentialsService;
@Override
protected void setRelatedEntities(TenantId tenantId, Device device, DeviceExportData exportData) {
protected void setRelatedEntities(TenantId tenantId, Device device, DeviceExportData exportData, EntityExportSettings settings) {
if (settings.isExportCredentials()) {
exportData.setCredentials(deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, device.getId()));
}
}
@Override
protected DeviceExportData newExportData() {

3
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java
View File

@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.common.data.sync.ie.RuleChainExportData;
@ -35,7 +36,7 @@ public class RuleChainExportService extends BaseEntityExportService<RuleChainId,
private final RuleChainService ruleChainService;
@Override
protected void setRelatedEntities(TenantId tenantId, RuleChain ruleChain, RuleChainExportData exportData) {
protected void setRelatedEntities(TenantId tenantId, RuleChain ruleChain, RuleChainExportData exportData, EntityExportSettings exportSettings) {
exportData.setMetaData(ruleChainService.loadRuleChainMetaData(tenantId, ruleChain.getId()));
}

3
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java
View File

@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
@ -42,7 +43,7 @@ public class AssetImportService extends BaseEntityImportService<AssetId, Asset,
}
@Override
protected Asset prepareAndSave(TenantId tenantId, Asset asset, EntityExportData<Asset> exportData, IdProvider idProvider) {
protected Asset prepareAndSave(TenantId tenantId, Asset asset, EntityExportData<Asset> exportData, IdProvider idProvider, EntityImportSettings importSettings) {
return assetService.saveAsset(asset);
}

12
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java
View File

@ -64,7 +64,8 @@ import java.util.stream.Collectors;
@Slf4j
public abstract class BaseEntityImportService<I extends EntityId, E extends ExportableEntity<I>, D extends EntityExportData<E>> implements EntityImportService<I, E, D> {
@Autowired @Lazy
@Autowired
@Lazy
private ExportableEntitiesService exportableEntitiesService;
@Autowired
private RelationService relationService;
@ -94,7 +95,7 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
exportableEntitiesService.checkPermission(user, existingEntity, getEntityType(), Operation.WRITE);
}
E savedEntity = prepareAndSave(user.getTenantId(), entity, exportData, idProvider);
E savedEntity = prepareAndSave(user.getTenantId(), entity, exportData, idProvider, importSettings);
EntityImportResult<E> importResult = new EntityImportResult<>();
importResult.setSavedEntity(savedEntity);
@ -108,7 +109,7 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
protected abstract void setOwner(TenantId tenantId, E entity, IdProvider idProvider);
protected abstract E prepareAndSave(TenantId tenantId, E entity, D exportData, IdProvider idProvider);
protected abstract E prepareAndSave(TenantId tenantId, E entity, D exportData, IdProvider idProvider, EntityImportSettings importSettings);
protected void processAfterSaved(SecurityUser user, EntityImportResult<E> importResult, D exportData,
@ -210,7 +211,8 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
// fixme: attributes are saved outside the transaction
tsSubService.saveAndNotify(user.getTenantId(), entity.getId(), scope, attributeKvEntries, new FutureCallback<Void>() {
@Override
public void onSuccess(@Nullable Void unused) {}
public void onSuccess(@Nullable Void unused) {
}
@Override
public void onFailure(Throwable thr) {
@ -246,7 +248,7 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
private <ID extends EntityId> HasId<ID> findInternalEntity(TenantId tenantId, ID externalId) {
return (HasId<ID>) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId))
.or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId)))
.orElseThrow(() -> new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId));
.orElseThrow(() -> new MissingEntityException(externalId));
}

3
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java
View File

@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
@ -41,7 +42,7 @@ public class CustomerImportService extends BaseEntityImportService<CustomerId, C
}
@Override
protected Customer prepareAndSave(TenantId tenantId, Customer customer, EntityExportData<Customer> exportData, IdProvider idProvider) {
protected Customer prepareAndSave(TenantId tenantId, Customer customer, EntityExportData<Customer> exportData, IdProvider idProvider, EntityImportSettings importSettings) {
if (customer.isPublic()) {
return customerService.findOrCreatePublicCustomer(tenantId);
} else {

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java
View File

@ -64,7 +64,7 @@ public class DashboardImportService extends BaseEntityImportService<DashboardId,
}
@Override
protected Dashboard prepareAndSave(TenantId tenantId, Dashboard dashboard, EntityExportData<Dashboard> exportData, IdProvider idProvider) {
protected Dashboard prepareAndSave(TenantId tenantId, Dashboard dashboard, EntityExportData<Dashboard> exportData, IdProvider idProvider, EntityImportSettings importSettings) {
JsonNode configuration = dashboard.getConfiguration();
String newConfigurationJson = RegexUtils.replace(configuration.toString(), RegexUtils.UUID_PATTERN, uuid -> {
return idProvider.getInternalIdByUuid(UUID.fromString(uuid))

5
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java
View File

@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
@ -41,11 +42,11 @@ public class DeviceImportService extends BaseEntityImportService<DeviceId, Devic
}
@Override
protected Device prepareAndSave(TenantId tenantId, Device device, DeviceExportData exportData, IdProvider idProvider) {
protected Device prepareAndSave(TenantId tenantId, Device device, DeviceExportData exportData, IdProvider idProvider, EntityImportSettings importSettings) {
device.setDeviceProfileId(idProvider.getInternalId(device.getDeviceProfileId()));
device.setFirmwareId(idProvider.getInternalId(device.getFirmwareId()));
device.setSoftwareId(idProvider.getInternalId(device.getSoftwareId()));
if (exportData.getCredentials() != null) {
if (exportData.getCredentials() != null && importSettings.isSaveCredentials()) {
exportData.getCredentials().setId(null);
exportData.getCredentials().setDeviceId(null);
return deviceService.saveDeviceWithCredentials(device, exportData.getCredentials());

3
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java
View File

@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.ota.OtaPackageStateService;
@ -46,7 +47,7 @@ public class DeviceProfileImportService extends BaseEntityImportService<DevicePr
}
@Override
protected DeviceProfile prepareAndSave(TenantId tenantId, DeviceProfile deviceProfile, EntityExportData<DeviceProfile> exportData, IdProvider idProvider) {
protected DeviceProfile prepareAndSave(TenantId tenantId, DeviceProfile deviceProfile, EntityExportData<DeviceProfile> exportData, IdProvider idProvider, EntityImportSettings importSettings) {
deviceProfile.setDefaultRuleChainId(idProvider.getInternalId(deviceProfile.getDefaultRuleChainId()));
deviceProfile.setDefaultDashboardId(idProvider.getInternalId(deviceProfile.getDefaultDashboardId()));
deviceProfile.setFirmwareId(idProvider.getInternalId(deviceProfile.getFirmwareId()));

20
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java Normal file
View File

@ -0,0 +1,20 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.importing.impl;
public class ImportServiceException extends RuntimeException{
private static final long serialVersionUID = -4932715239522125041L;
}

30
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java Normal file
View File

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.importing.impl;
import lombok.Getter;
import org.thingsboard.server.common.data.id.EntityId;
public class MissingEntityException extends ImportServiceException {
private static final long serialVersionUID = 3669135386955906022L;
@Getter
private final EntityId entityId;
public MissingEntityException(EntityId entityId) {
this.entityId = entityId;
}
}

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java
View File

@ -62,7 +62,7 @@ public class RuleChainImportService extends BaseEntityImportService<RuleChainId,
}
@Override
protected RuleChain prepareAndSave(TenantId tenantId, RuleChain ruleChain, RuleChainExportData exportData, IdProvider idProvider) {
protected RuleChain prepareAndSave(TenantId tenantId, RuleChain ruleChain, RuleChainExportData exportData, IdProvider idProvider, EntityImportSettings importSettings) {
RuleChainMetaData metaData = exportData.getMetaData();
Optional.ofNullable(metaData.getNodes()).orElse(Collections.emptyList())
.forEach(ruleNode -> {

103
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
View File

@ -22,8 +22,10 @@ import com.google.common.util.concurrent.MoreExecutors;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ObjectUtils;
import org.jetbrains.annotations.NotNull;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionCallback;
import org.springframework.transaction.support.TransactionTemplate;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardExecutors;
@ -45,10 +47,12 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
import org.thingsboard.server.common.data.sync.vc.EntityDataInfo;
import org.thingsboard.server.common.data.sync.vc.EntityLoadError;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.EntityDataDiff;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig;
@ -64,12 +68,14 @@ import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersi
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.TbNotificationEntityService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.sync.ie.EntitiesExportImportService;
import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
import org.thingsboard.server.service.sync.ie.importing.impl.MissingEntityException;
import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService;
import org.thingsboard.server.service.sync.vc.data.CommitGitRequest;
import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService;
@ -83,8 +89,10 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@ -172,6 +180,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
EntityExportData<ExportableEntity<EntityId>> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder()
.exportRelations(config.isSaveRelations())
.exportAttributes(config.isSaveAttributes())
.exportCredentials(config.isSaveCredentials())
.build());
return gitServiceQueue.addToCommit(commit, entityData);
}
@ -203,54 +212,75 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
@SuppressWarnings({"UnstableApiUsage", "rawtypes"})
@Override
public ListenableFuture<List<VersionLoadResult>> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception {
public ListenableFuture<VersionLoadResult> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception {
switch (request.getType()) {
case SINGLE_ENTITY: {
SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request;
VersionLoadConfig config = versionLoadRequest.getConfig();
ListenableFuture<EntityExportData> future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId());
return Futures.transform(future, entityData -> {
EntityImportResult<?> importResult = transactionTemplate.execute(status -> {
return Futures.transform(future, entityData -> doInTemplate(status -> loadSingleEntity(user, config, entityData)), executor);
}
case ENTITY_TYPE: {
EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request;
return executor.submit(() -> doInTemplate(status -> loadMultipleEntities(user, versionLoadRequest)));
}
default:
throw new IllegalArgumentException("Unsupported version load request");
}
}
private VersionLoadResult doInTemplate(TransactionCallback<VersionLoadResult> result) {
try {
return exportImportService.importEntity(user, entityData, EntityImportSettings.builder()
return transactionTemplate.execute(result);
} catch (LoadEntityException e) {
return onError(e.getData(), e.getCause());
}
}
private VersionLoadResult loadSingleEntity(SecurityUser user, VersionLoadConfig config, EntityExportData entityData) {
try {
EntityImportResult<?> importResult = exportImportService.importEntity(user, entityData,
EntityImportSettings.builder()
.updateRelations(config.isLoadRelations())
.saveAttributes(config.isLoadAttributes())
.saveCredentials(config.isLoadCredentials())
.findExistingByName(false)
.build(), true, true);
} catch (Exception e) {
throw new RuntimeException(e);
}
});
return List.of(VersionLoadResult.builder()
return VersionLoadResult.success(EntityTypeLoadResult.builder()
.entityType(importResult.getEntityType())
.created(importResult.getOldEntity() == null ? 1 : 0)
.updated(importResult.getOldEntity() != null ? 1 : 0)
.deleted(0)
.build());
}, executor);
} catch (Exception e) {
throw new LoadEntityException(entityData, e);
}
case ENTITY_TYPE: {
EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request;
return executor.submit(() -> transactionTemplate.execute(status -> {
Map<EntityType, VersionLoadResult> results = new HashMap<>();
}
private VersionLoadResult loadMultipleEntities(SecurityUser user, EntityTypeVersionLoadRequest versionLoadRequest) {
Map<EntityType, EntityTypeLoadResult> results = new HashMap<>();
Map<EntityType, Set<EntityId>> importedEntities = new HashMap<>();
List<ThrowingRunnable> saveReferencesCallbacks = new ArrayList<>();
List<ThrowingRunnable> sendEventsCallbacks = new ArrayList<>();
versionLoadRequest.getEntityTypes().keySet().stream()
.sorted(exportImportService.getEntityTypeComparatorForImport())
.forEach(entityType -> {
List<EntityType> entityTypes = versionLoadRequest.getEntityTypes().keySet().stream()
.sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList());
for (EntityType entityType : entityTypes) {
EntityTypeVersionLoadConfig config = versionLoadRequest.getEntityTypes().get(entityType);
AtomicInteger created = new AtomicInteger();
AtomicInteger updated = new AtomicInteger();
try {
int limit = 100;
int offset = 0;
List<EntityExportData> entityDataList;
do {
entityDataList = gitServiceQueue.getEntities(user.getTenantId(), request.getVersionId(), entityType, offset, limit).get();
try {
entityDataList = gitServiceQueue.getEntities(user.getTenantId(), versionLoadRequest.getVersionId(), entityType, offset, limit).get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
for (EntityExportData entityData : entityDataList) {
try {
EntityImportResult<?> importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder()
.updateRelations(config.isLoadRelations())
.saveAttributes(config.isLoadAttributes())
@ -261,20 +291,20 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
else updated.incrementAndGet();
saveReferencesCallbacks.add(importResult.getSaveReferencesCallback());
sendEventsCallbacks.add(importResult.getSendEventsCallback());
} catch (Exception e) {
throw new LoadEntityException(entityData, e);
}
}
offset += limit;
importedEntities.computeIfAbsent(entityType, t -> new HashSet<>())
.addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getExternalId()).collect(Collectors.toSet()));
} while (entityDataList.size() == limit);
} catch (Exception e) {
throw new RuntimeException(e);
}
results.put(entityType, VersionLoadResult.builder()
results.put(entityType, EntityTypeLoadResult.builder()
.entityType(entityType)
.created(created.get())
.updated(updated.get())
.build());
});
}
versionLoadRequest.getEntityTypes().keySet().stream()
.filter(entityType -> versionLoadRequest.getEntityTypes().get(entityType).isRemoveOtherEntities())
@ -295,7 +325,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
entityNotificationService.notifyDeleteEntity(user.getTenantId(), entity.getId(),
entity, null, ActionType.DELETED, null, user);
});
VersionLoadResult result = results.get(entityType);
EntityTypeLoadResult result = results.get(entityType);
result.setDeleted(result.getDeleted() + 1);
}
});
@ -315,11 +345,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
log.error("Failed to send events for entity", e);
}
}
return new ArrayList<>(results.values());
}));
return VersionLoadResult.success(new ArrayList<>(results.values()));
}
private VersionLoadResult onError(EntityExportData<?> entityData, Throwable e) {
return analyze(e, entityData).orElseThrow(() -> new RuntimeException(e));
}
private Optional<VersionLoadResult> analyze(Throwable e, EntityExportData<?> entityData) {
if (e == null) {
return Optional.empty();
} else {
if (e instanceof DeviceCredentialsValidationException) {
return Optional.of(VersionLoadResult.error(EntityLoadError.credentialsError(entityData.getExternalId())));
} else if (e instanceof MissingEntityException) {
return Optional.of(VersionLoadResult.error(EntityLoadError.referenceEntityError(entityData.getExternalId(), ((MissingEntityException) e).getEntityId())));
} else {
return analyze(e.getCause(), entityData);
}
default:
throw new IllegalArgumentException("Unsupported version load request");
}
}
@ -347,7 +390,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
@Override
public ListenableFuture<EntityDataInfo> getEntityDataInfo(SecurityUser user, EntityId entityId, String versionId) {
return Futures.transform(gitServiceQueue.getEntity(user.getTenantId(), versionId, entityId),
entity -> new EntityDataInfo(entity.getRelations() != null, entity.getAttributes() != null), MoreExecutors.directExecutor());
entity -> new EntityDataInfo(entity.getRelations() != null, entity.getAttributes() != null, false), MoreExecutors.directExecutor());
}

6
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java
View File

@ -108,7 +108,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
public ListenableFuture<Void> addToCommit(CommitGitRequest commit, EntityExportData<ExportableEntity<EntityId>> entityData) {
SettableFuture<Void> future = SettableFuture.create();
String path = getRelativePath(entityData.getEntityType(), getExternalId(entityData.getEntity()));
String path = getRelativePath(entityData.getEntityType(), entityData.getExternalId());
String entityDataJson = JacksonUtil.toPrettyString(entityData.sort());
registerAndSend(commit, builder -> builder.setCommitRequest(
@ -120,10 +120,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
return future;
}
private EntityId getExternalId(ExportableEntity<EntityId> entity) {
return entity.getExternalId() != null ? entity.getExternalId() : entity.getId();
}
@Override
public ListenableFuture<Void> deleteAll(CommitGitRequest commit, EntityType entityType) {
SettableFuture<Void> future = SettableFuture.create();

5
application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java
View File

@ -23,11 +23,12 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.EntityDataDiff;
import org.thingsboard.server.common.data.sync.vc.EntityDataInfo;
import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
@ -49,7 +50,7 @@ public interface EntitiesVersionControlService {
ListenableFuture<List<VersionedEntityInfo>> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception;
ListenableFuture<List<VersionLoadResult>> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception;
ListenableFuture<VersionLoadResult> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception;
ListenableFuture<EntityDataDiff> compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception;

32
application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java Normal file
View File

@ -0,0 +1,32 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc;
import lombok.Getter;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
@SuppressWarnings("rawtypes")
public class LoadEntityException extends RuntimeException {
private static final long serialVersionUID = -1749719992370409504L;
@Getter
private final EntityExportData data;
public LoadEntityException(EntityExportData data, Throwable cause) {
super(cause);
this.data = data;
}
}

6
common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java
View File

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.common.data.sync.ie;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -75,4 +76,9 @@ public class EntityExportData<E extends ExportableEntity<? extends EntityId>> {
return this;
}
@JsonIgnore
public EntityId getExternalId() {
return entity.getExternalId() != null ? entity.getExternalId() : entity.getId();
}
}

1
common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java
View File

@ -27,4 +27,5 @@ import lombok.NoArgsConstructor;
public class EntityExportSettings {
private boolean exportRelations;
private boolean exportAttributes;
private boolean exportCredentials;
}

1
common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java
View File

@ -28,4 +28,5 @@ public class EntityImportSettings {
private boolean findExistingByName;
private boolean updateRelations;
private boolean saveAttributes;
private boolean saveCredentials;
}

1
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java
View File

@ -25,4 +25,5 @@ import lombok.NoArgsConstructor;
public class EntityDataInfo {
boolean hasRelations;
boolean hasAttributes;
boolean hasCredentials;
}

42
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java Normal file
View File

@ -0,0 +1,42 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.sync.vc;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Builder;
import lombok.Data;
import org.thingsboard.server.common.data.id.EntityId;
import java.util.List;
@Data
@Builder
@JsonInclude(JsonInclude.Include.NON_NULL)
public class EntityLoadError {
private String type;
private EntityId source;
private EntityId target;
public static EntityLoadError credentialsError(EntityId sourceId) {
return EntityLoadError.builder().type("DEVICE_CREDENTIALS_CONFLICT").source(sourceId).build();
}
public static EntityLoadError referenceEntityError(EntityId sourceId, EntityId targetId) {
return EntityLoadError.builder().type("MISSING_REFERENCED_ENTITY").source(sourceId).target(targetId).build();
}
}

33
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java Normal file
View File

@ -0,0 +1,33 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.sync.vc;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.thingsboard.server.common.data.EntityType;
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class EntityTypeLoadResult {
private EntityType entityType;
private int created;
private int updated;
private int deleted;
}

29
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java
View File

@ -15,19 +15,30 @@
*/
package org.thingsboard.server.common.data.sync.vc;
import lombok.AllArgsConstructor;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.thingsboard.server.common.data.EntityType;
import java.util.List;
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
@JsonInclude(JsonInclude.Include.NON_NULL)
public class VersionLoadResult {
private EntityType entityType;
private int created;
private int updated;
private int deleted;
private List<EntityTypeLoadResult> result;
private EntityLoadError error;
public static VersionLoadResult success(List<EntityTypeLoadResult> result) {
return VersionLoadResult.builder().result(result).build();
}
public static VersionLoadResult success(EntityTypeLoadResult result) {
return VersionLoadResult.builder().result(List.of(result)).build();
}
public static VersionLoadResult error(EntityLoadError error) {
return VersionLoadResult.builder().error(error).build();
}
}

1
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java
View File

@ -25,4 +25,5 @@ public class VersionCreateConfig implements Serializable {
private boolean saveRelations;
private boolean saveAttributes;
private boolean saveCredentials;
}

1
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java
View File

@ -22,5 +22,6 @@ public class VersionLoadConfig {
private boolean loadRelations;
private boolean loadAttributes;
private boolean loadCredentials;
}