Merge pull request #5328 from ViacheslavKlimov/bulk-import-improvements

Concurrent bulk import processing
This commit is contained in:
Andrew Shvayka 2021-10-19 15:47:17 +03:00 committed by GitHub
commit dda6138393
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 152 additions and 71 deletions

View File

@ -133,7 +133,7 @@ public class AssetController extends BaseController {
Asset savedAsset = checkNotNull(assetService.saveAsset(asset));
onAssetCreatedOrUpdated(savedAsset, asset.getId() != null);
onAssetCreatedOrUpdated(savedAsset, asset.getId() != null, getCurrentUser());
return savedAsset;
} catch (Exception e) {
@ -143,9 +143,9 @@ public class AssetController extends BaseController {
}
}
private void onAssetCreatedOrUpdated(Asset asset, boolean updated) {
private void onAssetCreatedOrUpdated(Asset asset, boolean updated, SecurityUser user) {
try {
logEntityAction(asset.getId(), asset,
logEntityAction(user, asset.getId(), asset,
asset.getCustomerId(),
updated ? ActionType.UPDATED : ActionType.ADDED, null);
} catch (ThingsboardException e) {
@ -648,8 +648,9 @@ public class AssetController extends BaseController {
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
@PostMapping("/asset/bulk_import")
public BulkImportResult<Asset> processAssetsBulkImport(@RequestBody BulkImportRequest request) throws Exception {
return assetBulkImportService.processBulkImport(request, getCurrentUser(), importedAssetInfo -> {
onAssetCreatedOrUpdated(importedAssetInfo.getEntity(), importedAssetInfo.isUpdated());
SecurityUser user = getCurrentUser();
return assetBulkImportService.processBulkImport(request, user, importedAssetInfo -> {
onAssetCreatedOrUpdated(importedAssetInfo.getEntity(), importedAssetInfo.isUpdated(), user);
});
}

View File

@ -161,7 +161,7 @@ public class DeviceController extends BaseController {
Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken));
onDeviceCreatedOrUpdated(savedDevice, oldDevice, !created);
onDeviceCreatedOrUpdated(savedDevice, oldDevice, !created, getCurrentUser());
return savedDevice;
} catch (Exception e) {
@ -172,11 +172,11 @@ public class DeviceController extends BaseController {
}
private void onDeviceCreatedOrUpdated(Device savedDevice, Device oldDevice, boolean updated) {
private void onDeviceCreatedOrUpdated(Device savedDevice, Device oldDevice, boolean updated, SecurityUser user) {
tbClusterService.onDeviceUpdated(savedDevice, oldDevice);
try {
logEntityAction(savedDevice.getId(), savedDevice,
logEntityAction(user, savedDevice.getId(), savedDevice,
savedDevice.getCustomerId(),
updated ? ActionType.UPDATED : ActionType.ADDED, null);
} catch (ThingsboardException e) {
@ -941,8 +941,9 @@ public class DeviceController extends BaseController {
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
@PostMapping("/device/bulk_import")
public BulkImportResult<Device> processDevicesBulkImport(@RequestBody BulkImportRequest request) throws Exception {
return deviceBulkImportService.processBulkImport(request, getCurrentUser(), importedDeviceInfo -> {
onDeviceCreatedOrUpdated(importedDeviceInfo.getEntity(), importedDeviceInfo.getOldEntity(), importedDeviceInfo.isUpdated());
SecurityUser user = getCurrentUser();
return deviceBulkImportService.processBulkImport(request, user, importedDeviceInfo -> {
onDeviceCreatedOrUpdated(importedDeviceInfo.getEntity(), importedDeviceInfo.getOldEntity(), importedDeviceInfo.isUpdated(), user);
});
}

View File

@ -140,7 +140,7 @@ public class EdgeController extends BaseController {
edge.getId(), edge);
Edge savedEdge = checkNotNull(edgeService.saveEdge(edge, true));
onEdgeCreatedOrUpdated(tenantId, savedEdge, edgeTemplateRootRuleChain, !created);
onEdgeCreatedOrUpdated(tenantId, savedEdge, edgeTemplateRootRuleChain, !created, getCurrentUser());
return savedEdge;
} catch (Exception e) {
@ -150,7 +150,7 @@ public class EdgeController extends BaseController {
}
}
private void onEdgeCreatedOrUpdated(TenantId tenantId, Edge edge, RuleChain edgeTemplateRootRuleChain, boolean updated) throws IOException, ThingsboardException {
private void onEdgeCreatedOrUpdated(TenantId tenantId, Edge edge, RuleChain edgeTemplateRootRuleChain, boolean updated, SecurityUser user) throws IOException, ThingsboardException {
if (!updated) {
ruleChainService.assignRuleChainToEdge(tenantId, edgeTemplateRootRuleChain.getId(), edge.getId());
edgeNotificationService.setEdgeRootRuleChain(tenantId, edge, edgeTemplateRootRuleChain.getId());
@ -160,7 +160,7 @@ public class EdgeController extends BaseController {
tbClusterService.broadcastEntityStateChangeEvent(edge.getTenantId(), edge.getId(),
updated ? ComponentLifecycleEvent.UPDATED : ComponentLifecycleEvent.CREATED);
logEntityAction(edge.getId(), edge, null, updated ? ActionType.UPDATED : ActionType.ADDED, null);
logEntityAction(user, edge.getId(), edge, null, updated ? ActionType.UPDATED : ActionType.ADDED, null);
}
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@ -586,7 +586,7 @@ public class EdgeController extends BaseController {
return edgeBulkImportService.processBulkImport(request, user, importedAssetInfo -> {
try {
onEdgeCreatedOrUpdated(user.getTenantId(), importedAssetInfo.getEntity(), edgeTemplateRootRuleChain, importedAssetInfo.isUpdated());
onEdgeCreatedOrUpdated(user.getTenantId(), importedAssetInfo.getEntity(), edgeTemplateRootRuleChain, importedAssetInfo.isUpdated(), user);
} catch (Exception e) {
throw new RuntimeException(e);
}

View File

@ -63,6 +63,8 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@Service
@TbCoreComponent
@ -71,6 +73,8 @@ public class DeviceBulkImportService extends AbstractBulkImportService<Device> {
protected final DeviceCredentialsService deviceCredentialsService;
protected final DeviceProfileService deviceProfileService;
private final Lock findOrCreateDeviceProfileLock = new ReentrantLock();
public DeviceBulkImportService(TelemetrySubscriptionService tsSubscriptionService, TbTenantProfileCache tenantProfileCache,
AccessControlService accessControlService, AccessValidator accessValidator,
EntityActionService entityActionService, TbClusterService clusterService,
@ -106,9 +110,13 @@ public class DeviceBulkImportService extends AbstractBulkImportService<Device> {
throw new DeviceCredentialsValidationException("Invalid device credentials: " + e.getMessage());
}
DeviceProfile deviceProfile;
if (deviceCredentials.getCredentialsType() == DeviceCredentialsType.LWM2M_CREDENTIALS) {
setUpLwM2mDeviceProfile(user.getTenantId(), device);
deviceProfile = setUpLwM2mDeviceProfile(user.getTenantId(), device);
} else {
deviceProfile = deviceProfileService.findOrCreateDeviceProfile(user.getTenantId(), device.getType());
}
device.setDeviceProfileId(deviceProfile.getId());
device = deviceService.saveDeviceWithCredentials(device, deviceCredentials);
@ -215,36 +223,43 @@ public class DeviceBulkImportService extends AbstractBulkImportService<Device> {
credentials.setCredentialsValue(lwm2mCredentials.toString());
}
private void setUpLwM2mDeviceProfile(TenantId tenantId, Device device) {
private DeviceProfile setUpLwM2mDeviceProfile(TenantId tenantId, Device device) {
DeviceProfile deviceProfile = deviceProfileService.findDeviceProfileByName(tenantId, device.getType());
if (deviceProfile != null) {
if (deviceProfile.getTransportType() != DeviceTransportType.LWM2M) {
deviceProfile.setTransportType(DeviceTransportType.LWM2M);
deviceProfile.getProfileData().setTransportConfiguration(new Lwm2mDeviceProfileTransportConfiguration());
deviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile);
device.setDeviceProfileId(deviceProfile.getId());
}
} else {
deviceProfile = new DeviceProfile();
deviceProfile.setTenantId(tenantId);
deviceProfile.setType(DeviceProfileType.DEFAULT);
deviceProfile.setName(device.getType());
deviceProfile.setTransportType(DeviceTransportType.LWM2M);
deviceProfile.setProvisionType(DeviceProfileProvisionType.DISABLED);
findOrCreateDeviceProfileLock.lock();
try {
deviceProfile = deviceProfileService.findDeviceProfileByName(tenantId, device.getType());
if (deviceProfile == null) {
deviceProfile = new DeviceProfile();
deviceProfile.setTenantId(tenantId);
deviceProfile.setType(DeviceProfileType.DEFAULT);
deviceProfile.setName(device.getType());
deviceProfile.setTransportType(DeviceTransportType.LWM2M);
deviceProfile.setProvisionType(DeviceProfileProvisionType.DISABLED);
DeviceProfileData deviceProfileData = new DeviceProfileData();
DefaultDeviceProfileConfiguration configuration = new DefaultDeviceProfileConfiguration();
DeviceProfileTransportConfiguration transportConfiguration = new Lwm2mDeviceProfileTransportConfiguration();
DisabledDeviceProfileProvisionConfiguration provisionConfiguration = new DisabledDeviceProfileProvisionConfiguration(null);
DeviceProfileData deviceProfileData = new DeviceProfileData();
DefaultDeviceProfileConfiguration configuration = new DefaultDeviceProfileConfiguration();
DeviceProfileTransportConfiguration transportConfiguration = new Lwm2mDeviceProfileTransportConfiguration();
DisabledDeviceProfileProvisionConfiguration provisionConfiguration = new DisabledDeviceProfileProvisionConfiguration(null);
deviceProfileData.setConfiguration(configuration);
deviceProfileData.setTransportConfiguration(transportConfiguration);
deviceProfileData.setProvisionConfiguration(provisionConfiguration);
deviceProfile.setProfileData(deviceProfileData);
deviceProfileData.setConfiguration(configuration);
deviceProfileData.setTransportConfiguration(transportConfiguration);
deviceProfileData.setProvisionConfiguration(provisionConfiguration);
deviceProfile.setProfileData(deviceProfileData);
deviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile);
device.setDeviceProfileId(deviceProfile.getId());
deviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile);
}
} finally {
findOrCreateDeviceProfileLock.unlock();
}
}
return deviceProfile;
}
private void setValues(ObjectNode objectNode, Map<BulkImportColumnType, String> data, Collection<BulkImportColumnType> columns) {

View File

@ -22,6 +22,9 @@ import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.thingsboard.common.util.DonAsynchron;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.BaseData;
import org.thingsboard.server.common.data.TenantProfile;
@ -47,11 +50,16 @@ import org.thingsboard.server.utils.CsvUtils;
import org.thingsboard.server.utils.TypeCastUtil;
import javax.annotation.Nullable;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
@ -67,39 +75,49 @@ public abstract class AbstractBulkImportService<E extends BaseData<? extends Ent
protected final EntityActionService entityActionService;
protected final TbClusterService clusterService;
public final BulkImportResult<E> processBulkImport(BulkImportRequest request, SecurityUser user, Consumer<ImportedEntityInfo<E>> onEntityImported) throws Exception {
BulkImportResult<E> result = new BulkImportResult<>();
private static ThreadPoolExecutor executor;
AtomicInteger i = new AtomicInteger(0);
if (request.getMapping().getHeader()) {
i.incrementAndGet();
@PostConstruct
private void initExecutor() {
if (executor == null) {
executor = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors(), Runtime.getRuntime().availableProcessors(),
60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(150_000),
ThingsBoardThreadFactory.forName("bulk-import"), new ThreadPoolExecutor.CallerRunsPolicy());
executor.allowCoreThreadTimeOut(true);
}
}
parseData(request).forEach(entityData -> {
i.incrementAndGet();
try {
ImportedEntityInfo<E> importedEntityInfo = saveEntity(request, entityData.getFields(), user);
onEntityImported.accept(importedEntityInfo);
public final BulkImportResult<E> processBulkImport(BulkImportRequest request, SecurityUser user, Consumer<ImportedEntityInfo<E>> onEntityImported) throws Exception {
List<EntityData> entitiesData = parseData(request);
E entity = importedEntityInfo.getEntity();
BulkImportResult<E> result = new BulkImportResult<>();
CountDownLatch completionLatch = new CountDownLatch(entitiesData.size());
saveKvs(user, entity, entityData.getKvs());
entitiesData.forEach(entityData -> DonAsynchron.submit(() -> {
ImportedEntityInfo<E> importedEntityInfo = saveEntity(request, entityData.getFields(), user);
E entity = importedEntityInfo.getEntity();
if (importedEntityInfo.getRelatedError() != null) {
throw new RuntimeException(importedEntityInfo.getRelatedError());
}
onEntityImported.accept(importedEntityInfo);
saveKvs(user, entity, entityData.getKvs());
if (importedEntityInfo.isUpdated()) {
result.setUpdated(result.getUpdated() + 1);
} else {
result.setCreated(result.getCreated() + 1);
}
} catch (Exception e) {
result.setErrors(result.getErrors() + 1);
result.getErrorsList().add(String.format("Line %d: %s", i.get(), e.getMessage()));
}
});
return importedEntityInfo;
},
importedEntityInfo -> {
if (importedEntityInfo.isUpdated()) {
result.getUpdated().incrementAndGet();
} else {
result.getCreated().incrementAndGet();
}
completionLatch.countDown();
},
throwable -> {
result.getErrors().incrementAndGet();
result.getErrorsList().add(String.format("Line %d: %s", entityData.getLineNumber(), ExceptionUtils.getRootCauseMessage(throwable)));
completionLatch.countDown();
},
executor));
completionLatch.await();
return result;
}
@ -186,8 +204,11 @@ public abstract class AbstractBulkImportService<E extends BaseData<? extends Ent
private List<EntityData> parseData(BulkImportRequest request) throws Exception {
List<List<String>> records = CsvUtils.parseCsv(request.getFile(), request.getMapping().getDelimiter());
AtomicInteger linesCounter = new AtomicInteger(0);
if (request.getMapping().getHeader()) {
records.remove(0);
linesCounter.incrementAndGet();
}
List<ColumnMapping> columnsMappings = request.getMapping().getColumns();
@ -205,15 +226,24 @@ public abstract class AbstractBulkImportService<E extends BaseData<? extends Ent
entityData.getKvs().put(entry.getKey(), new ParsedValue(castResult.getValue(), castResult.getKey()));
}
});
entityData.setLineNumber(linesCounter.incrementAndGet());
return entityData;
})
.collect(Collectors.toList());
}
@PreDestroy
private void shutdownExecutor() {
if (!executor.isTerminating()) {
executor.shutdown();
}
}
@Data
protected static class EntityData {
private final Map<BulkImportColumnType, String> fields = new LinkedHashMap<>();
private final Map<ColumnMapping, ParsedValue> kvs = new LinkedHashMap<>();
private int lineNumber;
}
@Data

View File

@ -17,14 +17,14 @@ package org.thingsboard.server.service.importing;
import lombok.Data;
import java.util.LinkedList;
import java.util.List;
import java.util.Collection;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.atomic.AtomicInteger;
@Data
public class BulkImportResult<E> {
private int created = 0;
private int updated = 0;
private int errors = 0;
private List<String> errorsList = new LinkedList<>();
private AtomicInteger created = new AtomicInteger();
private AtomicInteger updated = new AtomicInteger();
private AtomicInteger errors = new AtomicInteger();
private Collection<String> errorsList = new ConcurrentLinkedDeque<>();
}

View File

@ -22,5 +22,4 @@ public class ImportedEntityInfo<E> {
private E entity;
private boolean isUpdated;
private E oldEntity;
private String relatedError;
}

View File

@ -20,6 +20,7 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import java.util.concurrent.Callable;
import java.util.concurrent.Executor;
import java.util.function.Consumer;
@ -53,4 +54,15 @@ public class DonAsynchron {
Futures.addCallback(future, callback, MoreExecutors.directExecutor());
}
}
public static <T> ListenableFuture<T> submit(Callable<T> task, Consumer<T> onSuccess, Consumer<Throwable> onFailure, Executor executor) {
return submit(task, onSuccess, onFailure, executor, null);
}
public static <T> ListenableFuture<T> submit(Callable<T> task, Consumer<T> onSuccess, Consumer<Throwable> onFailure, Executor executor, Executor callbackExecutor) {
ListenableFuture<T> future = Futures.submit(task, executor);
withCallback(future, onSuccess, onFailure, callbackExecutor);
return future;
}
}

View File

@ -35,6 +35,8 @@ public interface DeviceCredentialsDao extends Dao<DeviceCredentials> {
*/
DeviceCredentials save(TenantId tenantId, DeviceCredentials deviceCredentials);
DeviceCredentials saveAndFlush(TenantId tenantId, DeviceCredentials deviceCredentials);
/**
* Find device credentials by device id.
*

View File

@ -96,7 +96,7 @@ public class DeviceCredentialsServiceImpl extends AbstractEntityService implemen
log.trace("Executing updateDeviceCredentials [{}]", deviceCredentials);
credentialsValidator.validate(deviceCredentials, id -> tenantId);
try {
return deviceCredentialsDao.save(tenantId, deviceCredentials);
return deviceCredentialsDao.saveAndFlush(tenantId, deviceCredentials);
} catch (Exception t) {
ConstraintViolationException e = extractConstraintViolationException(t).orElse(null);
if (e != null && e.getConstraintName() != null

View File

@ -30,6 +30,8 @@ public interface DeviceProfileDao extends Dao<DeviceProfile> {
DeviceProfile save(TenantId tenantId, DeviceProfile deviceProfile);
DeviceProfile saveAndFlush(TenantId tenantId, DeviceProfile deviceProfile);
PageData<DeviceProfile> findDeviceProfiles(TenantId tenantId, PageLink pageLink);
PageData<DeviceProfileInfo> findDeviceProfileInfos(TenantId tenantId, PageLink pageLink, String transportType);

View File

@ -167,7 +167,7 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D
}
DeviceProfile savedDeviceProfile;
try {
savedDeviceProfile = deviceProfileDao.save(deviceProfile.getTenantId(), deviceProfile);
savedDeviceProfile = deviceProfileDao.saveAndFlush(deviceProfile.getTenantId(), deviceProfile);
} catch (Exception t) {
ConstraintViolationException e = extractConstraintViolationException(t).orElse(null);
if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("device_profile_name_unq_key")) {

View File

@ -15,7 +15,7 @@
*/
package org.thingsboard.server.dao.sql.device;
import org.springframework.data.repository.CrudRepository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.thingsboard.server.dao.model.sql.DeviceCredentialsEntity;
import java.util.UUID;
@ -23,7 +23,7 @@ import java.util.UUID;
/**
* Created by Valerii Sosliuk on 5/6/2017.
*/
public interface DeviceCredentialsRepository extends CrudRepository<DeviceCredentialsEntity, UUID> {
public interface DeviceCredentialsRepository extends JpaRepository<DeviceCredentialsEntity, UUID> {
DeviceCredentialsEntity findByDeviceId(UUID deviceId);

View File

@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.device;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.PagingAndSortingRepository;
import org.springframework.data.repository.query.Param;
@ -26,7 +27,7 @@ import org.thingsboard.server.dao.model.sql.DeviceProfileEntity;
import java.util.UUID;
public interface DeviceProfileRepository extends PagingAndSortingRepository<DeviceProfileEntity, UUID> {
public interface DeviceProfileRepository extends JpaRepository<DeviceProfileEntity, UUID> {
@Query("SELECT new org.thingsboard.server.common.data.DeviceProfileInfo(d.id, d.name, d.image, d.defaultDashboardId, d.type, d.transportType) " +
"FROM DeviceProfileEntity d " +

View File

@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.device;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.repository.CrudRepository;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.dao.DaoUtil;
@ -46,6 +47,14 @@ public class JpaDeviceCredentialsDao extends JpaAbstractDao<DeviceCredentialsEnt
return deviceCredentialsRepository;
}
@Transactional
@Override
public DeviceCredentials saveAndFlush(TenantId tenantId, DeviceCredentials deviceCredentials) {
DeviceCredentials result = save(tenantId, deviceCredentials);
deviceCredentialsRepository.flush();
return result;
}
@Override
public DeviceCredentials findByDeviceId(TenantId tenantId, UUID deviceId) {
return DaoUtil.getData(deviceCredentialsRepository.findByDeviceId(deviceId));

View File

@ -19,6 +19,7 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.repository.CrudRepository;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.DeviceProfileInfo;
import org.thingsboard.server.common.data.DeviceTransportType;
@ -54,6 +55,14 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao<DeviceProfileE
return deviceProfileRepository.findDeviceProfileInfoById(deviceProfileId);
}
@Transactional
@Override
public DeviceProfile saveAndFlush(TenantId tenantId, DeviceProfile deviceProfile) {
DeviceProfile result = save(tenantId, deviceProfile);
deviceProfileRepository.flush();
return result;
}
@Override
public PageData<DeviceProfile> findDeviceProfiles(TenantId tenantId, PageLink pageLink) {
return DaoUtil.toPageData(