dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Micro improvement: fix logic for filteringKeys containsAll latestValues && entityFields. Improve test

Browse Source
This commit is contained in:
Andrii Landiak 2024-04-04 16:00:19 +03:00
parent 74fed02b8b
commit 0252ab851a
2 changed files with 47 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java
View File

@ -95,7 +95,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id);
validateEntityDataQuery(query); validateEntityDataQuery(query);
if (isOptimizationExcluded(query)) { if (!isValidForOptimization(query)) {
return this.entityQueryDao.findEntityDataByQuery(tenantId, customerId, query); return this.entityQueryDao.findEntityDataByQuery(tenantId, customerId, query);
} }
@ -204,25 +204,25 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
} }
} }
private boolean isOptimizationExcluded(EntityDataQuery query) { private boolean isValidForOptimization(EntityDataQuery query) {
if (StringUtils.isNotEmpty(query.getPageLink().getTextSearch())) { if (StringUtils.isNotEmpty(query.getPageLink().getTextSearch())) {
return true; return false;
} }
if (EXCLUDED_TYPES_FROM_OPTIMIZATION.contains(query.getEntityFilter().getType())) { if (EXCLUDED_TYPES_FROM_OPTIMIZATION.contains(query.getEntityFilter().getType())) {
return true; return false;
} }
if ((query.getEntityFields() == null || query.getEntityFields().isEmpty()) && if ((query.getEntityFields() == null || query.getEntityFields().isEmpty()) &&
(query.getLatestValues() == null || query.getLatestValues().isEmpty())) { (query.getLatestValues() == null || query.getLatestValues().isEmpty())) {
return true; return false;
} }
Set<EntityKey> entityKeys = new HashSet<>(Optional.ofNullable(query.getKeyFilters()).orElse(Collections.emptyList()).stream().map(KeyFilter::getKey).toList()); Set<EntityKey> filteringKeys = new HashSet<>(Optional.ofNullable(query.getKeyFilters()).orElse(Collections.emptyList()).stream().map(KeyFilter::getKey).toList());
Set<EntityKey> entityFields = new HashSet<>(Optional.ofNullable(query.getEntityFields()).orElse(Collections.emptyList())); Set<EntityKey> entityFields = new HashSet<>(Optional.ofNullable(query.getEntityFields()).orElse(Collections.emptyList()));
Set<EntityKey> latestValues = new HashSet<>(Optional.ofNullable(query.getLatestValues()).orElse(Collections.emptyList())); Set<EntityKey> latestValues = new HashSet<>(Optional.ofNullable(query.getLatestValues()).orElse(Collections.emptyList()));
return entityKeys.equals(entityFields) && entityKeys.equals(latestValues); return !(filteringKeys.containsAll(entityFields) && filteringKeys.containsAll(latestValues));
} }
private PageData<EntityData> findEntityIdsByFilterAndSorterColumns(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { private PageData<EntityData> findEntityIdsByFilterAndSorterColumns(TenantId tenantId, CustomerId customerId, EntityDataQuery query) {

67
dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java
View File

@ -98,6 +98,7 @@ import java.util.stream.Stream;
import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.thingsboard.server.common.data.query.EntityKeyType.ATTRIBUTE;
import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD; import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD;
@Slf4j @Slf4j
@ -2114,37 +2115,40 @@ public class EntityServiceTest extends AbstractServiceTest {
} }
@Test @Test
public void testFindEntityQueryWith_3000_pageSize() throws InterruptedException { public void testFindEntityQuery_for_5000_devices_with_3000_pageSize() {
int pageSize = 3000; int pageSize = 3000;
int expectedDevicesSize = 4000;
int unexpectedDevicesSize = 1000;
List<Device> devices = new ArrayList<>(); for (int i = 0; i < expectedDevicesSize + unexpectedDevicesSize; i++) {
for (int i = 0; i < pageSize; i++) {
Device device = new Device(); Device device = new Device();
device.setTenantId(tenantId); device.setTenantId(tenantId);
device.setName("Device_" + i); if (i < expectedDevicesSize) {
device.setName("Device_" + i); // match deviceNameFilter 'D%'
} else {
device.setName("Test_" + i); // does not match deviceNameFilter 'D%'
}
device.setType("default"); device.setType("default");
device.setLabel("testLabel" + (int) (Math.random() * 1000)); device.setLabel("testLabel" + (int) (Math.random() * 1000));
devices.add(deviceService.saveDevice(device)); Device savedDevice = deviceService.saveDevice(device);
//TO make sure devices have different created time
Thread.sleep(1); attributesService.save(tenantId, savedDevice.getId(), AttributeScope.CLIENT_SCOPE,
new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("telemetry", (long) i)));
} }
DeviceTypeFilter filter = new DeviceTypeFilter(); DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceTypes(List.of("default")); filter.setDeviceTypes(List.of("default"));
filter.setDeviceNameFilter("D%"); filter.setDeviceNameFilter("D%");
EntityDataSortOrder sortOrder = new EntityDataSortOrder(new EntityKey(ENTITY_FIELD, "name"), EntityDataSortOrder.Direction.DESC); EntityDataSortOrder sortOrder = new EntityDataSortOrder(new EntityKey(ATTRIBUTE, "telemetry"), EntityDataSortOrder.Direction.DESC);
List<KeyFilter> deviceTypeFilters = createStringKeyFilters("type", ENTITY_FIELD, StringFilterPredicate.StringOperation.EQUAL, "default"); List<KeyFilter> deviceTypeFilters = createStringKeyFilters("type", ENTITY_FIELD, StringFilterPredicate.StringOperation.EQUAL, "default");
KeyFilter createdTimeFilter = createNumericKeyFilter("createdTime", ENTITY_FIELD, NumericFilterPredicate.NumericOperation.GREATER, 1L); List<KeyFilter> attributeFilters = Collections.singletonList(createNumericKeyFilter("telemetry", ATTRIBUTE, NumericFilterPredicate.NumericOperation.LESS, expectedDevicesSize));
List<KeyFilter> createdTimeFilters = Collections.singletonList(createdTimeFilter);
List<KeyFilter> nameFilters = createStringKeyFilters("name", ENTITY_FIELD, StringFilterPredicate.StringOperation.CONTAINS, "Device"); List<KeyFilter> nameFilters = createStringKeyFilters("name", ENTITY_FIELD, StringFilterPredicate.StringOperation.CONTAINS, "Device");
List<EntityKey> entityFields = Arrays.asList(new EntityKey(ENTITY_FIELD, "name"), List<EntityKey> entityFields = Arrays.asList(new EntityKey(ENTITY_FIELD, "name"), new EntityKey(ENTITY_FIELD, "type"));
new EntityKey(ENTITY_FIELD, "type"));
// 1. Device type filters: // 1. Device type filters:
@ -2158,11 +2162,14 @@ public class EntityServiceTest extends AbstractServiceTest {
EntityDataQuery optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, deviceTypeFilters); EntityDataQuery optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, deviceTypeFilters);
PageData<EntityData> optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); PageData<EntityData> optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery);
List<EntityData> loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); List<EntityData> loadedEntities = getLoadedEntities(optimizedData, optimizedQuery);
Assert.assertEquals(devices.size(), loadedEntities.size()); Assert.assertEquals(expectedDevicesSize, loadedEntities.size());
loadedEntities = getLoadedEntities(originalData, originalQuery);
Assert.assertEquals(expectedDevicesSize, loadedEntities.size());
Assert.assertEquals(pageSize, optimizedData.getData().size());
for (int i = 0; i < devices.size(); i++) { for (int i = 0; i < pageSize; i++) {
var originalElement = originalData.getData().get(i); EntityData originalElement = originalData.getData().get(i);
var optimizedElement = optimizedData.getData().get(i); EntityData optimizedElement = optimizedData.getData().get(i);
Assert.assertEquals(originalElement.getEntityId(), optimizedElement.getEntityId()); Assert.assertEquals(originalElement.getEntityId(), optimizedElement.getEntityId());
originalElement.getLatest().get(ENTITY_FIELD).forEach((key, value) -> { originalElement.getLatest().get(ENTITY_FIELD).forEach((key, value) -> {
Assert.assertEquals(value.getValue(), optimizedElement.getLatest().get(EntityKeyType.ENTITY_FIELD).get(key).getValue()); Assert.assertEquals(value.getValue(), optimizedElement.getLatest().get(EntityKeyType.ENTITY_FIELD).get(key).getValue());
@ -2176,19 +2183,22 @@ public class EntityServiceTest extends AbstractServiceTest {
// query with textSearch - optimization is not performing // query with textSearch - optimization is not performing
originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder); originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder);
originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, createdTimeFilters); originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, attributeFilters);
originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery); originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery);
// query without textSearch - optimization is performing // query without textSearch - optimization is performing
optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder); optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder);
optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, createdTimeFilters); optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, attributeFilters);
optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery);
loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); loadedEntities = getLoadedEntities(optimizedData, optimizedQuery);
Assert.assertEquals(devices.size(), loadedEntities.size()); Assert.assertEquals(expectedDevicesSize, loadedEntities.size());
loadedEntities = getLoadedEntities(originalData, originalQuery);
Assert.assertEquals(expectedDevicesSize, loadedEntities.size());
Assert.assertEquals(pageSize, optimizedData.getData().size());
for (int i = 0; i < devices.size(); i++) { for (int i = 0; i < pageSize; i++) {
var originalElement = originalData.getData().get(i); EntityData originalElement = originalData.getData().get(i);
var optimizedElement = optimizedData.getData().get(i); EntityData optimizedElement = optimizedData.getData().get(i);
Assert.assertEquals(originalElement.getEntityId(), optimizedElement.getEntityId()); Assert.assertEquals(originalElement.getEntityId(), optimizedElement.getEntityId());
originalElement.getLatest().get(ENTITY_FIELD).forEach((key, value) -> { originalElement.getLatest().get(ENTITY_FIELD).forEach((key, value) -> {
Assert.assertEquals(value.getValue(), optimizedElement.getLatest().get(EntityKeyType.ENTITY_FIELD).get(key).getValue()); Assert.assertEquals(value.getValue(), optimizedElement.getLatest().get(EntityKeyType.ENTITY_FIELD).get(key).getValue());
@ -2210,11 +2220,14 @@ public class EntityServiceTest extends AbstractServiceTest {
optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, nameFilters); optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, nameFilters);
optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery);
loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); loadedEntities = getLoadedEntities(optimizedData, optimizedQuery);
Assert.assertEquals(devices.size(), loadedEntities.size()); Assert.assertEquals(expectedDevicesSize, loadedEntities.size());
loadedEntities = getLoadedEntities(originalData, originalQuery);
Assert.assertEquals(expectedDevicesSize, loadedEntities.size());
Assert.assertEquals(pageSize, optimizedData.getData().size());
for (int i = 0; i < devices.size(); i++) { for (int i = 0; i < pageSize; i++) {
var originalElement = originalData.getData().get(i); EntityData originalElement = originalData.getData().get(i);
var optimizedElement = optimizedData.getData().get(i); EntityData optimizedElement = optimizedData.getData().get(i);
Assert.assertEquals(originalElement.getEntityId(), optimizedElement.getEntityId()); Assert.assertEquals(originalElement.getEntityId(), optimizedElement.getEntityId());
originalElement.getLatest().get(ENTITY_FIELD).forEach((key, value) -> { originalElement.getLatest().get(ENTITY_FIELD).forEach((key, value) -> {
Assert.assertEquals(value.getValue(), optimizedElement.getLatest().get(EntityKeyType.ENTITY_FIELD).get(key).getValue()); Assert.assertEquals(value.getValue(), optimizedElement.getLatest().get(EntityKeyType.ENTITY_FIELD).get(key).getValue());