diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 59475213b3..3cdcf5e874 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -95,7 +95,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityDataQuery(query); - if (isOptimizationExcluded(query)) { + if (!isValidForOptimization(query)) { return this.entityQueryDao.findEntityDataByQuery(tenantId, customerId, query); } @@ -204,25 +204,25 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe } } - private boolean isOptimizationExcluded(EntityDataQuery query) { + private boolean isValidForOptimization(EntityDataQuery query) { if (StringUtils.isNotEmpty(query.getPageLink().getTextSearch())) { - return true; + return false; } if (EXCLUDED_TYPES_FROM_OPTIMIZATION.contains(query.getEntityFilter().getType())) { - return true; + return false; } if ((query.getEntityFields() == null || query.getEntityFields().isEmpty()) && (query.getLatestValues() == null || query.getLatestValues().isEmpty())) { - return true; + return false; } - Set entityKeys = new HashSet<>(Optional.ofNullable(query.getKeyFilters()).orElse(Collections.emptyList()).stream().map(KeyFilter::getKey).toList()); + Set filteringKeys = new HashSet<>(Optional.ofNullable(query.getKeyFilters()).orElse(Collections.emptyList()).stream().map(KeyFilter::getKey).toList()); Set entityFields = new HashSet<>(Optional.ofNullable(query.getEntityFields()).orElse(Collections.emptyList())); Set latestValues = new HashSet<>(Optional.ofNullable(query.getLatestValues()).orElse(Collections.emptyList())); - return entityKeys.equals(entityFields) && entityKeys.equals(latestValues); + return !(filteringKeys.containsAll(entityFields) && filteringKeys.containsAll(latestValues)); } private PageData findEntityIdsByFilterAndSorterColumns(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java index 58ee94f8b7..f1efe9f730 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java @@ -98,6 +98,7 @@ import java.util.stream.Stream; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; +import static org.thingsboard.server.common.data.query.EntityKeyType.ATTRIBUTE; import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD; @Slf4j @@ -2114,37 +2115,40 @@ public class EntityServiceTest extends AbstractServiceTest { } @Test - public void testFindEntityQueryWith_3000_pageSize() throws InterruptedException { + public void testFindEntityQuery_for_5000_devices_with_3000_pageSize() { int pageSize = 3000; + int expectedDevicesSize = 4000; + int unexpectedDevicesSize = 1000; - List devices = new ArrayList<>(); - - for (int i = 0; i < pageSize; i++) { + for (int i = 0; i < expectedDevicesSize + unexpectedDevicesSize; i++) { Device device = new Device(); device.setTenantId(tenantId); - device.setName("Device_" + i); + if (i < expectedDevicesSize) { + device.setName("Device_" + i); // match deviceNameFilter 'D%' + } else { + device.setName("Test_" + i); // does not match deviceNameFilter 'D%' + } device.setType("default"); device.setLabel("testLabel" + (int) (Math.random() * 1000)); - devices.add(deviceService.saveDevice(device)); - //TO make sure devices have different created time - Thread.sleep(1); + Device savedDevice = deviceService.saveDevice(device); + + attributesService.save(tenantId, savedDevice.getId(), AttributeScope.CLIENT_SCOPE, + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("telemetry", (long) i))); } DeviceTypeFilter filter = new DeviceTypeFilter(); filter.setDeviceTypes(List.of("default")); filter.setDeviceNameFilter("D%"); - EntityDataSortOrder sortOrder = new EntityDataSortOrder(new EntityKey(ENTITY_FIELD, "name"), EntityDataSortOrder.Direction.DESC); + EntityDataSortOrder sortOrder = new EntityDataSortOrder(new EntityKey(ATTRIBUTE, "telemetry"), EntityDataSortOrder.Direction.DESC); List deviceTypeFilters = createStringKeyFilters("type", ENTITY_FIELD, StringFilterPredicate.StringOperation.EQUAL, "default"); - KeyFilter createdTimeFilter = createNumericKeyFilter("createdTime", ENTITY_FIELD, NumericFilterPredicate.NumericOperation.GREATER, 1L); - List createdTimeFilters = Collections.singletonList(createdTimeFilter); + List attributeFilters = Collections.singletonList(createNumericKeyFilter("telemetry", ATTRIBUTE, NumericFilterPredicate.NumericOperation.LESS, expectedDevicesSize)); List nameFilters = createStringKeyFilters("name", ENTITY_FIELD, StringFilterPredicate.StringOperation.CONTAINS, "Device"); - List entityFields = Arrays.asList(new EntityKey(ENTITY_FIELD, "name"), - new EntityKey(ENTITY_FIELD, "type")); + List entityFields = Arrays.asList(new EntityKey(ENTITY_FIELD, "name"), new EntityKey(ENTITY_FIELD, "type")); // 1. Device type filters: @@ -2158,11 +2162,14 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataQuery optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, deviceTypeFilters); PageData optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); List loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); - Assert.assertEquals(devices.size(), loadedEntities.size()); + Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); + loadedEntities = getLoadedEntities(originalData, originalQuery); + Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); + Assert.assertEquals(pageSize, optimizedData.getData().size()); - for (int i = 0; i < devices.size(); i++) { - var originalElement = originalData.getData().get(i); - var optimizedElement = optimizedData.getData().get(i); + for (int i = 0; i < pageSize; i++) { + EntityData originalElement = originalData.getData().get(i); + EntityData optimizedElement = optimizedData.getData().get(i); Assert.assertEquals(originalElement.getEntityId(), optimizedElement.getEntityId()); originalElement.getLatest().get(ENTITY_FIELD).forEach((key, value) -> { Assert.assertEquals(value.getValue(), optimizedElement.getLatest().get(EntityKeyType.ENTITY_FIELD).get(key).getValue()); @@ -2176,19 +2183,22 @@ public class EntityServiceTest extends AbstractServiceTest { // query with textSearch - optimization is not performing originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder); - originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, createdTimeFilters); + originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, attributeFilters); originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery); // query without textSearch - optimization is performing optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder); - optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, createdTimeFilters); + optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, attributeFilters); optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); - Assert.assertEquals(devices.size(), loadedEntities.size()); + Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); + loadedEntities = getLoadedEntities(originalData, originalQuery); + Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); + Assert.assertEquals(pageSize, optimizedData.getData().size()); - for (int i = 0; i < devices.size(); i++) { - var originalElement = originalData.getData().get(i); - var optimizedElement = optimizedData.getData().get(i); + for (int i = 0; i < pageSize; i++) { + EntityData originalElement = originalData.getData().get(i); + EntityData optimizedElement = optimizedData.getData().get(i); Assert.assertEquals(originalElement.getEntityId(), optimizedElement.getEntityId()); originalElement.getLatest().get(ENTITY_FIELD).forEach((key, value) -> { Assert.assertEquals(value.getValue(), optimizedElement.getLatest().get(EntityKeyType.ENTITY_FIELD).get(key).getValue()); @@ -2210,11 +2220,14 @@ public class EntityServiceTest extends AbstractServiceTest { optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, nameFilters); optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); - Assert.assertEquals(devices.size(), loadedEntities.size()); + Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); + loadedEntities = getLoadedEntities(originalData, originalQuery); + Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); + Assert.assertEquals(pageSize, optimizedData.getData().size()); - for (int i = 0; i < devices.size(); i++) { - var originalElement = originalData.getData().get(i); - var optimizedElement = optimizedData.getData().get(i); + for (int i = 0; i < pageSize; i++) { + EntityData originalElement = originalData.getData().get(i); + EntityData optimizedElement = optimizedData.getData().get(i); Assert.assertEquals(originalElement.getEntityId(), optimizedElement.getEntityId()); originalElement.getLatest().get(ENTITY_FIELD).forEach((key, value) -> { Assert.assertEquals(value.getValue(), optimizedElement.getLatest().get(EntityKeyType.ENTITY_FIELD).get(key).getValue());