diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index aee8cc41c0..c057ce477c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -114,25 +114,23 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { entityFieldsSelection = String.format("e.id, '%s'", entityType.name()); } String latestSelection = EntityKeyMapping.buildSelections(latestSelectionMapping); - String selection = entityFieldsSelection; + String topSelection = "entities.*"; if (!StringUtils.isEmpty(latestSelection)) { - selection = entityFieldsSelection + ", " + latestSelection; + topSelection = topSelection + ", " + latestSelection; } - String fromClause = String.format("from (select %s from %s e where %s) entities %s", - selection, + String fromClause = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result", + topSelection, + entityFieldsSelection, entityTableMap.get(entityType), entityWhereClause, - latestJoins); - - if (!StringUtils.isEmpty(whereClause)) { - fromClause = String.format("%s where %s", fromClause, whereClause); - } + latestJoins, + whereClause); int totalElements = ((BigInteger)entityManager.createNativeQuery(String.format("select count(*) %s", fromClause)) .getSingleResult()).intValue(); - String dataQuery = String.format("select entities.* %s", fromClause); + String dataQuery = String.format("select * %s", fromClause); EntityDataSortOrder sortOrder = pageLink.getSortOrder(); if (sortOrder != null) { @@ -198,12 +196,18 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { private String buildWhere(List selectionMapping, List latestFiltersMapping, String searchText) { String latestFilters = EntityKeyMapping.buildQuery(latestFiltersMapping); String textSearchQuery = this.buildTextSearchQuery(selectionMapping, searchText); + String query; if (!StringUtils.isEmpty(latestFilters) && !StringUtils.isEmpty(textSearchQuery)) { - return String.join(" AND ", latestFilters, textSearchQuery); + query = String.join(" AND ", latestFilters, textSearchQuery); } else if (!StringUtils.isEmpty(latestFilters)) { - return latestFilters; + query = latestFilters; } else { - return textSearchQuery; + query = textSearchQuery; + } + if (!StringUtils.isEmpty(query)) { + return String.format("where %s", query); + } else { + return ""; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java index 914f6d4ec5..be3365fbb6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java @@ -56,20 +56,22 @@ public class EntityDataAdapter { Map timeseries = new HashMap<>(); EntityData entityData = new EntityData(entityId, latest, timeseries); for (EntityKeyMapping mapping: selectionMapping) { - EntityKey entityKey = mapping.getEntityKey(); - Object value = row[mapping.getIndex()]; - String strValue; - long ts; - if (entityKey.getType().equals(EntityKeyType.ENTITY_FIELD)) { - strValue = value != null ? value.toString() : null; - ts = System.currentTimeMillis(); - } else { - strValue = convertValue(value); - Object tsObject = row[mapping.getIndex()+1]; - ts = Long.parseLong(tsObject.toString()); + if (!mapping.isIgnore()) { + EntityKey entityKey = mapping.getEntityKey(); + Object value = row[mapping.getIndex()]; + String strValue; + long ts; + if (entityKey.getType().equals(EntityKeyType.ENTITY_FIELD)) { + strValue = value != null ? value.toString() : null; + ts = System.currentTimeMillis(); + } else { + strValue = convertValue(value); + Object tsObject = row[mapping.getIndex() + 1]; + ts = Long.parseLong(tsObject.toString()); + } + TsValue tsValue = new TsValue(ts, strValue); + latest.computeIfAbsent(entityKey.getType(), entityKeyType -> new HashMap<>()).put(entityKey.getKey(), tsValue); } - TsValue tsValue = new TsValue(ts, strValue); - latest.computeIfAbsent(entityKey.getType(), entityKeyType -> new HashMap<>()).put(entityKey.getKey(), tsValue); } return entityData; } @@ -80,8 +82,8 @@ public class EntityDataAdapter { // check number if (strVal.length() > 0) { try { - int intVal = Integer.parseInt(strVal); - return Integer.toString(intVal); + long longVal = Long.parseLong(strVal); + return Long.toString(longVal); } catch (NumberFormatException ignored) { } try { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index aa734f1166..f764c28f91 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -35,6 +35,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -65,6 +66,7 @@ public class EntityKeyMapping { private boolean isLatest; private boolean isSelection; private boolean isSortOrder; + private boolean ignore = false; private List keyFilters; private EntityKey entityKey; @@ -151,42 +153,62 @@ public class EntityKeyMapping { query.getKeyFilters().stream().collect(Collectors.groupingBy(KeyFilter::getKey)) : Collections.emptyMap(); EntityDataSortOrder sortOrder = query.getPageLink().getSortOrder(); EntityKey sortOrderKey = sortOrder != null ? sortOrder.getKey() : null; - EntityKeyMapping sortOrderMapping = null; int index = 2; - List mappings = new ArrayList<>(); - for (EntityKey entityField : entityFields) { - EntityKeyMapping mapping = new EntityKeyMapping(); - mapping.setIndex(index); - mapping.setAlias(String.format("alias%s", index)); - mapping.setKeyFilters(filters.remove(entityField)); - mapping.setLatest(false); - mapping.setSelection(true); - mapping.setEntityKey(entityField); - if (entityField.equals(sortOrderKey)) { - mapping.setSortOrder(true); - sortOrderMapping = mapping; + List entityFieldsMappings = entityFields.stream().map( + key -> { + EntityKeyMapping mapping = new EntityKeyMapping(); + mapping.setLatest(false); + mapping.setSelection(true); + mapping.setEntityKey(key); + return mapping; + } + ).collect(Collectors.toList()); + List latestMappings = latestValues.stream().map( + key -> { + EntityKeyMapping mapping = new EntityKeyMapping(); + mapping.setLatest(true); + mapping.setSelection(true); + mapping.setEntityKey(key); + return mapping; + } + ).collect(Collectors.toList()); + if (sortOrderKey != null) { + Optional existing; + if (sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD)) { + existing = + entityFieldsMappings.stream().filter(mapping -> mapping.entityKey.equals(sortOrderKey)).findFirst(); } else { - mapping.setSortOrder(false); + existing = + latestMappings.stream().filter(mapping -> mapping.entityKey.equals(sortOrderKey)).findFirst(); + } + if (existing.isPresent()) { + existing.get().setSortOrder(true); + } else { + EntityKeyMapping sortOrderMapping = new EntityKeyMapping(); + sortOrderMapping.setLatest(!sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD)); + sortOrderMapping.setSelection(true); + sortOrderMapping.setEntityKey(sortOrderKey); + sortOrderMapping.setSortOrder(true); + sortOrderMapping.setIgnore(true); + if (sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD)) { + entityFieldsMappings.add(sortOrderMapping); + } else { + latestMappings.add(sortOrderMapping); + } } - mappings.add(mapping); - index++; } - for (EntityKey latestField : latestValues) { - EntityKeyMapping mapping = new EntityKeyMapping(); + List mappings = new ArrayList<>(); + mappings.addAll(entityFieldsMappings); + mappings.addAll(latestMappings); + for (EntityKeyMapping mapping : mappings) { mapping.setIndex(index); mapping.setAlias(String.format("alias%s", index)); - mapping.setKeyFilters(filters.remove(latestField)); - mapping.setLatest(true); - mapping.setSelection(true); - mapping.setEntityKey(latestField); - if (latestField.equals(sortOrderKey)) { - mapping.setSortOrder(true); - sortOrderMapping = mapping; + mapping.setKeyFilters(filters.remove(mapping.entityKey)); + if (mapping.getEntityKey().getType().equals(EntityKeyType.ENTITY_FIELD)) { + index++; } else { - mapping.setSortOrder(false); + index +=2; } - mappings.add(mapping); - index +=2; } if (!filters.isEmpty()) { for (EntityKey filterField : filters.keySet()) { @@ -201,37 +223,28 @@ public class EntityKeyMapping { index +=1; } } - if (sortOrderKey != null && sortOrderMapping == null) { - sortOrderMapping = new EntityKeyMapping(); - sortOrderMapping.setIndex(index); - sortOrderMapping.setAlias(String.format("alias%s", index)); - sortOrderMapping.setLatest(!sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD)); - sortOrderMapping.setSelection(true); - sortOrderMapping.setEntityKey(sortOrderKey); - sortOrderMapping.setSortOrder(true); - mappings.add(sortOrderMapping); - } + return mappings; } private String buildAttributeSelection() { String attrValAlias = getValueAlias(); String attrTsAlias = getTsAlias(); - String attrTsSelection = String.format("%s.last_update_ts as %s", alias, attrTsAlias); String attrValSelection = - String.format("coalesce(cast(%s.bool_v as varchar), '') || " + + String.format("(coalesce(cast(%s.bool_v as varchar), '') || " + "coalesce(%s.str_v, '') || " + "coalesce(cast(%s.long_v as varchar), '') || " + "coalesce(cast(%s.dbl_v as varchar), '') || " + "coalesce(cast(%s.json_v as varchar), '')) as %s", alias, alias, alias, alias, alias, attrValAlias); - return String.join(", ", attrTsSelection, attrValSelection); + String attrTsSelection = String.format("%s.last_update_ts as %s", alias, attrTsAlias); + return String.join(", ", attrValSelection, attrTsSelection); } private String buildTimeseriesSelection() { // TODO: String attrValAlias = getValueAlias(); String attrTsAlias = getTsAlias(); - return String.format("(select 1) as %s, (select '') as %s", attrTsAlias, attrValAlias); + return String.format("(select '') as %s, (select 1) as %s", attrValAlias, attrTsAlias); } private String buildKeyQuery(String alias, KeyFilter keyFilter) { diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index cc0c77da2a..220aaf32e7 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -15,10 +15,14 @@ */ package org.thingsboard.server.dao.service; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; @@ -26,6 +30,10 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.DeviceTypeFilter; import org.thingsboard.server.common.data.query.EntityCountQuery; @@ -37,14 +45,21 @@ import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.NumericFilterPredicate; +import org.thingsboard.server.dao.attributes.AttributesService; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; public abstract class BaseEntityServiceTest extends AbstractServiceTest { + @Autowired + private AttributesService attributesService; + private TenantId tenantId; @Before @@ -105,7 +120,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } @Test - public void testFindEntityDataByQuery() { + public void testSimpleFindEntityDataByQuery() { List devices = new ArrayList<>(); for (int i = 0; i < 97; i++) { Device device = new Device(); @@ -135,7 +150,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { Assert.assertEquals(10, data.getData().size()); List loadedEntities = new ArrayList<>(data.getData()); - while(data.hasNext()) { + while (data.hasNext()) { query = query.next(); data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); loadedEntities.addAll(data.getData()); @@ -162,5 +177,96 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); Assert.assertEquals(11, data.getTotalElements()); Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + + deviceService.deleteDevicesByTenantId(tenantId); + } + + @Test + public void testFindEntityDataByQueryWithAttributes() throws ExecutionException, InterruptedException { + + List devices = new ArrayList<>(); + List temperatures = new ArrayList<>(); + List highTemperatures = new ArrayList<>(); + for (int i=0;i<67;i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device"+i); + device.setType("default"); + device.setLabel("testLabel"+(int)(Math.random()*1000)); + devices.add(deviceService.saveDevice(device)); + long temperature = (long)(Math.random()*100); + temperatures.add(temperature); + if (temperature > 45) { + highTemperatures.add(temperature); + } + } + + List>> attributeFutures = new ArrayList<>(); + for (int i=0;i entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); + + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); + PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + + List loadedEntities = new ArrayList<>(data.getData()); + while (data.hasNext()) { + query = query.next(); + data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + loadedEntities.addAll(data.getData()); + } + Assert.assertEquals(67, loadedEntities.size()); + List loadedTemperatures = loadedEntities.stream().map(entityData -> + entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); + List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); + Assert.assertEquals(deviceTemperatures, loadedTemperatures); + + pageLink = new EntityDataPageLink(10, 0, null, sortOrder); + KeyFilter highTemperatureFilter = new KeyFilter(); + highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); + NumericFilterPredicate predicate = new NumericFilterPredicate(); + predicate.setValue(45); + predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + highTemperatureFilter.setPredicate(predicate); + List keyFilters = Collections.singletonList(highTemperatureFilter); + + query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + + data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + + loadedEntities = new ArrayList<>(data.getData()); + while (data.hasNext()) { + query = query.next(); + data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + loadedEntities.addAll(data.getData()); + } + Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); + + List loadedHighTemperatures = loadedEntities.stream().map(entityData -> + entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); + List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); + + Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + + deviceService.deleteDevicesByTenantId(tenantId); + } + + private ListenableFuture> saveLongAttribute(EntityId entityId, String key, long value, String scope) { + KvEntry attrValue = new LongDataEntry(key, value); + AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); + return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); } }