Improve data query

dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java

@@ -114,25 +114,23 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
             entityFieldsSelection = String.format("e.id, '%s'", entityType.name());
         }
         String latestSelection = EntityKeyMapping.buildSelections(latestSelectionMapping);
-        String selection = entityFieldsSelection;
+        String topSelection = "entities.*";
         if (!StringUtils.isEmpty(latestSelection)) {
-            selection = entityFieldsSelection + ", " + latestSelection;
+            topSelection = topSelection + ", " + latestSelection;
         }
 
-        String fromClause = String.format("from (select %s from %s e where %s) entities %s",
+        String fromClause = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result",
-                selection,
+                topSelection,
+                entityFieldsSelection,
                 entityTableMap.get(entityType),
                 entityWhereClause,
-                latestJoins);
+                latestJoins,
-
+                whereClause);
-        if (!StringUtils.isEmpty(whereClause)) {
-            fromClause = String.format("%s where %s", fromClause, whereClause);
-        }
 
         int totalElements = ((BigInteger)entityManager.createNativeQuery(String.format("select count(*) %s", fromClause))
                 .getSingleResult()).intValue();
 
-        String dataQuery = String.format("select entities.* %s", fromClause);
+        String dataQuery = String.format("select * %s", fromClause);
 
         EntityDataSortOrder sortOrder = pageLink.getSortOrder();
         if (sortOrder != null) {
@@ -198,12 +196,18 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
     private String buildWhere(List<EntityKeyMapping> selectionMapping, List<EntityKeyMapping> latestFiltersMapping, String searchText) {
         String latestFilters = EntityKeyMapping.buildQuery(latestFiltersMapping);
         String textSearchQuery = this.buildTextSearchQuery(selectionMapping, searchText);
+        String query;
         if (!StringUtils.isEmpty(latestFilters) && !StringUtils.isEmpty(textSearchQuery)) {
-            return String.join(" AND ", latestFilters, textSearchQuery);
+            query = String.join(" AND ", latestFilters, textSearchQuery);
         } else if (!StringUtils.isEmpty(latestFilters)) {
-            return latestFilters;
+            query = latestFilters;
         } else {
-            return textSearchQuery;
+            query = textSearchQuery;
+        }
+        if (!StringUtils.isEmpty(query)) {
+            return String.format("where %s", query);
+        } else {
+            return "";
         }
     }
 

dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java

@@ -56,20 +56,22 @@ public class EntityDataAdapter {
         Map<String, TsValue[]> timeseries = new HashMap<>();
         EntityData entityData = new EntityData(entityId, latest, timeseries);
         for (EntityKeyMapping mapping: selectionMapping) {
-            EntityKey entityKey = mapping.getEntityKey();
+            if (!mapping.isIgnore()) {
-            Object value = row[mapping.getIndex()];
+                EntityKey entityKey = mapping.getEntityKey();
-            String strValue;
+                Object value = row[mapping.getIndex()];
-            long ts;
+                String strValue;
-            if (entityKey.getType().equals(EntityKeyType.ENTITY_FIELD)) {
+                long ts;
-                strValue = value != null ? value.toString() : null;
+                if (entityKey.getType().equals(EntityKeyType.ENTITY_FIELD)) {
-                ts = System.currentTimeMillis();
+                    strValue = value != null ? value.toString() : null;
-            } else {
+                    ts = System.currentTimeMillis();
-                strValue = convertValue(value);
+                } else {
-                Object tsObject = row[mapping.getIndex()+1];
+                    strValue = convertValue(value);
-                ts = Long.parseLong(tsObject.toString());
+                    Object tsObject = row[mapping.getIndex() + 1];
+                    ts = Long.parseLong(tsObject.toString());
+                }
+                TsValue tsValue = new TsValue(ts, strValue);
+                latest.computeIfAbsent(entityKey.getType(), entityKeyType -> new HashMap<>()).put(entityKey.getKey(), tsValue);
             }
-            TsValue tsValue = new TsValue(ts, strValue);
-            latest.computeIfAbsent(entityKey.getType(), entityKeyType -> new HashMap<>()).put(entityKey.getKey(), tsValue);
         }
         return entityData;
     }
@@ -80,8 +82,8 @@ public class EntityDataAdapter {
             // check number
             if (strVal.length() > 0) {
                 try {
-                    int intVal = Integer.parseInt(strVal);
+                    long longVal = Long.parseLong(strVal);
-                    return Integer.toString(intVal);
+                    return Long.toString(longVal);
                 } catch (NumberFormatException ignored) {
                 }
                 try {

dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java

@@ -35,6 +35,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -65,6 +66,7 @@ public class EntityKeyMapping {
     private boolean isLatest;
     private boolean isSelection;
     private boolean isSortOrder;
+    private boolean ignore = false;
     private List<KeyFilter> keyFilters;
     private EntityKey entityKey;
 
@@ -151,42 +153,62 @@ public class EntityKeyMapping {
                         query.getKeyFilters().stream().collect(Collectors.groupingBy(KeyFilter::getKey)) : Collections.emptyMap();
         EntityDataSortOrder sortOrder = query.getPageLink().getSortOrder();
         EntityKey sortOrderKey = sortOrder != null ? sortOrder.getKey() : null;
-        EntityKeyMapping sortOrderMapping = null;
         int index = 2;
-        List<EntityKeyMapping> mappings = new ArrayList<>();
+        List<EntityKeyMapping> entityFieldsMappings = entityFields.stream().map(
-        for (EntityKey entityField : entityFields) {
+                key -> {
-            EntityKeyMapping mapping = new EntityKeyMapping();
+                    EntityKeyMapping mapping = new EntityKeyMapping();
-            mapping.setIndex(index);
+                    mapping.setLatest(false);
-            mapping.setAlias(String.format("alias%s", index));
+                    mapping.setSelection(true);
-            mapping.setKeyFilters(filters.remove(entityField));
+                    mapping.setEntityKey(key);
-            mapping.setLatest(false);
+                    return mapping;
-            mapping.setSelection(true);
+                }
-            mapping.setEntityKey(entityField);
+        ).collect(Collectors.toList());
-            if (entityField.equals(sortOrderKey)) {
+        List<EntityKeyMapping> latestMappings = latestValues.stream().map(
-                mapping.setSortOrder(true);
+                key -> {
-                sortOrderMapping = mapping;
+                    EntityKeyMapping mapping = new EntityKeyMapping();
+                    mapping.setLatest(true);
+                    mapping.setSelection(true);
+                    mapping.setEntityKey(key);
+                    return mapping;
+                }
+        ).collect(Collectors.toList());
+        if (sortOrderKey != null) {
+            Optional<EntityKeyMapping> existing;
+            if (sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD)) {
+                existing =
+                        entityFieldsMappings.stream().filter(mapping -> mapping.entityKey.equals(sortOrderKey)).findFirst();
             } else {
-                mapping.setSortOrder(false);
+                existing =
+                        latestMappings.stream().filter(mapping -> mapping.entityKey.equals(sortOrderKey)).findFirst();
+            }
+            if (existing.isPresent()) {
+                existing.get().setSortOrder(true);
+            } else {
+                EntityKeyMapping sortOrderMapping = new EntityKeyMapping();
+                sortOrderMapping.setLatest(!sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD));
+                sortOrderMapping.setSelection(true);
+                sortOrderMapping.setEntityKey(sortOrderKey);
+                sortOrderMapping.setSortOrder(true);
+                sortOrderMapping.setIgnore(true);
+                if (sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD)) {
+                    entityFieldsMappings.add(sortOrderMapping);
+                } else {
+                    latestMappings.add(sortOrderMapping);
+                }
             }
-            mappings.add(mapping);
-            index++;
         }
-        for (EntityKey latestField : latestValues) {
+        List<EntityKeyMapping> mappings = new ArrayList<>();
-            EntityKeyMapping mapping = new EntityKeyMapping();
+        mappings.addAll(entityFieldsMappings);
+        mappings.addAll(latestMappings);
+        for (EntityKeyMapping mapping : mappings) {
             mapping.setIndex(index);
             mapping.setAlias(String.format("alias%s", index));
-            mapping.setKeyFilters(filters.remove(latestField));
+            mapping.setKeyFilters(filters.remove(mapping.entityKey));
-            mapping.setLatest(true);
+            if (mapping.getEntityKey().getType().equals(EntityKeyType.ENTITY_FIELD)) {
-            mapping.setSelection(true);
+                index++;
-            mapping.setEntityKey(latestField);
-            if (latestField.equals(sortOrderKey)) {
-                mapping.setSortOrder(true);
-                sortOrderMapping = mapping;
             } else {
-                mapping.setSortOrder(false);
+                index +=2;
             }
-            mappings.add(mapping);
-            index +=2;
         }
         if (!filters.isEmpty()) {
             for (EntityKey filterField : filters.keySet()) {
@@ -201,37 +223,28 @@ public class EntityKeyMapping {
                 index +=1;
             }
         }
-        if (sortOrderKey != null && sortOrderMapping == null) {
+
-            sortOrderMapping = new EntityKeyMapping();
-            sortOrderMapping.setIndex(index);
-            sortOrderMapping.setAlias(String.format("alias%s", index));
-            sortOrderMapping.setLatest(!sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD));
-            sortOrderMapping.setSelection(true);
-            sortOrderMapping.setEntityKey(sortOrderKey);
-            sortOrderMapping.setSortOrder(true);
-            mappings.add(sortOrderMapping);
-        }
         return mappings;
     }
 
     private String buildAttributeSelection() {
         String attrValAlias = getValueAlias();
         String attrTsAlias = getTsAlias();
-        String attrTsSelection = String.format("%s.last_update_ts as %s", alias, attrTsAlias);
         String attrValSelection =
-                String.format("coalesce(cast(%s.bool_v as varchar), '') || " +
+                String.format("(coalesce(cast(%s.bool_v as varchar), '') || " +
                         "coalesce(%s.str_v, '') || " +
                         "coalesce(cast(%s.long_v as varchar), '') || " +
                         "coalesce(cast(%s.dbl_v as varchar), '') || " +
                         "coalesce(cast(%s.json_v as varchar), '')) as %s", alias, alias, alias, alias, alias, attrValAlias);
-        return String.join(", ", attrTsSelection, attrValSelection);
+        String attrTsSelection = String.format("%s.last_update_ts as %s", alias, attrTsAlias);
+        return String.join(", ", attrValSelection, attrTsSelection);
     }
 
     private String buildTimeseriesSelection() {
         // TODO:
         String attrValAlias = getValueAlias();
         String attrTsAlias = getTsAlias();
-        return String.format("(select 1) as %s, (select '') as %s", attrTsAlias, attrValAlias);
+        return String.format("(select '') as %s, (select 1) as %s", attrValAlias, attrTsAlias);
     }
 
     private String buildKeyQuery(String alias, KeyFilter keyFilter) {

dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java

@@ -15,10 +15,14 @@
  */
 package org.thingsboard.server.dao.service;
 
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.thingsboard.server.common.data.DataConstants;
 import org.thingsboard.server.common.data.Device;
 import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.Tenant;
@@ -26,6 +30,10 @@ import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.DeviceId;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.kv.AttributeKvEntry;
+import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
+import org.thingsboard.server.common.data.kv.KvEntry;
+import org.thingsboard.server.common.data.kv.LongDataEntry;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.query.DeviceTypeFilter;
 import org.thingsboard.server.common.data.query.EntityCountQuery;
@@ -37,14 +45,21 @@ import org.thingsboard.server.common.data.query.EntityFilter;
 import org.thingsboard.server.common.data.query.EntityKey;
 import org.thingsboard.server.common.data.query.EntityKeyType;
 import org.thingsboard.server.common.data.query.EntityListFilter;
+import org.thingsboard.server.common.data.query.KeyFilter;
+import org.thingsboard.server.common.data.query.NumericFilterPredicate;
+import org.thingsboard.server.dao.attributes.AttributesService;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
 
 public abstract class BaseEntityServiceTest extends AbstractServiceTest {
 
+    @Autowired
+    private AttributesService attributesService;
+
     private TenantId tenantId;
 
     @Before
@@ -105,7 +120,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
     }
 
     @Test
-    public void testFindEntityDataByQuery() {
+    public void testSimpleFindEntityDataByQuery() {
         List<Device> devices = new ArrayList<>();
         for (int i = 0; i < 97; i++) {
             Device device = new Device();
@@ -135,7 +150,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
         Assert.assertEquals(10, data.getData().size());
 
         List<EntityData> loadedEntities = new ArrayList<>(data.getData());
-        while(data.hasNext()) {
+        while (data.hasNext()) {
             query = query.next();
             data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
             loadedEntities.addAll(data.getData());
@@ -162,5 +177,96 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
         data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
         Assert.assertEquals(11, data.getTotalElements());
         Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue());
+
+        deviceService.deleteDevicesByTenantId(tenantId);
+    }
+
+    @Test
+    public void testFindEntityDataByQueryWithAttributes() throws ExecutionException, InterruptedException {
+
+        List<Device> devices = new ArrayList<>();
+        List<Long> temperatures = new ArrayList<>();
+        List<Long> highTemperatures = new ArrayList<>();
+        for (int i=0;i<67;i++) {
+            Device device = new Device();
+            device.setTenantId(tenantId);
+            device.setName("Device"+i);
+            device.setType("default");
+            device.setLabel("testLabel"+(int)(Math.random()*1000));
+            devices.add(deviceService.saveDevice(device));
+            long temperature = (long)(Math.random()*100);
+            temperatures.add(temperature);
+            if (temperature > 45) {
+                highTemperatures.add(temperature);
+            }
+        }
+
+        List<ListenableFuture<List<Void>>> attributeFutures = new ArrayList<>();
+        for (int i=0;i<devices.size();i++) {
+            Device device = devices.get(i);
+            attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), DataConstants.CLIENT_SCOPE));
+        }
+        Futures.successfulAsList(attributeFutures).get();
+
+        DeviceTypeFilter filter = new DeviceTypeFilter();
+        filter.setDeviceType("default");
+        filter.setDeviceNameFilter("");
+
+        EntityDataSortOrder sortOrder = new EntityDataSortOrder(
+                new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC
+        );
+        EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
+        List<EntityKey> entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
+        List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature"));
+
+        EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null);
+        PageData<EntityData> data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
+
+        List<EntityData> loadedEntities = new ArrayList<>(data.getData());
+        while (data.hasNext()) {
+            query = query.next();
+            data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
+            loadedEntities.addAll(data.getData());
+        }
+        Assert.assertEquals(67, loadedEntities.size());
+        List<String> loadedTemperatures = loadedEntities.stream().map(entityData ->
+                entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList());
+        List<String> deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
+        Assert.assertEquals(deviceTemperatures, loadedTemperatures);
+
+        pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
+        KeyFilter highTemperatureFilter = new KeyFilter();
+        highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature"));
+        NumericFilterPredicate predicate = new NumericFilterPredicate();
+        predicate.setValue(45);
+        predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
+        highTemperatureFilter.setPredicate(predicate);
+        List<KeyFilter> keyFilters = Collections.singletonList(highTemperatureFilter);
+
+        query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters);
+
+        data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
+
+        loadedEntities = new ArrayList<>(data.getData());
+        while (data.hasNext()) {
+            query = query.next();
+            data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
+            loadedEntities.addAll(data.getData());
+        }
+        Assert.assertEquals(highTemperatures.size(), loadedEntities.size());
+
+        List<String> loadedHighTemperatures = loadedEntities.stream().map(entityData ->
+                entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList());
+        List<String> deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
+
+        Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures);
+
+        deviceService.deleteDevicesByTenantId(tenantId);
+    }
+
+    private ListenableFuture<List<Void>> saveLongAttribute(EntityId entityId, String key, long value, String scope) {
+        KvEntry attrValue = new LongDataEntry(key, value);
+        AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L);
+        return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr));
     }
 }