Fix for sort order in search by attributes or telemetry

application/src/main/data/upgrade/3.0.1/schema_update_to_uuid.sql

@@ -86,6 +86,7 @@ BEGIN
     DROP INDEX IF EXISTS idx_asset_customer_id;
     DROP INDEX IF EXISTS idx_asset_customer_id_and_type;
     DROP INDEX IF EXISTS idx_asset_type;
+    DROP INDEX IF EXISTS idx_attribute_kv_by_key_and_last_update_ts;
 END;
 $$;
 
@@ -105,6 +106,7 @@ BEGIN
     CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id);
     CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type);
     CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type);
+    CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc);
 END;
 $$;
 

dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java

@@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.query.EntityDataQuery;
 import org.thingsboard.server.common.data.query.EntityDataSortOrder;
 import org.thingsboard.server.common.data.query.EntityFilter;
 import org.thingsboard.server.common.data.query.EntityFilterType;
+import org.thingsboard.server.common.data.query.EntityKeyType;
 import org.thingsboard.server.common.data.query.EntityListFilter;
 import org.thingsboard.server.common.data.query.EntityNameFilter;
 import org.thingsboard.server.common.data.query.EntitySearchQueryFilter;
@@ -265,7 +266,8 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
 
 
             String entityWhereClause = DefaultEntityQueryRepository.this.buildEntityWhere(ctx, query.getEntityFilter(), entityFieldsFiltersMapping);
-            String latestJoins = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings);
+            String latestJoinsCnt = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings, true);
+            String latestJoinsData = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings, false);
             String whereClause = DefaultEntityQueryRepository.this.buildWhere(ctx, latestFiltersMapping, query.getEntityFilter().getType());
             String textSearchQuery = DefaultEntityQueryRepository.this.buildTextSearchQuery(ctx, selectionMapping, pageLink.getTextSearch());
             String entityFieldsSelection = EntityKeyMapping.buildSelections(entityFieldsSelectionMapping, query.getEntityFilter().getType(), entityType);
@@ -286,29 +288,44 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
                 topSelection = topSelection + ", " + latestSelection;
             }
 
-            String fromClause = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result %s",
+            String fromClauseCount = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result %s",
+                    "entities.*",
+                    entityFieldsSelection,
+                    addEntityTableQuery(ctx, query.getEntityFilter()),
+                    entityWhereClause,
+                    latestJoinsCnt,
+                    whereClause,
+                    textSearchQuery);
+
+            String fromClauseData = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result %s",
                     topSelection,
                     entityFieldsSelection,
                     addEntityTableQuery(ctx, query.getEntityFilter()),
                     entityWhereClause,
-                    latestJoins,
+                    latestJoinsData,
                     whereClause,
                     textSearchQuery);
 
-            int totalElements = jdbcTemplate.queryForObject(String.format("select count(*) %s", fromClause), ctx, Integer.class);
+            if (!StringUtils.isEmpty(pageLink.getTextSearch())) {
+                //Unfortunately, we need to sacrifice performance in case of full text search, because it is applied to all joined records.
+                fromClauseCount = fromClauseData;
+            }
+            String countQuery = String.format("select count(id) %s", fromClauseCount);
+            int totalElements = jdbcTemplate.queryForObject(countQuery, ctx, Integer.class);
 
-            String dataQuery = String.format("select * %s", fromClause);
+            String dataQuery = String.format("select * %s", fromClauseData);
 
             EntityDataSortOrder sortOrder = pageLink.getSortOrder();
             if (sortOrder != null) {
                 Optional<EntityKeyMapping> sortOrderMappingOpt = mappings.stream().filter(EntityKeyMapping::isSortOrder).findFirst();
                 if (sortOrderMappingOpt.isPresent()) {
                     EntityKeyMapping sortOrderMapping = sortOrderMappingOpt.get();
-                    dataQuery = String.format("%s order by %s", dataQuery, sortOrderMapping.getValueAlias());
+                    String direction = sortOrder.getDirection() == EntityDataSortOrder.Direction.ASC ? "asc" : "desc";
-                    if (sortOrder.getDirection() == EntityDataSortOrder.Direction.ASC) {
+                    if (sortOrderMapping.getEntityKey().getType() == EntityKeyType.ENTITY_FIELD) {
-                        dataQuery += " asc";
+                        dataQuery = String.format("%s order by %s %s", dataQuery, sortOrderMapping.getValueAlias(), direction);
                     } else {
-                        dataQuery += " desc";
+                        dataQuery = String.format("%s order by %s %s, %s %s", dataQuery,
+                                sortOrderMapping.getSortOrderNumAlias(), direction, sortOrderMapping.getSortOrderStrAlias(), direction);
                     }
                 }
             }

dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java

@@ -270,9 +270,10 @@ public class EntityKeyMapping {
                 Collectors.joining(", "));
     }
 
-    public static String buildLatestJoins(QueryContext ctx, EntityFilter entityFilter, EntityType entityType, List<EntityKeyMapping> latestMappings) {
+    public static String buildLatestJoins(QueryContext ctx, EntityFilter entityFilter, EntityType entityType, List<EntityKeyMapping> latestMappings, boolean countQuery) {
-        return latestMappings.stream().map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)).collect(
+        return latestMappings.stream().filter(mapping -> !countQuery || mapping.hasFilter())
-                Collectors.joining(" "));
+                .map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)).collect(
+                        Collectors.joining(" "));
     }
 
     public static String buildQuery(QueryContext ctx, List<EntityKeyMapping> mappings, EntityFilterType filterType) {
@@ -363,19 +364,14 @@ public class EntityKeyMapping {
     }
 
     private String buildAttributeSelection() {
-        String attrValAlias = getValueAlias();
+        return buildTimeSeriesOrAttrSelection(true);
-        String attrTsAlias = getTsAlias();
-        String attrValSelection =
-                String.format("(coalesce(cast(%s.bool_v as varchar), '') || " +
-                        "coalesce(%s.str_v, '') || " +
-                        "coalesce(cast(%s.long_v as varchar), '') || " +
-                        "coalesce(cast(%s.dbl_v as varchar), '') || " +
-                        "coalesce(cast(%s.json_v as varchar), '')) as %s", alias, alias, alias, alias, alias, attrValAlias);
-        String attrTsSelection = String.format("%s.last_update_ts as %s", alias, attrTsAlias);
-        return String.join(", ", attrValSelection, attrTsSelection);
     }
 
     private String buildTimeSeriesSelection() {
+        return buildTimeSeriesOrAttrSelection(false);
+    }
+
+    private String buildTimeSeriesOrAttrSelection(boolean attr) {
         String attrValAlias = getValueAlias();
         String attrTsAlias = getTsAlias();
         String attrValSelection =
@@ -384,8 +380,25 @@ public class EntityKeyMapping {
                         "coalesce(cast(%s.long_v as varchar), '') || " +
                         "coalesce(cast(%s.dbl_v as varchar), '') || " +
                         "coalesce(cast(%s.json_v as varchar), '')) as %s", alias, alias, alias, alias, alias, attrValAlias);
-        String attrTsSelection = String.format("%s.ts as %s", alias, attrTsAlias);
+        String attrTsSelection = String.format("%s.%s as %s", alias, attr ? "last_update_ts" : "ts", attrTsAlias);
-        return String.join(", ", attrValSelection, attrTsSelection);
+        if (this.isSortOrder) {
+            String attrNumAlias = getSortOrderNumAlias();
+            String attrVarcharAlias = getSortOrderStrAlias();
+            String attrSortOrderSelection =
+                    String.format("coalesce(%s.dbl_v, cast(%s.long_v as double precision), (case when %s.bool_v then 1 else 0 end)) %s," +
+                            "coalesce(%s.str_v, cast(%s.json_v as varchar), '') %s", alias, alias, alias, attrNumAlias, alias, alias, attrVarcharAlias);
+            return String.join(", ", attrValSelection, attrTsSelection, attrSortOrderSelection);
+        } else {
+            return String.join(", ", attrValSelection, attrTsSelection);
+        }
+    }
+
+    public String getSortOrderStrAlias() {
+        return getValueAlias() + "_so_varchar";
+    }
+
+    public String getSortOrderNumAlias() {
+        return getValueAlias() + "_so_num";
     }
 
     private String buildKeyQuery(QueryContext ctx, String alias, KeyFilter keyFilter,
@@ -425,11 +438,11 @@ public class EntityKeyMapping {
                 if (predicate.getType().equals(FilterPredicateType.NUMERIC)) {
                     return this.buildNumericPredicateQuery(ctx, field, (NumericFilterPredicate) predicate);
                 } else if (predicate.getType().equals(FilterPredicateType.STRING)) {
-                    if (key.getKey().equals("entityType") && !filterType.equals(EntityFilterType.RELATIONS_QUERY)){
+                    if (key.getKey().equals("entityType") && !filterType.equals(EntityFilterType.RELATIONS_QUERY)) {
                         field = ctx.getEntityType().toString();
                         return this.buildStringPredicateQuery(ctx, field, (StringFilterPredicate) predicate)
                                 .replace("lower(" + field, "lower('" + field + "'")
-                                .replace(field + " ","'" + field + "' ");
+                                .replace(field + " ", "'" + field + "' ");
                     } else {
                         return this.buildStringPredicateQuery(ctx, field, (StringFilterPredicate) predicate);
                     }
@@ -481,13 +494,13 @@ public class EntityKeyMapping {
                 stringOperationQuery = String.format("%s like :%s) or (%s is null and :%s = '%%')", operationField, paramName, operationField, paramName);
                 break;
             case CONTAINS:
-                if (value.length()>0) {
+                if (value.length() > 0) {
                     value = "%" + value + "%";
                 }
                 stringOperationQuery = String.format("%s like :%s) or (%s is null and :%s = '')", operationField, paramName, operationField, paramName);
                 break;
             case NOT_CONTAINS:
-                if (value.length()>0) {
+                if (value.length() > 0) {
                     value = "%" + value + "%";
                 }
                 stringOperationQuery = String.format("%s not like :%s) or (%s is null and :%s != '')", operationField, paramName, operationField, paramName);

dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java

@@ -41,8 +41,9 @@ public class QueryContext implements SqlParameterSource {
     }
 
     void addParameter(String name, Object value, int type, String typeName) {
-        Parameter existing = params.put(name, new Parameter(value, type, typeName));
+        Parameter newParam = new Parameter(value, type, typeName);
-        if (existing != null) {
+        Parameter oldParam = params.put(name, newParam);
+        if (oldParam != null && !oldParam.value.equals(newParam.value)) {
             throw new RuntimeException("Parameter with name: " + name + " was already registered!");
         }
     }

dao/src/main/resources/sql/schema-entities-idx.sql

@@ -36,4 +36,6 @@ CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id
 
 CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type);
 
-CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type);
+CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type);
+
+CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc);