Merge pull request #12991 from dashevchenko/edqs_numeric_sort_fix

Fixed edqs sorting for numeric values

application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java

@@ -110,7 +110,7 @@ public class EdqsEntityServiceTest extends EntityServiceTest {
 
     @Override
     protected List<EntityData> findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List<String> expectedTelemetries) {
-        return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetries),
+        return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> loadAllData(query, expectedTelemetries.size()),
                 loadedEntities -> loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList().containsAll(expectedTelemetries));
     }
 

application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java

@@ -1698,6 +1698,19 @@ public class EntityServiceTest extends AbstractControllerTest {
         query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters);
         findByQueryAndCheckTelemetry(query, EntityKeyType.TIME_SERIES, "temperature", deviceHighTemperatures);
 
+        // change sort order to sort by temperature
+        temperatures.sort(Comparator.naturalOrder());
+        List<String> expectedSortedList = temperatures.stream().map(aDouble -> Double.toString(aDouble)).collect(Collectors.toList());
+
+        EntityDataSortOrder sortByTempOrder = new EntityDataSortOrder(
+                new EntityKey(EntityKeyType.TIME_SERIES, "temperature"), EntityDataSortOrder.Direction.ASC);
+        EntityDataPageLink sortByTempPageLink = new EntityDataPageLink(10, 0, null, sortByTempOrder);
+        EntityDataQuery querySortByTemp = new EntityDataQuery(filter, sortByTempPageLink, entityFields, latestValues, null);
+
+        List<EntityData> loadedEntities = loadAllData(querySortByTemp, deviceTemperatures.size());
+        List<String> entitiesTelemetry = loadedEntities.stream().map(entityData -> entityData.getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()).toList();
+        assertThat(entitiesTelemetry).containsExactlyElementsOf(expectedSortedList);
+
         deviceService.deleteDevicesByTenantId(tenantId);
     }
 
@@ -2377,14 +2390,14 @@ public class EntityServiceTest extends AbstractControllerTest {
     }
 
     protected List<EntityData> findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List<String> expectedTelemetry) {
-        List<EntityData> loadedEntities = findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetry);
+        List<EntityData> loadedEntities = loadAllData(query, expectedTelemetry.size());
         List<String> entitiesTelemetry = loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList();
         assertThat(entitiesTelemetry).containsExactlyInAnyOrderElementsOf(expectedTelemetry);
         return loadedEntities;
     }
 
-    protected List<EntityData> findEntitiesTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List<String> expectedTelemetries) {
+    protected List<EntityData> loadAllData(EntityDataQuery query, int expectedSize) {
-        PageData<EntityData> data = findByQueryAndCheck(query, expectedTelemetries.size());
+        PageData<EntityData> data = findByQueryAndCheck(query, expectedSize);
         List<EntityData> loadedEntities = new ArrayList<>(data.getData());
         while (data.hasNext()) {
             query = query.next();

common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java

@@ -17,7 +17,7 @@ package org.thingsboard.server.common.data.edqs;
 
 import org.thingsboard.server.common.data.kv.DataType;
 
-public interface DataPoint {
+public interface DataPoint extends Comparable<DataPoint> {
 
     String NOT_SUPPORTED = "Not supported!";
 

common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java

@@ -17,6 +17,7 @@ package org.thingsboard.server.edqs.data.dp;
 
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
+import org.apache.commons.lang3.StringUtils;
 import org.thingsboard.server.common.data.edqs.DataPoint;
 
 @RequiredArgsConstructor
@@ -54,4 +55,9 @@ public abstract class AbstractDataPoint implements DataPoint {
         return valueToString();
     }
 
+    @Override
+    public int compareTo(DataPoint dataPoint) {
+        return StringUtils.compareIgnoreCase(valueToString(), dataPoint.valueToString());
+    }
+
 }

common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java

@@ -16,6 +16,7 @@
 package org.thingsboard.server.edqs.data.dp;
 
 import lombok.Getter;
+import org.thingsboard.server.common.data.edqs.DataPoint;
 import org.thingsboard.server.common.data.kv.DataType;
 
 public class BoolDataPoint extends AbstractDataPoint {
@@ -43,4 +44,8 @@ public class BoolDataPoint extends AbstractDataPoint {
         return Boolean.toString(value);
     }
 
+    @Override
+    public int compareTo(DataPoint dataPoint) {
+        return Boolean.compare(value, dataPoint.getBool());
+    }
 }

common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java

@@ -16,6 +16,7 @@
 package org.thingsboard.server.edqs.data.dp;
 
 import lombok.Getter;
+import org.thingsboard.server.common.data.edqs.DataPoint;
 import org.thingsboard.server.common.data.kv.DataType;
 
 public class DoubleDataPoint extends AbstractDataPoint {
@@ -43,4 +44,8 @@ public class DoubleDataPoint extends AbstractDataPoint {
         return Double.toString(value);
     }
 
+    @Override
+    public int compareTo(DataPoint dataPoint) {
+        return Double.compare(value, dataPoint.getDouble());
+    }
 }

common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java

@@ -16,6 +16,7 @@
 package org.thingsboard.server.edqs.data.dp;
 
 import lombok.Getter;
+import org.thingsboard.server.common.data.edqs.DataPoint;
 import org.thingsboard.server.common.data.kv.DataType;
 
 public class LongDataPoint extends AbstractDataPoint {
@@ -47,4 +48,9 @@ public class LongDataPoint extends AbstractDataPoint {
     public String valueToString() {
         return Long.toString(value);
     }
+
+    @Override
+    public int compareTo(DataPoint dataPoint) {
+        return Long.compare(value, dataPoint.getLong());
+    }
 }

common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java

@@ -16,6 +16,7 @@
 package org.thingsboard.server.edqs.query;
 
 import lombok.Data;
+import org.thingsboard.server.common.data.edqs.DataPoint;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.EntityIdFactory;
 import org.thingsboard.server.edqs.data.EntityData;
@@ -26,7 +27,7 @@ import java.util.UUID;
 public class SortableEntityData {
 
     private final EntityData entityData;
-    private String sortValue;
+    private DataPoint sortValue;
 
     public UUID getId(){
         return entityData.getId();

common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java

@@ -16,6 +16,7 @@
 package org.thingsboard.server.edqs.query.processor;
 
 import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.edqs.DataPoint;
 import org.thingsboard.server.common.data.permission.QueryContext;
 import org.thingsboard.server.common.data.query.EntityFilter;
 import org.thingsboard.server.edqs.data.EntityData;
@@ -50,7 +51,7 @@ public abstract class AbstractQueryProcessor<T extends EntityFilter> implements
 
     protected SortableEntityData toSortData(EntityData<?> ed) {
         SortableEntityData sortData = new SortableEntityData(ed);
-        sortData.setSortValue(getSortValue(ed, sortKey));
+        sortData.setSortValue(getSortValue(ed, sortKey, ctx));
         return sortData;
     }
 

common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java

@@ -34,6 +34,7 @@ import java.util.List;
 import java.util.Queue;
 import java.util.Set;
 import java.util.UUID;
+import java.util.stream.Collectors;
 
 
 public abstract class AbstractRelationQueryProcessor<T extends EntityFilter> extends AbstractQueryProcessor<T> {
@@ -89,7 +90,7 @@ public abstract class AbstractRelationQueryProcessor<T extends EntityFilter> ext
     private List<SortableEntityData> processTenantQuery(Set<EntityData<?>> entities) {
         return entities.stream()
                 .map(this::toSortData)
-                .toList();
+                .collect(Collectors.toList());
     }
 
     private List<SortableEntityData> processCustomerQuery(Set<EntityData<?>> entities) {

common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java

@@ -67,10 +67,10 @@ import static org.thingsboard.server.common.data.query.ComplexFilterPredicate.Co
 @Slf4j
 public class RepositoryUtils {
 
-    public static final Comparator<SortableEntityData> SORT_ASC = Comparator.comparing((SortableEntityData sed) -> Optional.ofNullable(sed.getSortValue()).orElse(""), String.CASE_INSENSITIVE_ORDER)
+    public static final Comparator<SortableEntityData> SORT_ASC = Comparator.comparing(SortableEntityData::getSortValue, Comparator.nullsFirst(Comparator.naturalOrder()))
             .thenComparing(sp -> sp.getId().toString());
 
-    public static final Comparator<SortableEntityData> SORT_DESC = Comparator.comparing((SortableEntityData sed) -> Optional.ofNullable(sed.getSortValue()).orElse(""), String.CASE_INSENSITIVE_ORDER)
+    public static final Comparator<SortableEntityData> SORT_DESC =  Comparator.comparing(SortableEntityData::getSortValue, Comparator.nullsFirst(Comparator.naturalOrder()))
             .thenComparing(sp -> sp.getId().toString()).reversed();
 
     public static EntityType resolveEntityType(EntityFilter entityFilter) {
@@ -348,24 +348,11 @@ public class RepositoryUtils {
         }
     }
 
-    public static String getSortValue(EntityData entity, DataKey sortKey) {
+    public static DataPoint getSortValue(EntityData entity, DataKey sortKey, QueryContext queryContext) {
         if (sortKey == null) {
             return null;
         }
-        switch (sortKey.type()) {
+       return entity.getDataPoint(sortKey, queryContext);
-            case ENTITY_FIELD -> {
-                return entity.getField(sortKey.key());
-            }
-            case ATTRIBUTE, CLIENT_ATTRIBUTE, SHARED_ATTRIBUTE, SERVER_ATTRIBUTE -> {
-                var dp = entity.getAttr(sortKey.keyId(), sortKey.type());
-                return dp != null ? dp.valueToString() : "";
-            }
-            case TIME_SERIES -> {
-                var dp = entity.getTs(sortKey.keyId());
-                return dp != null ? dp.valueToString() : "";
-            }
-            default -> throw new IllegalStateException("toSortKey is not implemented for type: " + sortKey.type());
-        }
     }
 
     public static boolean checkFilters(EdqsQuery query, EntityData entity) {