From d83d9ec3df6e0a1fc87520a53fd7bc4deff87d54 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 27 Jul 2020 16:29:51 +0300 Subject: [PATCH] Fix for sort order in search by attributes or telemetry --- .../upgrade/3.0.1/schema_update_to_uuid.sql | 2 + .../query/DefaultEntityQueryRepository.java | 35 +++++++++---- .../dao/sql/query/EntityKeyMapping.java | 51 ++++++++++++------- .../server/dao/sql/query/QueryContext.java | 5 +- .../resources/sql/schema-entities-idx.sql | 4 +- 5 files changed, 66 insertions(+), 31 deletions(-) diff --git a/application/src/main/data/upgrade/3.0.1/schema_update_to_uuid.sql b/application/src/main/data/upgrade/3.0.1/schema_update_to_uuid.sql index 48806665a3..0e4e84d7b1 100644 --- a/application/src/main/data/upgrade/3.0.1/schema_update_to_uuid.sql +++ b/application/src/main/data/upgrade/3.0.1/schema_update_to_uuid.sql @@ -86,6 +86,7 @@ BEGIN DROP INDEX IF EXISTS idx_asset_customer_id; DROP INDEX IF EXISTS idx_asset_customer_id_and_type; DROP INDEX IF EXISTS idx_asset_type; + DROP INDEX IF EXISTS idx_attribute_kv_by_key_and_last_update_ts; END; $$; @@ -105,6 +106,7 @@ BEGIN CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id); CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type); CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type); + CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc); END; $$; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 6bf1b71a43..a5d7d570fd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.common.data.query.EntityFilterType; +import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.query.EntityNameFilter; import org.thingsboard.server.common.data.query.EntitySearchQueryFilter; @@ -265,7 +266,8 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { String entityWhereClause = DefaultEntityQueryRepository.this.buildEntityWhere(ctx, query.getEntityFilter(), entityFieldsFiltersMapping); - String latestJoins = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings); + String latestJoinsCnt = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings, true); + String latestJoinsData = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings, false); String whereClause = DefaultEntityQueryRepository.this.buildWhere(ctx, latestFiltersMapping, query.getEntityFilter().getType()); String textSearchQuery = DefaultEntityQueryRepository.this.buildTextSearchQuery(ctx, selectionMapping, pageLink.getTextSearch()); String entityFieldsSelection = EntityKeyMapping.buildSelections(entityFieldsSelectionMapping, query.getEntityFilter().getType(), entityType); @@ -286,29 +288,44 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { topSelection = topSelection + ", " + latestSelection; } - String fromClause = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result %s", + String fromClauseCount = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result %s", + "entities.*", + entityFieldsSelection, + addEntityTableQuery(ctx, query.getEntityFilter()), + entityWhereClause, + latestJoinsCnt, + whereClause, + textSearchQuery); + + String fromClauseData = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result %s", topSelection, entityFieldsSelection, addEntityTableQuery(ctx, query.getEntityFilter()), entityWhereClause, - latestJoins, + latestJoinsData, whereClause, textSearchQuery); - int totalElements = jdbcTemplate.queryForObject(String.format("select count(*) %s", fromClause), ctx, Integer.class); + if (!StringUtils.isEmpty(pageLink.getTextSearch())) { + //Unfortunately, we need to sacrifice performance in case of full text search, because it is applied to all joined records. + fromClauseCount = fromClauseData; + } + String countQuery = String.format("select count(id) %s", fromClauseCount); + int totalElements = jdbcTemplate.queryForObject(countQuery, ctx, Integer.class); - String dataQuery = String.format("select * %s", fromClause); + String dataQuery = String.format("select * %s", fromClauseData); EntityDataSortOrder sortOrder = pageLink.getSortOrder(); if (sortOrder != null) { Optional sortOrderMappingOpt = mappings.stream().filter(EntityKeyMapping::isSortOrder).findFirst(); if (sortOrderMappingOpt.isPresent()) { EntityKeyMapping sortOrderMapping = sortOrderMappingOpt.get(); - dataQuery = String.format("%s order by %s", dataQuery, sortOrderMapping.getValueAlias()); - if (sortOrder.getDirection() == EntityDataSortOrder.Direction.ASC) { - dataQuery += " asc"; + String direction = sortOrder.getDirection() == EntityDataSortOrder.Direction.ASC ? "asc" : "desc"; + if (sortOrderMapping.getEntityKey().getType() == EntityKeyType.ENTITY_FIELD) { + dataQuery = String.format("%s order by %s %s", dataQuery, sortOrderMapping.getValueAlias(), direction); } else { - dataQuery += " desc"; + dataQuery = String.format("%s order by %s %s, %s %s", dataQuery, + sortOrderMapping.getSortOrderNumAlias(), direction, sortOrderMapping.getSortOrderStrAlias(), direction); } } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index 19d4051f87..b95125dd18 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -270,9 +270,10 @@ public class EntityKeyMapping { Collectors.joining(", ")); } - public static String buildLatestJoins(QueryContext ctx, EntityFilter entityFilter, EntityType entityType, List latestMappings) { - return latestMappings.stream().map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)).collect( - Collectors.joining(" ")); + public static String buildLatestJoins(QueryContext ctx, EntityFilter entityFilter, EntityType entityType, List latestMappings, boolean countQuery) { + return latestMappings.stream().filter(mapping -> !countQuery || mapping.hasFilter()) + .map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)).collect( + Collectors.joining(" ")); } public static String buildQuery(QueryContext ctx, List mappings, EntityFilterType filterType) { @@ -363,19 +364,14 @@ public class EntityKeyMapping { } private String buildAttributeSelection() { - String attrValAlias = getValueAlias(); - String attrTsAlias = getTsAlias(); - String attrValSelection = - String.format("(coalesce(cast(%s.bool_v as varchar), '') || " + - "coalesce(%s.str_v, '') || " + - "coalesce(cast(%s.long_v as varchar), '') || " + - "coalesce(cast(%s.dbl_v as varchar), '') || " + - "coalesce(cast(%s.json_v as varchar), '')) as %s", alias, alias, alias, alias, alias, attrValAlias); - String attrTsSelection = String.format("%s.last_update_ts as %s", alias, attrTsAlias); - return String.join(", ", attrValSelection, attrTsSelection); + return buildTimeSeriesOrAttrSelection(true); } private String buildTimeSeriesSelection() { + return buildTimeSeriesOrAttrSelection(false); + } + + private String buildTimeSeriesOrAttrSelection(boolean attr) { String attrValAlias = getValueAlias(); String attrTsAlias = getTsAlias(); String attrValSelection = @@ -384,8 +380,25 @@ public class EntityKeyMapping { "coalesce(cast(%s.long_v as varchar), '') || " + "coalesce(cast(%s.dbl_v as varchar), '') || " + "coalesce(cast(%s.json_v as varchar), '')) as %s", alias, alias, alias, alias, alias, attrValAlias); - String attrTsSelection = String.format("%s.ts as %s", alias, attrTsAlias); - return String.join(", ", attrValSelection, attrTsSelection); + String attrTsSelection = String.format("%s.%s as %s", alias, attr ? "last_update_ts" : "ts", attrTsAlias); + if (this.isSortOrder) { + String attrNumAlias = getSortOrderNumAlias(); + String attrVarcharAlias = getSortOrderStrAlias(); + String attrSortOrderSelection = + String.format("coalesce(%s.dbl_v, cast(%s.long_v as double precision), (case when %s.bool_v then 1 else 0 end)) %s," + + "coalesce(%s.str_v, cast(%s.json_v as varchar), '') %s", alias, alias, alias, attrNumAlias, alias, alias, attrVarcharAlias); + return String.join(", ", attrValSelection, attrTsSelection, attrSortOrderSelection); + } else { + return String.join(", ", attrValSelection, attrTsSelection); + } + } + + public String getSortOrderStrAlias() { + return getValueAlias() + "_so_varchar"; + } + + public String getSortOrderNumAlias() { + return getValueAlias() + "_so_num"; } private String buildKeyQuery(QueryContext ctx, String alias, KeyFilter keyFilter, @@ -425,11 +438,11 @@ public class EntityKeyMapping { if (predicate.getType().equals(FilterPredicateType.NUMERIC)) { return this.buildNumericPredicateQuery(ctx, field, (NumericFilterPredicate) predicate); } else if (predicate.getType().equals(FilterPredicateType.STRING)) { - if (key.getKey().equals("entityType") && !filterType.equals(EntityFilterType.RELATIONS_QUERY)){ + if (key.getKey().equals("entityType") && !filterType.equals(EntityFilterType.RELATIONS_QUERY)) { field = ctx.getEntityType().toString(); return this.buildStringPredicateQuery(ctx, field, (StringFilterPredicate) predicate) .replace("lower(" + field, "lower('" + field + "'") - .replace(field + " ","'" + field + "' "); + .replace(field + " ", "'" + field + "' "); } else { return this.buildStringPredicateQuery(ctx, field, (StringFilterPredicate) predicate); } @@ -481,13 +494,13 @@ public class EntityKeyMapping { stringOperationQuery = String.format("%s like :%s) or (%s is null and :%s = '%%')", operationField, paramName, operationField, paramName); break; case CONTAINS: - if (value.length()>0) { + if (value.length() > 0) { value = "%" + value + "%"; } stringOperationQuery = String.format("%s like :%s) or (%s is null and :%s = '')", operationField, paramName, operationField, paramName); break; case NOT_CONTAINS: - if (value.length()>0) { + if (value.length() > 0) { value = "%" + value + "%"; } stringOperationQuery = String.format("%s not like :%s) or (%s is null and :%s != '')", operationField, paramName, operationField, paramName); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java index 1399967b27..c0ce89e2fa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java @@ -41,8 +41,9 @@ public class QueryContext implements SqlParameterSource { } void addParameter(String name, Object value, int type, String typeName) { - Parameter existing = params.put(name, new Parameter(value, type, typeName)); - if (existing != null) { + Parameter newParam = new Parameter(value, type, typeName); + Parameter oldParam = params.put(name, newParam); + if (oldParam != null && !oldParam.value.equals(newParam.value)) { throw new RuntimeException("Parameter with name: " + name + " was already registered!"); } } diff --git a/dao/src/main/resources/sql/schema-entities-idx.sql b/dao/src/main/resources/sql/schema-entities-idx.sql index c1490415ac..c59d87a950 100644 --- a/dao/src/main/resources/sql/schema-entities-idx.sql +++ b/dao/src/main/resources/sql/schema-entities-idx.sql @@ -36,4 +36,6 @@ CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type); -CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type); \ No newline at end of file +CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type); + +CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc); \ No newline at end of file