dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix for sort order in search by attributes or telemetry

Browse Source
This commit is contained in:
Andrii Shvaika 2020-07-27 16:29:51 +03:00
parent 112b92503e
commit d83d9ec3df
5 changed files with 66 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
application/src/main/data/upgrade/3.0.1/schema_update_to_uuid.sql
View File

@ -86,6 +86,7 @@ BEGIN
DROP INDEX IF EXISTS idx_asset_customer_id;
DROP INDEX IF EXISTS idx_asset_customer_id_and_type;
DROP INDEX IF EXISTS idx_asset_type;
DROP INDEX IF EXISTS idx_attribute_kv_by_key_and_last_update_ts;
END;
$$;
@ -105,6 +106,7 @@ BEGIN
CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id);
CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type);
CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type);
CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc);
END;
$$;

35
dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java
View File

@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityDataSortOrder;
import org.thingsboard.server.common.data.query.EntityFilter;
import org.thingsboard.server.common.data.query.EntityFilterType;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.EntityListFilter;
import org.thingsboard.server.common.data.query.EntityNameFilter;
import org.thingsboard.server.common.data.query.EntitySearchQueryFilter;
@ -265,7 +266,8 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
String entityWhereClause = DefaultEntityQueryRepository.this.buildEntityWhere(ctx, query.getEntityFilter(), entityFieldsFiltersMapping);
String latestJoins = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings);
String latestJoinsCnt = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings, true);
String latestJoinsData = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings, false);
String whereClause = DefaultEntityQueryRepository.this.buildWhere(ctx, latestFiltersMapping, query.getEntityFilter().getType());
String textSearchQuery = DefaultEntityQueryRepository.this.buildTextSearchQuery(ctx, selectionMapping, pageLink.getTextSearch());
String entityFieldsSelection = EntityKeyMapping.buildSelections(entityFieldsSelectionMapping, query.getEntityFilter().getType(), entityType);
@ -286,29 +288,44 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
topSelection = topSelection + ", " + latestSelection;
}
String fromClause = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result %s",
String fromClauseCount = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result %s",
"entities.*",
entityFieldsSelection,
addEntityTableQuery(ctx, query.getEntityFilter()),
entityWhereClause,
latestJoinsCnt,
whereClause,
textSearchQuery);
String fromClauseData = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result %s",
topSelection,
entityFieldsSelection,
addEntityTableQuery(ctx, query.getEntityFilter()),
entityWhereClause,
latestJoins,
latestJoinsData,
whereClause,
textSearchQuery);
int totalElements = jdbcTemplate.queryForObject(String.format("select count(*) %s", fromClause), ctx, Integer.class);
if (!StringUtils.isEmpty(pageLink.getTextSearch())) {
//Unfortunately, we need to sacrifice performance in case of full text search, because it is applied to all joined records.
fromClauseCount = fromClauseData;
}
String countQuery = String.format("select count(id) %s", fromClauseCount);
int totalElements = jdbcTemplate.queryForObject(countQuery, ctx, Integer.class);
String dataQuery = String.format("select * %s", fromClause);
String dataQuery = String.format("select * %s", fromClauseData);
EntityDataSortOrder sortOrder = pageLink.getSortOrder();
if (sortOrder != null) {
Optional<EntityKeyMapping> sortOrderMappingOpt = mappings.stream().filter(EntityKeyMapping::isSortOrder).findFirst();
if (sortOrderMappingOpt.isPresent()) {
EntityKeyMapping sortOrderMapping = sortOrderMappingOpt.get();
dataQuery = String.format("%s order by %s", dataQuery, sortOrderMapping.getValueAlias());
if (sortOrder.getDirection() == EntityDataSortOrder.Direction.ASC) {
dataQuery += " asc";
String direction = sortOrder.getDirection() == EntityDataSortOrder.Direction.ASC ? "asc" : "desc";
if (sortOrderMapping.getEntityKey().getType() == EntityKeyType.ENTITY_FIELD) {
dataQuery = String.format("%s order by %s %s", dataQuery, sortOrderMapping.getValueAlias(), direction);
} else {
dataQuery += " desc";
dataQuery = String.format("%s order by %s %s, %s %s", dataQuery,
sortOrderMapping.getSortOrderNumAlias(), direction, sortOrderMapping.getSortOrderStrAlias(), direction);
}
}
}

39
dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java
View File

@ -270,8 +270,9 @@ public class EntityKeyMapping {
Collectors.joining(", "));
}
public static String buildLatestJoins(QueryContext ctx, EntityFilter entityFilter, EntityType entityType, List<EntityKeyMapping> latestMappings) {
return latestMappings.stream().map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)).collect(
public static String buildLatestJoins(QueryContext ctx, EntityFilter entityFilter, EntityType entityType, List<EntityKeyMapping> latestMappings, boolean countQuery) {
return latestMappings.stream().filter(mapping -> !countQuery || mapping.hasFilter())
.map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)).collect(
Collectors.joining(" "));
}
@ -363,19 +364,14 @@ public class EntityKeyMapping {
}
private String buildAttributeSelection() {
String attrValAlias = getValueAlias();
String attrTsAlias = getTsAlias();
String attrValSelection =
String.format("(coalesce(cast(%s.bool_v as varchar), '') || " +
"coalesce(%s.str_v, '') || " +
"coalesce(cast(%s.long_v as varchar), '') || " +
"coalesce(cast(%s.dbl_v as varchar), '') || " +
"coalesce(cast(%s.json_v as varchar), '')) as %s", alias, alias, alias, alias, alias, attrValAlias);
String attrTsSelection = String.format("%s.last_update_ts as %s", alias, attrTsAlias);
return String.join(", ", attrValSelection, attrTsSelection);
return buildTimeSeriesOrAttrSelection(true);
}
private String buildTimeSeriesSelection() {
return buildTimeSeriesOrAttrSelection(false);
}
private String buildTimeSeriesOrAttrSelection(boolean attr) {
String attrValAlias = getValueAlias();
String attrTsAlias = getTsAlias();
String attrValSelection =
@ -384,9 +380,26 @@ public class EntityKeyMapping {
"coalesce(cast(%s.long_v as varchar), '') || " +
"coalesce(cast(%s.dbl_v as varchar), '') || " +
"coalesce(cast(%s.json_v as varchar), '')) as %s", alias, alias, alias, alias, alias, attrValAlias);
String attrTsSelection = String.format("%s.ts as %s", alias, attrTsAlias);
String attrTsSelection = String.format("%s.%s as %s", alias, attr ? "last_update_ts" : "ts", attrTsAlias);
if (this.isSortOrder) {
String attrNumAlias = getSortOrderNumAlias();
String attrVarcharAlias = getSortOrderStrAlias();
String attrSortOrderSelection =
String.format("coalesce(%s.dbl_v, cast(%s.long_v as double precision), (case when %s.bool_v then 1 else 0 end)) %s," +
"coalesce(%s.str_v, cast(%s.json_v as varchar), '') %s", alias, alias, alias, attrNumAlias, alias, alias, attrVarcharAlias);
return String.join(", ", attrValSelection, attrTsSelection, attrSortOrderSelection);
} else {
return String.join(", ", attrValSelection, attrTsSelection);
}
}
public String getSortOrderStrAlias() {
return getValueAlias() + "_so_varchar";
}
public String getSortOrderNumAlias() {
return getValueAlias() + "_so_num";
}
private String buildKeyQuery(QueryContext ctx, String alias, KeyFilter keyFilter,
EntityFilterType filterType) {

5
dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java
View File

@ -41,8 +41,9 @@ public class QueryContext implements SqlParameterSource {
}
void addParameter(String name, Object value, int type, String typeName) {
Parameter existing = params.put(name, new Parameter(value, type, typeName));
if (existing != null) {
Parameter newParam = new Parameter(value, type, typeName);
Parameter oldParam = params.put(name, newParam);
if (oldParam != null && !oldParam.value.equals(newParam.value)) {
throw new RuntimeException("Parameter with name: " + name + " was already registered!");
}
}

2
dao/src/main/resources/sql/schema-entities-idx.sql
View File

@ -37,3 +37,5 @@ CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id
CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type);
CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type);
CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc);