dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve data query

Browse Source
This commit is contained in:
Igor Kulikov 2020-06-15 15:09:56 +03:00
parent 42bf0a37a2
commit eebcccc6fd
4 changed files with 197 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java
View File

@ -114,25 +114,23 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
entityFieldsSelection = String.format("e.id, '%s'", entityType.name());
}
String latestSelection = EntityKeyMapping.buildSelections(latestSelectionMapping);
String selection = entityFieldsSelection;
String topSelection = "entities.*";
if (!StringUtils.isEmpty(latestSelection)) {
selection = entityFieldsSelection + ", " + latestSelection;
topSelection = topSelection + ", " + latestSelection;
}
String fromClause = String.format("from (select %s from %s e where %s) entities %s",
selection,
String fromClause = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result",
topSelection,
entityFieldsSelection,
entityTableMap.get(entityType),
entityWhereClause,
latestJoins);
if (!StringUtils.isEmpty(whereClause)) {
fromClause = String.format("%s where %s", fromClause, whereClause);
}
latestJoins,
whereClause);
int totalElements = ((BigInteger)entityManager.createNativeQuery(String.format("select count(*) %s", fromClause))
.getSingleResult()).intValue();
String dataQuery = String.format("select entities.* %s", fromClause);
String dataQuery = String.format("select * %s", fromClause);
EntityDataSortOrder sortOrder = pageLink.getSortOrder();
if (sortOrder != null) {
@ -198,12 +196,18 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
private String buildWhere(List<EntityKeyMapping> selectionMapping, List<EntityKeyMapping> latestFiltersMapping, String searchText) {
String latestFilters = EntityKeyMapping.buildQuery(latestFiltersMapping);
String textSearchQuery = this.buildTextSearchQuery(selectionMapping, searchText);
String query;
if (!StringUtils.isEmpty(latestFilters) && !StringUtils.isEmpty(textSearchQuery)) {
return String.join(" AND ", latestFilters, textSearchQuery);
query = String.join(" AND ", latestFilters, textSearchQuery);
} else if (!StringUtils.isEmpty(latestFilters)) {
return latestFilters;
query = latestFilters;
} else {
return textSearchQuery;
query = textSearchQuery;
}
if (!StringUtils.isEmpty(query)) {
return String.format("where %s", query);
} else {
return "";
}
}

6
dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java
View File

@ -56,6 +56,7 @@ public class EntityDataAdapter {
Map<String, TsValue[]> timeseries = new HashMap<>();
EntityData entityData = new EntityData(entityId, latest, timeseries);
for (EntityKeyMapping mapping: selectionMapping) {
if (!mapping.isIgnore()) {
EntityKey entityKey = mapping.getEntityKey();
Object value = row[mapping.getIndex()];
String strValue;
@ -71,6 +72,7 @@ public class EntityDataAdapter {
TsValue tsValue = new TsValue(ts, strValue);
latest.computeIfAbsent(entityKey.getType(), entityKeyType -> new HashMap<>()).put(entityKey.getKey(), tsValue);
}
}
return entityData;
}
@ -80,8 +82,8 @@ public class EntityDataAdapter {
// check number
if (strVal.length() > 0) {
try {
int intVal = Integer.parseInt(strVal);
return Integer.toString(intVal);
long longVal = Long.parseLong(strVal);
return Long.toString(longVal);
} catch (NumberFormatException ignored) {
}
try {

93
dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java
View File

@ -35,6 +35,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -65,6 +66,7 @@ public class EntityKeyMapping {
private boolean isLatest;
private boolean isSelection;
private boolean isSortOrder;
private boolean ignore = false;
private List<KeyFilter> keyFilters;
private EntityKey entityKey;
@ -151,43 +153,63 @@ public class EntityKeyMapping {
query.getKeyFilters().stream().collect(Collectors.groupingBy(KeyFilter::getKey)) : Collections.emptyMap();
EntityDataSortOrder sortOrder = query.getPageLink().getSortOrder();
EntityKey sortOrderKey = sortOrder != null ? sortOrder.getKey() : null;
EntityKeyMapping sortOrderMapping = null;
int index = 2;
List<EntityKeyMapping> mappings = new ArrayList<>();
for (EntityKey entityField : entityFields) {
List<EntityKeyMapping> entityFieldsMappings = entityFields.stream().map(
key -> {
EntityKeyMapping mapping = new EntityKeyMapping();
mapping.setIndex(index);
mapping.setAlias(String.format("alias%s", index));
mapping.setKeyFilters(filters.remove(entityField));
mapping.setLatest(false);
mapping.setSelection(true);
mapping.setEntityKey(entityField);
if (entityField.equals(sortOrderKey)) {
mapping.setSortOrder(true);
sortOrderMapping = mapping;
} else {
mapping.setSortOrder(false);
mapping.setEntityKey(key);
return mapping;
}
mappings.add(mapping);
index++;
}
for (EntityKey latestField : latestValues) {
).collect(Collectors.toList());
List<EntityKeyMapping> latestMappings = latestValues.stream().map(
key -> {
EntityKeyMapping mapping = new EntityKeyMapping();
mapping.setIndex(index);
mapping.setAlias(String.format("alias%s", index));
mapping.setKeyFilters(filters.remove(latestField));
mapping.setLatest(true);
mapping.setSelection(true);
mapping.setEntityKey(latestField);
if (latestField.equals(sortOrderKey)) {
mapping.setSortOrder(true);
sortOrderMapping = mapping;
} else {
mapping.setSortOrder(false);
mapping.setEntityKey(key);
return mapping;
}
mappings.add(mapping);
).collect(Collectors.toList());
if (sortOrderKey != null) {
Optional<EntityKeyMapping> existing;
if (sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD)) {
existing =
entityFieldsMappings.stream().filter(mapping -> mapping.entityKey.equals(sortOrderKey)).findFirst();
} else {
existing =
latestMappings.stream().filter(mapping -> mapping.entityKey.equals(sortOrderKey)).findFirst();
}
if (existing.isPresent()) {
existing.get().setSortOrder(true);
} else {
EntityKeyMapping sortOrderMapping = new EntityKeyMapping();
sortOrderMapping.setLatest(!sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD));
sortOrderMapping.setSelection(true);
sortOrderMapping.setEntityKey(sortOrderKey);
sortOrderMapping.setSortOrder(true);
sortOrderMapping.setIgnore(true);
if (sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD)) {
entityFieldsMappings.add(sortOrderMapping);
} else {
latestMappings.add(sortOrderMapping);
}
}
}
List<EntityKeyMapping> mappings = new ArrayList<>();
mappings.addAll(entityFieldsMappings);
mappings.addAll(latestMappings);
for (EntityKeyMapping mapping : mappings) {
mapping.setIndex(index);
mapping.setAlias(String.format("alias%s", index));
mapping.setKeyFilters(filters.remove(mapping.entityKey));
if (mapping.getEntityKey().getType().equals(EntityKeyType.ENTITY_FIELD)) {
index++;
} else {
index +=2;
}
}
if (!filters.isEmpty()) {
for (EntityKey filterField : filters.keySet()) {
EntityKeyMapping mapping = new EntityKeyMapping();
@ -201,37 +223,28 @@ public class EntityKeyMapping {
index +=1;
}
}
if (sortOrderKey != null && sortOrderMapping == null) {
sortOrderMapping = new EntityKeyMapping();
sortOrderMapping.setIndex(index);
sortOrderMapping.setAlias(String.format("alias%s", index));
sortOrderMapping.setLatest(!sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD));
sortOrderMapping.setSelection(true);
sortOrderMapping.setEntityKey(sortOrderKey);
sortOrderMapping.setSortOrder(true);
mappings.add(sortOrderMapping);
}
return mappings;
}
private String buildAttributeSelection() {
String attrValAlias = getValueAlias();
String attrTsAlias = getTsAlias();
String attrTsSelection = String.format("%s.last_update_ts as %s", alias, attrTsAlias);
String attrValSelection =
String.format("coalesce(cast(%s.bool_v as varchar), '') || " +
String.format("(coalesce(cast(%s.bool_v as varchar), '') || " +
"coalesce(%s.str_v, '') || " +
"coalesce(cast(%s.long_v as varchar), '') || " +
"coalesce(cast(%s.dbl_v as varchar), '') || " +
"coalesce(cast(%s.json_v as varchar), '')) as %s", alias, alias, alias, alias, alias, attrValAlias);
return String.join(", ", attrTsSelection, attrValSelection);
String attrTsSelection = String.format("%s.last_update_ts as %s", alias, attrTsAlias);
return String.join(", ", attrValSelection, attrTsSelection);
}
private String buildTimeseriesSelection() {
// TODO:
String attrValAlias = getValueAlias();
String attrTsAlias = getTsAlias();
return String.format("(select 1) as %s, (select '') as %s", attrTsAlias, attrValAlias);
return String.format("(select '') as %s, (select 1) as %s", attrValAlias, attrTsAlias);
}
private String buildKeyQuery(String alias, KeyFilter keyFilter) {

108
dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java
View File

@ -15,10 +15,14 @@
*/
package org.thingsboard.server.dao.service;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.Tenant;
@ -26,6 +30,10 @@ import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.DeviceTypeFilter;
import org.thingsboard.server.common.data.query.EntityCountQuery;
@ -37,14 +45,21 @@ import org.thingsboard.server.common.data.query.EntityFilter;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.EntityListFilter;
import org.thingsboard.server.common.data.query.KeyFilter;
import org.thingsboard.server.common.data.query.NumericFilterPredicate;
import org.thingsboard.server.dao.attributes.AttributesService;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
public abstract class BaseEntityServiceTest extends AbstractServiceTest {
@Autowired
private AttributesService attributesService;
private TenantId tenantId;
@Before
@ -105,7 +120,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
}
@Test
public void testFindEntityDataByQuery() {
public void testSimpleFindEntityDataByQuery() {
List<Device> devices = new ArrayList<>();
for (int i = 0; i < 97; i++) {
Device device = new Device();
@ -162,5 +177,96 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
Assert.assertEquals(11, data.getTotalElements());
Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue());
deviceService.deleteDevicesByTenantId(tenantId);
}
@Test
public void testFindEntityDataByQueryWithAttributes() throws ExecutionException, InterruptedException {
List<Device> devices = new ArrayList<>();
List<Long> temperatures = new ArrayList<>();
List<Long> highTemperatures = new ArrayList<>();
for (int i=0;i<67;i++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Device"+i);
device.setType("default");
device.setLabel("testLabel"+(int)(Math.random()*1000));
devices.add(deviceService.saveDevice(device));
long temperature = (long)(Math.random()*100);
temperatures.add(temperature);
if (temperature > 45) {
highTemperatures.add(temperature);
}
}
List<ListenableFuture<List<Void>>> attributeFutures = new ArrayList<>();
for (int i=0;i<devices.size();i++) {
Device device = devices.get(i);
attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), DataConstants.CLIENT_SCOPE));
}
Futures.successfulAsList(attributeFutures).get();
DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceType("default");
filter.setDeviceNameFilter("");
EntityDataSortOrder sortOrder = new EntityDataSortOrder(
new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC
);
EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
List<EntityKey> entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature"));
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null);
PageData<EntityData> data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
List<EntityData> loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) {
query = query.next();
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities.addAll(data.getData());
}
Assert.assertEquals(67, loadedEntities.size());
List<String> loadedTemperatures = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList());
List<String> deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
Assert.assertEquals(deviceTemperatures, loadedTemperatures);
pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
KeyFilter highTemperatureFilter = new KeyFilter();
highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature"));
NumericFilterPredicate predicate = new NumericFilterPredicate();
predicate.setValue(45);
predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
highTemperatureFilter.setPredicate(predicate);
List<KeyFilter> keyFilters = Collections.singletonList(highTemperatureFilter);
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) {
query = query.next();
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities.addAll(data.getData());
}
Assert.assertEquals(highTemperatures.size(), loadedEntities.size());
List<String> loadedHighTemperatures = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList());
List<String> deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures);
deviceService.deleteDevicesByTenantId(tenantId);
}
private ListenableFuture<List<Void>> saveLongAttribute(EntityId entityId, String key, long value, String scope) {
KvEntry attrValue = new LongDataEntry(key, value);
AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L);
return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr));
}
}