UUID refactoring

This commit is contained in:
Andrii Shvaika 2020-06-22 15:35:11 +03:00
parent 1f8d790fbe
commit ac1ded94b7
9 changed files with 337 additions and 156 deletions

View File

@ -20,12 +20,14 @@ import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity;
import org.thingsboard.server.dao.util.HsqlDao;
import org.thingsboard.server.dao.util.SqlDao;
import javax.persistence.Query;
@SqlDao
@HsqlDao
@Repository
public class HsqlComponentDescriptorInsertRepository extends AbstractComponentDescriptorInsertRepository {
private static final String P_KEY_CONFLICT_STATEMENT = "(component_descriptor.id=I.id)";
private static final String P_KEY_CONFLICT_STATEMENT = "(component_descriptor.id=UUID(I.id))";
private static final String UNQ_KEY_CONFLICT_STATEMENT = "(component_descriptor.clazz=I.clazz)";
private static final String INSERT_OR_UPDATE_ON_P_KEY_CONFLICT = getInsertString(P_KEY_CONFLICT_STATEMENT);
@ -36,6 +38,20 @@ public class HsqlComponentDescriptorInsertRepository extends AbstractComponentDe
return saveAndGet(entity, INSERT_OR_UPDATE_ON_P_KEY_CONFLICT, INSERT_OR_UPDATE_ON_UNQ_KEY_CONFLICT);
}
@Override
protected Query getQuery(ComponentDescriptorEntity entity, String query) {
return entityManager.createNativeQuery(query, ComponentDescriptorEntity.class)
.setParameter("id", entity.getUuid().toString())
.setParameter("created_time", entity.getCreatedTime())
.setParameter("actions", entity.getActions())
.setParameter("clazz", entity.getClazz())
.setParameter("configuration_descriptor", entity.getConfigurationDescriptor().toString())
.setParameter("name", entity.getName())
.setParameter("scope", entity.getScope().name())
.setParameter("search_text", entity.getSearchText())
.setParameter("type", entity.getType().name());
}
@Override
protected ComponentDescriptorEntity doProcessSaveOrUpdate(ComponentDescriptorEntity entity, String query) {
getQuery(entity, query).executeUpdate();
@ -43,7 +59,9 @@ public class HsqlComponentDescriptorInsertRepository extends AbstractComponentDe
}
private static String getInsertString(String conflictStatement) {
return "MERGE INTO component_descriptor USING (VALUES :id, :created_time, :actions, :clazz, :configuration_descriptor, :name, :scope, :search_text, :type) I (id, craeted_time, actions, clazz, configuration_descriptor, name, scope, search_text, type) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET component_descriptor.id = I.id, component_descriptor.actions = I.actions, component_descriptor.clazz = I.clazz, component_descriptor.configuration_descriptor = I.configuration_descriptor, component_descriptor.name = I.name, component_descriptor.scope = I.scope, component_descriptor.search_text = I.search_text, component_descriptor.type = I.type" +
" WHEN NOT MATCHED THEN INSERT (id, created_time, actions, clazz, configuration_descriptor, name, scope, search_text, type) VALUES (I.id, I.created_time, I.actions, I.clazz, I.configuration_descriptor, I.name, I.scope, I.search_text, I.type)";
return "MERGE INTO component_descriptor USING (VALUES :id, :created_time, :actions, :clazz, :configuration_descriptor, :name, :scope, :search_text, :type) I (id, created_time, actions, clazz, configuration_descriptor, name, scope, search_text, type) ON "
+ conflictStatement
+ " WHEN MATCHED THEN UPDATE SET component_descriptor.id = UUID(I.id), component_descriptor.actions = I.actions, component_descriptor.clazz = I.clazz, component_descriptor.configuration_descriptor = I.configuration_descriptor, component_descriptor.name = I.name, component_descriptor.scope = I.scope, component_descriptor.search_text = I.search_text, component_descriptor.type = I.type" +
" WHEN NOT MATCHED THEN INSERT (id, created_time, actions, clazz, configuration_descriptor, name, scope, search_text, type) VALUES (UUID(I.id), I.created_time, I.actions, I.clazz, I.configuration_descriptor, I.name, I.scope, I.search_text, I.type)";
}
}

View File

@ -17,6 +17,8 @@ package org.thingsboard.server.dao.sql.query;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.stereotype.Repository;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.CustomerId;
@ -44,8 +46,6 @@ import org.thingsboard.server.common.data.relation.EntitySearchDirection;
import org.thingsboard.server.common.data.relation.EntityTypeFilter;
import org.thingsboard.server.dao.util.SqlDao;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import java.math.BigInteger;
import java.util.Collections;
import java.util.HashMap;
@ -75,7 +75,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
public static final String HIERARCHICAL_QUERY_TEMPLATE = " FROM (WITH RECURSIVE related_entities(from_id, from_type, to_id, to_type, relation_type, lvl) AS (" +
" SELECT from_id, from_type, to_id, to_type, relation_type, 1 as lvl" +
" FROM relation" +
" WHERE $in_id = '%s' and $in_type = '%s' and relation_type_group = 'COMMON'" +
" WHERE $in_id = :relation_root_id and $in_type = :relation_root_type and relation_type_group = 'COMMON'" +
" UNION ALL" +
" SELECT r.from_id, r.from_type, r.to_id, r.to_type, r.relation_type, lvl + 1" +
" FROM relation r" +
@ -88,21 +88,23 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
public static final String HIERARCHICAL_TO_QUERY_TEMPLATE = HIERARCHICAL_QUERY_TEMPLATE.replace("$in", "to").replace("$out", "from");
public static final String HIERARCHICAL_FROM_QUERY_TEMPLATE = HIERARCHICAL_QUERY_TEMPLATE.replace("$in", "from").replace("$out", "to");
@PersistenceContext
private EntityManager entityManager;
@Autowired
protected NamedParameterJdbcTemplate jdbcTemplate;
@Override
public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) {
EntityType entityType = resolveEntityType(query.getEntityFilter());
String countQuery = String.format("select count(e.id) from %s e where %s",
getEntityTableQuery(query.getEntityFilter(), entityType), this.buildEntityWhere(tenantId, customerId, query.getEntityFilter(),
Collections.emptyList(), entityType));
return ((BigInteger) entityManager.createNativeQuery(countQuery)
.getSingleResult()).longValue();
EntityQueryContext ctx = new EntityQueryContext();
ctx.append("select count(e.id) from ");
ctx.append(addEntityTableQuery(ctx, query.getEntityFilter(), entityType));
ctx.append(" e where ");
ctx.append(buildEntityWhere(ctx, tenantId, customerId, query.getEntityFilter(), Collections.emptyList(), entityType));
return jdbcTemplate.queryForObject(ctx.getQuery(), ctx, Long.class);
}
@Override
public PageData<EntityData> findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query) {
EntityQueryContext ctx = new EntityQueryContext();
EntityType entityType = resolveEntityType(query.getEntityFilter());
EntityDataPageLink pageLink = query.getPageLink();
@ -126,9 +128,9 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
.collect(Collectors.toList());
String entityWhereClause = this.buildEntityWhere(tenantId, customerId, query.getEntityFilter(), entityFieldsFiltersMapping, entityType);
String latestJoins = EntityKeyMapping.buildLatestJoins(query.getEntityFilter(), entityType, allLatestMappings);
String whereClause = this.buildWhere(selectionMapping, latestFiltersMapping, pageLink.getTextSearch());
String entityWhereClause = this.buildEntityWhere(ctx, tenantId, customerId, query.getEntityFilter(), entityFieldsFiltersMapping, entityType);
String latestJoins = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings);
String whereClause = this.buildWhere(ctx, selectionMapping, latestFiltersMapping, pageLink.getTextSearch());
String entityFieldsSelection = EntityKeyMapping.buildSelections(entityFieldsSelectionMapping);
String entityTypeStr;
if (query.getEntityFilter().getType().equals(EntityFilterType.RELATIONS_QUERY)) {
@ -137,9 +139,9 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
entityTypeStr = "'" + entityType.name() + "'";
}
if (!StringUtils.isEmpty(entityFieldsSelection)) {
entityFieldsSelection = String.format("e.id, %s, %s", entityTypeStr, entityFieldsSelection);
entityFieldsSelection = String.format("e.id id, %s entity_type, %s", entityTypeStr, entityFieldsSelection);
} else {
entityFieldsSelection = String.format("e.id, %s", entityTypeStr);
entityFieldsSelection = String.format("e.id id, %s entity_type", entityTypeStr);
}
String latestSelection = EntityKeyMapping.buildSelections(latestSelectionMapping);
String topSelection = "entities.*";
@ -150,13 +152,13 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
String fromClause = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result",
topSelection,
entityFieldsSelection,
getEntityTableQuery(query.getEntityFilter(), entityType),
addEntityTableQuery(ctx, query.getEntityFilter(), entityType),
entityWhereClause,
latestJoins,
whereClause);
int totalElements = ((BigInteger) entityManager.createNativeQuery(String.format("select count(*) %s", fromClause))
.getSingleResult()).intValue();
int totalElements = jdbcTemplate.queryForObject(String.format("select count(*) %s", fromClause), ctx, Integer.class);
String dataQuery = String.format("select * %s", fromClause);
@ -177,17 +179,18 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
if (pageLink.getPageSize() > 0) {
dataQuery = String.format("%s limit %s offset %s", dataQuery, pageLink.getPageSize(), startIndex);
}
List rows = entityManager.createNativeQuery(dataQuery).getResultList();
List<Map<String, Object>> rows = jdbcTemplate.queryForList(dataQuery, ctx);
return EntityDataAdapter.createEntityData(pageLink, selectionMapping, rows, totalElements);
}
private String buildEntityWhere(TenantId tenantId,
private String buildEntityWhere(EntityQueryContext ctx,
TenantId tenantId,
CustomerId customerId,
EntityFilter entityFilter,
List<EntityKeyMapping> entityFieldsFilters,
EntityType entityType) {
String permissionQuery = this.buildPermissionQuery(entityFilter, tenantId, customerId, entityType);
String entityFilterQuery = this.buildEntityFilterQuery(entityFilter);
String permissionQuery = this.buildPermissionQuery(ctx, entityFilter, tenantId, customerId, entityType);
String entityFilterQuery = this.buildEntityFilterQuery(ctx, entityFilter);
String result = permissionQuery;
if (!entityFilterQuery.isEmpty()) {
result += " and " + entityFilterQuery;
@ -198,35 +201,42 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
return result;
}
private String buildPermissionQuery(EntityFilter entityFilter, TenantId tenantId, CustomerId customerId, EntityType entityType) {
private String buildPermissionQuery(EntityQueryContext ctx, EntityFilter entityFilter, TenantId tenantId, CustomerId customerId, EntityType entityType) {
switch (entityFilter.getType()) {
case RELATIONS_QUERY:
case DEVICE_SEARCH_QUERY:
case ASSET_SEARCH_QUERY:
return String.format("e.tenant_id='%s' and e.customer_id='%s'", tenantId.getId(), customerId.getId());
ctx.addUuidParameter("permissions_tenant_id", tenantId.getId());
ctx.addUuidParameter("permissions_customer_id", customerId.getId());
return "e.tenant_id=:permissions_tenant_id and e.customer_id=:permissions_customer_id";
default:
if (entityType == EntityType.TENANT) {
return String.format("e.id='%s'", tenantId.getId());
ctx.addUuidParameter("permissions_tenant_id", tenantId.getId());
return "e.id=:permissions_tenant_id";
} else if (entityType == EntityType.CUSTOMER) {
return String.format("e.tenant_id='%s' and e.id='%s'", tenantId.getId(), customerId.getId());
ctx.addUuidParameter("permissions_tenant_id", tenantId.getId());
ctx.addUuidParameter("permissions_customer_id", customerId.getId());
return "e.tenant_id=:permissions_tenant_id and e.id=:permissions_customer_id";
} else {
return String.format("e.tenant_id='%s' and e.customer_id='%s'", tenantId.getId(), customerId.getId());
ctx.addUuidParameter("permissions_tenant_id", tenantId.getId());
ctx.addUuidParameter("permissions_customer_id", customerId.getId());
return "e.tenant_id=:permissions_tenant_id and e.customer_id=:permissions_customer_id";
}
}
}
private String buildEntityFilterQuery(EntityFilter entityFilter) {
private String buildEntityFilterQuery(EntityQueryContext ctx, EntityFilter entityFilter) {
switch (entityFilter.getType()) {
case SINGLE_ENTITY:
return this.singleEntityQuery((SingleEntityFilter) entityFilter);
return this.singleEntityQuery(ctx, (SingleEntityFilter) entityFilter);
case ENTITY_LIST:
return this.entityListQuery((EntityListFilter) entityFilter);
return this.entityListQuery(ctx, (EntityListFilter) entityFilter);
case ENTITY_NAME:
return this.entityNameQuery((EntityNameFilter) entityFilter);
return this.entityNameQuery(ctx, (EntityNameFilter) entityFilter);
case ASSET_TYPE:
case DEVICE_TYPE:
case ENTITY_VIEW_TYPE:
return this.typeQuery(entityFilter);
return this.typeQuery(ctx, entityFilter);
case RELATIONS_QUERY:
case DEVICE_SEARCH_QUERY:
case ASSET_SEARCH_QUERY:
@ -236,53 +246,60 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
}
}
private String getEntityTableQuery(EntityFilter entityFilter, EntityType entityType) {
private String addEntityTableQuery(EntityQueryContext ctx, EntityFilter entityFilter, EntityType entityType) {
switch (entityFilter.getType()) {
case RELATIONS_QUERY:
return relationQuery((RelationsQueryFilter) entityFilter);
return relationQuery(ctx, (RelationsQueryFilter) entityFilter);
case DEVICE_SEARCH_QUERY:
DeviceSearchQueryFilter deviceQuery = (DeviceSearchQueryFilter) entityFilter;
return entitySearchQuery(deviceQuery, EntityType.DEVICE, deviceQuery.getDeviceTypes());
return entitySearchQuery(ctx, deviceQuery, EntityType.DEVICE, deviceQuery.getDeviceTypes());
case ASSET_SEARCH_QUERY:
AssetSearchQueryFilter assetQuery = (AssetSearchQueryFilter) entityFilter;
return entitySearchQuery(assetQuery, EntityType.ASSET, assetQuery.getAssetTypes());
return entitySearchQuery(ctx, assetQuery, EntityType.ASSET, assetQuery.getAssetTypes());
default:
return entityTableMap.get(entityType);
}
}
private String entitySearchQuery(EntitySearchQueryFilter entityFilter, EntityType entityType, List<String> types) {
private String entitySearchQuery(EntityQueryContext ctx, EntitySearchQueryFilter entityFilter, EntityType entityType, List<String> types) {
EntityId rootId = entityFilter.getRootEntity();
//TODO: fetch last level only.
//TODO: fetch distinct records.
String lvlFilter = getLvlFilter(entityFilter.getMaxLevel());
String selectFields = "SELECT tenant_id, customer_id, id, type, name, label FROM " + entityType.name() + " WHERE id in ( SELECT entity_id";
String from = getQueryTemplate(entityFilter.getDirection());
String whereFilter = " WHERE re.relation_type = :where_relation_type AND re.to_type = :where_entity_type";
String whereFilter = " WHERE " + " re.relation_type = '" + entityFilter.getRelationType() + "'" +
" AND re.to_type = '" + entityType.name() + "'";
from = String.format(from, rootId.getId(), rootId.getEntityType().name(), lvlFilter, whereFilter);
from = String.format(from, lvlFilter, whereFilter);
String query = "( " + selectFields + from + ")";
if (types != null && !types.isEmpty()) {
query += " and type in (" + types.stream().map(type -> "'" + type + "'").collect(Collectors.joining(", ")) + ")";
query += " and type in (:relation_sub_types)";
ctx.addStringListParameter("relation_sub_types", types);
}
query += " )";
ctx.addUuidParameter("relation_root_id", rootId.getId());
ctx.addStringParameter("relation_root_type", rootId.getEntityType().name());
ctx.addStringParameter("where_relation_type", entityFilter.getRelationType());
ctx.addStringParameter("where_entity_type", entityType.name());
return query;
}
private String relationQuery(RelationsQueryFilter entityFilter) {
private String relationQuery(EntityQueryContext ctx, RelationsQueryFilter entityFilter) {
EntityId rootId = entityFilter.getRootEntity();
String lvlFilter = getLvlFilter(entityFilter.getMaxLevel());
String selectFields = getSelectTenantId() + ", " + getSelectCustomerId() + ", " +
" entity.entity_id as id," + getSelectType() + ", " + getSelectName() + ", " +
getSelectLabel() + ", entity.entity_type as entity_type";
String from = getQueryTemplate(entityFilter.getDirection());
ctx.addUuidParameter("relation_root_id", rootId.getId());
ctx.addStringParameter("relation_root_type", rootId.getEntityType().name());
StringBuilder whereFilter;
if (entityFilter.getFilters() != null && !entityFilter.getFilters().isEmpty()) {
whereFilter = new StringBuilder(" WHERE ");
boolean first = true;
boolean single = entityFilter.getFilters().size() == 1;
int entityTypeFilterIdx = 0;
for (EntityTypeFilter etf : entityFilter.getFilters()) {
if (first) {
first = false;
@ -290,21 +307,23 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
whereFilter.append(" AND ");
}
String relationType = etf.getRelationType();
String entityTypes = etf.getEntityTypes().stream().map(type -> "'" + type + "'").collect(Collectors.joining(", "));
if (!single) {
whereFilter.append(" (");
}
whereFilter.append(" re.relation_type = '").append(relationType).append("' and re.")
whereFilter.append(" re.relation_type = :where_relation_type").append(entityTypeFilterIdx).append(" and re.")
.append(entityFilter.getDirection().equals(EntitySearchDirection.FROM) ? "to" : "from")
.append("_type in (").append(entityTypes).append(")");
.append("_type in (:where_entity_types").append(entityTypeFilterIdx).append(")");
if (!single) {
whereFilter.append(" )");
}
ctx.addStringParameter("where_relation_type" + entityTypeFilterIdx, relationType);
ctx.addStringListParameter("where_entity_types" + entityTypeFilterIdx, etf.getEntityTypes().stream().map(EntityType::name).collect(Collectors.toList()));
entityTypeFilterIdx++;
}
} else {
whereFilter = new StringBuilder();
}
from = String.format(from, rootId.getId(), rootId.getEntityType().name(), lvlFilter, whereFilter);
from = String.format(from, lvlFilter, whereFilter);
return "( " + selectFields + from + ")";
}
@ -410,10 +429,9 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
" END as label";
}
private String buildWhere
(List<EntityKeyMapping> selectionMapping, List<EntityKeyMapping> latestFiltersMapping, String searchText) {
String latestFilters = EntityKeyMapping.buildQuery(latestFiltersMapping);
String textSearchQuery = this.buildTextSearchQuery(selectionMapping, searchText);
private String buildWhere(EntityQueryContext ctx, List<EntityKeyMapping> selectionMapping, List<EntityKeyMapping> latestFiltersMapping, String searchText) {
String latestFilters = EntityKeyMapping.buildQuery(ctx, latestFiltersMapping);
String textSearchQuery = this.buildTextSearchQuery(ctx, selectionMapping, searchText);
String query;
if (!StringUtils.isEmpty(latestFilters) && !StringUtils.isEmpty(textSearchQuery)) {
query = String.join(" AND ", latestFilters, textSearchQuery);
@ -429,32 +447,38 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
}
}
private String buildTextSearchQuery(List<EntityKeyMapping> selectionMapping, String searchText) {
private String buildTextSearchQuery(EntityQueryContext ctx, List<EntityKeyMapping> selectionMapping, String searchText) {
if (!StringUtils.isEmpty(searchText) && !selectionMapping.isEmpty()) {
String lowerSearchText = searchText.toLowerCase() + "%";
List<String> searchPredicates = selectionMapping.stream().map(mapping -> String.format("LOWER(%s) LIKE '%s'",
mapping.getValueAlias(), lowerSearchText)).collect(Collectors.toList());
List<String> searchPredicates = selectionMapping.stream().map(mapping -> {
String paramName = mapping.getValueAlias() + "_lowerSearchText";
ctx.addStringParameter(paramName, lowerSearchText);
return String.format("LOWER(%s) LIKE :%s", mapping.getValueAlias(), paramName);
}
).collect(Collectors.toList());
return String.format("(%s)", String.join(" or ", searchPredicates));
} else {
return null;
}
}
private String singleEntityQuery(SingleEntityFilter filter) {
return String.format("e.id='%s'", filter.getSingleEntity().getId());
private String singleEntityQuery(EntityQueryContext ctx, SingleEntityFilter filter) {
ctx.addUuidParameter("entity_filter_single_entity_id", filter.getSingleEntity().getId());
return "e.id=:entity_filter_single_entity_id";
}
private String entityListQuery(EntityListFilter filter) {
return String.format("e.id in (%s)",
filter.getEntityList().stream().map(UUID::fromString)
.map(s -> String.format("'%s'", s)).collect(Collectors.joining(",")));
private String entityListQuery(EntityQueryContext ctx, EntityListFilter filter) {
ctx.addUuidListParameter("entity_filter_entity_ids", filter.getEntityList().stream().map(UUID::fromString).collect(Collectors.toList()));
return "e.id in (:entity_filter_entity_ids)";
}
private String entityNameQuery(EntityNameFilter filter) {
return String.format("lower(e.search_text) like lower(concat(%s, '%%'))", filter.getEntityNameFilter());
private String entityNameQuery(EntityQueryContext ctx, EntityNameFilter filter) {
ctx.addStringParameter("entity_filter_name_filter", filter.getEntityNameFilter());
return "lower(e.search_text) like lower(concat(:entity_filter_name_filter, '%%'))";
}
private String typeQuery(EntityFilter filter) {
private String typeQuery(EntityQueryContext ctx, EntityFilter filter) {
String type;
String name;
switch (filter.getType()) {
@ -473,7 +497,9 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
default:
throw new RuntimeException("Not supported!");
}
return String.format("e.type = '%s' and lower(e.search_text) like lower(concat('%s', '%%'))", type, name);
ctx.addStringParameter("entity_filter_type_query_type", type);
ctx.addStringParameter("entity_filter_type_query_name", name);
return "e.type = :entity_filter_type_query_type and lower(e.search_text) like lower(concat(:entity_filter_type_query_name, '%%'))";
}
private EntityType resolveEntityType(EntityFilter entityFilter) {

View File

@ -37,7 +37,7 @@ public class EntityDataAdapter {
public static PageData<EntityData> createEntityData(EntityDataPageLink pageLink,
List<EntityKeyMapping> selectionMapping,
List<Object[]> rows,
List<Map<String, Object>> rows,
int totalElements) {
int totalPages = pageLink.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageLink.getPageSize()) : 1;
int startIndex = pageLink.getPageSize() * pageLink.getPage();
@ -46,14 +46,13 @@ public class EntityDataAdapter {
return new PageData<>(entitiesData, totalPages, totalElements, hasNext);
}
private static List<EntityData> convertListToEntityData(List<Object[]> result, List<EntityKeyMapping> selectionMapping) {
private static List<EntityData> convertListToEntityData(List<Map<String, Object>> result, List<EntityKeyMapping> selectionMapping) {
return result.stream().map(row -> toEntityData(row, selectionMapping)).collect(Collectors.toList());
}
private static EntityData toEntityData(Object[] row, List<EntityKeyMapping> selectionMapping) {
ByteBuffer bb = ByteBuffer.wrap((byte[])row[0]);
UUID id = new UUID(bb.getLong(), bb.getLong());
EntityType entityType = EntityType.valueOf((String)row[1]);
private static EntityData toEntityData(Map<String, Object> row, List<EntityKeyMapping> selectionMapping) {
UUID id = (UUID)row.get("id");
EntityType entityType = EntityType.valueOf((String) row.get("entity_type"));
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id);
Map<EntityKeyType, Map<String, TsValue>> latest = new HashMap<>();
Map<String, TsValue[]> timeseries = new HashMap<>();
@ -61,7 +60,7 @@ public class EntityDataAdapter {
for (EntityKeyMapping mapping : selectionMapping) {
if (!mapping.isIgnore()) {
EntityKey entityKey = mapping.getEntityKey();
Object value = row[mapping.getIndex()];
Object value = row.get(mapping.getValueAlias());
String strValue;
long ts;
if (entityKey.getType().equals(EntityKeyType.ENTITY_FIELD)) {
@ -69,7 +68,7 @@ public class EntityDataAdapter {
ts = System.currentTimeMillis();
} else {
strValue = convertValue(value);
Object tsObject = row[mapping.getIndex() + 1];
Object tsObject = row.get(mapping.getTsAlias());
ts = tsObject != null ? Long.parseLong(tsObject.toString()) : 0;
}
TsValue tsValue = new TsValue(ts, strValue);

View File

@ -72,6 +72,7 @@ public class EntityKeyMapping {
private boolean ignore = false;
private List<KeyFilter> keyFilters;
private EntityKey entityKey;
private int paramIdx = 0;
public boolean hasFilter() {
return keyFilters != null && !keyFilters.isEmpty();
@ -100,17 +101,17 @@ public class EntityKeyMapping {
}
}
public Stream<String> toQueries() {
public Stream<String> toQueries(EntityQueryContext ctx) {
if (hasFilter()) {
String keyAlias = entityKey.getType().equals(EntityKeyType.ENTITY_FIELD) ? "e" : alias;
return keyFilters.stream().map(keyFilter ->
this.buildKeyQuery(keyAlias, keyFilter));
this.buildKeyQuery(ctx, keyAlias, keyFilter));
} else {
return null;
}
}
public String toLatestJoin(EntityFilter entityFilter, EntityType entityType) {
public String toLatestJoin(EntityQueryContext ctx, EntityFilter entityFilter, EntityType entityType) {
String entityTypeStr;
if (entityFilter.getType().equals(EntityFilterType.RELATIONS_QUERY)) {
entityTypeStr = "entities.entity_type";
@ -118,12 +119,13 @@ public class EntityKeyMapping {
entityTypeStr = "'" + entityType.name() + "'";
}
String join = hasFilter() ? "left join" : "left outer join";
ctx.addStringParameter(alias + "_key_id", entityKey.getKey());
if (entityKey.getType().equals(EntityKeyType.TIME_SERIES)) {
return String.format("%s ts_kv_latest %s ON %s.entity_id=to_uuid(entities.id) AND %s.key = (select key_id from ts_kv_dictionary where key = '%s')",
join, alias, alias, alias, entityKey.getKey());
return String.format("%s ts_kv_latest %s ON %s.entity_id=to_uuid(entities.id) AND %s.key = (select key_id from ts_kv_dictionary where key = :%s_key_id)",
join, alias, alias, alias, alias);
} else {
String query = String.format("%s attribute_kv %s ON %s.entity_id=entities.id AND %s.entity_type=%s AND %s.attribute_key='%s'",
join, alias, alias, alias, entityTypeStr, alias, entityKey.getKey());
String query = String.format("%s attribute_kv %s ON %s.entity_id=entities.id AND %s.entity_type=%s AND %s.attribute_key=:%s_key_id",
join, alias, alias, alias, entityTypeStr, alias, alias);
if (!entityKey.getType().equals(EntityKeyType.ATTRIBUTE)) {
String scope;
if (entityKey.getType().equals(EntityKeyType.CLIENT_ATTRIBUTE)) {
@ -144,13 +146,13 @@ public class EntityKeyMapping {
Collectors.joining(", "));
}
public static String buildLatestJoins(EntityFilter entityFilter, EntityType entityType, List<EntityKeyMapping> latestMappings) {
return latestMappings.stream().map(mapping -> mapping.toLatestJoin(entityFilter, entityType)).collect(
public static String buildLatestJoins(EntityQueryContext ctx, EntityFilter entityFilter, EntityType entityType, List<EntityKeyMapping> latestMappings) {
return latestMappings.stream().map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)).collect(
Collectors.joining(" "));
}
public static String buildQuery(List<EntityKeyMapping> mappings) {
return mappings.stream().flatMap(EntityKeyMapping::toQueries).collect(
public static String buildQuery(EntityQueryContext ctx, List<EntityKeyMapping> mappings) {
return mappings.stream().flatMap(mapping -> mapping.toQueries(ctx)).collect(
Collectors.joining(" AND "));
}
@ -262,33 +264,33 @@ public class EntityKeyMapping {
return String.join(", ", attrValSelection, attrTsSelection);
}
private String buildKeyQuery(String alias, KeyFilter keyFilter) {
return this.buildPredicateQuery(alias, keyFilter.getKey(), keyFilter.getPredicate());
private String buildKeyQuery(EntityQueryContext ctx, String alias, KeyFilter keyFilter) {
return this.buildPredicateQuery(ctx, alias, keyFilter.getKey(), keyFilter.getPredicate());
}
private String buildPredicateQuery(String alias, EntityKey key, KeyFilterPredicate predicate) {
private String buildPredicateQuery(EntityQueryContext ctx, String alias, EntityKey key, KeyFilterPredicate predicate) {
if (predicate.getType().equals(FilterPredicateType.COMPLEX)) {
return this.buildComplexPredicateQuery(alias, key, (ComplexFilterPredicate) predicate);
return this.buildComplexPredicateQuery(ctx, alias, key, (ComplexFilterPredicate) predicate);
} else {
return this.buildSimplePredicateQuery(alias, key, predicate);
return this.buildSimplePredicateQuery(ctx, alias, key, predicate);
}
}
private String buildComplexPredicateQuery(String alias, EntityKey key, ComplexFilterPredicate predicate) {
private String buildComplexPredicateQuery(EntityQueryContext ctx, String alias, EntityKey key, ComplexFilterPredicate predicate) {
return predicate.getPredicates().stream()
.map(keyFilterPredicate -> this.buildPredicateQuery(alias, key, keyFilterPredicate)).collect(Collectors.joining(
.map(keyFilterPredicate -> this.buildPredicateQuery(ctx, alias, key, keyFilterPredicate)).collect(Collectors.joining(
" " + predicate.getOperation().name() + " "
));
}
private String buildSimplePredicateQuery(String alias, EntityKey key, KeyFilterPredicate predicate) {
private String buildSimplePredicateQuery(EntityQueryContext ctx, String alias, EntityKey key, KeyFilterPredicate predicate) {
if (predicate.getType().equals(FilterPredicateType.NUMERIC)) {
if (key.getType().equals(EntityKeyType.ENTITY_FIELD)) {
String column = entityFieldColumnMap.get(key.getKey());
return this.buildNumericPredicateQuery(alias + "." + column, (NumericFilterPredicate) predicate);
return this.buildNumericPredicateQuery(ctx, alias + "." + column, (NumericFilterPredicate) predicate);
} else {
String longQuery = this.buildNumericPredicateQuery(alias + ".long_v", (NumericFilterPredicate) predicate);
String doubleQuery = this.buildNumericPredicateQuery(alias + ".dbl_v", (NumericFilterPredicate) predicate);
String longQuery = this.buildNumericPredicateQuery(ctx, alias + ".long_v", (NumericFilterPredicate) predicate);
String doubleQuery = this.buildNumericPredicateQuery(ctx, alias + ".dbl_v", (NumericFilterPredicate) predicate);
return String.format("(%s or %s)", longQuery, doubleQuery);
}
} else {
@ -300,15 +302,16 @@ public class EntityKeyMapping {
}
String field = alias + "." + column;
if (predicate.getType().equals(FilterPredicateType.STRING)) {
return this.buildStringPredicateQuery(field, (StringFilterPredicate) predicate);
return this.buildStringPredicateQuery(ctx, field, (StringFilterPredicate) predicate);
} else {
return this.buildBooleanPredicateQuery(field, (BooleanFilterPredicate) predicate);
return this.buildBooleanPredicateQuery(ctx, field, (BooleanFilterPredicate) predicate);
}
}
}
private String buildStringPredicateQuery(String field, StringFilterPredicate stringFilterPredicate) {
private String buildStringPredicateQuery(EntityQueryContext ctx, String field, StringFilterPredicate stringFilterPredicate) {
String operationField = field;
String paramName = getNextParameterName(field);
String value = stringFilterPredicate.getValue();
String stringOperationQuery = "";
if (stringFilterPredicate.isIgnoreCase()) {
@ -317,65 +320,77 @@ public class EntityKeyMapping {
}
switch (stringFilterPredicate.getOperation()) {
case EQUAL:
stringOperationQuery = String.format("%s = '%s'", operationField, value);
stringOperationQuery = String.format("%s = :%s", operationField, paramName);
break;
case NOT_EQUAL:
stringOperationQuery = String.format("%s != '%s'", operationField, value);
stringOperationQuery = String.format("%s != :%s", operationField, paramName);
break;
case STARTS_WITH:
stringOperationQuery = String.format("%s like '%s%%'", operationField, value);
value += "%";
stringOperationQuery = String.format("%s like :%s", operationField, paramName);
break;
case ENDS_WITH:
stringOperationQuery = String.format("%s like '%%%s'", operationField, value);
value = "%" + value;
stringOperationQuery = String.format("%s like :%s", operationField, paramName);
break;
case CONTAINS:
stringOperationQuery = String.format("%s like '%%%s%%'", operationField, value);
value = "%" + value + "%";
stringOperationQuery = String.format("%s like :%s", operationField, paramName);
break;
case NOT_CONTAINS:
stringOperationQuery = String.format("%s not like '%%%s%%'", operationField, value);
value = "%" + value + "%";
stringOperationQuery = String.format("%s not like :%s", operationField, paramName);
break;
}
ctx.addStringParameter(paramName, value);
return String.format("(%s is not null and %s)", field, stringOperationQuery);
}
private String buildNumericPredicateQuery(String field, NumericFilterPredicate numericFilterPredicate) {
double value = numericFilterPredicate.getValue();
private String buildNumericPredicateQuery(EntityQueryContext ctx, String field, NumericFilterPredicate numericFilterPredicate) {
String paramName = getNextParameterName(field);
ctx.addDoubleParameter(paramName, numericFilterPredicate.getValue());
String numericOperationQuery = "";
switch (numericFilterPredicate.getOperation()) {
case EQUAL:
numericOperationQuery = String.format("%s = %s", field, value);
numericOperationQuery = String.format("%s = :%s", field, paramName);
break;
case NOT_EQUAL:
numericOperationQuery = String.format("%s != '%s'", field, value);
numericOperationQuery = String.format("%s != :%s", field, paramName);
break;
case GREATER:
numericOperationQuery = String.format("%s > %s", field, value);
numericOperationQuery = String.format("%s > :%s", field, paramName);
break;
case GREATER_OR_EQUAL:
numericOperationQuery = String.format("%s >= %s", field, value);
numericOperationQuery = String.format("%s >= :%s", field, paramName);
break;
case LESS:
numericOperationQuery = String.format("%s < %s", field, value);
numericOperationQuery = String.format("%s < :%s", field, paramName);
break;
case LESS_OR_EQUAL:
numericOperationQuery = String.format("%s <= %s", field, value);
numericOperationQuery = String.format("%s <= :%s", field, paramName);
break;
}
return String.format("(%s is not null and %s)", field, numericOperationQuery);
}
private String buildBooleanPredicateQuery(String field,
private String buildBooleanPredicateQuery(EntityQueryContext ctx, String field,
BooleanFilterPredicate booleanFilterPredicate) {
boolean value = booleanFilterPredicate.isValue();
String paramName = getNextParameterName(field);
ctx.addBooleanParameter(paramName, booleanFilterPredicate.isValue());
String booleanOperationQuery = "";
switch (booleanFilterPredicate.getOperation()) {
case EQUAL:
booleanOperationQuery = String.format("%s = %s", field, value);
booleanOperationQuery = String.format("%s = :%s", field, paramName);
break;
case NOT_EQUAL:
booleanOperationQuery = String.format("%s != %s", field, value);
booleanOperationQuery = String.format("%s != :%s", field, paramName);
break;
}
return String.format("(%s is not null and %s)", field, booleanOperationQuery);
}
private String getNextParameterName(String field) {
paramIdx++;
return field.replace(".", "_") + "_" + paramIdx;
}
}

View File

@ -0,0 +1,122 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sql.query;
import org.hibernate.type.PostgresUUIDType;
import org.springframework.jdbc.core.namedparam.SqlParameterSource;
import java.sql.Types;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
public class EntityQueryContext implements SqlParameterSource {
private static final PostgresUUIDType UUID_TYPE = new PostgresUUIDType();
private final StringBuilder query;
private final Map<String, Parameter> params;
public EntityQueryContext() {
query = new StringBuilder();
params = new HashMap<>();
}
void addParameter(String name, Object value, int type, String typeName) {
Parameter existing = params.put(name, new Parameter(value, type, typeName));
if (existing != null) {
throw new RuntimeException("Parameter with name: " + name + " was already registered!");
}
}
public void append(String s) {
query.append(s);
}
@Override
public boolean hasValue(String paramName) {
return params.containsKey(paramName);
}
@Override
public Object getValue(String paramName) throws IllegalArgumentException {
return checkParameter(paramName).value;
}
@Override
public int getSqlType(String paramName) {
return checkParameter(paramName).type;
}
private Parameter checkParameter(String paramName) {
Parameter param = params.get(paramName);
if (param == null) {
throw new RuntimeException("Parameter with name: " + paramName + " is not set!");
}
return param;
}
@Override
public String getTypeName(String paramName) {
return params.get(paramName).name;
}
@Override
public String[] getParameterNames() {
return params.keySet().toArray(new String[]{});
}
public void addUuidParameter(String name, UUID value) {
addParameter(name, value, UUID_TYPE.sqlType(), UUID_TYPE.getName());
}
public void addStringParameter(String name, String value) {
addParameter(name, value, Types.VARCHAR, "VARCHAR");
}
public void addDoubleParameter(String name, double value) {
addParameter(name, value, Types.DOUBLE, "DOUBLE");
}
public void addStringListParameter(String name, List<String> value) {
addParameter(name, value, Types.VARCHAR, "VARCHAR");
}
public void addBooleanParameter(String name, boolean value) {
addParameter(name, value, Types.BOOLEAN, "BOOLEAN");
}
public void addUuidListParameter(String name, List<UUID> value) {
addParameter(name, value, UUID_TYPE.sqlType(), UUID_TYPE.getName());
}
public String getQuery() {
return query.toString();
}
public static class Parameter {
private final Object value;
private final int type;
private final String name;
public Parameter(Object value, int type, String name) {
this.value = value;
this.type = type;
this.name = name;
}
}
}

View File

@ -24,30 +24,30 @@ import java.util.Arrays;
@RunWith(ClasspathSuite.class)
@ClassnameFilters({
"org.thingsboard.server.dao.service.sql.EntityServiceSqlTest"
"org.thingsboard.server.dao.service.sql.*SqlTest"
})
public class SqlDaoServiceTestSuite {
// @ClassRule
// public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
// Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/schema-entities-idx.sql"
// , "sql/system-data.sql"
// , "sql/system-test.sql"
// ),
// "sql/hsql/drop-all-tables.sql",
// "sql-test.properties"
// );
@ClassRule
public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
Arrays.asList("sql/schema-ts-psql.sql"
, "sql/schema-entities.sql", "sql/schema-entities-idx.sql"
, "sql/system-data.sql", "sql/system-test.sql"
Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/schema-entities-idx.sql"
, "sql/system-data.sql"
, "sql/system-test.sql"
),
"sql/psql/drop-all-tables.sql",
"sql/hsql/drop-all-tables.sql",
"sql-test.properties"
);
// @ClassRule
// public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
// Arrays.asList("sql/schema-ts-psql.sql"
// , "sql/schema-entities.sql", "sql/schema-entities-idx.sql"
// , "sql/system-data.sql", "sql/system-test.sql"
// ),
// "sql/psql/drop-all-tables.sql",
// "sql-test.properties"
// );
// @ClassRule
// public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
// Arrays.asList("sql/schema-timescale.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"),

View File

@ -20,6 +20,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.server.common.data.DataConstants;

View File

@ -3,18 +3,18 @@ database.ts.type=sql
sql.ts_inserts_executor_type=fixed
sql.ts_inserts_fixed_thread_pool_size=200
sql.ts_key_value_partitioning=MONTHS
#spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true
#spring.jpa.properties.hibernate.order_by.default_null_ordering=last
#spring.jpa.show-sql=false
#spring.jpa.hibernate.ddl-auto=validate
#spring.jpa.database-platform=org.hibernate.dialect.HSQLDialect
#
#spring.datasource.username=sa
#spring.datasource.password=
#spring.datasource.url=jdbc:hsqldb:file:/tmp/testDb;sql.enforce_size=false
#spring.datasource.driverClassName=org.hsqldb.jdbc.JDBCDriver
#spring.datasource.hikari.maximumPoolSize = 50
spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true
spring.jpa.properties.hibernate.order_by.default_null_ordering=last
spring.jpa.show-sql=false
spring.jpa.hibernate.ddl-auto=validate
spring.jpa.database-platform=org.hibernate.dialect.HSQLDialect
spring.datasource.username=sa
spring.datasource.password=
spring.datasource.url=jdbc:hsqldb:file:/tmp/testDb;sql.enforce_size=false
spring.datasource.driverClassName=org.hsqldb.jdbc.JDBCDriver
spring.datasource.hikari.maximumPoolSize = 50
service.type=monolith
@ -26,16 +26,16 @@ service.type=monolith
#sql.ts_inserts_fixed_thread_pool_size=200
#sql.ts_key_value_partitioning=MONTHS
#
spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true
spring.jpa.show-sql=false
spring.jpa.hibernate.ddl-auto=none
spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect
spring.datasource.username=postgres
spring.datasource.password=postgres
spring.datasource.url=jdbc:postgresql://localhost:5432/sqltest
spring.datasource.driverClassName=org.postgresql.Driver
spring.datasource.hikari.maximumPoolSize = 50
#spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true
#spring.jpa.show-sql=false
#spring.jpa.hibernate.ddl-auto=none
#spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect
#
#spring.datasource.username=postgres
#spring.datasource.password=postgres
#spring.datasource.url=jdbc:postgresql://localhost:5432/sqltest
#spring.datasource.driverClassName=org.postgresql.Driver
#spring.datasource.hikari.maximumPoolSize = 50
queue.core.pack-processing-timeout=3000
queue.rule-engine.pack-processing-timeout=3000