added ability to get attributes and timeseries keys by device profile id (#3817)

* added ability to get attributes and timeseries keys by entity ids

* refactored

* refactored

* get attributes and timeseries keys by device profile improvements

* added limit

* added device_profile_id index and upgrade script

* improvements

* refactored
This commit is contained in:
Yevhen Bondarenko 2020-12-10 15:18:28 +02:00 committed by GitHub
parent 24ccd2a2b5
commit 5d30243eea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 160 additions and 14 deletions

View File

@ -16,6 +16,8 @@
package org.thingsboard.server.controller;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PathVariable;
@ -35,21 +37,30 @@ import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
import java.util.List;
import java.util.UUID;
@RestController
@TbCoreComponent
@RequestMapping("/api")
@Slf4j
public class DeviceProfileController extends BaseController {
private static final String DEVICE_PROFILE_ID = "deviceProfileId";
@Autowired
private TimeseriesService timeseriesService;
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/deviceProfile/{deviceProfileId}", method = RequestMethod.GET)
@ResponseBody
public DeviceProfile getDeviceProfileById(@PathVariable("deviceProfileId") String strDeviceProfileId) throws ThingsboardException {
checkParameter("deviceProfileId", strDeviceProfileId);
public DeviceProfile getDeviceProfileById(@PathVariable(DEVICE_PROFILE_ID) String strDeviceProfileId) throws ThingsboardException {
checkParameter(DEVICE_PROFILE_ID, strDeviceProfileId);
try {
DeviceProfileId deviceProfileId = new DeviceProfileId(toUUID(strDeviceProfileId));
return checkDeviceProfileId(deviceProfileId, Operation.READ);
@ -61,8 +72,8 @@ public class DeviceProfileController extends BaseController {
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/deviceProfileInfo/{deviceProfileId}", method = RequestMethod.GET)
@ResponseBody
public DeviceProfileInfo getDeviceProfileInfoById(@PathVariable("deviceProfileId") String strDeviceProfileId) throws ThingsboardException {
checkParameter("deviceProfileId", strDeviceProfileId);
public DeviceProfileInfo getDeviceProfileInfoById(@PathVariable(DEVICE_PROFILE_ID) String strDeviceProfileId) throws ThingsboardException {
checkParameter(DEVICE_PROFILE_ID, strDeviceProfileId);
try {
DeviceProfileId deviceProfileId = new DeviceProfileId(toUUID(strDeviceProfileId));
return checkNotNull(deviceProfileService.findDeviceProfileInfoById(getTenantId(), deviceProfileId));
@ -82,6 +93,46 @@ public class DeviceProfileController extends BaseController {
}
}
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/deviceProfile/devices/keys/timeseries", method = RequestMethod.GET)
@ResponseBody
public List<String> getTimeseriesKeys(
@RequestParam(name = DEVICE_PROFILE_ID, required = false) String deviceProfileIdStr) throws ThingsboardException {
DeviceProfileId deviceProfileId;
if (StringUtils.isNotEmpty(deviceProfileIdStr)) {
deviceProfileId = new DeviceProfileId(UUID.fromString(deviceProfileIdStr));
checkDeviceProfileId(deviceProfileId, Operation.READ);
} else {
deviceProfileId = null;
}
try {
return timeseriesService.findAllKeysByDeviceProfileId(getTenantId(), deviceProfileId);
} catch (Exception e) {
throw handleException(e);
}
}
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/deviceProfile/devices/keys/attributes", method = RequestMethod.GET)
@ResponseBody
public List<String> getAttributesKeys(
@RequestParam(name = DEVICE_PROFILE_ID, required = false) String deviceProfileIdStr) throws ThingsboardException {
DeviceProfileId deviceProfileId;
if (StringUtils.isNotEmpty(deviceProfileIdStr)) {
deviceProfileId = new DeviceProfileId(UUID.fromString(deviceProfileIdStr));
checkDeviceProfileId(deviceProfileId, Operation.READ);
} else {
deviceProfileId = null;
}
try {
return attributesService.findAllKeysByDeviceProfileId(getTenantId(), deviceProfileId);
} catch (Exception e) {
throw handleException(e);
}
}
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/deviceProfile", method = RequestMethod.POST)
@ResponseBody
@ -113,8 +164,8 @@ public class DeviceProfileController extends BaseController {
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/deviceProfile/{deviceProfileId}", method = RequestMethod.DELETE)
@ResponseStatus(value = HttpStatus.OK)
public void deleteDeviceProfile(@PathVariable("deviceProfileId") String strDeviceProfileId) throws ThingsboardException {
checkParameter("deviceProfileId", strDeviceProfileId);
public void deleteDeviceProfile(@PathVariable(DEVICE_PROFILE_ID) String strDeviceProfileId) throws ThingsboardException {
checkParameter(DEVICE_PROFILE_ID, strDeviceProfileId);
try {
DeviceProfileId deviceProfileId = new DeviceProfileId(toUUID(strDeviceProfileId));
DeviceProfile deviceProfile = checkDeviceProfileId(deviceProfileId, Operation.DELETE);
@ -139,8 +190,8 @@ public class DeviceProfileController extends BaseController {
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/deviceProfile/{deviceProfileId}/default", method = RequestMethod.POST)
@ResponseBody
public DeviceProfile setDefaultDeviceProfile(@PathVariable("deviceProfileId") String strDeviceProfileId) throws ThingsboardException {
checkParameter("deviceProfileId", strDeviceProfileId);
public DeviceProfile setDefaultDeviceProfile(@PathVariable(DEVICE_PROFILE_ID) String strDeviceProfileId) throws ThingsboardException {
checkParameter(DEVICE_PROFILE_ID, strDeviceProfileId);
try {
DeviceProfileId deviceProfileId = new DeviceProfileId(toUUID(strDeviceProfileId));
DeviceProfile deviceProfile = checkDeviceProfileId(deviceProfileId, Operation.WRITE);

View File

@ -186,6 +186,10 @@ public class ThingsboardInstallService {
systemDataLoaderService.updateSystemWidgets();
systemDataLoaderService.createOAuth2Templates();
break;
case "3.2.0":
log.info("Upgrading ThingsBoard from version 3.2.0 to 3.2.1 ...");
databaseEntitiesUpgradeService.upgradeDatabase("3.2.0");
break;
default:
throw new RuntimeException("Unable to upgrade ThingsBoard, unsupported fromVersion: " + upgradeFromVersion);

View File

@ -421,6 +421,18 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
log.error("Failed updating schema!!!", e);
}
break;
case "3.2.0":
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
log.info("Updating schema ...");
try {
conn.createStatement().execute("CREATE INDEX IF NOT EXISTS idx_device_device_profile_id ON device(tenant_id, device_profile_id);");
conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3002001;");
} catch (Exception e) {
log.error("Failed updating schema!!!", e);
}
log.info("Schema updated.");
}
break;
default:
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
}

View File

@ -16,6 +16,7 @@
package org.thingsboard.server.dao.attributes;
import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
@ -38,4 +39,7 @@ public interface AttributesService {
ListenableFuture<List<Void>> save(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes);
ListenableFuture<List<Void>> removeAll(TenantId tenantId, EntityId entityId, String scope, List<String> attributeKeys);
List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId);
}

View File

@ -16,6 +16,7 @@
package org.thingsboard.server.dao.timeseries;
import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
@ -47,4 +48,6 @@ public interface TimeseriesService {
ListenableFuture<List<Void>> removeLatest(TenantId tenantId, EntityId entityId, Collection<String> keys);
ListenableFuture<Collection<String>> removeAllLatest(TenantId tenantId, EntityId entityId);
List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId);
}

View File

@ -16,6 +16,7 @@
package org.thingsboard.server.dao.attributes;
import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
@ -38,4 +39,6 @@ public interface AttributesDao {
ListenableFuture<Void> save(TenantId tenantId, EntityId entityId, String attributeType, AttributeKvEntry attribute);
ListenableFuture<List<Void>> removeAll(TenantId tenantId, EntityId entityId, String attributeType, List<String> keys);
List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId);
}

View File

@ -20,6 +20,7 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
@ -59,6 +60,11 @@ public class BaseAttributesService implements AttributesService {
return attributesDao.findAll(tenantId, entityId, scope);
}
@Override
public List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId) {
return attributesDao.findAllKeysByDeviceProfileId(tenantId, deviceProfileId);
}
@Override
public ListenableFuture<List<Void>> save(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes) {
validate(entityId, scope);

View File

@ -46,5 +46,15 @@ public interface AttributeKvRepository extends CrudRepository<AttributeKvEntity,
@Param("entityId") UUID entityId,
@Param("attributeType") String attributeType,
@Param("attributeKey") String attributeKey);
@Query(value = "SELECT DISTINCT attribute_key FROM attribute_kv WHERE entity_type = 'DEVICE' " +
"AND entity_id in (SELECT id FROM device WHERE tenant_id = :tenantId and device_profile_id = :deviceProfileId limit 100) ORDER BY attribute_key", nativeQuery = true)
List<String> findAllKeysByDeviceProfileId(@Param("tenantId") UUID tenantId,
@Param("deviceProfileId") UUID deviceProfileId);
@Query(value = "SELECT DISTINCT attribute_key FROM attribute_kv WHERE entity_type = 'DEVICE' " +
"AND entity_id in (SELECT id FROM device WHERE tenant_id = :tenantId limit 100) ORDER BY attribute_key", nativeQuery = true)
List<String> findAllKeysByTenantId(@Param("tenantId") UUID tenantId);
}

View File

@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
@ -135,6 +136,15 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl
attributeType))));
}
@Override
public List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId) {
if (deviceProfileId != null) {
return attributeKvRepository.findAllKeysByDeviceProfileId(tenantId.getId(), deviceProfileId.getId());
} else {
return attributeKvRepository.findAllKeysByTenantId(tenantId.getId());
}
}
@Override
public ListenableFuture<Void> save(TenantId tenantId, EntityId entityId, String attributeType, AttributeKvEntry attribute) {
AttributeKvEntity entity = new AttributeKvEntity();

View File

@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.Aggregation;
@ -51,10 +52,15 @@ import org.thingsboard.server.dao.util.SqlTsLatestAnyDao;
import javax.annotation.Nullable;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.*;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
import java.util.stream.Collectors;
@Slf4j
@Component
@ -154,6 +160,15 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme
return getFindAllLatestFuture(entityId);
}
@Override
public List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId) {
if (deviceProfileId != null) {
return tsKvLatestRepository.getKeysByDeviceProfileId(tenantId.getId(), deviceProfileId.getId());
} else {
return tsKvLatestRepository.getKeysByTenantId(tenantId.getId());
}
}
private ListenableFuture<Void> getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
ListenableFuture<List<TsKvEntry>> future = findNewLatestEntryFuture(tenantId, entityId, query);
return Futures.transformAsync(future, entryList -> {

View File

@ -15,10 +15,25 @@
*/
package org.thingsboard.server.dao.sqlts.latest;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.CrudRepository;
import org.springframework.data.repository.query.Param;
import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey;
import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity;
import java.util.List;
import java.util.UUID;
public interface TsKvLatestRepository extends CrudRepository<TsKvLatestEntity, TsKvLatestCompositeKey> {
@Query(value = "SELECT DISTINCT ts_kv_dictionary.key AS strKey FROM ts_kv_latest " +
"INNER JOIN ts_kv_dictionary ON ts_kv_latest.key = ts_kv_dictionary.key_id " +
"WHERE ts_kv_latest.entity_id IN (SELECT id FROM device WHERE device_profile_id = :device_profile_id AND tenant_id = :tenant_id limit 100) ORDER BY ts_kv_dictionary.key", nativeQuery = true)
List<String> getKeysByDeviceProfileId(@Param("tenant_id") UUID tenantId, @Param("device_profile_id") UUID deviceProfileId);
@Query(value = "SELECT DISTINCT ts_kv_dictionary.key AS strKey FROM ts_kv_latest " +
"INNER JOIN ts_kv_dictionary ON ts_kv_latest.key = ts_kv_dictionary.key_id " +
"WHERE ts_kv_latest.entity_id IN (SELECT id FROM device WHERE tenant_id = :tenant_id limit 100) ORDER BY ts_kv_dictionary.key", nativeQuery = true)
List<String> getKeysByTenantId(@Param("tenant_id") UUID tenantId);
}

View File

@ -27,6 +27,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EntityView;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.TenantId;
@ -40,7 +41,6 @@ import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.service.Validator;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@ -116,6 +116,11 @@ public class BaseTimeseriesService implements TimeseriesService {
return timeseriesLatestDao.findAllLatest(tenantId, entityId);
}
@Override
public List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId) {
return timeseriesLatestDao.findAllKeysByDeviceProfileId(tenantId, deviceProfileId);
}
@Override
public ListenableFuture<Integer> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) {
validate(entityId);

View File

@ -18,7 +18,6 @@ package org.thingsboard.server.dao.timeseries;
import com.datastax.oss.driver.api.core.cql.BoundStatement;
import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder;
import com.datastax.oss.driver.api.core.cql.PreparedStatement;
import com.datastax.oss.driver.api.core.cql.Row;
import com.datastax.oss.driver.api.core.cql.Statement;
import com.datastax.oss.driver.api.querybuilder.QueryBuilder;
import com.google.common.util.concurrent.FutureCallback;
@ -28,14 +27,13 @@ import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.Aggregation;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.StringDataEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.nosql.TbResultSet;
@ -43,6 +41,7 @@ import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao;
import org.thingsboard.server.dao.util.NoSqlTsLatestDao;
import javax.annotation.Nullable;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
@ -82,6 +81,11 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes
return getFutureAsync(executeAsyncRead(tenantId, stmt), rs -> convertAsyncResultSetToTsKvEntryList(rs));
}
@Override
public List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId) {
return Collections.emptyList();
}
@Override
public ListenableFuture<Void> saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) {
BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getLatestStmt().bind());

View File

@ -16,6 +16,7 @@
package org.thingsboard.server.dao.timeseries;
import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
@ -33,4 +34,5 @@ public interface TimeseriesLatestDao {
ListenableFuture<Void> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query);
List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId);
}

View File

@ -34,6 +34,8 @@ CREATE INDEX IF NOT EXISTS idx_device_customer_id_and_type ON device(tenant_id,
CREATE INDEX IF NOT EXISTS idx_device_type ON device(tenant_id, type);
CREATE INDEX IF NOT EXISTS idx_device_device_profile_id ON device(tenant_id, device_profile_id);
CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id);
CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type);