dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1615 from dmytro-landiak/feature/set-nulls-cassandra

Browse Source 
set data types to null cassandra
This commit is contained in:
Andrew Shvayka 2019-04-09 16:38:21 +03:00 committed by GitHub
commit d35d302a50
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 50 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
application/src/main/resources/thingsboard.yml
View File

@ -173,6 +173,8 @@ cassandra:
callback_threads: "${CASSANDRA_QUERY_CALLBACK_THREADS:4}" callback_threads: "${CASSANDRA_QUERY_CALLBACK_THREADS:4}"
poll_ms: "${CASSANDRA_QUERY_POLL_MS:50}" poll_ms: "${CASSANDRA_QUERY_POLL_MS:50}"
rate_limit_print_interval_ms: "${CASSANDRA_QUERY_RATE_LIMIT_PRINT_MS:10000}" rate_limit_print_interval_ms: "${CASSANDRA_QUERY_RATE_LIMIT_PRINT_MS:10000}"
# set all data types values except target to null for the same ts on save
set_null_values_enabled: "${CASSANDRA_QUERY_SET_NULL_VALUES_ENABLED:false}"
tenant_rate_limits: tenant_rate_limits:
enabled: "${CASSANDRA_QUERY_TENANT_RATE_LIMITS_ENABLED:false}" enabled: "${CASSANDRA_QUERY_TENANT_RATE_LIMITS_ENABLED:false}"
configuration: "${CASSANDRA_QUERY_TENANT_RATE_LIMITS_CONFIGURATION:1000:1,30000:60}" configuration: "${CASSANDRA_QUERY_TENANT_RATE_LIMITS_CONFIGURATION:1000:1,30000:60}"

47
dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java
View File

@ -94,6 +94,9 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem
@Value("${cassandra.query.ts_key_value_ttl}") @Value("${cassandra.query.ts_key_value_ttl}")
private long systemTtl; private long systemTtl;
@Value("${cassandra.query.set_null_values_enabled}")
private boolean setNullValuesEnabled;
private TsPartitionDate tsFormat; private TsPartitionDate tsFormat;
private PreparedStatement partitionInsertStmt; private PreparedStatement partitionInsertStmt;
@ -307,9 +310,13 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem
@Override @Override
public ListenableFuture<Void> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { public ListenableFuture<Void> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) {
List<ListenableFuture<Void>> futures = new ArrayList<>();
ttl = computeTtl(ttl); ttl = computeTtl(ttl);
long partition = toPartitionTs(tsKvEntry.getTs()); long partition = toPartitionTs(tsKvEntry.getTs());
DataType type = tsKvEntry.getDataType(); DataType type = tsKvEntry.getDataType();
if (setNullValuesEnabled) {
processSetNullValues(tenantId, entityId, tsKvEntry, ttl, futures, partition, type);
}
BoundStatement stmt = (ttl == 0 ? getSaveStmt(type) : getSaveTtlStmt(type)).bind(); BoundStatement stmt = (ttl == 0 ? getSaveStmt(type) : getSaveTtlStmt(type)).bind();
stmt.setString(0, entityId.getEntityType().name()) stmt.setString(0, entityId.getEntityType().name())
.setUUID(1, entityId.getId()) .setUUID(1, entityId.getId())
@ -320,6 +327,46 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem
if (ttl > 0) { if (ttl > 0) {
stmt.setInt(6, (int) ttl); stmt.setInt(6, (int) ttl);
} }
futures.add(getFuture(executeAsyncWrite(tenantId, stmt), rs -> null));
return Futures.transform(Futures.allAsList(futures), result -> null);
}
private void processSetNullValues(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl, List<ListenableFuture<Void>> futures, long partition, DataType type) {
switch (type) {
case LONG:
futures.add(saveNull(tenantId, entityId, tsKvEntry, ttl, partition, DataType.BOOLEAN));
futures.add(saveNull(tenantId, entityId, tsKvEntry, ttl, partition, DataType.DOUBLE));
futures.add(saveNull(tenantId, entityId, tsKvEntry, ttl, partition, DataType.STRING));
break;
case BOOLEAN:
futures.add(saveNull(tenantId, entityId, tsKvEntry, ttl, partition, DataType.DOUBLE));
futures.add(saveNull(tenantId, entityId, tsKvEntry, ttl, partition, DataType.LONG));
futures.add(saveNull(tenantId, entityId, tsKvEntry, ttl, partition, DataType.STRING));
break;
case DOUBLE:
futures.add(saveNull(tenantId, entityId, tsKvEntry, ttl, partition, DataType.BOOLEAN));
futures.add(saveNull(tenantId, entityId, tsKvEntry, ttl, partition, DataType.LONG));
futures.add(saveNull(tenantId, entityId, tsKvEntry, ttl, partition, DataType.STRING));
break;
case STRING:
futures.add(saveNull(tenantId, entityId, tsKvEntry, ttl, partition, DataType.BOOLEAN));
futures.add(saveNull(tenantId, entityId, tsKvEntry, ttl, partition, DataType.DOUBLE));
futures.add(saveNull(tenantId, entityId, tsKvEntry, ttl, partition, DataType.LONG));
break;
}
}
private ListenableFuture<Void> saveNull(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl, long partition, DataType type) {
BoundStatement stmt = (ttl == 0 ? getSaveStmt(type) : getSaveTtlStmt(type)).bind();
stmt.setString(0, entityId.getEntityType().name())
.setUUID(1, entityId.getId())
.setString(2, tsKvEntry.getKey())
.setLong(3, partition)
.setLong(4, tsKvEntry.getTs());
stmt.setToNull(getColumnName(type));
if (ttl > 0) {
stmt.setInt(6, (int) ttl);
}
return getFuture(executeAsyncWrite(tenantId, stmt), rs -> null); return getFuture(executeAsyncWrite(tenantId, stmt), rs -> null);
} }

1
dao/src/test/resources/cassandra-test.properties
View File

@ -53,6 +53,7 @@ cassandra.query.buffer_size=100000
cassandra.query.concurrent_limit=1000 cassandra.query.concurrent_limit=1000
cassandra.query.permit_max_wait_time=20000 cassandra.query.permit_max_wait_time=20000
cassandra.query.rate_limit_print_interval_ms=30000 cassandra.query.rate_limit_print_interval_ms=30000
cassandra.query.set_null_values_enabled=false
cassandra.query.tenant_rate_limits.enabled=false cassandra.query.tenant_rate_limits.enabled=false
cassandra.query.tenant_rate_limits.configuration=5000:1,100000:60 cassandra.query.tenant_rate_limits.configuration=5000:1,100000:60
cassandra.query.tenant_rate_limits.print_tenant_names=false cassandra.query.tenant_rate_limits.print_tenant_names=false