dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1091 from thingsboard/feature/hybrid-db

Browse Source 
Feature/hybrid db
This commit is contained in:
VoBa 2018-09-29 11:01:43 +03:00 committed by GitHub
commit 5c6df92e1f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
44 changed files with 406 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
View File

@ -24,9 +24,10 @@ import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.install.DataUpdateService;
import org.thingsboard.server.service.install.DatabaseSchemaService;
import org.thingsboard.server.service.install.DatabaseUpgradeService;
import org.thingsboard.server.service.install.EntityDatabaseSchemaService;
import org.thingsboard.server.service.install.SystemDataLoaderService;
import org.thingsboard.server.service.install.TsDatabaseSchemaService;
@Service
@Profile("install")
@ -43,7 +44,10 @@ public class ThingsboardInstallService {
private Boolean loadDemo;
@Autowired
private DatabaseSchemaService databaseSchemaService;
private EntityDatabaseSchemaService entityDatabaseSchemaService;
@Autowired
private TsDatabaseSchemaService tsDatabaseSchemaService;
@Autowired
private DatabaseUpgradeService databaseUpgradeService;
@ -114,9 +118,13 @@ public class ThingsboardInstallService {
log.info("Starting ThingsBoard Installation...");
log.info("Installing DataBase schema...");
log.info("Installing DataBase schema for entities...");
databaseSchemaService.createDatabaseSchema();
entityDatabaseSchemaService.createDatabaseSchema();
log.info("Installing DataBase schema for timeseries...");
tsDatabaseSchemaService.createDatabaseSchema();
log.info("Loading system data...");

19
application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseSchemaService.java → application/src/main/java/org/thingsboard/server/service/install/CassandraAbstractDatabaseSchemaService.java
View File

@ -17,24 +17,17 @@ package org.thingsboard.server.service.install;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.cassandra.CassandraInstallCluster;
import org.thingsboard.server.dao.util.NoSqlDao;
import org.thingsboard.server.service.install.cql.CQLStatementsParser;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
@Service
@NoSqlDao
@Profile("install")
@Slf4j
public class CassandraDatabaseSchemaService implements DatabaseSchemaService {
public abstract class CassandraAbstractDatabaseSchemaService implements DatabaseSchemaService {
private static final String CASSANDRA_DIR = "cassandra";
private static final String SCHEMA_CQL = "schema.cql";
@Autowired
private CassandraInstallCluster cluster;
@ -42,10 +35,16 @@ public class CassandraDatabaseSchemaService implements DatabaseSchemaService {
@Autowired
private InstallScripts installScripts;
private final String schemaCql;
protected CassandraAbstractDatabaseSchemaService(String schemaCql) {
this.schemaCql = schemaCql;
}
@Override
public void createDatabaseSchema() throws Exception {
log.info("Installing Cassandra DataBase schema...");
Path schemaFile = Paths.get(installScripts.getDataDir(), CASSANDRA_DIR, SCHEMA_CQL);
log.info("Installing Cassandra DataBase schema part: " + schemaCql);
Path schemaFile = Paths.get(installScripts.getDataDir(), CASSANDRA_DIR, schemaCql);
loadCql(schemaFile);
}

30
application/src/main/java/org/thingsboard/server/service/install/CassandraEntityDatabaseSchemaService.java Normal file
View File

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.NoSqlDao;
@Service
@NoSqlDao
@Profile("install")
public class CassandraEntityDatabaseSchemaService extends CassandraAbstractDatabaseSchemaService
implements EntityDatabaseSchemaService {
public CassandraEntityDatabaseSchemaService() {
super("schema-entities.cql");
}
}

30
application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseSchemaService.java Normal file
View File

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.NoSqlTsDao;
@Service
@NoSqlTsDao
@Profile("install")
public class CassandraTsDatabaseSchemaService extends CassandraAbstractDatabaseSchemaService
implements TsDatabaseSchemaService {
public CassandraTsDatabaseSchemaService() {
super("schema-ts.cql");
}
}

19
application/src/main/java/org/thingsboard/server/service/install/EntityDatabaseSchemaService.java Normal file
View File

@ -0,0 +1,19 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
public interface EntityDatabaseSchemaService extends DatabaseSchemaService {
}

19
application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseSchemaService.java → application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java
View File

@ -18,9 +18,6 @@ package org.thingsboard.server.service.install;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.SqlDao;
import java.nio.charset.Charset;
import java.nio.file.Files;
@ -29,14 +26,10 @@ import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.DriverManager;
@Service
@Profile("install")
@Slf4j
@SqlDao
public class SqlDatabaseSchemaService implements DatabaseSchemaService {
public abstract class SqlAbstractDatabaseSchemaService implements DatabaseSchemaService {
private static final String SQL_DIR = "sql";
private static final String SCHEMA_SQL = "schema.sql";
@Value("${spring.datasource.url}")
private String dbUrl;
@ -50,12 +43,18 @@ public class SqlDatabaseSchemaService implements DatabaseSchemaService {
@Autowired
private InstallScripts installScripts;
private final String schemaSql;
protected SqlAbstractDatabaseSchemaService(String schemaSql) {
this.schemaSql = schemaSql;
}
@Override
public void createDatabaseSchema() throws Exception {
log.info("Installing SQL DataBase schema...");
log.info("Installing SQL DataBase schema part: " + schemaSql);
Path schemaFile = Paths.get(installScripts.getDataDir(), SQL_DIR, SCHEMA_SQL);
Path schemaFile = Paths.get(installScripts.getDataDir(), SQL_DIR, schemaSql);
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
String sql = new String(Files.readAllBytes(schemaFile), Charset.forName("UTF-8"));
conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to load initial thingsboard database schema

30
application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java Normal file
View File

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.SqlDao;
@Service
@SqlDao
@Profile("install")
public class SqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService
implements EntityDatabaseSchemaService {
public SqlEntityDatabaseSchemaService() {
super("schema-entities.sql");
}
}

30
application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseSchemaService.java Normal file
View File

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.SqlTsDao;
@Service
@SqlTsDao
@Profile("install")
public class SqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService
implements TsDatabaseSchemaService {
public SqlTsDatabaseSchemaService() {
super("schema-ts.sql");
}
}

19
application/src/main/java/org/thingsboard/server/service/install/TsDatabaseSchemaService.java Normal file
View File

@ -0,0 +1,19 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
public interface TsDatabaseSchemaService extends DatabaseSchemaService {
}

8
application/src/main/resources/thingsboard.yml
View File

@ -159,7 +159,11 @@ quota:
intervalMin: 2
database:
type: "${DATABASE_TYPE:sql}" # cassandra OR sql
entities:
type: "${DATABASE_TS_TYPE:sql}" # cassandra OR sql
ts:
type: "${DATABASE_CASSANDRA_TYPE:sql}" # cassandra OR sql (for hybrid mode, only this value should be cassandra)
# Cassandra driver configuration parameters
cassandra:
@ -206,7 +210,7 @@ cassandra:
write_consistency_level: "${CASSANDRA_WRITE_CONSISTENCY_LEVEL:ONE}"
default_fetch_size: "${CASSANDRA_DEFAULT_FETCH_SIZE:2000}"
# Specify partitioning size for timestamp key-value storage. Example MINUTES, HOURS, DAYS, MONTHS,INDEFINITE
ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}"
ts_key_value_partitioning: "${TS_KV_PARTITIONING:INDEFINITE}"
ts_key_value_ttl: "${TS_KV_TTL:0}"
buffer_size: "${CASSANDRA_QUERY_BUFFER_SIZE:200000}"
concurrent_limit: "${CASSANDRA_QUERY_CONCURRENT_LIMIT:1000}"

3
application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java
View File

@ -32,7 +32,8 @@ public class ControllerNoSqlTestSuite {
public static CustomCassandraCQLUnit cassandraUnit =
new CustomCassandraCQLUnit(
Arrays.asList(
new ClassPathCQLDataSet("cassandra/schema.cql", false, false),
new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false),
new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false),
new ClassPathCQLDataSet("cassandra/system-data.cql", false, false),
new ClassPathCQLDataSet("cassandra/system-test.cql", false, false)),
"cassandra-test.yaml", 30000l);

2
application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java
View File

@ -30,7 +30,7 @@ public class ControllerSqlTestSuite {
@ClassRule
public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
Arrays.asList("sql/schema.sql", "sql/system-data.sql"),
Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
"sql/drop-all-tables.sql",
"sql-test.properties");
}

3
application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java
View File

@ -32,7 +32,8 @@ public class MqttNoSqlTestSuite {
public static CustomCassandraCQLUnit cassandraUnit =
new CustomCassandraCQLUnit(
Arrays.asList(
new ClassPathCQLDataSet("cassandra/schema.cql", false, false),
new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false),
new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false),
new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)),
"cassandra-test.yaml", 30000l);
}

4
application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java
View File

@ -15,11 +15,9 @@
*/
package org.thingsboard.server.mqtt;
import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;
import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomCassandraCQLUnit;
import org.thingsboard.server.dao.CustomSqlUnit;
import java.util.Arrays;
@ -31,7 +29,7 @@ public class MqttSqlTestSuite {
@ClassRule
public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
Arrays.asList("sql/schema.sql", "sql/system-data.sql"),
Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
"sql/drop-all-tables.sql",
"sql-test.properties");
}

3
application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java
View File

@ -35,7 +35,8 @@ public class RuleEngineNoSqlTestSuite {
public static CustomCassandraCQLUnit cassandraUnit =
new CustomCassandraCQLUnit(
Arrays.asList(
new ClassPathCQLDataSet("cassandra/schema.cql", false, false),
new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false),
new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false),
new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)),
"cassandra-test.yaml", 30000l);

2
application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java
View File

@ -30,7 +30,7 @@ public class RuleEngineSqlTestSuite {
@ClassRule
public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
Arrays.asList("sql/schema.sql", "sql/system-data.sql"),
Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
"sql/drop-all-tables.sql",
"sql-test.properties");
}

3
application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java
View File

@ -34,7 +34,8 @@ public class SystemNoSqlTestSuite {
public static CustomCassandraCQLUnit cassandraUnit =
new CustomCassandraCQLUnit(
Arrays.asList(
new ClassPathCQLDataSet("cassandra/schema.cql", false, false),
new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false),
new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false),
new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)),
"cassandra-test.yaml", 30000l);
}

2
application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java
View File

@ -31,7 +31,7 @@ public class SystemSqlTestSuite {
@ClassRule
public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
Arrays.asList("sql/schema.sql", "sql/system-data.sql"),
Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
"sql/drop-all-tables.sql",
"sql-test.properties");

4
dao/src/main/java/org/thingsboard/server/dao/cassandra/CassandraCluster.java
View File

@ -17,12 +17,12 @@ package org.thingsboard.server.dao.cassandra;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.dao.util.NoSqlDao;
import org.thingsboard.server.dao.util.NoSqlAnyDao;
import javax.annotation.PostConstruct;
@Component
@NoSqlDao
@NoSqlAnyDao
public class CassandraCluster extends AbstractCassandraCluster {
@Value("${cassandra.keyspace_name}")

4
dao/src/main/java/org/thingsboard/server/dao/cassandra/CassandraInstallCluster.java
View File

@ -17,12 +17,12 @@ package org.thingsboard.server.dao.cassandra;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Component;
import org.thingsboard.server.dao.util.NoSqlDao;
import org.thingsboard.server.dao.util.NoSqlAnyDao;
import javax.annotation.PostConstruct;
@Component
@NoSqlDao
@NoSqlAnyDao
@Profile("install")
public class CassandraInstallCluster extends AbstractCassandraCluster {

4
dao/src/main/java/org/thingsboard/server/dao/cassandra/CassandraQueryOptions.java
View File

@ -21,14 +21,14 @@ import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Component;
import org.thingsboard.server.dao.util.NoSqlDao;
import org.thingsboard.server.dao.util.NoSqlAnyDao;
import javax.annotation.PostConstruct;
@Component
@Configuration
@Data
@NoSqlDao
@NoSqlAnyDao
public class CassandraQueryOptions {
@Value("${cassandra.query.default_fetch_size}")

4
dao/src/main/java/org/thingsboard/server/dao/cassandra/CassandraSocketOptions.java
View File

@ -20,14 +20,14 @@ import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Component;
import org.thingsboard.server.dao.util.NoSqlDao;
import org.thingsboard.server.dao.util.NoSqlAnyDao;
import javax.annotation.PostConstruct;
@Component
@Configuration
@Data
@NoSqlDao
@NoSqlAnyDao
public class CassandraSocketOptions {
@Value("${cassandra.socket.connect_timeout}")

3
dao/src/main/java/org/thingsboard/server/dao/sql/timeseries/JpaTimeseriesDao.java
View File

@ -44,6 +44,7 @@ import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService;
import org.thingsboard.server.dao.timeseries.TimeseriesDao;
import org.thingsboard.server.dao.timeseries.TsInsertExecutorType;
import org.thingsboard.server.dao.util.SqlDao;
import org.thingsboard.server.dao.util.SqlTsDao;
import javax.annotation.Nullable;
import javax.annotation.PostConstruct;
@ -60,7 +61,7 @@ import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID;
@Component
@Slf4j
@SqlDao
@SqlTsDao
public class JpaTimeseriesDao extends JpaAbstractDaoListeningExecutorService implements TimeseriesDao {
@Value("${sql.ts_inserts_executor_type}")

3
dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java
View File

@ -49,6 +49,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.nosql.CassandraAbstractAsyncDao;
import org.thingsboard.server.dao.util.NoSqlDao;
import org.thingsboard.server.dao.util.NoSqlTsDao;
import javax.annotation.Nullable;
import javax.annotation.PostConstruct;
@ -70,7 +71,7 @@ import static com.datastax.driver.core.querybuilder.QueryBuilder.eq;
*/
@Component
@Slf4j
@NoSqlDao
@NoSqlTsDao
public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implements TimeseriesDao {
private static final int MIN_AGGREGATION_STEP_MS = 1000;

2
dao/src/main/java/org/thingsboard/server/dao/util/BufferedRateLimiter.java
View File

@ -34,7 +34,7 @@ import java.util.concurrent.atomic.AtomicInteger;
@Component
@Slf4j
@NoSqlDao
@NoSqlAnyDao
public class BufferedRateLimiter implements AsyncRateLimiter {
private final ListeningExecutorService pool = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(10));

22
dao/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java Normal file
View File

@ -0,0 +1,22 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.util;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
@ConditionalOnExpression("'${database.ts.type}'=='cassandra' || '${database.entities.type}'=='cassandra'")
public @interface NoSqlAnyDao {
}

2
dao/src/main/java/org/thingsboard/server/dao/util/NoSqlDao.java
View File

@ -17,6 +17,6 @@ package org.thingsboard.server.dao.util;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@ConditionalOnProperty(prefix = "database", value = "type", havingValue = "cassandra")
@ConditionalOnProperty(prefix = "database.entities", value = "type", havingValue = "cassandra")
public @interface NoSqlDao {
}

22
dao/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java Normal file
View File

@ -0,0 +1,22 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.util;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "cassandra")
public @interface NoSqlTsDao {
}

2
dao/src/main/java/org/thingsboard/server/dao/util/SqlDao.java
View File

@ -17,6 +17,6 @@ package org.thingsboard.server.dao.util;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@ConditionalOnProperty(prefix = "database", value = "type", havingValue = "sql")
@ConditionalOnProperty(prefix = "database.entities", value = "type", havingValue = "sql")
public @interface SqlDao {
}

22
dao/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java Normal file
View File

@ -0,0 +1,22 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.util;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "sql")
public @interface SqlTsDao {
}

35
dao/src/main/resources/cassandra/schema.cql → dao/src/main/resources/cassandra/schema-entities.cql
View File

@ -378,41 +378,6 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.dashboard_by_tenant_and_searc
PRIMARY KEY ( tenant_id, search_text, id )
WITH CLUSTERING ORDER BY ( search_text ASC, id DESC );
CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_cf (
entity_type text, // (DEVICE, CUSTOMER, TENANT)
entity_id timeuuid,
key text,
partition bigint,
ts bigint,
bool_v boolean,
str_v text,
long_v bigint,
dbl_v double,
PRIMARY KEY (( entity_type, entity_id, key, partition ), ts)
);
CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_partitions_cf (
entity_type text, // (DEVICE, CUSTOMER, TENANT)
entity_id timeuuid,
key text,
partition bigint,
PRIMARY KEY (( entity_type, entity_id, key ), partition)
) WITH CLUSTERING ORDER BY ( partition ASC )
AND compaction = { 'class' : 'LeveledCompactionStrategy' };
CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_latest_cf (
entity_type text, // (DEVICE, CUSTOMER, TENANT)
entity_id timeuuid,
key text,
ts bigint,
bool_v boolean,
str_v text,
long_v bigint,
dbl_v double,
PRIMARY KEY (( entity_type, entity_id ), key)
) WITH compaction = { 'class' : 'LeveledCompactionStrategy' };
CREATE TABLE IF NOT EXISTS thingsboard.attributes_kv_cf (
entity_type text, // (DEVICE, CUSTOMER, TENANT)
entity_id timeuuid,

55
dao/src/main/resources/cassandra/schema-ts.cql Normal file
View File

@ -0,0 +1,55 @@
--
-- Copyright © 2016-2018 The Thingsboard Authors
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
CREATE KEYSPACE IF NOT EXISTS thingsboard
WITH replication = {
'class' : 'SimpleStrategy',
'replication_factor' : 1
};
CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_cf (
entity_type text, // (DEVICE, CUSTOMER, TENANT)
entity_id timeuuid,
key text,
partition bigint,
ts bigint,
bool_v boolean,
str_v text,
long_v bigint,
dbl_v double,
PRIMARY KEY (( entity_type, entity_id, key, partition ), ts)
);
CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_partitions_cf (
entity_type text, // (DEVICE, CUSTOMER, TENANT)
entity_id timeuuid,
key text,
partition bigint,
PRIMARY KEY (( entity_type, entity_id, key ), partition)
) WITH CLUSTERING ORDER BY ( partition ASC )
AND compaction = { 'class' : 'LeveledCompactionStrategy' };
CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_latest_cf (
entity_type text, // (DEVICE, CUSTOMER, TENANT)
entity_id timeuuid,
key text,
ts bigint,
bool_v boolean,
str_v text,
long_v bigint,
dbl_v double,
PRIMARY KEY (( entity_type, entity_id ), key)
) WITH compaction = { 'class' : 'LeveledCompactionStrategy' };

24
dao/src/main/resources/sql/schema.sql → dao/src/main/resources/sql/schema-entities.sql
View File

@ -179,30 +179,6 @@ CREATE TABLE IF NOT EXISTS tenant (
zip varchar(255)
);
CREATE TABLE IF NOT EXISTS ts_kv (
entity_type varchar(255) NOT NULL,
entity_id varchar(31) NOT NULL,
key varchar(255) NOT NULL,
ts bigint NOT NULL,
bool_v boolean,
str_v varchar(10000000),
long_v bigint,
dbl_v double precision,
CONSTRAINT ts_kv_unq_key UNIQUE (entity_type, entity_id, key, ts)
);
CREATE TABLE IF NOT EXISTS ts_kv_latest (
entity_type varchar(255) NOT NULL,
entity_id varchar(31) NOT NULL,
key varchar(255) NOT NULL,
ts bigint NOT NULL,
bool_v boolean,
str_v varchar(10000000),
long_v bigint,
dbl_v double precision,
CONSTRAINT ts_kv_latest_unq_key UNIQUE (entity_type, entity_id, key)
);
CREATE TABLE IF NOT EXISTS user_credentials (
id varchar(31) NOT NULL CONSTRAINT user_credentials_pkey PRIMARY KEY,
activate_token varchar(255) UNIQUE,

39
dao/src/main/resources/sql/schema-ts.sql Normal file
View File

@ -0,0 +1,39 @@
--
-- Copyright © 2016-2018 The Thingsboard Authors
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
CREATE TABLE IF NOT EXISTS ts_kv (
entity_type varchar(255) NOT NULL,
entity_id varchar(31) NOT NULL,
key varchar(255) NOT NULL,
ts bigint NOT NULL,
bool_v boolean,
str_v varchar(10000000),
long_v bigint,
dbl_v double precision,
CONSTRAINT ts_kv_unq_key UNIQUE (entity_type, entity_id, key, ts)
);
CREATE TABLE IF NOT EXISTS ts_kv_latest (
entity_type varchar(255) NOT NULL,
entity_id varchar(31) NOT NULL,
key varchar(255) NOT NULL,
ts bigint NOT NULL,
bool_v boolean,
str_v varchar(10000000),
long_v bigint,
dbl_v double precision,
CONSTRAINT ts_kv_latest_unq_key UNIQUE (entity_type, entity_id, key)
);

2
dao/src/test/java/org/thingsboard/server/dao/JpaDaoTestSuite.java
View File

@ -30,7 +30,7 @@ public class JpaDaoTestSuite {
@ClassRule
public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
Arrays.asList("sql/schema.sql", "sql/system-data.sql"),
Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
"sql/drop-all-tables.sql",
"sql-test.properties"
);

4
dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java
View File

@ -34,7 +34,9 @@ public class NoSqlDaoServiceTestSuite {
@ClassRule
public static CustomCassandraCQLUnit cassandraUnit =
new CustomCassandraCQLUnit(
Arrays.asList(new ClassPathCQLDataSet("cassandra/schema.cql", false, false),
Arrays.asList(
new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false),
new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false),
new ClassPathCQLDataSet("cassandra/system-data.cql", false, false),
new ClassPathCQLDataSet("cassandra/system-test.cql", false, false)),
"cassandra-test.yaml", 30000L);

2
dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java
View File

@ -30,7 +30,7 @@ public class SqlDaoServiceTestSuite {
@ClassRule
public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
Arrays.asList("sql/schema.sql", "sql/system-data.sql", "sql/system-test.sql"),
Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql", "sql/system-test.sql"),
"sql/drop-all-tables.sql",
"sql-test.properties"
);

3
dao/src/test/resources/nosql-test.properties
View File

@ -1,4 +1,5 @@
database.type=cassandra
database.entities.type=cassandra
database.ts.type=cassandra
cassandra.queue.partitioning=HOURS
cassandra.queue.ack.ttl=3600

3
dao/src/test/resources/sql-test.properties
View File

@ -1,4 +1,5 @@
database.type=sql
database.ts.type=sql
database.entities.type=sql
sql.ts_inserts_executor_type=fixed
sql.ts_inserts_fixed_thread_pool_size=10

4
docker/k8s/cassandra-setup.yaml
View File

@ -30,7 +30,9 @@ spec:
value: "cassandra-headless"
- name : CASSANDRA_PORT
value: "9042"
- name : DATABASE_TYPE
- name : DATABASE_ENTITIES_TYPE
value: "cassandra"
- name : DATABASE_TS_TYPE
value: "cassandra"
- name : CASSANDRA_URL
value: "cassandra-headless:9042"

4
docker/k8s/cassandra-upgrade.yaml
View File

@ -30,7 +30,9 @@ spec:
value: "cassandra-headless"
- name : CASSANDRA_PORT
value: "9042"
- name : DATABASE_TYPE
- name : DATABASE_ENTITIES_TYPE
value: "cassandra"
- name : DATABASE_TS_TYPE
value: "cassandra"
- name : CASSANDRA_URL
value: "cassandra-headless:9042"

7
docker/k8s/tb.yaml
View File

@ -120,7 +120,12 @@ spec:
configMapKeyRef:
name: tb-config
key: cassandra.url
- name: DATABASE_TYPE
- name: DATABASE_ENTITIES_TYPE
valueFrom:
configMapKeyRef:
name: tb-config
key: database.type
- name: DATABASE_TS_TYPE
valueFrom:
configMapKeyRef:
name: tb-config

3
docker/tb.env
View File

@ -8,7 +8,8 @@ COAP_BIND_PORT=5683
ZOOKEEPER_URL=zk:2181
# type of database to use: sql[DEFAULT] or cassandra
DATABASE_TYPE=sql
DATABASE_TS_TYPE=sql
DATABASE_ENTITIES_TYPE=sql
# cassandra db config
CASSANDRA_URL=cassandra:9042

4
docker/tb/run-application.sh
View File

@ -23,7 +23,7 @@ printenv | awk -F "=" '{print "export " $1 "='\''" $2 "'\''"}' >> /usr/share/thi
cat /usr/share/thingsboard/conf/thingsboard.conf
if [ "$DATABASE_TYPE" == "cassandra" ]; then
if [ "$DATABASE_ENTITIES_TYPE" == "cassandra" ]; then
until nmap $CASSANDRA_HOST -p $CASSANDRA_PORT | grep "$CASSANDRA_PORT/tcp open\|filtered"
do
echo "Wait for cassandra db to start..."
@ -31,7 +31,7 @@ if [ "$DATABASE_TYPE" == "cassandra" ]; then
done
fi
if [ "$DATABASE_TYPE" == "sql" ]; then
if [ "$DATABASE_ENTITIES_TYPE" == "sql" ]; then
if [ "$SPRING_DRIVER_CLASS_NAME" == "org.postgresql.Driver" ]; then
until nmap $POSTGRES_HOST -p $POSTGRES_PORT | grep "$POSTGRES_PORT/tcp open"
do