dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cassandra cloud connect to Datastax Astra DB using SECURE_BUNDLE, CLIENT_ID and CLIENT_SECRET

Browse Source
This commit is contained in:
Sergey Matvienko 2022-08-18 00:43:54 +03:00
parent 5891a1df65
commit 3c496f4749
14 changed files with 143 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
View File

@ -26,6 +26,7 @@ import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService;
import org.thingsboard.server.service.install.DatabaseTsUpgradeService;
import org.thingsboard.server.service.install.EntityDatabaseSchemaService;
import org.thingsboard.server.service.install.NoSqlKeyspaceService;
import org.thingsboard.server.service.install.SystemDataLoaderService;
import org.thingsboard.server.service.install.TsDatabaseSchemaService;
import org.thingsboard.server.service.install.TsLatestDatabaseSchemaService;
@ -51,6 +52,9 @@ public class ThingsboardInstallService {
@Autowired
private EntityDatabaseSchemaService entityDatabaseSchemaService;
@Autowired(required = false)
private NoSqlKeyspaceService noSqlKeyspaceService;
@Autowired
private TsDatabaseSchemaService tsDatabaseSchemaService;
@ -248,6 +252,10 @@ public class ThingsboardInstallService {
log.info("Installing DataBase schema for timeseries...");
if (noSqlKeyspaceService != null) {
noSqlKeyspaceService.createDatabaseSchema();
}
tsDatabaseSchemaService.createDatabaseSchema();
if (tsLatestDatabaseSchemaService != null) {

37
application/src/main/java/org/thingsboard/server/service/install/CassandraKeyspaceService.java Normal file
View File

@ -0,0 +1,37 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.NoSqlAnyDaoNonCloud;
/*
* Create keyspace for Cassandra NoSQL database for non-cloud deployment.
* For cloud service like Astra DBaas admin have to create keyspace manually on cloud UI.
* Then create tokens with database admin role and put it on Thingsboard parameters.
* Without this service cloud DB will end up with exception like
* UnauthorizedException: Missing correct permission on thingsboard
* */
@Service
@NoSqlAnyDaoNonCloud
@Profile("install")
public class CassandraKeyspaceService extends CassandraAbstractDatabaseSchemaService
implements NoSqlKeyspaceService {
public CassandraKeyspaceService() {
super("schema-keyspace.cql");
}
}

19
application/src/main/java/org/thingsboard/server/service/install/NoSqlKeyspaceService.java Normal file
View File

@ -0,0 +1,19 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
public interface NoSqlKeyspaceService extends DatabaseSchemaService {
}

8
application/src/main/resources/thingsboard.yml
View File

@ -200,6 +200,14 @@ cassandra:
username: "${CASSANDRA_USERNAME:}"
# Specify your password
password: "${CASSANDRA_PASSWORD:}"
# Astra DB connect https://astra.datastax.com/
cloud:
# /etc/thingsboard/astra/secure-connect-thingsboard.zip
secure_connect_bundle_path: "${CASSANDRA_CLOUD_SECURE_BUNDLE_PATH:}"
# DucitQPHMzPCBOZqFYexAfKk
client_id: "${CASSANDRA_CLOUD_CLIENT_ID:}"
# ZnF7FpuHp43FP5BzM+KY8wGmSb4Ql6BhT4Z7sOU13ze+gXQ-n7OkFpNuB,oACUIQObQnK0g4bSPoZhK5ejkcF9F.j6f64j71Sr.tiRe0Fsq2hPS1ZCGSfAaIgg63IydG
client_secret: "${CASSANDRA_CLOUD_CLIENT_SECRET:}"
# Cassandra cluster connection socket parameters #
socket:

1
application/src/test/java/org/thingsboard/server/transport/TransportNoSqlTestSuite.java
View File

@ -35,6 +35,7 @@ public class TransportNoSqlTestSuite {
public static CustomCassandraCQLUnit cassandraUnit =
new CustomCassandraCQLUnit(
Arrays.asList(
new ClassPathCQLDataSet("cassandra/schema-keyspace.cql", false, false),
new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false),
new ClassPathCQLDataSet("cassandra/schema-ts-latest.cql", false, false)
),

16
common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/AbstractCassandraCluster.java
View File

@ -24,11 +24,13 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.env.Environment;
import org.springframework.core.env.Profiles;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.dao.cassandra.guava.GuavaSession;
import org.thingsboard.server.dao.cassandra.guava.GuavaSessionBuilder;
import org.thingsboard.server.dao.cassandra.guava.GuavaSessionUtils;
import javax.annotation.PreDestroy;
import java.nio.file.Paths;
@Slf4j
public abstract class AbstractCassandraCluster {
@ -40,6 +42,13 @@ public abstract class AbstractCassandraCluster {
@Value("${cassandra.local_datacenter:datacenter1}")
private String localDatacenter;
@Value("${cassandra.cloud.secure_connect_bundle_path:}")
private String cloudSecureConnectBundlePath;
@Value("${cassandra.cloud.client_id:}")
private String cloudClientId;
@Value("${cassandra.cloud.client_secret:}")
private String cloudClientSecret;
@Autowired
private CassandraDriverOptions driverOptions;
@ -86,7 +95,14 @@ public abstract class AbstractCassandraCluster {
this.sessionBuilder.withKeyspace(this.keyspaceName);
}
this.sessionBuilder.withLocalDatacenter(localDatacenter);
if (StringUtils.isNotBlank(cloudSecureConnectBundlePath)) {
this.sessionBuilder.withCloudSecureConnectBundle(Paths.get(cloudSecureConnectBundlePath));
this.sessionBuilder.withAuthCredentials(cloudClientId, cloudClientSecret);
}
session = sessionBuilder.build();
if (this.metrics && this.jmx) {
MetricRegistry registry =
session.getMetrics().orElseThrow(

22
common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/guava/GuavaDriverContext.java
View File

@ -40,26 +40,8 @@ import java.util.function.Predicate;
*/
public class GuavaDriverContext extends DefaultDriverContext {
public GuavaDriverContext(
DriverConfigLoader configLoader,
List<TypeCodec<?>> typeCodecs,
NodeStateListener nodeStateListener,
SchemaChangeListener schemaChangeListener,
RequestTracker requestTracker,
Map<String, String> localDatacenters,
Map<String, Predicate<Node>> nodeFilters,
ClassLoader classLoader) {
super(
configLoader,
ProgrammaticArguments.builder()
.addTypeCodecs(typeCodecs.toArray(new TypeCodec<?>[0]))
.withNodeStateListener(nodeStateListener)
.withSchemaChangeListener(schemaChangeListener)
.withRequestTracker(requestTracker)
.withLocalDatacenters(localDatacenters)
.withNodeFilters(nodeFilters)
.withClassLoader(classLoader)
.build());
public GuavaDriverContext(DriverConfigLoader configLoader, ProgrammaticArguments programmaticArguments) {
super(configLoader, programmaticArguments);
}
@Override

14
common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/guava/GuavaSessionBuilder.java
View File

@ -25,18 +25,8 @@ import edu.umd.cs.findbugs.annotations.NonNull;
public class GuavaSessionBuilder extends SessionBuilder<GuavaSessionBuilder, GuavaSession> {
@Override
protected DriverContext buildContext(
DriverConfigLoader configLoader,
ProgrammaticArguments programmaticArguments) {
return new GuavaDriverContext(
configLoader,
programmaticArguments.getTypeCodecs(),
programmaticArguments.getNodeStateListener(),
programmaticArguments.getSchemaChangeListener(),
programmaticArguments.getRequestTracker(),
programmaticArguments.getLocalDatacenters(),
programmaticArguments.getNodeFilters(),
programmaticArguments.getClassLoader());
protected DriverContext buildContext(DriverConfigLoader configLoader, ProgrammaticArguments programmaticArguments) {
return new GuavaDriverContext(configLoader, programmaticArguments);
}
@Override

27
common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDaoNonCloud.java Normal file
View File

@ -0,0 +1,27 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.util;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@Retention(RetentionPolicy.RUNTIME)
@ConditionalOnExpression("('${database.ts.type}'=='cassandra' || '${database.ts_latest.type}'=='cassandra') " +
"&& ('${cassandra.cloud.secure_connect_bundle_path}' == null || '${cassandra.cloud.secure_connect_bundle_path}'.isBlank() )")
public @interface NoSqlAnyDaoNonCloud {
}

21
dao/src/main/resources/cassandra/schema-keyspace.cql Normal file
View File

@ -0,0 +1,21 @@
--
-- Copyright © 2016-2022 The Thingsboard Authors
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
CREATE KEYSPACE IF NOT EXISTS thingsboard
WITH replication = {
'class' : 'SimpleStrategy',
'replication_factor' : 1
};

6
dao/src/main/resources/cassandra/schema-ts-latest.cql
View File

@ -14,12 +14,6 @@
-- limitations under the License.
--
CREATE KEYSPACE IF NOT EXISTS thingsboard
WITH replication = {
'class' : 'SimpleStrategy',
'replication_factor' : 1
};
CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_latest_cf (
entity_type text, // (DEVICE, CUSTOMER, TENANT)
entity_id timeuuid,

6
dao/src/main/resources/cassandra/schema-ts.cql
View File

@ -14,12 +14,6 @@
-- limitations under the License.
--
CREATE KEYSPACE IF NOT EXISTS thingsboard
WITH replication = {
'class' : 'SimpleStrategy',
'replication_factor' : 1
};
CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_cf (
entity_type text, // (DEVICE, CUSTOMER, TENANT)
entity_id timeuuid,

1
dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java
View File

@ -33,6 +33,7 @@ public class NoSqlDaoServiceTestSuite {
public static CustomCassandraCQLUnit cassandraUnit =
new CustomCassandraCQLUnit(
Arrays.asList(
new ClassPathCQLDataSet("cassandra/schema-keyspace.cql", false, false),
new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false),
new ClassPathCQLDataSet("cassandra/schema-ts-latest.cql", false, false)
),

2
tools/src/main/java/org/thingsboard/client/tools/migrator/README.md
View File

@ -63,7 +63,7 @@ Tool execution time depends on DB size, CPU resources and Disk throughput
* Note that this this part works only for single node Cassandra Cluster. If you have more nodes - it is better to use `sstableloader` tool.
1. [Optional] install Cassandra on the instance
2. [Optional] Using `cqlsh` create `thingsboard` keyspace and requred tables from this files `schema-ts.cql` and `schema-ts-latest.cql` using `source` command
2. [Optional] Using `cqlsh` create `thingsboard` keyspace and requred tables from this files `schema-keyspace.cql`, `schema-ts.cql` and `schema-ts-latest.cql` using `source` command
3. Stop Cassandra
4. Look at `/var/lib/cassandra/data/thingsboard` and check for names of data folders
5. Copy generated SSTable files into cassandra data dir using next command: