From c2a87aba6097028553bc35857f825bff252ba09a Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Thu, 1 Jun 2023 14:47:47 +0300 Subject: [PATCH 1/4] Add redis-sentinel mode support --- .../src/main/resources/thingsboard.yml | 12 +- .../cache/TBRedisCacheConfiguration.java | 23 ++++ .../cache/TBRedisClusterConfiguration.java | 26 +--- .../cache/TBRedisSentinelConfiguration.java | 62 +++++++++ docker/.env | 2 +- docker/.gitignore | 3 + docker/README.md | 3 +- docker/cache-redis-sentinel.env | 7 ++ docker/compose-utils.sh | 19 ++- .../docker-compose.redis-sentinel.volumes.yml | 40 ++++++ docker/docker-compose.redis-sentinel.yml | 119 ++++++++++++++++++ msa/black-box-tests/README.md | 4 + .../server/msa/ContainerTestSuite.java | 29 ++++- .../server/msa/ThingsBoardDbInstaller.java | 58 +++++++-- .../src/main/resources/tb-coap-transport.yml | 12 +- .../src/main/resources/tb-http-transport.yml | 12 +- .../src/main/resources/tb-lwm2m-transport.yml | 12 +- .../src/main/resources/tb-mqtt-transport.yml | 12 +- .../src/main/resources/tb-snmp-transport.yml | 12 +- 19 files changed, 417 insertions(+), 50 deletions(-) create mode 100644 common/cache/src/main/java/org/thingsboard/server/cache/TBRedisSentinelConfiguration.java create mode 100644 docker/cache-redis-sentinel.env create mode 100644 docker/docker-compose.redis-sentinel.volumes.yml create mode 100644 docker/docker-compose.redis-sentinel.yml diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index fa7565b048..2525956d35 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -502,7 +502,7 @@ cache: spring.data.redis.repositories.enabled: false redis: - # standalone or cluster + # standalone or cluster or sentinel connection: type: "${REDIS_CONNECTION_TYPE:standalone}" standalone: @@ -522,6 +522,16 @@ redis: nodes: "${REDIS_NODES:}" # Maximum number of redirects to follow when executing commands across the cluster. max-redirects: "${REDIS_MAX_REDIRECTS:12}" + # if set false will be used pool config build from values of the pool config section + useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" + sentinel: + # name of master node + master: "${REDIS_MASTER:}" + # comma-separated list of "host:port" pairs of sentinels + sentinels: "${REDIS_SENTINELS:}" + # password to authenticate with sentinel + password: "${REDIS_SENTINEL_PASSWORD:}" + # if set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" # db index db: "${REDIS_DB:0}" diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisCacheConfiguration.java b/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisCacheConfiguration.java index 9c4c962dc2..36f3dddc3d 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisCacheConfiguration.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisCacheConfiguration.java @@ -26,14 +26,19 @@ import org.springframework.core.convert.converter.ConverterRegistry; import org.springframework.data.redis.cache.RedisCacheConfiguration; import org.springframework.data.redis.cache.RedisCacheManager; import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.connection.RedisNode; import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.format.support.DefaultFormattingConversionService; import org.springframework.util.Assert; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.EntityId; import redis.clients.jedis.JedisPoolConfig; import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; @Configuration @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @@ -41,6 +46,9 @@ import java.time.Duration; @Data public abstract class TBRedisCacheConfiguration { + private static final String COMMA = ","; + private static final String COLON = ":"; + @Value("${redis.evictTtlInMs:60000}") private int evictTtlInMs; @@ -126,4 +134,19 @@ public abstract class TBRedisCacheConfiguration { poolConfig.setBlockWhenExhausted(blockWhenExhausted); return poolConfig; } + + protected List getNodes(String nodes) { + List result; + if (StringUtils.isBlank(nodes)) { + result = Collections.emptyList(); + } else { + result = new ArrayList<>(); + for (String hostPort : nodes.split(COMMA)) { + String host = hostPort.split(COLON)[0]; + int port = Integer.parseInt(hostPort.split(COLON)[1]); + result.add(new RedisNode(host, port)); + } + } + return result; + } } diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisClusterConfiguration.java b/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisClusterConfiguration.java index dfc6ebd225..0a378103b0 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisClusterConfiguration.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisClusterConfiguration.java @@ -20,22 +20,13 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Configuration; import org.springframework.data.redis.connection.RedisClusterConfiguration; -import org.springframework.data.redis.connection.RedisNode; import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; -import org.thingsboard.server.common.data.StringUtils; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; @Configuration @ConditionalOnMissingBean(TbCaffeineCacheConfiguration.class) @ConditionalOnProperty(prefix = "redis.connection", value = "type", havingValue = "cluster") public class TBRedisClusterConfiguration extends TBRedisCacheConfiguration { - private static final String COMMA = ","; - private static final String COLON = ":"; - @Value("${redis.cluster.nodes:}") private String clusterNodes; @@ -59,19 +50,4 @@ public class TBRedisClusterConfiguration extends TBRedisCacheConfiguration { return new JedisConnectionFactory(clusterConfiguration, buildPoolConfig()); } } - - private List getNodes(String nodes) { - List result; - if (StringUtils.isBlank(nodes)) { - result = Collections.emptyList(); - } else { - result = new ArrayList<>(); - for (String hostPort : nodes.split(COMMA)) { - String host = hostPort.split(COLON)[0]; - Integer port = Integer.valueOf(hostPort.split(COLON)[1]); - result.add(new RedisNode(host, port)); - } - } - return result; - } -} \ No newline at end of file +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisSentinelConfiguration.java b/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisSentinelConfiguration.java new file mode 100644 index 0000000000..78cb445d82 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisSentinelConfiguration.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.connection.RedisSentinelConfiguration; +import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; + +@Configuration +@ConditionalOnMissingBean(TbCaffeineCacheConfiguration.class) +@ConditionalOnProperty(prefix = "redis.connection", value = "type", havingValue = "sentinel") +public class TBRedisSentinelConfiguration extends TBRedisCacheConfiguration { + + @Value("${redis.sentinel.master:}") + private String master; + + @Value("${redis.sentinel.sentinels:}") + private String sentinels; + + @Value("${redis.sentinel.password:}") + private String sentinelPassword; + + @Value("${redis.sentinel.useDefaultPoolConfig:true}") + private boolean useDefaultPoolConfig; + + @Value("${redis.db:}") + private Integer database; + + @Value("${redis.password:}") + private String password; + + public JedisConnectionFactory loadFactory() { + RedisSentinelConfiguration redisSentinelConfiguration = new RedisSentinelConfiguration(); + redisSentinelConfiguration.setMaster(master); + redisSentinelConfiguration.setSentinels(getNodes(sentinels)); + redisSentinelConfiguration.setSentinelPassword(sentinelPassword); + redisSentinelConfiguration.setPassword(password); + redisSentinelConfiguration.setDatabase(database); + if (useDefaultPoolConfig) { + return new JedisConnectionFactory(redisSentinelConfiguration); + } else { + return new JedisConnectionFactory(redisSentinelConfiguration, buildPoolConfig()); + } + } + +} diff --git a/docker/.env b/docker/.env index f33ed50da5..37c9768296 100644 --- a/docker/.env +++ b/docker/.env @@ -1,6 +1,6 @@ TB_QUEUE_TYPE=kafka -# redis or redis-cluster +# redis or redis-cluster or redis-sentinel CACHE=redis DOCKER_REPO=thingsboard diff --git a/docker/.gitignore b/docker/.gitignore index c9172ae6ce..b6c9fcf18c 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -12,6 +12,9 @@ tb-node/redis-cluster-data-2/** tb-node/redis-cluster-data-3/** tb-node/redis-cluster-data-4/** tb-node/redis-cluster-data-5/** +tb-node/redis-sentinel-data-master/** +tb-node/redis-sentinel-data-slave/** +tb-node/redis-sentinel-data-sentinel/** tb-node/redis-data/** !.env diff --git a/docker/README.md b/docker/README.md index 71ea87f353..437e583a99 100644 --- a/docker/README.md +++ b/docker/README.md @@ -21,8 +21,9 @@ In order to set cache type change the value of `CACHE` variable in `.env` file t - `redis` - use Redis standalone cache (1 node - 1 master); - `redis-cluster` - use Redis cluster cache (6 nodes - 3 masters, 3 slaves); +- `redis-sentinel` - use Redis cluster in a sentinel mode (3 nodes - 1 master, 1 slave, 1 sentinel) -**NOTE**: According to the cache type corresponding docker service will be deployed (see `docker-compose.redis.yml`, `docker-compose.redis-cluster.yml` for details). +**NOTE**: According to the cache type corresponding docker service will be deployed (see `docker-compose.redis.yml`, `docker-compose.redis-cluster.yml`, `docker-compose.redis-sentinel.yml` for details). Execute the following command to create log folders for the services and chown of these folders to the docker container users. To be able to change user, **chown** command is used, which requires sudo permissions (script will request password for a sudo access): diff --git a/docker/cache-redis-sentinel.env b/docker/cache-redis-sentinel.env new file mode 100644 index 0000000000..39a1246d9c --- /dev/null +++ b/docker/cache-redis-sentinel.env @@ -0,0 +1,7 @@ +CACHE_TYPE=redis +REDIS_CONNECTION_TYPE=sentinel +REDIS_MASTER=mymaster +REDIS_SENTINELS=redis-sentinel:26379 +REDIS_SENTINEL_PASSWORD=sentinel +REDIS_USE_DEFAULT_POOL_CONFIG=false +REDIS_PASSWORD=thingsboard diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index 49f3e20e27..19ca342b9b 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -84,8 +84,11 @@ function additionalComposeCacheArgs() { redis-cluster) CACHE_COMPOSE_ARGS="-f docker-compose.redis-cluster.yml" ;; + redis-sentinel) + CACHE_COMPOSE_ARGS="-f docker-compose.redis-sentinel.yml" + ;; *) - echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 + echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster' or 'redis-sentinel'." >&2 exit 1 esac echo $CACHE_COMPOSE_ARGS @@ -114,8 +117,11 @@ function additionalStartupServices() { redis-cluster) ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5" ;; + redis-sentinel) + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis-master redis-slave redis-sentinel" + ;; *) - echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 + echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster' or 'redis-sentinel'." >&2 exit 1 esac @@ -160,8 +166,15 @@ function permissionList() { 1001 1001 tb-node/redis-cluster-data-5 " ;; + redis-sentinel) + PERMISSION_LIST="$PERMISSION_LIST + 1001 1001 tb-node/redis-sentinel-data-master + 1001 1001 tb-node/redis-sentinel-data-slave + 1001 1001 tb-node/redis-sentinel-data-sentinel + " + ;; *) - echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 + echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster' or 'redis-sentinel'." >&2 exit 1 esac diff --git a/docker/docker-compose.redis-sentinel.volumes.yml b/docker/docker-compose.redis-sentinel.volumes.yml new file mode 100644 index 0000000000..38534368ea --- /dev/null +++ b/docker/docker-compose.redis-sentinel.volumes.yml @@ -0,0 +1,40 @@ +# +# Copyright © 2016-2023 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '3.0' + +services: + # Redis sentinel + redis-master: + volumes: + - redis-sentinel-data-master:/bitnami/redis/data + redis-slave: + volumes: + - redis-sentinel-data-slave:/bitnami/redis/data + redis-sentinel: + volumes: + - redis-sentinel-data-sentinel:/bitnami/redis/data + +volumes: + redis-sentinel-data-master: + external: + name: ${REDIS_SENTINEL_DATA_VOLUME_MASTER} + redis-sentinel-data-slave: + external: + name: ${REDIS_SENTINEL_DATA_VOLUME_SLAVE} + redis-sentinel-data-sentinel: + external: + name: ${REDIS_SENTINEL_DATA_VOLUME_SENTINEL} diff --git a/docker/docker-compose.redis-sentinel.yml b/docker/docker-compose.redis-sentinel.yml new file mode 100644 index 0000000000..32b3f1e826 --- /dev/null +++ b/docker/docker-compose.redis-sentinel.yml @@ -0,0 +1,119 @@ +# +# Copyright © 2016-2023 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '3.0' + +services: + # Redis sentinel + redis-master: + image: 'bitnami/redis:7.0' + volumes: + - ./tb-node/redis-sentinel-data-master:/bitnami/redis/data + environment: + - 'REDIS_REPLICATION_MODE=master' + - 'REDIS_PASSWORD=thingsboard' + + redis-slave: + image: 'bitnami/redis:7.0' + volumes: + - ./tb-node/redis-sentinel-data-slave:/bitnami/redis/data + environment: + - 'REDIS_REPLICATION_MODE=slave' + - 'REDIS_MASTER_HOST=redis-master' + - 'REDIS_MASTER_PASSWORD=thingsboard' + - 'REDIS_PASSWORD=thingsboard' + depends_on: + - redis-master + + redis-sentinel: + image: 'bitnami/redis-sentinel:7.0' + volumes: + - ./tb-node/redis-sentinel-data-sentinel:/bitnami/redis/data + environment: + - 'REDIS_MASTER_HOST=redis-master' + - 'REDIS_MASTER_SET=mymaster' + - 'REDIS_SENTINEL_PASSWORD=sentinel' + - 'REDIS_MASTER_PASSWORD=thingsboard' + depends_on: + - redis-master + - redis-slave + + # ThingsBoard setup to use redis-sentinel + tb-core1: + env_file: + - cache-redis-sentinel.env + depends_on: + - redis-sentinel + tb-core2: + env_file: + - cache-redis-sentinel.env + depends_on: + - redis-sentinel + tb-rule-engine1: + env_file: + - cache-redis-sentinel.env + depends_on: + - redis-sentinel + tb-rule-engine2: + env_file: + - cache-redis-sentinel.env + depends_on: + - redis-sentinel + tb-mqtt-transport1: + env_file: + - cache-redis-sentinel.env + depends_on: + - redis-sentinel + tb-mqtt-transport2: + env_file: + - cache-redis-sentinel.env + depends_on: + - redis-sentinel + tb-http-transport1: + env_file: + - cache-redis-sentinel.env + depends_on: + - redis-sentinel + tb-http-transport2: + env_file: + - cache-redis-sentinel.env + depends_on: + - redis-sentinel + tb-coap-transport: + env_file: + - cache-redis-sentinel.env + depends_on: + - redis-sentinel + tb-lwm2m-transport: + env_file: + - cache-redis-sentinel.env + depends_on: + - redis-sentinel + tb-snmp-transport: + env_file: + - cache-redis-sentinel.env + depends_on: + - redis-sentinel + tb-vc-executor1: + env_file: + - cache-redis-sentinel.env + depends_on: + - redis-sentinel + tb-vc-executor2: + env_file: + - cache-redis-sentinel.env + depends_on: + - redis-sentinel diff --git a/msa/black-box-tests/README.md b/msa/black-box-tests/README.md index 4a88d5e8cd..340f0d0eb8 100644 --- a/msa/black-box-tests/README.md +++ b/msa/black-box-tests/README.md @@ -26,6 +26,10 @@ As result, in REPOSITORY column, next images should be present: mvn clean install -DblackBoxTests.skip=false -DblackBoxTests.redisCluster=true +- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory with Redis sentinel: + + mvn clean install -DblackBoxTests.skip=false -DblackBoxTests.redisSentinel=true + - Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory in Hybrid mode (postgres + cassandra): mvn clean install -DblackBoxTests.skip=false -DblackBoxTests.hybridMode=true diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index ffbe14ed52..c6be6ec271 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -43,6 +43,7 @@ import static org.testng.Assert.fail; @Slf4j public class ContainerTestSuite { final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); + final static boolean IS_REDIS_SENTINEL = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisSentinel")); final static boolean IS_HYBRID_MODE = Boolean.parseBoolean(System.getProperty("blackBoxTests.hybridMode")); final static String QUEUE_TYPE = System.getProperty("blackBoxTests.queue", "kafka"); private static final String SOURCE_DIR = "./../../docker/"; @@ -80,8 +81,9 @@ public class ContainerTestSuite { installTb = new ThingsBoardDbInstaller(); installTb.createVolumes(); log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); + log.info("System property of blackBoxTests.redisSentinel is {}", IS_REDIS_SENTINEL); log.info("System property of blackBoxTests.hybridMode is {}", IS_HYBRID_MODE); - boolean skipTailChildContainers = Boolean.valueOf(System.getProperty("blackBoxTests.skipTailChildContainers")); + boolean skipTailChildContainers = Boolean.parseBoolean(System.getProperty("blackBoxTests.skipTailChildContainers")); try { final String targetDir = FileUtils.getTempDirectoryPath() + "/" + "ContainerTestSuite-" + UUID.randomUUID() + "/"; log.info("targetDir {}", targetDir); @@ -109,8 +111,8 @@ public class ContainerTestSuite { new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid-test-extras.yml" : "docker-compose.postgres-test-extras.yml")), new File(targetDir + "docker-compose.postgres.volumes.yml"), new File(targetDir + "docker-compose." + QUEUE_TYPE + ".yml"), - new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.yml" : "docker-compose.redis.yml")), - new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.volumes.yml" : "docker-compose.redis.volumes.yml")), + new File(targetDir + resolveComposeFile()), + new File(targetDir + resolveComposeVolumesFile()), new File(targetDir + ("docker-selenium.yml")) )); @@ -175,6 +177,27 @@ public class ContainerTestSuite { fail("Failed to create test container"); } } + + private static String resolveComposeFile() { + if (IS_REDIS_CLUSTER) { + return "docker-compose.redis-cluster.yml"; + } + if (IS_REDIS_SENTINEL) { + return "docker-compose.redis-sentinel.yml"; + } + return "docker-compose.redis.yml"; + } + + private static String resolveComposeVolumesFile() { + if (IS_REDIS_CLUSTER) { + return "docker-compose.redis-cluster.volumes.yml"; + } + if (IS_REDIS_SENTINEL) { + return "docker-compose.redis-sentinel.volumes.yml"; + } + return "docker-compose.redis.volumes.yml"; + } + public void stop() { if (isActive) { testContainer.stop(); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index e6c9856094..40f9758f33 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -32,12 +32,14 @@ import java.util.stream.IntStream; public class ThingsBoardDbInstaller { final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); + final static boolean IS_REDIS_SENTINEL = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisSentinel")); final static boolean IS_HYBRID_MODE = Boolean.parseBoolean(System.getProperty("blackBoxTests.hybridMode")); private final static String POSTGRES_DATA_VOLUME = "tb-postgres-test-data-volume"; private final static String CASSANDRA_DATA_VOLUME = "tb-cassandra-test-data-volume"; private final static String REDIS_DATA_VOLUME = "tb-redis-data-volume"; private final static String REDIS_CLUSTER_DATA_VOLUME = "tb-redis-cluster-data-volume"; + private final static String REDIS_SENTINEL_DATA_VOLUME = "tb-redis-sentinel-data-volume"; private final static String TB_LOG_VOLUME = "tb-log-test-volume"; private final static String TB_COAP_TRANSPORT_LOG_VOLUME = "tb-coap-transport-log-test-volume"; private final static String TB_LWM2M_TRANSPORT_LOG_VOLUME = "tb-lwm2m-transport-log-test-volume"; @@ -54,6 +56,7 @@ public class ThingsBoardDbInstaller { private final String redisDataVolume; private final String redisClusterDataVolume; + private final String redisSentinelDataVolume; private final String tbLogVolume; private final String tbCoapTransportLogVolume; private final String tbLwm2mTransportLogVolume; @@ -73,12 +76,8 @@ public class ThingsBoardDbInstaller { ? new File("./../../docker/docker-compose.hybrid.yml") : new File("./../../docker/docker-compose.postgres.yml"), new File("./../../docker/docker-compose.postgres.volumes.yml"), - IS_REDIS_CLUSTER - ? new File("./../../docker/docker-compose.redis-cluster.yml") - : new File("./../../docker/docker-compose.redis.yml"), - IS_REDIS_CLUSTER - ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml") - : new File("./../../docker/docker-compose.redis.volumes.yml") + resolveComposeFile(), + resolveComposeVolumesFile() )); if (IS_HYBRID_MODE) { composeFiles.add(new File("./../../docker/docker-compose.cassandra.volumes.yml")); @@ -94,6 +93,7 @@ public class ThingsBoardDbInstaller { cassandraDataVolume = project + "_" + CASSANDRA_DATA_VOLUME; redisDataVolume = project + "_" + REDIS_DATA_VOLUME; redisClusterDataVolume = project + "_" + REDIS_CLUSTER_DATA_VOLUME; + redisSentinelDataVolume = project + "_" + REDIS_SENTINEL_DATA_VOLUME; tbLogVolume = project + "_" + TB_LOG_VOLUME; tbCoapTransportLogVolume = project + "_" + TB_COAP_TRANSPORT_LOG_VOLUME; tbLwm2mTransportLogVolume = project + "_" + TB_LWM2M_TRANSPORT_LOG_VOLUME; @@ -121,12 +121,36 @@ public class ThingsBoardDbInstaller { for (int i = 0; i < 6; i++) { env.put("REDIS_CLUSTER_DATA_VOLUME_" + i, redisClusterDataVolume + '-' + i); } + } else if (IS_REDIS_SENTINEL) { + env.put("REDIS_SENTINEL_DATA_VOLUME_MASTER", redisSentinelDataVolume + "-" + "master"); + env.put("REDIS_SENTINEL_DATA_VOLUME_SLAVE", redisSentinelDataVolume + "-" + "slave"); + env.put("REDIS_SENTINEL_DATA_VOLUME_SENTINEL", redisSentinelDataVolume + "-" + "sentinel"); } else { env.put("REDIS_DATA_VOLUME", redisDataVolume); } dockerCompose.withEnv(env); } + private static File resolveComposeVolumesFile() { + if (IS_REDIS_CLUSTER) { + return new File("./../../docker/docker-compose.redis-cluster.volumes.yml"); + } + if (IS_REDIS_SENTINEL) { + return new File("./../../docker/docker-compose.redis-sentinel.volumes.yml"); + } + return new File("./../../docker/docker-compose.redis.volumes.yml"); + } + + private static File resolveComposeFile() { + if (IS_REDIS_CLUSTER) { + return new File("./../../docker/docker-compose.redis-cluster.yml"); + } + if (IS_REDIS_SENTINEL) { + return new File("./../../docker/docker-compose.redis-sentinel.yml"); + } + return new File("./../../docker/docker-compose.redis.yml"); + } + public Map getEnv() { return env; } @@ -163,18 +187,30 @@ public class ThingsBoardDbInstaller { dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume); dockerCompose.invokeDocker(); - String additionalServices = ""; + StringBuilder additionalServices = new StringBuilder(); if (IS_HYBRID_MODE) { - additionalServices += " cassandra"; + additionalServices.append(" cassandra"); } if (IS_REDIS_CLUSTER) { for (int i = 0; i < 6; i++) { - additionalServices = additionalServices + " redis-node-" + i; + additionalServices.append(" redis-node-").append(i); dockerCompose.withCommand("volume create " + redisClusterDataVolume + '-' + i); dockerCompose.invokeDocker(); } + } else if (IS_REDIS_SENTINEL) { + additionalServices.append(" redis-master"); + dockerCompose.withCommand("volume create " + redisSentinelDataVolume +"-" + "master"); + dockerCompose.invokeDocker(); + + additionalServices.append(" redis-slave"); + dockerCompose.withCommand("volume create " + redisSentinelDataVolume + '-' + "slave"); + dockerCompose.invokeDocker(); + + additionalServices.append(" redis-sentinel"); + dockerCompose.withCommand("volume create " + redisSentinelDataVolume + '-' + "sentinel"); + dockerCompose.invokeDocker(); } else { - additionalServices += " redis"; + additionalServices.append(" redis"); dockerCompose.withCommand("volume create " + redisDataVolume); dockerCompose.invokeDocker(); } @@ -189,7 +225,7 @@ public class ThingsBoardDbInstaller { try { dockerCompose.withCommand("down -v"); dockerCompose.invokeCompose(); - } catch (Exception e) {} + } catch (Exception ignored) {} } } diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 7ea553fe5c..332709e3fa 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -46,7 +46,7 @@ cache: type: "${CACHE_TYPE:redis}" redis: - # standalone or cluster + # standalone or cluster or sentinel connection: type: "${REDIS_CONNECTION_TYPE:standalone}" standalone: @@ -66,6 +66,16 @@ redis: nodes: "${REDIS_NODES:}" # Maximum number of redirects to follow when executing commands across the cluster. max-redirects: "${REDIS_MAX_REDIRECTS:12}" + # if set false will be used pool config build from values of the pool config section + useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" + sentinel: + # name of master node + master: "${REDIS_MASTER:}" + # comma-separated list of "host:port" pairs of sentinels + sentinels: "${REDIS_SENTINELS:}" + # password to authenticate with sentinel + password: "${REDIS_SENTINEL_PASSWORD:}" + # if set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" # db index db: "${REDIS_DB:0}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 346ec48eae..c8a45c161b 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -73,7 +73,7 @@ cache: type: "${CACHE_TYPE:redis}" redis: - # standalone or cluster + # standalone or cluster or sentinel connection: type: "${REDIS_CONNECTION_TYPE:standalone}" standalone: @@ -93,6 +93,16 @@ redis: nodes: "${REDIS_NODES:}" # Maximum number of redirects to follow when executing commands across the cluster. max-redirects: "${REDIS_MAX_REDIRECTS:12}" + # if set false will be used pool config build from values of the pool config section + useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" + sentinel: + # name of master node + master: "${REDIS_MASTER:}" + # comma-separated list of "host:port" pairs of sentinels + sentinels: "${REDIS_SENTINELS:}" + # password to authenticate with sentinel + password: "${REDIS_SENTINEL_PASSWORD:}" + # if set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" # db index db: "${REDIS_DB:0}" diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 4e8167d89d..5de2484860 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -46,7 +46,7 @@ cache: type: "${CACHE_TYPE:redis}" redis: - # standalone or cluster + # standalone or cluster or sentinel connection: type: "${REDIS_CONNECTION_TYPE:standalone}" standalone: @@ -66,6 +66,16 @@ redis: nodes: "${REDIS_NODES:}" # Maximum number of redirects to follow when executing commands across the cluster. max-redirects: "${REDIS_MAX_REDIRECTS:12}" + # if set false will be used pool config build from values of the pool config section + useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" + sentinel: + # name of master node + master: "${REDIS_MASTER:}" + # comma-separated list of "host:port" pairs of sentinels + sentinels: "${REDIS_SENTINELS:}" + # password to authenticate with sentinel + password: "${REDIS_SENTINEL_PASSWORD:}" + # if set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" # db index db: "${REDIS_DB:0}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 1e0b1ebcd4..ed25d00732 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -46,7 +46,7 @@ cache: type: "${CACHE_TYPE:redis}" redis: - # standalone or cluster + # standalone or cluster or sentinel connection: type: "${REDIS_CONNECTION_TYPE:standalone}" standalone: @@ -66,6 +66,16 @@ redis: nodes: "${REDIS_NODES:}" # Maximum number of redirects to follow when executing commands across the cluster. max-redirects: "${REDIS_MAX_REDIRECTS:12}" + # if set false will be used pool config build from values of the pool config section + useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" + sentinel: + # name of master node + master: "${REDIS_MASTER:}" + # comma-separated list of "host:port" pairs of sentinels + sentinels: "${REDIS_SENTINELS:}" + # password to authenticate with sentinel + password: "${REDIS_SENTINEL_PASSWORD:}" + # if set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" # db index db: "${REDIS_DB:0}" diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 9f086bcbc5..f7fa3902b8 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -46,7 +46,7 @@ cache: type: "${CACHE_TYPE:redis}" redis: - # standalone or cluster + # standalone or cluster or sentinel connection: type: "${REDIS_CONNECTION_TYPE:standalone}" standalone: @@ -66,6 +66,16 @@ redis: nodes: "${REDIS_NODES:}" # Maximum number of redirects to follow when executing commands across the cluster. max-redirects: "${REDIS_MAX_REDIRECTS:12}" + # if set false will be used pool config build from values of the pool config section + useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" + sentinel: + # name of master node + master: "${REDIS_MASTER:}" + # comma-separated list of "host:port" pairs of sentinels + sentinels: "${REDIS_SENTINELS:}" + # password to authenticate with sentinel + password: "${REDIS_SENTINEL_PASSWORD:}" + # if set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" # db index db: "${REDIS_DB:0}" From 72423991b9f9b96c0f8bd332ab7874953b936549 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 5 Jun 2023 13:14:44 +0300 Subject: [PATCH 2/4] Add option compatibility for DockerComposeContainer to work locally with docker compose --- .../server/msa/ContainerTestSuite.java | 1 + .../server/msa/ThingsBoardDbInstaller.java | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index c6be6ec271..95b1b2129e 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -156,6 +156,7 @@ public class ContainerTestSuite { testContainer = new DockerComposeContainerImpl<>(composeFiles) .withPull(false) .withLocalCompose(true) + .withOptions("--compatibility") .withTailChildContainers(!skipTailChildContainers) .withEnv(installTb.getEnv()) .withEnv(queueEnv) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 40f9758f33..ea3c9c637a 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -68,6 +68,7 @@ public class ThingsBoardDbInstaller { public ThingsBoardDbInstaller() { log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); + log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_SENTINEL); log.info("System property of blackBoxTests.hybridMode is {}", IS_HYBRID_MODE); List composeFiles = new ArrayList<>(Arrays.asList( new File("./../../docker/docker-compose.yml"), @@ -240,13 +241,22 @@ public class ThingsBoardDbInstaller { dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume + " " + tbCoapTransportLogVolume + " " + tbLwm2mTransportLogVolume + " " + tbHttpTransportLogVolume + - " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume + - (IS_REDIS_CLUSTER - ? IntStream.range(0, 6).mapToObj(i -> " " + redisClusterDataVolume + '-' + i).collect(Collectors.joining()) - : redisDataVolume)); + " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume + resolveComposeVolumeLog()); dockerCompose.invokeDocker(); } + private String resolveComposeVolumeLog() { + if (IS_REDIS_CLUSTER) { + return IntStream.range(0, 6).mapToObj(i -> " " + redisClusterDataVolume + "-" + i).collect(Collectors.joining()); + } + if (IS_REDIS_SENTINEL) { + return redisSentinelDataVolume + "-" + "master " + " " + + redisSentinelDataVolume + "-" + "slave" + " " + + redisSentinelDataVolume + " " + "sentinel"; + } + return redisDataVolume; + } + private void copyLogs(String volumeName, String targetDir) { File tbLogsDir = new File(targetDir); tbLogsDir.mkdirs(); From 90bf0a0ace65f4d32eaf3d4007bc6fae80ce5a66 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 5 Jun 2023 13:39:06 +0300 Subject: [PATCH 3/4] Rewrite readme for redis sentinel --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 437e583a99..1a852c80eb 100644 --- a/docker/README.md +++ b/docker/README.md @@ -21,7 +21,7 @@ In order to set cache type change the value of `CACHE` variable in `.env` file t - `redis` - use Redis standalone cache (1 node - 1 master); - `redis-cluster` - use Redis cluster cache (6 nodes - 3 masters, 3 slaves); -- `redis-sentinel` - use Redis cluster in a sentinel mode (3 nodes - 1 master, 1 slave, 1 sentinel) +- `redis-sentinel` - use Redis sentinel cache (3 nodes - 1 master, 1 slave, 1 sentinel) **NOTE**: According to the cache type corresponding docker service will be deployed (see `docker-compose.redis.yml`, `docker-compose.redis-cluster.yml`, `docker-compose.redis-sentinel.yml` for details). From 4065c94131165ded41d38eccc43c4ec6fd1ede64 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 5 Jun 2023 17:15:32 +0300 Subject: [PATCH 4/4] Refactor naming to resolve redis compose in black-box tests --- .../thingsboard/server/msa/ContainerTestSuite.java | 8 ++++---- .../server/msa/ThingsBoardDbInstaller.java | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 95b1b2129e..dd3948c495 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -111,8 +111,8 @@ public class ContainerTestSuite { new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid-test-extras.yml" : "docker-compose.postgres-test-extras.yml")), new File(targetDir + "docker-compose.postgres.volumes.yml"), new File(targetDir + "docker-compose." + QUEUE_TYPE + ".yml"), - new File(targetDir + resolveComposeFile()), - new File(targetDir + resolveComposeVolumesFile()), + new File(targetDir + resolveRedisComposeFile()), + new File(targetDir + resolveRedisComposeVolumesFile()), new File(targetDir + ("docker-selenium.yml")) )); @@ -179,7 +179,7 @@ public class ContainerTestSuite { } } - private static String resolveComposeFile() { + private static String resolveRedisComposeFile() { if (IS_REDIS_CLUSTER) { return "docker-compose.redis-cluster.yml"; } @@ -189,7 +189,7 @@ public class ContainerTestSuite { return "docker-compose.redis.yml"; } - private static String resolveComposeVolumesFile() { + private static String resolveRedisComposeVolumesFile() { if (IS_REDIS_CLUSTER) { return "docker-compose.redis-cluster.volumes.yml"; } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index ea3c9c637a..e41bbc3449 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -77,8 +77,8 @@ public class ThingsBoardDbInstaller { ? new File("./../../docker/docker-compose.hybrid.yml") : new File("./../../docker/docker-compose.postgres.yml"), new File("./../../docker/docker-compose.postgres.volumes.yml"), - resolveComposeFile(), - resolveComposeVolumesFile() + resolveRedisComposeFile(), + resolveRedisComposeVolumesFile() )); if (IS_HYBRID_MODE) { composeFiles.add(new File("./../../docker/docker-compose.cassandra.volumes.yml")); @@ -132,7 +132,7 @@ public class ThingsBoardDbInstaller { dockerCompose.withEnv(env); } - private static File resolveComposeVolumesFile() { + private static File resolveRedisComposeVolumesFile() { if (IS_REDIS_CLUSTER) { return new File("./../../docker/docker-compose.redis-cluster.volumes.yml"); } @@ -142,7 +142,7 @@ public class ThingsBoardDbInstaller { return new File("./../../docker/docker-compose.redis.volumes.yml"); } - private static File resolveComposeFile() { + private static File resolveRedisComposeFile() { if (IS_REDIS_CLUSTER) { return new File("./../../docker/docker-compose.redis-cluster.yml"); } @@ -241,11 +241,11 @@ public class ThingsBoardDbInstaller { dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume + " " + tbCoapTransportLogVolume + " " + tbLwm2mTransportLogVolume + " " + tbHttpTransportLogVolume + - " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume + resolveComposeVolumeLog()); + " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume + resolveRedisComposeVolumeLog()); dockerCompose.invokeDocker(); } - private String resolveComposeVolumeLog() { + private String resolveRedisComposeVolumeLog() { if (IS_REDIS_CLUSTER) { return IntStream.range(0, 6).mapToObj(i -> " " + redisClusterDataVolume + "-" + i).collect(Collectors.joining()); }