Support of aws sqs and rabbitmq for bbt

docker/docker-compose.rabbitmq-server.yml

@@ -0,0 +1,69 @@
+#
+# Copyright © 2016-2022 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: '2.2'
+
+services:
+  rabbitmq:
+    restart: always
+    image: rabbitmq:3
+    ports:
+      - '5672:5672'
+    environment:
+      RABBITMQ_DEFAULT_USER: YOUR_USERNAME
+      RABBITMQ_DEFAULT_PASS: YOUR_PASSWORD
+  tb-js-executor:
+    depends_on:
+      - rabbitmq
+  tb-core1:
+    depends_on:
+      - rabbitmq
+  tb-core2:
+    depends_on:
+      - rabbitmq
+  tb-rule-engine1:
+    depends_on:
+      - rabbitmq
+  tb-rule-engine2:
+    depends_on:
+      - rabbitmq
+  tb-mqtt-transport1:
+    depends_on:
+      - rabbitmq
+  tb-mqtt-transport2:
+    depends_on:
+      - rabbitmq
+  tb-http-transport1:
+    depends_on:
+      - rabbitmq
+  tb-http-transport2:
+    depends_on:
+      - rabbitmq
+  tb-coap-transport:
+    depends_on:
+      - rabbitmq
+  tb-lwm2m-transport:
+    depends_on:
+      - rabbitmq
+  tb-snmp-transport:
+    depends_on:
+      - rabbitmq
+  tb-vc-executor1:
+    depends_on:
+      - rabbitmq
+  tb-vc-executor2:
+    depends_on:
+      - rabbitmq

msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java

@@ -22,14 +22,20 @@ import org.junit.extensions.cpsuite.ClasspathSuite;
 import org.junit.runner.RunWith;
 import org.testcontainers.containers.DockerComposeContainer;
 import org.testcontainers.containers.wait.strategy.Wait;
+import org.thingsboard.server.common.data.StringUtils;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.UUID;
 
 import static org.hamcrest.CoreMatchers.containsString;
@@ -44,10 +50,12 @@ import static org.junit.Assert.fail;
 public class ContainerTestSuite {
     final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster"));
     final static boolean IS_HYBRID_MODE = Boolean.parseBoolean(System.getProperty("blackBoxTests.hybridMode"));
+    final static String QUEUE_TYPE = System.getProperty("blackBoxTests.queue", "kafka");
     private static final String SOURCE_DIR = "./../../docker/";
     private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*";
     private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*";
     private static final String TB_VC_LOG_REGEXP = TRANSPORTS_LOG_REGEXP;
+    private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(120);
 
     private static DockerComposeContainer<?> testContainer;
 
@@ -81,17 +89,33 @@ public class ContainerTestSuite {
                 List<File> composeFiles = new ArrayList<>(Arrays.asList(
                         new File(targetDir + "docker-compose.yml"),
                         new File(targetDir + "docker-compose.volumes.yml"),
-                        IS_HYBRID_MODE
+                        new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")),
-                                ? new File(targetDir + "docker-compose.hybrid.yml")
-                                : new File(targetDir + "docker-compose.postgres.yml"),
                         new File(targetDir + "docker-compose.postgres.volumes.yml"),
-                        new File(targetDir + "docker-compose.kafka.yml"),
+                        new File(targetDir + "docker-compose." + QUEUE_TYPE + ".yml"),
-                        IS_REDIS_CLUSTER
+                        new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.yml" : "docker-compose.redis.yml")),
-                                ? new File(targetDir + "docker-compose.redis-cluster.yml")
+                        new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.redis-cluster.volumes.yml" : "docker-compose.redis.volumes.yml"))
-                                : new File(targetDir + "docker-compose.redis.yml"),
+                ));
-                        IS_REDIS_CLUSTER
+
-                                ? new File(targetDir + "docker-compose.redis-cluster.volumes.yml")
+                Map<String, String> queueEnv = new HashMap<>();
-                                : new File(targetDir + "docker-compose.redis.volumes.yml")));
+                queueEnv.put("TB_QUEUE_TYPE", QUEUE_TYPE);
+                switch (QUEUE_TYPE) {
+                    case "kafka":
+                        composeFiles.add(new File(targetDir + "docker-compose.kafka.yml"));
+                        break;
+                    case "aws-sqs":
+                        replaceInFile(targetDir, "queue-aws-sqs.env",
+                                Map.of("YOUR_KEY", getSysProp("blackBoxTests.awsKey"),
+                                        "YOUR_SECRET", "blackBoxTests.awsSecret",
+                                        "YOUR_REGION", "blackBoxTests.awsRegion"));
+                        break;
+                    case "rabbitmq":
+                        composeFiles.add(new File(targetDir + "docker-compose.rabbitmq-server.yml"));
+                        replaceInFile(targetDir, "queue-rabbitmq.env",
+                                Map.of("localhost", "rabbitmq"));
+                        break;
+                    default:
+                        throw new RuntimeException("Unsupported queue type: " + QUEUE_TYPE);
+                }
 
                 if (IS_HYBRID_MODE) {
                     composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml"));
@@ -102,16 +126,17 @@ public class ContainerTestSuite {
                         .withLocalCompose(true)
                         .withTailChildContainers(!skipTailChildContainers)
                         .withEnv(installTb.getEnv())
+                        .withEnv(queueEnv)
                         .withEnv("LOAD_BALANCER_NAME", "")
-                        .withExposedService("haproxy", 80, Wait.forHttp("/swagger-ui.html").withStartupTimeout(Duration.ofSeconds(400)))
+                        .withExposedService("haproxy", 80, Wait.forHttp("/swagger-ui.html").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT))
-                        .waitingFor("tb-core1", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400)))
+                        .waitingFor("tb-core1", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT))
-                        .waitingFor("tb-core2", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400)))
+                        .waitingFor("tb-core2", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT))
-                        .waitingFor("tb-http-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400)))
+                        .waitingFor("tb-http-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT))
-                        .waitingFor("tb-http-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400)))
+                        .waitingFor("tb-http-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT))
-                        .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400)))
+                        .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT))
-                        .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400)))
+                        .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT))
-                        .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400)))
+                        .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT))
-                        .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400)));
+                        .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT));
             } catch (Exception e) {
                 log.error("Failed to create test container", e);
                 fail("Failed to create test container");
@@ -120,6 +145,23 @@ public class ContainerTestSuite {
         return testContainer;
     }
 
+    private static void replaceInFile(String targetDir, String fileName, Map<String, String> replacements) throws IOException {
+        Path envFilePath = Path.of(targetDir, fileName);
+        String data = Files.readString(envFilePath);
+        for (var entry : replacements.entrySet()) {
+            data = data.replace(entry.getKey(), entry.getValue());
+        }
+        Files.write(envFilePath, data.getBytes(StandardCharsets.UTF_8));
+    }
+
+    private static String getSysProp(String propertyName) {
+        var value = System.getProperty(propertyName);
+        if (StringUtils.isEmpty(value)) {
+            throw new RuntimeException("Please define system property: " + propertyName + "!");
+        }
+        return value;
+    }
+
     private static void tryDeleteDir(String targetDir) {
         try {
             log.info("Trying to delete temp dir {}", targetDir);
@@ -135,7 +177,7 @@ public class ContainerTestSuite {
      * docker-compose files which contain container_name are not supported and the creation of DockerComposeContainer fails due to IllegalStateException.
      * This has been introduced in #1151 as a quick fix for unintuitive feedback. https://github.com/testcontainers/testcontainers-java/issues/1151
      * Using the latest testcontainers and waiting for the fix...
-     * */
+     */
     private static void replaceInFile(String sourceFilename, String target, String replacement, String verifyPhrase) {
         try {
             File file = new File(sourceFilename);