MSA Docker TB Cluster configuration.

2018-10-25 19:08:06 +03:00 · 2018-10-25 19:08:06 +03:00 · dc61052bd5
commit dc61052bd5
parent 00b4f4708b
19 changed files with 107 additions and 74 deletions
--- a/docker/.env
+++ b/docker/.env
@ -10,9 +10,9 @@ COAP_TRANSPORT_DOCKER_NAME=tb-coap-transport

 TB_VERSION=latest

-# Database used by ThingsBoard, can be either local (local HSQLDB), postgres (PostgreSQL), cassandra (Cassandra).
-# In case of postgres or cassandra corresponding docker service will be deployed (see docker-compose.postgres.yml, docker-compose.cassandra.yml for details).
+# Database used by ThingsBoard, can be either postgres (PostgreSQL) or cassandra (Cassandra).
+# According to the database type corresponding docker service will be deployed (see docker-compose.postgres.yml, docker-compose.cassandra.yml for details).

-DATABASE=local
+DATABASE=postgres

 KAFKA_TOPICS="js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1"
--- a/docker/README.md
+++ b/docker/README.md
@ -12,11 +12,10 @@ Before starting please make sure [Docker CE](https://docs.docker.com/install/) a
 Before performing initial installation you can configure the type of database to be used with ThinsBoard.
 In order to set database type change the value of `DATABASE` variable in `.env` file to one of the following:

- `local` - use local embedded HSQLDB database;
 - `postgres` - use PostgreSQL database;
 - `cassandra` - use Cassandra database;
 
-**NOTE**: In case of `postgres` or `cassandra` corresponding docker service will be deployed (see `docker-compose.postgres.yml`, `docker-compose.cassandra.yml` for details).  
+**NOTE**: According to the database type corresponding docker service will be deployed (see `docker-compose.postgres.yml`, `docker-compose.cassandra.yml` for details).  

 Execute the following command to run installation:

@ -52,7 +51,7 @@ In case of any issues you can examine service logs for errors.
 For example to see ThingsBoard node logs execute the following command:

 ` 
-$ docker-compose logs -f tb
+$ docker-compose logs -f tb1
 ` 

 Or use `docker-compose ps` to see the state of all the containers.
--- a/docker/check-dirs.sh
+++ b/docker/check-dirs.sh
@ -15,7 +15,7 @@
 # limitations under the License.
 #

-dirsArray=("./haproxy/certs.d" "./haproxy/letsencrypt" "./tb-node/db" "./tb-node/postgres" "./tb-node/cassandra" "./tb-node/log")
+dirsArray=("./haproxy/certs.d" "./haproxy/letsencrypt" "./tb-node/postgres" "./tb-node/cassandra" "./tb-node/log/tb1" "./tb-node/log/tb2")

 for dir in ${dirsArray[@]}
 do
--- a/docker/compose-utils.sh
+++ b/docker/compose-utils.sh
@ -19,8 +19,6 @@ function additionalComposeArgs() {
    source .env
    ADDITIONAL_COMPOSE_ARGS=""
    case $DATABASE in
-        local)
-        ;;
        postgres)
        ADDITIONAL_COMPOSE_ARGS="-f docker-compose.postgres.yml"
        ;;
@ -28,7 +26,7 @@ function additionalComposeArgs() {
        ADDITIONAL_COMPOSE_ARGS="-f docker-compose.cassandra.yml"
        ;;
        *)
-        echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either local, postgres or cassandra." >&2
+        echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or cassandra." >&2
        exit 1
    esac
    echo $ADDITIONAL_COMPOSE_ARGS
@ -38,8 +36,6 @@ function additionalStartupServices() {
    source .env
    ADDITIONAL_STARTUP_SERVICES=""
    case $DATABASE in
-        local)
-        ;;
        postgres)
        ADDITIONAL_STARTUP_SERVICES=postgres
        ;;
@ -47,7 +43,7 @@ function additionalStartupServices() {
        ADDITIONAL_STARTUP_SERVICES=cassandra
        ;;
        *)
-        echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either local, postgres or cassandra." >&2
+        echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or cassandra." >&2
        exit 1
    esac
    echo $ADDITIONAL_STARTUP_SERVICES
--- a/docker/docker-compose.cassandra.yml
+++ b/docker/docker-compose.cassandra.yml
@ -24,11 +24,17 @@ services:
    - "9042"
    volumes:
      - ./tb-node/cassandra:/var/lib/cassandra
-  tb:
-    environment:
-      DATABASE_TS_TYPE: cassandra
-      DATABASE_ENTITIES_TYPE: cassandra
-      CASSANDRA_URL: cassandra:9042
+  tb1:
+    env_file:
+      - tb-node.cassandra.env
    depends_on:
      - kafka
+      - redis
+      - cassandra
+  tb2:
+    env_file:
+      - tb-node.cassandra.env
+    depends_on:
+      - kafka
+      - redis
      - cassandra
--- a/docker/docker-compose.postgres.yml
+++ b/docker/docker-compose.postgres.yml
@ -26,15 +26,17 @@ services:
      POSTGRES_DB: thingsboard
    volumes:
      - ./tb-node/postgres:/var/lib/postgresql/data
-  tb:
-    environment:
-      DATABASE_TS_TYPE: sql
-      DATABASE_ENTITIES_TYPE: sql
-      SPRING_JPA_DATABASE_PLATFORM: org.hibernate.dialect.PostgreSQLDialect
-      SPRING_DRIVER_CLASS_NAME: org.postgresql.Driver
-      SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/thingsboard
-      SPRING_DATASOURCE_USERNAME: postgres
-      SPRING_DATASOURCE_PASSWORD: postgres
+  tb1:
+    env_file:
+      - tb-node.postgres.env
    depends_on:
      - kafka
+      - redis
+      - postgres
+  tb2:
+    env_file:
+      - tb-node.postgres.env
+    depends_on:
+      - kafka
+      - redis
      - postgres
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@ -20,7 +20,7 @@ version: '2.2'
 services:
  zookeeper:
    restart: always
-    image: "wurstmeister/zookeeper"
+    image: "zookeeper:3.5"
    ports:
      - "2181"
  kafka:
@ -28,31 +28,23 @@ services:
    image: "wurstmeister/kafka"
    ports:
      - "9092:9092"
-    environment:
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_LISTENERS: INSIDE://:9093,OUTSIDE://:9092
-      KAFKA_ADVERTISED_LISTENERS: INSIDE://:9093,OUTSIDE://kafka:9092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
-      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
-      KAFKA_CREATE_TOPICS: "${KAFKA_TOPICS}"
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
-      KAFKA_LOG_RETENTION_BYTES: 1073741824
-      KAFKA_LOG_SEGMENT_BYTES: 268435456
-      KAFKA_LOG_RETENTION_MS: 300000
-      KAFKA_LOG_CLEANUP_POLICY: delete
+    env_file:
+      - kafka.env
    depends_on:
      - zookeeper
+  redis:
+    image: redis:4.0
+    ports:
+      - "6379"
  tb-js-executor:
    restart: always
    image: "${DOCKER_REPO}/${JS_EXECUTOR_DOCKER_NAME}:${TB_VERSION}"
    scale: 20
-    environment:
-      TB_KAFKA_SERVERS: kafka:9092
    env_file:
      - tb-js-executor.env
    depends_on:
      - kafka
-  tb:
+  tb1:
    restart: always
    image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}"
    ports:
@ -62,28 +54,41 @@ services:
      options:
        max-size: "200m"
        max-file: "30"
+    environment:
+      TB_HOST: tb1
    env_file:
      - tb-node.env
-    environment:
-      ZOOKEEPER_URL: zk:2181
-      TB_KAFKA_SERVERS: kafka:9092
-      JS_EVALUATOR: remote
-      DATABASE_TS_TYPE: sql
-      DATABASE_ENTITIES_TYPE: sql
-      SQL_DATA_FOLDER: /usr/share/thingsboard/data/db
    volumes:
-      - ./tb-node/db:/usr/share/thingsboard/data/db
      - ./tb-node/conf:/config
      - ./tb-node/log:/var/log/thingsboard
    depends_on:
      - kafka
+      - redis
+  tb2:
+    restart: always
+    image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}"
+    ports:
+      - "8080"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "200m"
+        max-file: "30"
+    environment:
+      TB_HOST: tb2
+    env_file:
+      - tb-node.env
+    volumes:
+      - ./tb-node/conf:/config
+      - ./tb-node/log:/var/log/thingsboard
+    depends_on:
+      - kafka
+      - redis
  tb-mqtt-transport1:
    restart: always
    image: "${DOCKER_REPO}/${MQTT_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
    ports:
      - "1883"
-    environment:
-      TB_KAFKA_SERVERS: kafka:9092
    env_file:
      - tb-mqtt-transport.env
    depends_on:
@ -93,8 +98,6 @@ services:
    image: "${DOCKER_REPO}/${MQTT_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
    ports:
      - "1883"
-    environment:
-      TB_KAFKA_SERVERS: kafka:9092
    env_file:
      - tb-mqtt-transport.env
    depends_on:
@ -104,8 +107,6 @@ services:
    image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
    ports:
      - "8081"
-    environment:
-      TB_KAFKA_SERVERS: kafka:9092
    env_file:
      - tb-http-transport.env
    depends_on:
@ -115,8 +116,6 @@ services:
    image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
    ports:
      - "8081"
-    environment:
-      TB_KAFKA_SERVERS: kafka:9092
    env_file:
      - tb-http-transport.env
    depends_on:
@ -126,8 +125,6 @@ services:
    image: "${DOCKER_REPO}/${COAP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
    ports:
      - "5683:5683/udp"
-    environment:
-      TB_KAFKA_SERVERS: kafka:9092
    env_file:
      - tb-coap-transport.env
    depends_on:
@ -138,7 +135,7 @@ services:
    ports:
      - "8080"
    environment:
-      TB_HOST: tb
+      TB_HOST: tb1
      TB_PORT: 8080
    env_file:
      - tb-web-ui.env
@ -148,7 +145,7 @@ services:
    ports:
      - "8080"
    environment:
-      TB_HOST: tb
+      TB_HOST: tb2
      TB_PORT: 8080
    env_file:
      - tb-web-ui.env
--- a/docker/docker-install-tb.sh
+++ b/docker/docker-install-tb.sh
@ -48,9 +48,9 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
 ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $?

 if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then
-    docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES
+    docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES
 fi

-docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb
+docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb1


--- a/docker/docker-upgrade-tb.sh
+++ b/docker/docker-upgrade-tb.sh
@ -46,10 +46,10 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?

 ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $?

-docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull tb
+docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull tb1

 if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then
-    docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES
+    docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES
 fi

-docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb
+docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb1
--- a/docker/kafka.env
+++ b/docker/kafka.env
@ -0,0 +1,12 @@
+
+KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+KAFKA_LISTENERS=INSIDE://:9093,OUTSIDE://:9092
+KAFKA_ADVERTISED_LISTENERS=INSIDE://:9093,OUTSIDE://kafka:9092
+KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
+KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE
+KAFKA_CREATE_TOPICS=${KAFKA_TOPICS}
+KAFKA_AUTO_CREATE_TOPICS_ENABLE=false
+KAFKA_LOG_RETENTION_BYTES=1073741824
+KAFKA_LOG_SEGMENT_BYTES=268435456
+KAFKA_LOG_RETENTION_MS=300000
+KAFKA_LOG_CLEANUP_POLICY=delete
--- a/docker/tb-coap-transport.env
+++ b/docker/tb-coap-transport.env
@ -3,4 +3,4 @@ COAP_BIND_ADDRESS=0.0.0.0
 COAP_BIND_PORT=5683
 COAP_TIMEOUT=10000

-TB_KAFKA_SERVERS=localhost:9092
+TB_KAFKA_SERVERS=kafka:9092
--- a/docker/tb-http-transport.env
+++ b/docker/tb-http-transport.env
@ -3,4 +3,4 @@ HTTP_BIND_ADDRESS=0.0.0.0
 HTTP_BIND_PORT=8081
 HTTP_REQUEST_TIMEOUT=60000

-TB_KAFKA_SERVERS=localhost:9092
+TB_KAFKA_SERVERS=kafka:9092
--- a/docker/tb-js-executor.env
+++ b/docker/tb-js-executor.env
@ -1,6 +1,6 @@

 REMOTE_JS_EVAL_REQUEST_TOPIC=js.eval.requests
-TB_KAFKA_SERVERS=localhost:9092
+TB_KAFKA_SERVERS=kafka:9092
 LOGGER_LEVEL=info
 LOG_FOLDER=logs
 LOGGER_FILENAME=tb-js-executor-%DATE%.log
--- a/docker/tb-mqtt-transport.env
+++ b/docker/tb-mqtt-transport.env
@ -3,4 +3,4 @@ MQTT_BIND_ADDRESS=0.0.0.0
 MQTT_BIND_PORT=1883
 MQTT_TIMEOUT=10000

-TB_KAFKA_SERVERS=localhost:9092
+TB_KAFKA_SERVERS=kafka:9092
--- a/docker/tb-node.cassandra.env
+++ b/docker/tb-node.cassandra.env
@ -0,0 +1,5 @@
+# ThingsBoard server configuration for Cassandra database
+
+DATABASE_TS_TYPE=cassandra
+DATABASE_ENTITIES_TYPE=cassandra
+CASSANDRA_URL=cassandra:9042
--- a/docker/tb-node.env
+++ b/docker/tb-node.env
@ -1,3 +1,10 @@
 # ThingsBoard server configuration

+ZOOKEEPER_ENABLED=true
+ZOOKEEPER_URL=zookeeper:2181
+RPC_HOST=${TB_HOST}
+TB_KAFKA_SERVERS=kafka:9092
+JS_EVALUATOR=remote
 TRANSPORT_TYPE=remote
+CACHE_TYPE=redis
+REDIS_HOST=redis
--- a/docker/tb-node.postgres.env
+++ b/docker/tb-node.postgres.env
@ -0,0 +1,9 @@
+# ThingsBoard server configuration for PostgreSQL database
+
+DATABASE_TS_TYPE=sql
+DATABASE_ENTITIES_TYPE=sql
+SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.PostgreSQLDialect
+SPRING_DRIVER_CLASS_NAME=org.postgresql.Driver
+SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/thingsboard
+SPRING_DATASOURCE_USERNAME=postgres
+SPRING_DATASOURCE_PASSWORD=postgres
--- a/docker/tb-node/conf/logback.xml
+++ b/docker/tb-node/conf/logback.xml
@ -21,7 +21,7 @@

    <appender name="fileLogAppender"
              class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <file>/var/log/thingsboard/thingsboard.log</file>
+        <file>/var/log/thingsboard/${TB_HOST}/thingsboard.log</file>
        <rollingPolicy
                class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
            <fileNamePattern>/var/log/thingsboard/thingsboard.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
--- a/docker/tb-node/conf/thingsboard.conf
+++ b/docker/tb-node/conf/thingsboard.conf
@ -15,7 +15,7 @@
 #

 export JAVA_OPTS="$JAVA_OPTS -Dplatform=deb -Dinstall.data_dir=/usr/share/thingsboard/data"
-export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
+export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
 export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10"
 export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark"
 export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled"