From dc61052bd5f68b008dc4afbe81b080681e8fa78f Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 25 Oct 2018 19:08:06 +0300 Subject: [PATCH] MSA Docker TB Cluster configuration. --- docker/.env | 6 +-- docker/README.md | 5 +- docker/check-dirs.sh | 2 +- docker/compose-utils.sh | 8 +--- docker/docker-compose.cassandra.yml | 16 +++++-- docker/docker-compose.postgres.yml | 20 ++++---- docker/docker-compose.yml | 69 +++++++++++++--------------- docker/docker-install-tb.sh | 4 +- docker/docker-upgrade-tb.sh | 6 +-- docker/kafka.env | 12 +++++ docker/tb-coap-transport.env | 2 +- docker/tb-http-transport.env | 2 +- docker/tb-js-executor.env | 2 +- docker/tb-mqtt-transport.env | 2 +- docker/tb-node.cassandra.env | 5 ++ docker/tb-node.env | 7 +++ docker/tb-node.postgres.env | 9 ++++ docker/tb-node/conf/logback.xml | 2 +- docker/tb-node/conf/thingsboard.conf | 2 +- 19 files changed, 107 insertions(+), 74 deletions(-) create mode 100644 docker/kafka.env create mode 100644 docker/tb-node.cassandra.env create mode 100644 docker/tb-node.postgres.env diff --git a/docker/.env b/docker/.env index eb859b17c6..c03845dfe0 100644 --- a/docker/.env +++ b/docker/.env @@ -10,9 +10,9 @@ COAP_TRANSPORT_DOCKER_NAME=tb-coap-transport TB_VERSION=latest -# Database used by ThingsBoard, can be either local (local HSQLDB), postgres (PostgreSQL), cassandra (Cassandra). -# In case of postgres or cassandra corresponding docker service will be deployed (see docker-compose.postgres.yml, docker-compose.cassandra.yml for details). +# Database used by ThingsBoard, can be either postgres (PostgreSQL) or cassandra (Cassandra). +# According to the database type corresponding docker service will be deployed (see docker-compose.postgres.yml, docker-compose.cassandra.yml for details). -DATABASE=local +DATABASE=postgres KAFKA_TOPICS="js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1" diff --git a/docker/README.md b/docker/README.md index f48c81a50e..c43a136d09 100644 --- a/docker/README.md +++ b/docker/README.md @@ -12,11 +12,10 @@ Before starting please make sure [Docker CE](https://docs.docker.com/install/) a Before performing initial installation you can configure the type of database to be used with ThinsBoard. In order to set database type change the value of `DATABASE` variable in `.env` file to one of the following: -- `local` - use local embedded HSQLDB database; - `postgres` - use PostgreSQL database; - `cassandra` - use Cassandra database; -**NOTE**: In case of `postgres` or `cassandra` corresponding docker service will be deployed (see `docker-compose.postgres.yml`, `docker-compose.cassandra.yml` for details). +**NOTE**: According to the database type corresponding docker service will be deployed (see `docker-compose.postgres.yml`, `docker-compose.cassandra.yml` for details). Execute the following command to run installation: @@ -52,7 +51,7 @@ In case of any issues you can examine service logs for errors. For example to see ThingsBoard node logs execute the following command: ` -$ docker-compose logs -f tb +$ docker-compose logs -f tb1 ` Or use `docker-compose ps` to see the state of all the containers. diff --git a/docker/check-dirs.sh b/docker/check-dirs.sh index 6ae3f880a4..2042a44aea 100755 --- a/docker/check-dirs.sh +++ b/docker/check-dirs.sh @@ -15,7 +15,7 @@ # limitations under the License. # -dirsArray=("./haproxy/certs.d" "./haproxy/letsencrypt" "./tb-node/db" "./tb-node/postgres" "./tb-node/cassandra" "./tb-node/log") +dirsArray=("./haproxy/certs.d" "./haproxy/letsencrypt" "./tb-node/postgres" "./tb-node/cassandra" "./tb-node/log/tb1" "./tb-node/log/tb2") for dir in ${dirsArray[@]} do diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index 02a8e393e1..d1dc20e540 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -19,8 +19,6 @@ function additionalComposeArgs() { source .env ADDITIONAL_COMPOSE_ARGS="" case $DATABASE in - local) - ;; postgres) ADDITIONAL_COMPOSE_ARGS="-f docker-compose.postgres.yml" ;; @@ -28,7 +26,7 @@ function additionalComposeArgs() { ADDITIONAL_COMPOSE_ARGS="-f docker-compose.cassandra.yml" ;; *) - echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either local, postgres or cassandra." >&2 + echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or cassandra." >&2 exit 1 esac echo $ADDITIONAL_COMPOSE_ARGS @@ -38,8 +36,6 @@ function additionalStartupServices() { source .env ADDITIONAL_STARTUP_SERVICES="" case $DATABASE in - local) - ;; postgres) ADDITIONAL_STARTUP_SERVICES=postgres ;; @@ -47,7 +43,7 @@ function additionalStartupServices() { ADDITIONAL_STARTUP_SERVICES=cassandra ;; *) - echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either local, postgres or cassandra." >&2 + echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or cassandra." >&2 exit 1 esac echo $ADDITIONAL_STARTUP_SERVICES diff --git a/docker/docker-compose.cassandra.yml b/docker/docker-compose.cassandra.yml index 2bfb748b16..721ea3ef42 100644 --- a/docker/docker-compose.cassandra.yml +++ b/docker/docker-compose.cassandra.yml @@ -24,11 +24,17 @@ services: - "9042" volumes: - ./tb-node/cassandra:/var/lib/cassandra - tb: - environment: - DATABASE_TS_TYPE: cassandra - DATABASE_ENTITIES_TYPE: cassandra - CASSANDRA_URL: cassandra:9042 + tb1: + env_file: + - tb-node.cassandra.env depends_on: - kafka + - redis + - cassandra + tb2: + env_file: + - tb-node.cassandra.env + depends_on: + - kafka + - redis - cassandra diff --git a/docker/docker-compose.postgres.yml b/docker/docker-compose.postgres.yml index e2f2737b07..c81359aac2 100644 --- a/docker/docker-compose.postgres.yml +++ b/docker/docker-compose.postgres.yml @@ -26,15 +26,17 @@ services: POSTGRES_DB: thingsboard volumes: - ./tb-node/postgres:/var/lib/postgresql/data - tb: - environment: - DATABASE_TS_TYPE: sql - DATABASE_ENTITIES_TYPE: sql - SPRING_JPA_DATABASE_PLATFORM: org.hibernate.dialect.PostgreSQLDialect - SPRING_DRIVER_CLASS_NAME: org.postgresql.Driver - SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/thingsboard - SPRING_DATASOURCE_USERNAME: postgres - SPRING_DATASOURCE_PASSWORD: postgres + tb1: + env_file: + - tb-node.postgres.env depends_on: - kafka + - redis + - postgres + tb2: + env_file: + - tb-node.postgres.env + depends_on: + - kafka + - redis - postgres diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index fd0c80c28b..6d22046291 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -20,7 +20,7 @@ version: '2.2' services: zookeeper: restart: always - image: "wurstmeister/zookeeper" + image: "zookeeper:3.5" ports: - "2181" kafka: @@ -28,31 +28,23 @@ services: image: "wurstmeister/kafka" ports: - "9092:9092" - environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENERS: INSIDE://:9093,OUTSIDE://:9092 - KAFKA_ADVERTISED_LISTENERS: INSIDE://:9093,OUTSIDE://kafka:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE - KAFKA_CREATE_TOPICS: "${KAFKA_TOPICS}" - KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false' - KAFKA_LOG_RETENTION_BYTES: 1073741824 - KAFKA_LOG_SEGMENT_BYTES: 268435456 - KAFKA_LOG_RETENTION_MS: 300000 - KAFKA_LOG_CLEANUP_POLICY: delete + env_file: + - kafka.env depends_on: - zookeeper + redis: + image: redis:4.0 + ports: + - "6379" tb-js-executor: restart: always image: "${DOCKER_REPO}/${JS_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" scale: 20 - environment: - TB_KAFKA_SERVERS: kafka:9092 env_file: - tb-js-executor.env depends_on: - kafka - tb: + tb1: restart: always image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" ports: @@ -62,28 +54,41 @@ services: options: max-size: "200m" max-file: "30" + environment: + TB_HOST: tb1 env_file: - tb-node.env - environment: - ZOOKEEPER_URL: zk:2181 - TB_KAFKA_SERVERS: kafka:9092 - JS_EVALUATOR: remote - DATABASE_TS_TYPE: sql - DATABASE_ENTITIES_TYPE: sql - SQL_DATA_FOLDER: /usr/share/thingsboard/data/db volumes: - - ./tb-node/db:/usr/share/thingsboard/data/db - ./tb-node/conf:/config - ./tb-node/log:/var/log/thingsboard depends_on: - kafka + - redis + tb2: + restart: always + image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8080" + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "30" + environment: + TB_HOST: tb2 + env_file: + - tb-node.env + volumes: + - ./tb-node/conf:/config + - ./tb-node/log:/var/log/thingsboard + depends_on: + - kafka + - redis tb-mqtt-transport1: restart: always image: "${DOCKER_REPO}/${MQTT_TRANSPORT_DOCKER_NAME}:${TB_VERSION}" ports: - "1883" - environment: - TB_KAFKA_SERVERS: kafka:9092 env_file: - tb-mqtt-transport.env depends_on: @@ -93,8 +98,6 @@ services: image: "${DOCKER_REPO}/${MQTT_TRANSPORT_DOCKER_NAME}:${TB_VERSION}" ports: - "1883" - environment: - TB_KAFKA_SERVERS: kafka:9092 env_file: - tb-mqtt-transport.env depends_on: @@ -104,8 +107,6 @@ services: image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}" ports: - "8081" - environment: - TB_KAFKA_SERVERS: kafka:9092 env_file: - tb-http-transport.env depends_on: @@ -115,8 +116,6 @@ services: image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}" ports: - "8081" - environment: - TB_KAFKA_SERVERS: kafka:9092 env_file: - tb-http-transport.env depends_on: @@ -126,8 +125,6 @@ services: image: "${DOCKER_REPO}/${COAP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}" ports: - "5683:5683/udp" - environment: - TB_KAFKA_SERVERS: kafka:9092 env_file: - tb-coap-transport.env depends_on: @@ -138,7 +135,7 @@ services: ports: - "8080" environment: - TB_HOST: tb + TB_HOST: tb1 TB_PORT: 8080 env_file: - tb-web-ui.env @@ -148,7 +145,7 @@ services: ports: - "8080" environment: - TB_HOST: tb + TB_HOST: tb2 TB_PORT: 8080 env_file: - tb-web-ui.env diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 4d4b9b9732..9032e2fd15 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -48,9 +48,9 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then - docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES + docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES fi -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb1 diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index c594b9d3f9..d83b1b7579 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -46,10 +46,10 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull tb +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull tb1 if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then - docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES + docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES fi -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb1 diff --git a/docker/kafka.env b/docker/kafka.env new file mode 100644 index 0000000000..69fbdf6116 --- /dev/null +++ b/docker/kafka.env @@ -0,0 +1,12 @@ + +KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 +KAFKA_LISTENERS=INSIDE://:9093,OUTSIDE://:9092 +KAFKA_ADVERTISED_LISTENERS=INSIDE://:9093,OUTSIDE://kafka:9092 +KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT +KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE +KAFKA_CREATE_TOPICS=${KAFKA_TOPICS} +KAFKA_AUTO_CREATE_TOPICS_ENABLE=false +KAFKA_LOG_RETENTION_BYTES=1073741824 +KAFKA_LOG_SEGMENT_BYTES=268435456 +KAFKA_LOG_RETENTION_MS=300000 +KAFKA_LOG_CLEANUP_POLICY=delete diff --git a/docker/tb-coap-transport.env b/docker/tb-coap-transport.env index b3331fe849..ed8f78d116 100644 --- a/docker/tb-coap-transport.env +++ b/docker/tb-coap-transport.env @@ -3,4 +3,4 @@ COAP_BIND_ADDRESS=0.0.0.0 COAP_BIND_PORT=5683 COAP_TIMEOUT=10000 -TB_KAFKA_SERVERS=localhost:9092 \ No newline at end of file +TB_KAFKA_SERVERS=kafka:9092 \ No newline at end of file diff --git a/docker/tb-http-transport.env b/docker/tb-http-transport.env index 33c479104e..c8363230a2 100644 --- a/docker/tb-http-transport.env +++ b/docker/tb-http-transport.env @@ -3,4 +3,4 @@ HTTP_BIND_ADDRESS=0.0.0.0 HTTP_BIND_PORT=8081 HTTP_REQUEST_TIMEOUT=60000 -TB_KAFKA_SERVERS=localhost:9092 \ No newline at end of file +TB_KAFKA_SERVERS=kafka:9092 \ No newline at end of file diff --git a/docker/tb-js-executor.env b/docker/tb-js-executor.env index cf455243ee..f7f24acabf 100644 --- a/docker/tb-js-executor.env +++ b/docker/tb-js-executor.env @@ -1,6 +1,6 @@ REMOTE_JS_EVAL_REQUEST_TOPIC=js.eval.requests -TB_KAFKA_SERVERS=localhost:9092 +TB_KAFKA_SERVERS=kafka:9092 LOGGER_LEVEL=info LOG_FOLDER=logs LOGGER_FILENAME=tb-js-executor-%DATE%.log diff --git a/docker/tb-mqtt-transport.env b/docker/tb-mqtt-transport.env index 5ada3f786a..b024c7a1a5 100644 --- a/docker/tb-mqtt-transport.env +++ b/docker/tb-mqtt-transport.env @@ -3,4 +3,4 @@ MQTT_BIND_ADDRESS=0.0.0.0 MQTT_BIND_PORT=1883 MQTT_TIMEOUT=10000 -TB_KAFKA_SERVERS=localhost:9092 \ No newline at end of file +TB_KAFKA_SERVERS=kafka:9092 \ No newline at end of file diff --git a/docker/tb-node.cassandra.env b/docker/tb-node.cassandra.env new file mode 100644 index 0000000000..8d813b9059 --- /dev/null +++ b/docker/tb-node.cassandra.env @@ -0,0 +1,5 @@ +# ThingsBoard server configuration for Cassandra database + +DATABASE_TS_TYPE=cassandra +DATABASE_ENTITIES_TYPE=cassandra +CASSANDRA_URL=cassandra:9042 diff --git a/docker/tb-node.env b/docker/tb-node.env index 311428adc6..ca945ab6ed 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -1,3 +1,10 @@ # ThingsBoard server configuration +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 +RPC_HOST=${TB_HOST} +TB_KAFKA_SERVERS=kafka:9092 +JS_EVALUATOR=remote TRANSPORT_TYPE=remote +CACHE_TYPE=redis +REDIS_HOST=redis diff --git a/docker/tb-node.postgres.env b/docker/tb-node.postgres.env new file mode 100644 index 0000000000..9fa79e369c --- /dev/null +++ b/docker/tb-node.postgres.env @@ -0,0 +1,9 @@ +# ThingsBoard server configuration for PostgreSQL database + +DATABASE_TS_TYPE=sql +DATABASE_ENTITIES_TYPE=sql +SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.PostgreSQLDialect +SPRING_DRIVER_CLASS_NAME=org.postgresql.Driver +SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/thingsboard +SPRING_DATASOURCE_USERNAME=postgres +SPRING_DATASOURCE_PASSWORD=postgres diff --git a/docker/tb-node/conf/logback.xml b/docker/tb-node/conf/logback.xml index 87f9bf4c06..1c69f537bc 100644 --- a/docker/tb-node/conf/logback.xml +++ b/docker/tb-node/conf/logback.xml @@ -21,7 +21,7 @@ - /var/log/thingsboard/thingsboard.log + /var/log/thingsboard/${TB_HOST}/thingsboard.log /var/log/thingsboard/thingsboard.%d{yyyy-MM-dd}.%i.log diff --git a/docker/tb-node/conf/thingsboard.conf b/docker/tb-node/conf/thingsboard.conf index 85d7cfd871..aa430b4a79 100644 --- a/docker/tb-node/conf/thingsboard.conf +++ b/docker/tb-node/conf/thingsboard.conf @@ -15,7 +15,7 @@ # export JAVA_OPTS="$JAVA_OPTS -Dplatform=deb -Dinstall.data_dir=/usr/share/thingsboard/data" -export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:+PrintGCDetails -XX:+PrintGCDateStamps" +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled"