diff --git a/k8s/.env b/k8s/.env index e50e57d9b8..5490250d8d 100644 --- a/k8s/.env +++ b/k8s/.env @@ -1,5 +1,8 @@ +# Can be either basic (with single instance of Zookeeper, Kafka and Redis) or high-availability (with Zookeeper, Kafka and Redis in cluster modes). +# According to the deployment type corresponding kubernetes resources will be deployed (see content of the directories ./basic and ./high-availability for details). +DEPLOYMENT_TYPE=basic # Database used by ThingsBoard, can be either postgres (PostgreSQL) or cassandra (Cassandra). # According to the database type corresponding kubernetes resources will be deployed (see postgres.yml, cassandra.yml for details). +DATABASE=cassandra -DATABASE=postgres diff --git a/k8s/README.md b/k8s/README.md index 6e888352dc..90fdea6afb 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -20,7 +20,7 @@ $ minikube addons enable ingress ## Installation -Before performing initial installation you can configure the type of database to be used with ThingsBoard. +Before performing initial installation you can configure the type of database to be used with ThingsBoard and the type of deployment. In order to set database type change the value of `DATABASE` variable in `.env` file to one of the following: - `postgres` - use PostgreSQL database; @@ -28,6 +28,13 @@ In order to set database type change the value of `DATABASE` variable in `.env` **NOTE**: According to the database type corresponding kubernetes resources will be deployed (see `postgres.yml`, `cassandra.yml` for details). +In order to set deployment type change the value of `DEPLOYMENT_TYPE` variable in `.env` file to one of the following: + +- `basic` - start up with single instance of Zookeeper, Kafka and Redis; +- `cassandra` - start up with Zookeeper, Kafka and Redis in cluster modes; + +**NOTE**: According to the deployment type corresponding kubernetes resources will be deployed (see content of the directories `./basic` and `./high-availability` for details). + Execute the following command to run installation: ` @@ -52,7 +59,7 @@ Get list of the running tb-redis pods and verify that all of them are in running $ kubectl get pods -l app=tb-redis ` -Execute the following command to create redis cluster: +If you are running ThingsBoard in `high-availability` `DEPLOYMENT_TYPE` execute the following command to create redis cluster: ` $ kubectl exec -it tb-redis-0 -- redis-cli --cluster create --cluster-replicas 1 $(kubectl get pods -l app=tb-redis -o jsonpath='{range.items[*]}{.status.podIP}:6379 ') diff --git a/k8s/basic/tb-node-cache-configmap.yml b/k8s/basic/tb-node-cache-configmap.yml new file mode 100644 index 0000000000..707247cd66 --- /dev/null +++ b/k8s/basic/tb-node-cache-configmap.yml @@ -0,0 +1,26 @@ +# +# Copyright © 2016-2020 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: ConfigMap +metadata: + name: tb-node-cache-config + namespace: thingsboard + labels: + name: tb-node-cache-config +data: + CACHE_TYPE: redis + REDIS_HOST: tb-redis \ No newline at end of file diff --git a/k8s/basic/thirdparty.yml b/k8s/basic/thirdparty.yml new file mode 100644 index 0000000000..62818a1fb1 --- /dev/null +++ b/k8s/basic/thirdparty.yml @@ -0,0 +1,181 @@ +# +# Copyright © 2016-2020 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: zookeeper + namespace: thingsboard +spec: + selector: + matchLabels: + app: zookeeper + template: + metadata: + labels: + app: zookeeper + spec: + containers: + - name: server + imagePullPolicy: Always + image: zookeeper:3.5 + ports: + - containerPort: 2181 + readinessProbe: + periodSeconds: 5 + tcpSocket: + port: 2181 + livenessProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + tcpSocket: + port: 2181 + env: + - name: ZOO_MY_ID + value: "1" + - name: ZOO_SERVERS + value: "server.1=0.0.0.0:2888:3888;0.0.0.0:2181" + restartPolicy: Always +--- +apiVersion: v1 +kind: Service +metadata: + name: zookeeper + namespace: thingsboard +spec: + type: ClusterIP + selector: + app: zookeeper + ports: + - name: zk-port + port: 2181 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tb-kafka + namespace: thingsboard +spec: + selector: + matchLabels: + app: tb-kafka + template: + metadata: + labels: + app: tb-kafka + spec: + containers: + - name: server + imagePullPolicy: Always + image: wurstmeister/kafka:2.12-2.2.1 + ports: + - containerPort: 9092 + readinessProbe: + periodSeconds: 20 + tcpSocket: + port: 9092 + livenessProbe: + initialDelaySeconds: 25 + periodSeconds: 5 + tcpSocket: + port: 9092 + env: + - name: KAFKA_ZOOKEEPER_CONNECT + value: "zookeeper:2181" + - name: KAFKA_LISTENERS + value: "INSIDE://:9093,OUTSIDE://:9092" + - name: KAFKA_ADVERTISED_LISTENERS + value: "INSIDE://:9093,OUTSIDE://tb-kafka:9092" + - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP + value: "INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT" + - name: KAFKA_INTER_BROKER_LISTENER_NAME + value: "INSIDE" + - name: KAFKA_CREATE_TOPICS + value: "js_eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_rule_engine:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600" + - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE + value: "false" + - name: KAFKA_LOG_RETENTION_BYTES + value: "1073741824" + - name: KAFKA_LOG_SEGMENT_BYTES + value: "268435456" + - name: KAFKA_LOG_RETENTION_MS + value: "300000" + - name: KAFKA_LOG_CLEANUP_POLICY + value: "delete" + restartPolicy: Always +--- +apiVersion: v1 +kind: Service +metadata: + name: tb-kafka + namespace: thingsboard +spec: + type: ClusterIP + selector: + app: tb-kafka + ports: + - name: tb-kafka-port + port: 9092 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tb-redis + namespace: thingsboard +spec: + selector: + matchLabels: + app: tb-redis + template: + metadata: + labels: + app: tb-redis + spec: + containers: + - name: server + imagePullPolicy: Always + image: redis:4.0 + ports: + - containerPort: 6379 + readinessProbe: + periodSeconds: 5 + tcpSocket: + port: 6379 + livenessProbe: + periodSeconds: 5 + tcpSocket: + port: 6379 + volumeMounts: + - mountPath: /data + name: redis-data + volumes: + - name: redis-data + emptyDir: {} + restartPolicy: Always +--- +apiVersion: v1 +kind: Service +metadata: + name: tb-redis + namespace: thingsboard +spec: + type: ClusterIP + selector: + app: tb-redis + ports: + - name: tb-redis-port + port: 6379 +--- \ No newline at end of file diff --git a/k8s/cassandra.yml b/k8s/common/cassandra.yml similarity index 100% rename from k8s/cassandra.yml rename to k8s/common/cassandra.yml diff --git a/k8s/database-setup.yml b/k8s/common/database-setup.yml similarity index 100% rename from k8s/database-setup.yml rename to k8s/common/database-setup.yml diff --git a/k8s/postgres.yml b/k8s/common/postgres.yml similarity index 100% rename from k8s/postgres.yml rename to k8s/common/postgres.yml diff --git a/k8s/tb-coap-transport-configmap.yml b/k8s/common/tb-coap-transport-configmap.yml similarity index 100% rename from k8s/tb-coap-transport-configmap.yml rename to k8s/common/tb-coap-transport-configmap.yml diff --git a/k8s/tb-http-transport-configmap.yml b/k8s/common/tb-http-transport-configmap.yml similarity index 100% rename from k8s/tb-http-transport-configmap.yml rename to k8s/common/tb-http-transport-configmap.yml diff --git a/k8s/tb-mqtt-transport-configmap.yml b/k8s/common/tb-mqtt-transport-configmap.yml similarity index 100% rename from k8s/tb-mqtt-transport-configmap.yml rename to k8s/common/tb-mqtt-transport-configmap.yml diff --git a/k8s/tb-namespace.yml b/k8s/common/tb-namespace.yml similarity index 100% rename from k8s/tb-namespace.yml rename to k8s/common/tb-namespace.yml diff --git a/k8s/tb-node-cassandra-configmap.yml b/k8s/common/tb-node-cassandra-configmap.yml similarity index 100% rename from k8s/tb-node-cassandra-configmap.yml rename to k8s/common/tb-node-cassandra-configmap.yml diff --git a/k8s/tb-node-configmap.yml b/k8s/common/tb-node-configmap.yml similarity index 100% rename from k8s/tb-node-configmap.yml rename to k8s/common/tb-node-configmap.yml diff --git a/k8s/tb-node-postgres-configmap.yml b/k8s/common/tb-node-postgres-configmap.yml similarity index 100% rename from k8s/tb-node-postgres-configmap.yml rename to k8s/common/tb-node-postgres-configmap.yml diff --git a/k8s/common/tb-node.yml b/k8s/common/tb-node.yml new file mode 100644 index 0000000000..7d90153eb7 --- /dev/null +++ b/k8s/common/tb-node.yml @@ -0,0 +1,98 @@ +# +# Copyright © 2016-2020 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tb-node + namespace: thingsboard +spec: + replicas: 2 + selector: + matchLabels: + app: tb-node + template: + metadata: + labels: + app: tb-node + spec: + volumes: + - name: tb-node-config + configMap: + name: tb-node-config + items: + - key: conf + path: thingsboard.conf + - key: logback + path: logback.xml + containers: + - name: server + imagePullPolicy: Always + image: thingsboard/tb-node:latest + ports: + - containerPort: 8080 + name: http + - containerPort: 9001 + name: rpc + env: + - name: TB_SERVICE_ID + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: TB_SERVICE_TYPE + value: "monolith" + - name: TB_QUEUE_TYPE + value: "kafka" + - name: ZOOKEEPER_ENABLED + value: "true" + - name: ZOOKEEPER_URL + value: "zookeeper:2181" + - name: TB_KAFKA_SERVERS + value: "tb-kafka:9092" + - name: JS_EVALUATOR + value: "remote" + - name: TRANSPORT_TYPE + value: "remote" + - name: HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE + value: "false" + envFrom: + - configMapRef: + name: tb-node-db-config + - configMapRef: + name: tb-node-cache-config + volumeMounts: + - mountPath: /config + name: tb-node-config + livenessProbe: + httpGet: + path: /login + port: http + initialDelaySeconds: 300 + timeoutSeconds: 10 + restartPolicy: Always +--- +apiVersion: v1 +kind: Service +metadata: + name: tb-node + namespace: thingsboard +spec: + type: ClusterIP + selector: + app: tb-node + ports: + - port: 8080 + name: http \ No newline at end of file diff --git a/k8s/thingsboard.yml b/k8s/common/thingsboard.yml similarity index 80% rename from k8s/thingsboard.yml rename to k8s/common/thingsboard.yml index 67a4cdd52e..5e17142527 100644 --- a/k8s/thingsboard.yml +++ b/k8s/common/thingsboard.yml @@ -20,7 +20,7 @@ metadata: name: tb-js-executor namespace: thingsboard spec: - replicas: 20 + replicas: 2 selector: matchLabels: app: tb-js-executor @@ -52,95 +52,6 @@ spec: --- apiVersion: apps/v1 kind: Deployment -metadata: - name: tb-node - namespace: thingsboard -spec: - replicas: 2 - selector: - matchLabels: - app: tb-node - template: - metadata: - labels: - app: tb-node - spec: - volumes: - - name: tb-node-config - configMap: - name: tb-node-config - items: - - key: conf - path: thingsboard.conf - - key: logback - path: logback.xml - containers: - - name: server - imagePullPolicy: Always - image: thingsboard/tb-node:latest - ports: - - containerPort: 8080 - name: http - - containerPort: 9001 - name: rpc - env: - - name: TB_SERVICE_ID - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: TB_SERVICE_TYPE - value: "monolith" - - name: TB_QUEUE_TYPE - value: "kafka" - - name: ZOOKEEPER_ENABLED - value: "true" - - name: ZOOKEEPER_URL - value: "zookeeper:2181" - - name: TB_KAFKA_SERVERS - value: "tb-kafka:9092" - - name: JS_EVALUATOR - value: "remote" - - name: TRANSPORT_TYPE - value: "remote" - - name: CACHE_TYPE - value: "redis" - - name: REDIS_HOST - value: "tb-redis" - - name: REDIS_CONNECTION_TYPE - value: "cluster" - - name: REDIS_NODES - value: "tb-redis:6379" - - name: HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE - value: "false" - envFrom: - - configMapRef: - name: tb-node-db-config - volumeMounts: - - mountPath: /config - name: tb-node-config - livenessProbe: - httpGet: - path: /login - port: http - initialDelaySeconds: 120 - timeoutSeconds: 10 - restartPolicy: Always ---- -apiVersion: v1 -kind: Service -metadata: - name: tb-node - namespace: thingsboard -spec: - type: ClusterIP - selector: - app: tb-node - ports: - - port: 8080 - name: http ---- -apiVersion: apps/v1 -kind: Deployment metadata: name: tb-mqtt-transport namespace: thingsboard @@ -193,6 +104,7 @@ spec: tcpSocket: port: 1883 livenessProbe: + initialDelaySeconds: 120 periodSeconds: 20 tcpSocket: port: 1883 @@ -266,6 +178,7 @@ spec: tcpSocket: port: 8080 livenessProbe: + initialDelaySeconds: 120 periodSeconds: 20 tcpSocket: port: 8080 diff --git a/k8s/high-availability/tb-node-cache-configmap.yml b/k8s/high-availability/tb-node-cache-configmap.yml new file mode 100644 index 0000000000..d58d7688ae --- /dev/null +++ b/k8s/high-availability/tb-node-cache-configmap.yml @@ -0,0 +1,27 @@ +# +# Copyright © 2016-2020 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: ConfigMap +metadata: + name: tb-node-cache-config + namespace: thingsboard + labels: + name: tb-node-cache-config +data: + CACHE_TYPE: redis + REDIS_CONNECTION_TYPE: cluster + REDIS_NODES: tb-redis:6379 \ No newline at end of file diff --git a/k8s/thirdparty.yml b/k8s/high-availability/thirdparty.yml similarity index 98% rename from k8s/thirdparty.yml rename to k8s/high-availability/thirdparty.yml index 2638bf0af5..06ff285cf1 100644 --- a/k8s/thirdparty.yml +++ b/k8s/high-availability/thirdparty.yml @@ -169,7 +169,7 @@ spec: - name: KAFKA_CONTROLLER_SHUTDOWN_ENABLE value: "true" - name: KAFKA_CREATE_TOPICS - value: "js_eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600" + value: "js_eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_rule_engine:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600" - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE value: "false" - name: KAFKA_LOG_RETENTION_BYTES diff --git a/k8s/k8s-delete-resources.sh b/k8s/k8s-delete-resources.sh index 21efc56bc7..a815f8025c 100755 --- a/k8s/k8s-delete-resources.sh +++ b/k8s/k8s-delete-resources.sh @@ -17,6 +17,9 @@ set -e +source .env + kubectl config set-context $(kubectl config current-context) --namespace=thingsboard -kubectl delete -f thingsboard.yml -kubectl delete -f thirdparty.yml + +kubectl delete -f common/thingsboard.yml +kubectl delete -f common/tb-node.yml diff --git a/k8s/k8s-delete-thirdparty.sh b/k8s/k8s-delete-thirdparty.sh index a93b2b809e..e46e4973d5 100755 --- a/k8s/k8s-delete-thirdparty.sh +++ b/k8s/k8s-delete-thirdparty.sh @@ -17,5 +17,7 @@ set -e +source .env + kubectl config set-context $(kubectl config current-context) --namespace=thingsboard -kubectl delete -f thirdparty.yml +kubectl delete -f $DEPLOYMENT_TYPE/thirdparty.yml diff --git a/k8s/k8s-deploy-resources.sh b/k8s/k8s-deploy-resources.sh index 6a2deb31b4..14d12a6760 100755 --- a/k8s/k8s-deploy-resources.sh +++ b/k8s/k8s-deploy-resources.sh @@ -17,10 +17,14 @@ set -e -kubectl apply -f tb-namespace.yml +source .env + +kubectl apply -f common/tb-namespace.yml kubectl config set-context $(kubectl config current-context) --namespace=thingsboard -kubectl apply -f tb-node-configmap.yml -kubectl apply -f tb-mqtt-transport-configmap.yml -kubectl apply -f tb-http-transport-configmap.yml -kubectl apply -f tb-coap-transport-configmap.yml -kubectl apply -f thingsboard.yml +kubectl apply -f common/tb-node-configmap.yml +kubectl apply -f common/tb-mqtt-transport-configmap.yml +kubectl apply -f common/tb-http-transport-configmap.yml +kubectl apply -f common/tb-coap-transport-configmap.yml +kubectl apply -f common/thingsboard.yml +kubectl apply -f $DEPLOYMENT_TYPE/tb-node-cache-configmap.yml +kubectl apply -f common/tb-node.yml diff --git a/k8s/k8s-deploy-thirdparty.sh b/k8s/k8s-deploy-thirdparty.sh index adc526506a..17e9d3740f 100755 --- a/k8s/k8s-deploy-thirdparty.sh +++ b/k8s/k8s-deploy-thirdparty.sh @@ -17,6 +17,9 @@ set -e -kubectl apply -f tb-namespace.yml +source .env + +kubectl apply -f common/tb-namespace.yml kubectl config set-context $(kubectl config current-context) --namespace=thingsboard -kubectl apply -f thirdparty.yml + +kubectl apply -f $DEPLOYMENT_TYPE/thirdparty.yml diff --git a/k8s/k8s-install-tb.sh b/k8s/k8s-install-tb.sh index 1702a5b3b5..50ea6db443 100755 --- a/k8s/k8s-install-tb.sh +++ b/k8s/k8s-install-tb.sh @@ -19,8 +19,8 @@ function installTb() { loadDemo=$1 - kubectl apply -f tb-node-configmap.yml - kubectl apply -f database-setup.yml && + kubectl apply -f common/tb-node-configmap.yml + kubectl apply -f common/database-setup.yml && kubectl wait --for=condition=Ready pod/tb-db-setup --timeout=120s && kubectl exec tb-db-setup -- sh -c 'export INSTALL_TB=true; export LOAD_DEMO='"$loadDemo"'; start-tb-node.sh; touch /tmp/install-finished;' @@ -30,16 +30,16 @@ function installTb() { function installPostgres() { - kubectl apply -f postgres.yml - kubectl apply -f tb-node-postgres-configmap.yml + kubectl apply -f common/postgres.yml + kubectl apply -f common/tb-node-postgres-configmap.yml kubectl rollout status deployment/postgres } function installCassandra() { - kubectl apply -f cassandra.yml - kubectl apply -f tb-node-cassandra-configmap.yml + kubectl apply -f common/cassandra.yml + kubectl apply -f common/tb-node-cassandra-configmap.yml kubectl rollout status statefulset/cassandra @@ -75,9 +75,19 @@ fi source .env -kubectl apply -f tb-namespace.yml +kubectl apply -f common/tb-namespace.yml kubectl config set-context $(kubectl config current-context) --namespace=thingsboard +case $DEPLOYMENT_TYPE in + basic) + ;; + high-availability) + ;; + *) + echo "Unknown DEPLOYMENT_TYPE value specified: '${DEPLOYMENT_TYPE}'. Should be either basic or high-availability." >&2 + exit 1 +esac + case $DATABASE in postgres) installPostgres @@ -91,3 +101,4 @@ case $DATABASE in echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or cassandra." >&2 exit 1 esac +