Refactored k8s configs (made 'basic' and 'high-availability' deployment modes)

k8s/.env

@@ -1,5 +1,8 @@
+# Can be either basic (with single instance of Zookeeper, Kafka and Redis) or high-availability (with Zookeeper, Kafka and Redis in cluster modes).
+# According to the deployment type corresponding kubernetes resources will be deployed (see content of the directories ./basic and ./high-availability for details).
+DEPLOYMENT_TYPE=basic
 
 # Database used by ThingsBoard, can be either postgres (PostgreSQL) or cassandra (Cassandra).
 # According to the database type corresponding kubernetes resources will be deployed (see postgres.yml, cassandra.yml for details).
+DATABASE=cassandra
 
-DATABASE=postgres

k8s/README.md

@@ -20,7 +20,7 @@ $ minikube addons enable ingress
 
 ## Installation
 
-Before performing initial installation you can configure the type of database to be used with ThingsBoard.
+Before performing initial installation you can configure the type of database to be used with ThingsBoard and the type of deployment.
 In order to set database type change the value of `DATABASE` variable in `.env` file to one of the following:
 
 - `postgres` - use PostgreSQL database;
@@ -28,6 +28,13 @@ In order to set database type change the value of `DATABASE` variable in `.env`
 
 **NOTE**: According to the database type corresponding kubernetes resources will be deployed (see `postgres.yml`, `cassandra.yml` for details).
 
+In order to set deployment type change the value of `DEPLOYMENT_TYPE` variable in `.env` file to one of the following:
+
+- `basic` - start up with single instance of Zookeeper, Kafka and Redis;
+- `cassandra` - start up with Zookeeper, Kafka and Redis in cluster modes;
+
+**NOTE**: According to the deployment type corresponding kubernetes resources will be deployed (see content of the directories `./basic` and `./high-availability` for details).
+
 Execute the following command to run installation:
 
 `
@@ -52,7 +59,7 @@ Get list of the running tb-redis pods and verify that all of them are in running
 $ kubectl get pods -l app=tb-redis
 `
 
-Execute the following command to create redis cluster:
+If you are running ThingsBoard in `high-availability` `DEPLOYMENT_TYPE` execute the following command to create redis cluster:
 
 `
 $ kubectl exec -it tb-redis-0 -- redis-cli --cluster create --cluster-replicas 1 $(kubectl get pods -l app=tb-redis -o jsonpath='{range.items[*]}{.status.podIP}:6379 ')

k8s/basic/tb-node-cache-configmap.yml

@@ -0,0 +1,26 @@
+#
+# Copyright © 2016-2020 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tb-node-cache-config
+  namespace: thingsboard
+  labels:
+    name: tb-node-cache-config
+data:
+  CACHE_TYPE: redis
+  REDIS_HOST: tb-redis

k8s/basic/thirdparty.yml

@@ -0,0 +1,181 @@
+#
+# Copyright © 2016-2020 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: zookeeper
+  namespace: thingsboard
+spec:
+  selector:
+    matchLabels:
+      app: zookeeper
+  template:
+    metadata:
+      labels:
+        app: zookeeper
+    spec:
+      containers:
+        - name: server
+          imagePullPolicy: Always
+          image: zookeeper:3.5
+          ports:
+            - containerPort: 2181
+          readinessProbe:
+            periodSeconds: 5
+            tcpSocket:
+              port: 2181
+          livenessProbe:
+            initialDelaySeconds: 15
+            periodSeconds: 5
+            tcpSocket:
+              port: 2181
+          env:
+            - name: ZOO_MY_ID
+              value: "1"
+            - name: ZOO_SERVERS
+              value: "server.1=0.0.0.0:2888:3888;0.0.0.0:2181"
+      restartPolicy: Always
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: zookeeper
+  namespace: thingsboard
+spec:
+  type: ClusterIP
+  selector:
+    app: zookeeper
+  ports:
+    - name: zk-port
+      port: 2181
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tb-kafka
+  namespace: thingsboard
+spec:
+  selector:
+    matchLabels:
+      app: tb-kafka
+  template:
+    metadata:
+      labels:
+        app: tb-kafka
+    spec:
+      containers:
+        - name: server
+          imagePullPolicy: Always
+          image: wurstmeister/kafka:2.12-2.2.1
+          ports:
+            - containerPort: 9092
+          readinessProbe:
+            periodSeconds: 20
+            tcpSocket:
+              port: 9092
+          livenessProbe:
+            initialDelaySeconds: 25
+            periodSeconds: 5
+            tcpSocket:
+              port: 9092
+          env:
+            - name: KAFKA_ZOOKEEPER_CONNECT
+              value: "zookeeper:2181"
+            - name: KAFKA_LISTENERS
+              value: "INSIDE://:9093,OUTSIDE://:9092"
+            - name: KAFKA_ADVERTISED_LISTENERS
+              value: "INSIDE://:9093,OUTSIDE://tb-kafka:9092"
+            - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
+              value: "INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT"
+            - name: KAFKA_INTER_BROKER_LISTENER_NAME
+              value: "INSIDE"
+            - name: KAFKA_CREATE_TOPICS
+              value: "js_eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_rule_engine:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600"
+            - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE
+              value: "false"
+            - name: KAFKA_LOG_RETENTION_BYTES
+              value: "1073741824"
+            - name: KAFKA_LOG_SEGMENT_BYTES
+              value: "268435456"
+            - name: KAFKA_LOG_RETENTION_MS
+              value: "300000"
+            - name: KAFKA_LOG_CLEANUP_POLICY
+              value: "delete"
+      restartPolicy: Always
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: tb-kafka
+  namespace: thingsboard
+spec:
+  type: ClusterIP
+  selector:
+    app: tb-kafka
+  ports:
+    - name: tb-kafka-port
+      port: 9092
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tb-redis
+  namespace: thingsboard
+spec:
+  selector:
+    matchLabels:
+      app: tb-redis
+  template:
+    metadata:
+      labels:
+        app: tb-redis
+    spec:
+      containers:
+        - name: server
+          imagePullPolicy: Always
+          image: redis:4.0
+          ports:
+            - containerPort: 6379
+          readinessProbe:
+            periodSeconds: 5
+            tcpSocket:
+              port: 6379
+          livenessProbe:
+            periodSeconds: 5
+            tcpSocket:
+              port: 6379
+          volumeMounts:
+            - mountPath: /data
+              name: redis-data
+      volumes:
+        - name: redis-data
+          emptyDir: {}
+      restartPolicy: Always
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: tb-redis
+  namespace: thingsboard
+spec:
+  type: ClusterIP
+  selector:
+    app: tb-redis
+  ports:
+    - name: tb-redis-port
+      port: 6379
+---

k8s/common/tb-node.yml

@@ -0,0 +1,98 @@
+#
+# Copyright © 2016-2020 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tb-node
+  namespace: thingsboard
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: tb-node
+  template:
+    metadata:
+      labels:
+        app: tb-node
+    spec:
+      volumes:
+        - name: tb-node-config
+          configMap:
+            name: tb-node-config
+            items:
+              - key: conf
+                path:  thingsboard.conf
+              - key: logback
+                path:  logback.xml
+      containers:
+        - name: server
+          imagePullPolicy: Always
+          image: thingsboard/tb-node:latest
+          ports:
+            - containerPort: 8080
+              name: http
+            - containerPort: 9001
+              name: rpc
+          env:
+            - name: TB_SERVICE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: TB_SERVICE_TYPE
+              value: "monolith"
+            - name: TB_QUEUE_TYPE
+              value: "kafka"
+            - name: ZOOKEEPER_ENABLED
+              value: "true"
+            - name: ZOOKEEPER_URL
+              value: "zookeeper:2181"
+            - name: TB_KAFKA_SERVERS
+              value: "tb-kafka:9092"
+            - name: JS_EVALUATOR
+              value: "remote"
+            - name: TRANSPORT_TYPE
+              value: "remote"
+            - name: HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE
+              value: "false"
+          envFrom:
+            - configMapRef:
+                name: tb-node-db-config
+            - configMapRef:
+                name: tb-node-cache-config
+          volumeMounts:
+            - mountPath: /config
+              name: tb-node-config
+          livenessProbe:
+            httpGet:
+              path: /login
+              port: http
+            initialDelaySeconds: 300
+            timeoutSeconds: 10
+      restartPolicy: Always
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: tb-node
+  namespace: thingsboard
+spec:
+  type: ClusterIP
+  selector:
+    app: tb-node
+  ports:
+    - port: 8080
+      name: http

k8s/common/thingsboard.yml

@@ -20,7 +20,7 @@ metadata:
   name: tb-js-executor
   namespace: thingsboard
 spec:
-  replicas: 20
+  replicas: 2
   selector:
     matchLabels:
       app: tb-js-executor
@@ -52,95 +52,6 @@ spec:
 ---
 apiVersion: apps/v1
 kind: Deployment
-metadata:
-  name: tb-node
-  namespace: thingsboard
-spec:
-  replicas: 2
-  selector:
-    matchLabels:
-      app: tb-node
-  template:
-    metadata:
-      labels:
-        app: tb-node
-    spec:
-      volumes:
-        - name: tb-node-config
-          configMap:
-            name: tb-node-config
-            items:
-            - key: conf
-              path:  thingsboard.conf
-            - key: logback
-              path:  logback.xml
-      containers:
-      - name: server
-        imagePullPolicy: Always
-        image: thingsboard/tb-node:latest
-        ports:
-        - containerPort: 8080
-          name: http
-        - containerPort: 9001
-          name: rpc
-        env:
-        - name: TB_SERVICE_ID
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: TB_SERVICE_TYPE
-          value: "monolith"
-        - name: TB_QUEUE_TYPE
-          value: "kafka"
-        - name: ZOOKEEPER_ENABLED
-          value: "true"
-        - name: ZOOKEEPER_URL
-          value: "zookeeper:2181"
-        - name: TB_KAFKA_SERVERS
-          value: "tb-kafka:9092"
-        - name: JS_EVALUATOR
-          value: "remote"
-        - name: TRANSPORT_TYPE
-          value: "remote"
-        - name: CACHE_TYPE
-          value: "redis"
-        - name: REDIS_HOST
-          value: "tb-redis"
-        - name: REDIS_CONNECTION_TYPE
-          value: "cluster"
-        - name: REDIS_NODES
-          value: "tb-redis:6379"
-        - name: HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE
-          value: "false"
-        envFrom:
-        - configMapRef:
-            name: tb-node-db-config
-        volumeMounts:
-          - mountPath: /config
-            name: tb-node-config
-        livenessProbe:
-          httpGet:
-            path: /login
-            port: http
-          initialDelaySeconds: 120
-          timeoutSeconds: 10
-      restartPolicy: Always
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: tb-node
-  namespace: thingsboard
-spec:
-  type: ClusterIP
-  selector:
-    app: tb-node
-  ports:
-  - port: 8080
-    name: http
----
-apiVersion: apps/v1
-kind: Deployment
 metadata:
   name: tb-mqtt-transport
   namespace: thingsboard
@@ -193,6 +104,7 @@ spec:
           tcpSocket:
             port: 1883
         livenessProbe:
+          initialDelaySeconds: 120
           periodSeconds: 20
           tcpSocket:
             port: 1883
@@ -266,6 +178,7 @@ spec:
           tcpSocket:
             port: 8080
         livenessProbe:
+          initialDelaySeconds: 120
           periodSeconds: 20
           tcpSocket:
             port: 8080

k8s/high-availability/tb-node-cache-configmap.yml

@@ -0,0 +1,27 @@
+#
+# Copyright © 2016-2020 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tb-node-cache-config
+  namespace: thingsboard
+  labels:
+    name: tb-node-cache-config
+data:
+  CACHE_TYPE: redis
+  REDIS_CONNECTION_TYPE: cluster
+  REDIS_NODES: tb-redis:6379

k8s/high-availability/thirdparty.yml

@@ -169,7 +169,7 @@ spec:
             - name: KAFKA_CONTROLLER_SHUTDOWN_ENABLE
               value: "true"
             - name: KAFKA_CREATE_TOPICS
-              value: "js_eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600"
+              value: "js_eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_rule_engine:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600"
             - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE
               value: "false"
             - name: KAFKA_LOG_RETENTION_BYTES

k8s/k8s-delete-resources.sh

@@ -17,6 +17,9 @@
 
 set -e
 
+source .env
+
 kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
-kubectl delete -f thingsboard.yml
+
-kubectl delete -f thirdparty.yml
+kubectl delete -f common/thingsboard.yml
+kubectl delete -f common/tb-node.yml

k8s/k8s-delete-thirdparty.sh

@@ -17,5 +17,7 @@
 
 set -e
 
+source .env
+
 kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
-kubectl delete -f thirdparty.yml
+kubectl delete -f $DEPLOYMENT_TYPE/thirdparty.yml

k8s/k8s-deploy-resources.sh

@@ -17,10 +17,14 @@
 
 set -e
 
-kubectl apply -f tb-namespace.yml
+source .env
+
+kubectl apply -f common/tb-namespace.yml
 kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
-kubectl apply -f tb-node-configmap.yml
+kubectl apply -f common/tb-node-configmap.yml
-kubectl apply -f tb-mqtt-transport-configmap.yml
+kubectl apply -f common/tb-mqtt-transport-configmap.yml
-kubectl apply -f tb-http-transport-configmap.yml
+kubectl apply -f common/tb-http-transport-configmap.yml
-kubectl apply -f tb-coap-transport-configmap.yml
+kubectl apply -f common/tb-coap-transport-configmap.yml
-kubectl apply -f thingsboard.yml
+kubectl apply -f common/thingsboard.yml
+kubectl apply -f $DEPLOYMENT_TYPE/tb-node-cache-configmap.yml
+kubectl apply -f common/tb-node.yml

k8s/k8s-deploy-thirdparty.sh

@@ -17,6 +17,9 @@
 
 set -e
 
-kubectl apply -f tb-namespace.yml
+source .env
+
+kubectl apply -f common/tb-namespace.yml
 kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
-kubectl apply -f thirdparty.yml
+
+kubectl apply -f $DEPLOYMENT_TYPE/thirdparty.yml

k8s/k8s-install-tb.sh

@@ -19,8 +19,8 @@ function installTb() {
 
     loadDemo=$1
 
-    kubectl apply -f tb-node-configmap.yml
+    kubectl apply -f common/tb-node-configmap.yml
-    kubectl apply -f database-setup.yml &&
+    kubectl apply -f common/database-setup.yml &&
     kubectl wait --for=condition=Ready pod/tb-db-setup --timeout=120s &&
     kubectl exec tb-db-setup -- sh -c 'export INSTALL_TB=true; export LOAD_DEMO='"$loadDemo"'; start-tb-node.sh; touch /tmp/install-finished;'
 
@@ -30,16 +30,16 @@ function installTb() {
 
 function installPostgres() {
 
-    kubectl apply -f postgres.yml
+    kubectl apply -f common/postgres.yml
-    kubectl apply -f tb-node-postgres-configmap.yml
+    kubectl apply -f common/tb-node-postgres-configmap.yml
 
     kubectl rollout status deployment/postgres
 }
 
 function installCassandra() {
 
-    kubectl apply -f cassandra.yml
+    kubectl apply -f common/cassandra.yml
-    kubectl apply -f tb-node-cassandra-configmap.yml
+    kubectl apply -f common/tb-node-cassandra-configmap.yml
 
     kubectl rollout status statefulset/cassandra
 
@@ -75,9 +75,19 @@ fi
 
 source .env
 
-kubectl apply -f tb-namespace.yml
+kubectl apply -f common/tb-namespace.yml
 kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
 
+case $DEPLOYMENT_TYPE in
+        basic)
+        ;;
+        high-availability)
+        ;;
+        *)
+        echo "Unknown DEPLOYMENT_TYPE value specified: '${DEPLOYMENT_TYPE}'. Should be either basic or high-availability." >&2
+        exit 1
+esac
+
 case $DATABASE in
         postgres)
             installPostgres
@@ -91,3 +101,4 @@ case $DATABASE in
         echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or cassandra." >&2
         exit 1
 esac
+