Merge branch 'volodymyr-babak-feature/stateful-services'

k8s/README.md

@@ -40,6 +40,26 @@ Where:
 
 ## Running
 
+Execute the following command to deploy thirdparty resources:
+
+`
+$ ./k8s-deploy-thirdparty.sh
+`
+
+Get list of the running tb-redis pods and verify that all of them are in running state:
+
+`
+$ kubectl get pods -l app=tb-redis
+`
+
+Execute the following command to create redis cluster:
+
+`
+$ kubectl exec -it tb-redis-0 -- redis-cli --cluster create --cluster-replicas 1 $(kubectl get pods -l app=tb-redis -o jsonpath='{range.items[*]}{.status.podIP}:6379 ')
+`
+
+Type **'yes'** when prompted.
+
 Execute the following command to deploy resources:
 
 `

k8s/k8s-delete-resources.sh

@@ -19,3 +19,4 @@ set -e
 
 kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
 kubectl delete -f thingsboard.yml
+kubectl delete -f thirdparty.yml

k8s/k8s-delete-thirdparty.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright © 2016-2020 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
+kubectl delete -f thirdparty.yml

k8s/k8s-deploy-thirdparty.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# Copyright © 2016-2020 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+kubectl apply -f tb-namespace.yml
+kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
+kubectl apply -f thirdparty.yml

k8s/thingsboard.yml

@@ -14,169 +14,6 @@
 # limitations under the License.
 #
 
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: zookeeper
-  namespace: thingsboard
-spec:
-  selector:
-    matchLabels:
-      app: zookeeper
-  template:
-    metadata:
-      labels:
-        app: zookeeper
-    spec:
-      containers:
-      - name: server
-        imagePullPolicy: Always
-        image: zookeeper:3.5
-        ports:
-        - containerPort: 2181
-        readinessProbe:
-          periodSeconds: 5
-          tcpSocket:
-            port: 2181
-        livenessProbe:
-          periodSeconds: 5
-          tcpSocket:
-            port: 2181
-        env:
-        - name: ZOO_MY_ID
-          value: "1"
-        - name: ZOO_SERVERS
-          value: "server.1=0.0.0.0:2888:3888;0.0.0.0:2181"
-      restartPolicy: Always
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: zookeeper
-  namespace: thingsboard
-spec:
-  type: ClusterIP
-  selector:
-    app: zookeeper
-  ports:
-  - name: zk-port
-    port: 2181
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: tb-kafka
-  namespace: thingsboard
-spec:
-  selector:
-    matchLabels:
-      app: tb-kafka
-  template:
-    metadata:
-      labels:
-        app: tb-kafka
-    spec:
-      containers:
-      - name: server
-        imagePullPolicy: Always
-        image: wurstmeister/kafka:2.12-2.2.1
-        ports:
-        - containerPort: 9092
-        readinessProbe:
-          periodSeconds: 20
-          tcpSocket:
-            port: 9092
-        livenessProbe:
-          periodSeconds: 5
-          tcpSocket:
-            port: 9092
-        env:
-        - name: KAFKA_ZOOKEEPER_CONNECT
-          value: "zookeeper:2181"
-        - name: KAFKA_LISTENERS
-          value: "INSIDE://:9093,OUTSIDE://:9092"
-        - name: KAFKA_ADVERTISED_LISTENERS
-          value: "INSIDE://:9093,OUTSIDE://tb-kafka:9092"
-        - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
-          value: "INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT"
-        - name: KAFKA_INTER_BROKER_LISTENER_NAME
-          value: "INSIDE"
-        - name: KAFKA_CREATE_TOPICS
-          value: "js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600"
-        - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE
-          value: "false"
-        - name: KAFKA_LOG_RETENTION_BYTES
-          value: "1073741824"
-        - name: KAFKA_LOG_SEGMENT_BYTES
-          value: "268435456"
-        - name: KAFKA_LOG_RETENTION_MS
-          value: "300000"
-        - name: KAFKA_LOG_CLEANUP_POLICY
-          value: "delete"
-      restartPolicy: Always
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: tb-kafka
-  namespace: thingsboard
-spec:
-  type: ClusterIP
-  selector:
-    app: tb-kafka
-  ports:
-  - name: tb-kafka-port
-    port: 9092
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: tb-redis
-  namespace: thingsboard
-spec:
-  selector:
-    matchLabels:
-      app: tb-redis
-  template:
-    metadata:
-      labels:
-        app: tb-redis
-    spec:
-      containers:
-      - name: server
-        imagePullPolicy: Always
-        image: redis:4.0
-        ports:
-        - containerPort: 6379
-        readinessProbe:
-          periodSeconds: 5
-          tcpSocket:
-            port: 6379
-        livenessProbe:
-          periodSeconds: 5
-          tcpSocket:
-            port: 6379
-        volumeMounts:
-        - mountPath: /data
-          name: redis-data
-      volumes:
-      - name: redis-data
-        emptyDir: {}
-      restartPolicy: Always
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: tb-redis
-  namespace: thingsboard
-spec:
-  type: ClusterIP
-  selector:
-    app: tb-redis
-  ports:
-  - name: tb-redis-port
-    port: 6379
----
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -267,6 +104,10 @@ spec:
           value: "redis"
         - name: REDIS_HOST
           value: "tb-redis"
+        - name: REDIS_CONNECTION_TYPE
+          value: "cluster"
+        - name: REDIS_NODES
+          value: "tb-redis:6379"
         - name: HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE
           value: "false"
         envFrom:

k8s/thirdparty.yml

@@ -0,0 +1,301 @@
+#
+# Copyright © 2016-2020 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: zookeeper
+  namespace: thingsboard
+spec:
+  serviceName: "zookeeper"
+  replicas: 3
+  podManagementPolicy: Parallel
+  selector:
+    matchLabels:
+      app: zookeeper
+  template:
+    metadata:
+      labels:
+        app: zookeeper
+    spec:
+      containers:
+        - name: zookeeper
+          imagePullPolicy: Always
+          image: zookeeper:3.5
+          ports:
+            - containerPort: 2181
+              name: client
+            - containerPort: 2888
+              name: server
+            - containerPort: 3888
+              name: election
+          readinessProbe:
+            periodSeconds: 60
+            tcpSocket:
+              port: 2181
+          livenessProbe:
+            periodSeconds: 60
+            tcpSocket:
+              port: 2181
+          env:
+            - name: ZOO_SERVERS
+              value: "server.0=zookeeper-0.zookeeper:2888:3888;2181 server.1=zookeeper-1.zookeeper:2888:3888;2181 server.2=zookeeper-2.zookeeper:2888:3888;2181"
+            - name: JVMFLAGS
+              value: "-Dzookeeper.electionPortBindRetry=0"
+          volumeMounts:
+            - name: data
+              mountPath: /data
+              readOnly: false
+      initContainers:
+        - command:
+            - /bin/bash
+            - -c
+            - |-
+              set -ex;
+              mkdir -p "$ZOO_DATA_LOG_DIR" "$ZOO_DATA_DIR" "$ZOO_CONF_DIR";
+              chown "$ZOO_USER:$ZOO_USER" "$ZOO_DATA_LOG_DIR" "$ZOO_DATA_DIR" "$ZOO_CONF_DIR"
+              if [[ ! -f "$ZOO_DATA_DIR/myid" ]]; then
+                echo $HOSTNAME| rev | cut -d "-" -f1 | rev > "$ZOO_DATA_DIR/myid"
+              fi
+          env:
+            - name: HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          image: zookeeper:3.5
+          imagePullPolicy: IfNotPresent
+          name: zookeeper-init
+          securityContext:
+            runAsUser: 0
+          volumeMounts:
+            - name: data
+              mountPath: /data
+              readOnly: false
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: 100Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: zookeeper
+  namespace: thingsboard
+spec:
+  type: ClusterIP
+  ports:
+    - port: 2181
+      targetPort: 2181
+      name: client
+    - port: 2888
+      targetPort: 2888
+      name: server
+    - port: 3888
+      targetPort: 3888
+      name: election
+  selector:
+    app: zookeeper
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: tb-kafka
+  namespace: thingsboard
+spec:
+  serviceName: "tb-kafka"
+  replicas: 3
+  podManagementPolicy: Parallel
+  selector:
+    matchLabels:
+      app: tb-kafka
+  template:
+    metadata:
+      labels:
+        app: tb-kafka
+    spec:
+      containers:
+        - name: tb-kafka
+          imagePullPolicy: Always
+          image: wurstmeister/kafka:2.12-2.2.1
+          ports:
+            - containerPort: 9092
+              name: kafka-int
+          readinessProbe:
+            periodSeconds: 5
+            timeoutSeconds: 5
+            tcpSocket:
+              port: 9092
+            initialDelaySeconds: 60
+          livenessProbe:
+            timeoutSeconds: 5
+            periodSeconds: 5
+            tcpSocket:
+              port: 9092
+            initialDelaySeconds: 80
+          env:
+            - name: BROKER_ID_COMMAND
+              value: "hostname | cut -d'-' -f3"
+            - name: KAFKA_ZOOKEEPER_CONNECT
+              value: "zookeeper:2181"
+            - name: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS
+              value: "60000"
+            - name: KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE
+              value: "true"
+            - name: KAFKA_LISTENERS
+              value: "INSIDE://:9092"
+            - name: KAFKA_ADVERTISED_LISTENERS
+              value: "INSIDE://:9092"
+            - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
+              value: "INSIDE:PLAINTEXT"
+            - name: KAFKA_INTER_BROKER_LISTENER_NAME
+              value: "INSIDE"
+            - name: KAFKA_CONTROLLER_SHUTDOWN_ENABLE
+              value: "true"
+            - name: KAFKA_CREATE_TOPICS
+              value: "js_eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600"
+            - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE
+              value: "false"
+            - name: KAFKA_LOG_RETENTION_BYTES
+              value: "1073741824"
+            - name: KAFKA_LOG_SEGMENT_BYTES
+              value: "268435456"
+            - name: KAFKA_LOG_RETENTION_MS
+              value: "300000"
+            - name: KAFKA_LOG_CLEANUP_POLICY
+              value: "delete"
+            - name: KAFKA_PORT
+              value: "9092"
+            - name: KAFKA_LOG_DIRS
+              value: "/kafka-logs"
+          volumeMounts:
+            - name: logs
+              mountPath: /kafka-logs
+              subPath: logs
+  volumeClaimTemplates:
+    - metadata:
+        name: logs
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 1Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: tb-kafka
+  namespace: thingsboard
+spec:
+  type: ClusterIP
+  ports:
+    - port: 9092
+      targetPort: 9092
+      name: kafka-int
+  selector:
+    app: tb-kafka
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tb-redis
+  namespace: thingsboard
+data:
+  update-node.sh: |
+    #!/bin/sh
+    REDIS_NODES="/data/nodes.conf"
+    sed -i -e "/myself/ s/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/${POD_IP}/" ${REDIS_NODES}
+    exec "$@"
+  redis.conf: |+
+    cluster-enabled yes
+    cluster-require-full-coverage no
+    cluster-node-timeout 15000
+    cluster-config-file /data/nodes.conf
+    cluster-migration-barrier 1
+    appendonly yes
+    protected-mode no
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: tb-redis
+  namespace: thingsboard
+spec:
+  serviceName: server
+  replicas: 6
+  selector:
+    matchLabels:
+      app: tb-redis
+  template:
+    metadata:
+      labels:
+        app: tb-redis
+    spec:
+      containers:
+        - name: redis
+          image: redis:5.0.1-alpine
+          ports:
+            - containerPort: 6379
+              name: client
+            - containerPort: 16379
+              name: gossip
+          command: ["/conf/update-node.sh", "redis-server", "/conf/redis.conf"]
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+          volumeMounts:
+            - name: conf
+              mountPath: /conf
+              readOnly: false
+            - name: data
+              mountPath: /data
+              readOnly: false
+      volumes:
+        - name: conf
+          configMap:
+            name: tb-redis
+            defaultMode: 0755
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: 100Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: tb-redis
+  namespace: thingsboard
+spec:
+  type: ClusterIP
+  ports:
+    - port: 6379
+      targetPort: 6379
+      name: client
+    - port: 16379
+      targetPort: 16379
+      name: gossip
+  selector:
+    app: tb-redis