diff --git a/k8s/README.md b/k8s/README.md index 2a62329219..6e888352dc 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -40,6 +40,26 @@ Where: ## Running +Execute the following command to deploy thirdparty resources: + +` +$ ./k8s-deploy-thirdparty.sh +` + +Get list of the running tb-redis pods and verify that all of them are in running state: + +` +$ kubectl get pods -l app=tb-redis +` + +Execute the following command to create redis cluster: + +` +$ kubectl exec -it tb-redis-0 -- redis-cli --cluster create --cluster-replicas 1 $(kubectl get pods -l app=tb-redis -o jsonpath='{range.items[*]}{.status.podIP}:6379 ') +` + +Type **'yes'** when prompted. + Execute the following command to deploy resources: ` diff --git a/k8s/k8s-delete-resources.sh b/k8s/k8s-delete-resources.sh index 27be2b5074..21efc56bc7 100755 --- a/k8s/k8s-delete-resources.sh +++ b/k8s/k8s-delete-resources.sh @@ -19,3 +19,4 @@ set -e kubectl config set-context $(kubectl config current-context) --namespace=thingsboard kubectl delete -f thingsboard.yml +kubectl delete -f thirdparty.yml diff --git a/k8s/k8s-delete-thirdparty.sh b/k8s/k8s-delete-thirdparty.sh new file mode 100755 index 0000000000..a93b2b809e --- /dev/null +++ b/k8s/k8s-delete-thirdparty.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Copyright © 2016-2020 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +kubectl config set-context $(kubectl config current-context) --namespace=thingsboard +kubectl delete -f thirdparty.yml diff --git a/k8s/k8s-deploy-thirdparty.sh b/k8s/k8s-deploy-thirdparty.sh new file mode 100755 index 0000000000..adc526506a --- /dev/null +++ b/k8s/k8s-deploy-thirdparty.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Copyright © 2016-2020 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +kubectl apply -f tb-namespace.yml +kubectl config set-context $(kubectl config current-context) --namespace=thingsboard +kubectl apply -f thirdparty.yml diff --git a/k8s/thingsboard.yml b/k8s/thingsboard.yml index 8f50f34f8e..d1f598b6d3 100644 --- a/k8s/thingsboard.yml +++ b/k8s/thingsboard.yml @@ -14,169 +14,6 @@ # limitations under the License. # -apiVersion: apps/v1 -kind: Deployment -metadata: - name: zookeeper - namespace: thingsboard -spec: - selector: - matchLabels: - app: zookeeper - template: - metadata: - labels: - app: zookeeper - spec: - containers: - - name: server - imagePullPolicy: Always - image: zookeeper:3.5 - ports: - - containerPort: 2181 - readinessProbe: - periodSeconds: 5 - tcpSocket: - port: 2181 - livenessProbe: - periodSeconds: 5 - tcpSocket: - port: 2181 - env: - - name: ZOO_MY_ID - value: "1" - - name: ZOO_SERVERS - value: "server.1=0.0.0.0:2888:3888;0.0.0.0:2181" - restartPolicy: Always ---- -apiVersion: v1 -kind: Service -metadata: - name: zookeeper - namespace: thingsboard -spec: - type: ClusterIP - selector: - app: zookeeper - ports: - - name: zk-port - port: 2181 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tb-kafka - namespace: thingsboard -spec: - selector: - matchLabels: - app: tb-kafka - template: - metadata: - labels: - app: tb-kafka - spec: - containers: - - name: server - imagePullPolicy: Always - image: wurstmeister/kafka:2.12-2.2.1 - ports: - - containerPort: 9092 - readinessProbe: - periodSeconds: 20 - tcpSocket: - port: 9092 - livenessProbe: - periodSeconds: 5 - tcpSocket: - port: 9092 - env: - - name: KAFKA_ZOOKEEPER_CONNECT - value: "zookeeper:2181" - - name: KAFKA_LISTENERS - value: "INSIDE://:9093,OUTSIDE://:9092" - - name: KAFKA_ADVERTISED_LISTENERS - value: "INSIDE://:9093,OUTSIDE://tb-kafka:9092" - - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP - value: "INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT" - - name: KAFKA_INTER_BROKER_LISTENER_NAME - value: "INSIDE" - - name: KAFKA_CREATE_TOPICS - value: "js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600" - - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE - value: "false" - - name: KAFKA_LOG_RETENTION_BYTES - value: "1073741824" - - name: KAFKA_LOG_SEGMENT_BYTES - value: "268435456" - - name: KAFKA_LOG_RETENTION_MS - value: "300000" - - name: KAFKA_LOG_CLEANUP_POLICY - value: "delete" - restartPolicy: Always ---- -apiVersion: v1 -kind: Service -metadata: - name: tb-kafka - namespace: thingsboard -spec: - type: ClusterIP - selector: - app: tb-kafka - ports: - - name: tb-kafka-port - port: 9092 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tb-redis - namespace: thingsboard -spec: - selector: - matchLabels: - app: tb-redis - template: - metadata: - labels: - app: tb-redis - spec: - containers: - - name: server - imagePullPolicy: Always - image: redis:4.0 - ports: - - containerPort: 6379 - readinessProbe: - periodSeconds: 5 - tcpSocket: - port: 6379 - livenessProbe: - periodSeconds: 5 - tcpSocket: - port: 6379 - volumeMounts: - - mountPath: /data - name: redis-data - volumes: - - name: redis-data - emptyDir: {} - restartPolicy: Always ---- -apiVersion: v1 -kind: Service -metadata: - name: tb-redis - namespace: thingsboard -spec: - type: ClusterIP - selector: - app: tb-redis - ports: - - name: tb-redis-port - port: 6379 ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -267,6 +104,10 @@ spec: value: "redis" - name: REDIS_HOST value: "tb-redis" + - name: REDIS_CONNECTION_TYPE + value: "cluster" + - name: REDIS_NODES + value: "tb-redis:6379" - name: HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE value: "false" envFrom: diff --git a/k8s/thirdparty.yml b/k8s/thirdparty.yml new file mode 100644 index 0000000000..2638bf0af5 --- /dev/null +++ b/k8s/thirdparty.yml @@ -0,0 +1,301 @@ +# +# Copyright © 2016-2020 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zookeeper + namespace: thingsboard +spec: + serviceName: "zookeeper" + replicas: 3 + podManagementPolicy: Parallel + selector: + matchLabels: + app: zookeeper + template: + metadata: + labels: + app: zookeeper + spec: + containers: + - name: zookeeper + imagePullPolicy: Always + image: zookeeper:3.5 + ports: + - containerPort: 2181 + name: client + - containerPort: 2888 + name: server + - containerPort: 3888 + name: election + readinessProbe: + periodSeconds: 60 + tcpSocket: + port: 2181 + livenessProbe: + periodSeconds: 60 + tcpSocket: + port: 2181 + env: + - name: ZOO_SERVERS + value: "server.0=zookeeper-0.zookeeper:2888:3888;2181 server.1=zookeeper-1.zookeeper:2888:3888;2181 server.2=zookeeper-2.zookeeper:2888:3888;2181" + - name: JVMFLAGS + value: "-Dzookeeper.electionPortBindRetry=0" + volumeMounts: + - name: data + mountPath: /data + readOnly: false + initContainers: + - command: + - /bin/bash + - -c + - |- + set -ex; + mkdir -p "$ZOO_DATA_LOG_DIR" "$ZOO_DATA_DIR" "$ZOO_CONF_DIR"; + chown "$ZOO_USER:$ZOO_USER" "$ZOO_DATA_LOG_DIR" "$ZOO_DATA_DIR" "$ZOO_CONF_DIR" + if [[ ! -f "$ZOO_DATA_DIR/myid" ]]; then + echo $HOSTNAME| rev | cut -d "-" -f1 | rev > "$ZOO_DATA_DIR/myid" + fi + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: zookeeper:3.5 + imagePullPolicy: IfNotPresent + name: zookeeper-init + securityContext: + runAsUser: 0 + volumeMounts: + - name: data + mountPath: /data + readOnly: false + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 100Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: zookeeper + namespace: thingsboard +spec: + type: ClusterIP + ports: + - port: 2181 + targetPort: 2181 + name: client + - port: 2888 + targetPort: 2888 + name: server + - port: 3888 + targetPort: 3888 + name: election + selector: + app: zookeeper +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: tb-kafka + namespace: thingsboard +spec: + serviceName: "tb-kafka" + replicas: 3 + podManagementPolicy: Parallel + selector: + matchLabels: + app: tb-kafka + template: + metadata: + labels: + app: tb-kafka + spec: + containers: + - name: tb-kafka + imagePullPolicy: Always + image: wurstmeister/kafka:2.12-2.2.1 + ports: + - containerPort: 9092 + name: kafka-int + readinessProbe: + periodSeconds: 5 + timeoutSeconds: 5 + tcpSocket: + port: 9092 + initialDelaySeconds: 60 + livenessProbe: + timeoutSeconds: 5 + periodSeconds: 5 + tcpSocket: + port: 9092 + initialDelaySeconds: 80 + env: + - name: BROKER_ID_COMMAND + value: "hostname | cut -d'-' -f3" + - name: KAFKA_ZOOKEEPER_CONNECT + value: "zookeeper:2181" + - name: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS + value: "60000" + - name: KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE + value: "true" + - name: KAFKA_LISTENERS + value: "INSIDE://:9092" + - name: KAFKA_ADVERTISED_LISTENERS + value: "INSIDE://:9092" + - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP + value: "INSIDE:PLAINTEXT" + - name: KAFKA_INTER_BROKER_LISTENER_NAME + value: "INSIDE" + - name: KAFKA_CONTROLLER_SHUTDOWN_ENABLE + value: "true" + - name: KAFKA_CREATE_TOPICS + value: "js_eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600" + - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE + value: "false" + - name: KAFKA_LOG_RETENTION_BYTES + value: "1073741824" + - name: KAFKA_LOG_SEGMENT_BYTES + value: "268435456" + - name: KAFKA_LOG_RETENTION_MS + value: "300000" + - name: KAFKA_LOG_CLEANUP_POLICY + value: "delete" + - name: KAFKA_PORT + value: "9092" + - name: KAFKA_LOG_DIRS + value: "/kafka-logs" + volumeMounts: + - name: logs + mountPath: /kafka-logs + subPath: logs + volumeClaimTemplates: + - metadata: + name: logs + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: tb-kafka + namespace: thingsboard +spec: + type: ClusterIP + ports: + - port: 9092 + targetPort: 9092 + name: kafka-int + selector: + app: tb-kafka +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: tb-redis + namespace: thingsboard +data: + update-node.sh: | + #!/bin/sh + REDIS_NODES="/data/nodes.conf" + sed -i -e "/myself/ s/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/${POD_IP}/" ${REDIS_NODES} + exec "$@" + redis.conf: |+ + cluster-enabled yes + cluster-require-full-coverage no + cluster-node-timeout 15000 + cluster-config-file /data/nodes.conf + cluster-migration-barrier 1 + appendonly yes + protected-mode no +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: tb-redis + namespace: thingsboard +spec: + serviceName: server + replicas: 6 + selector: + matchLabels: + app: tb-redis + template: + metadata: + labels: + app: tb-redis + spec: + containers: + - name: redis + image: redis:5.0.1-alpine + ports: + - containerPort: 6379 + name: client + - containerPort: 16379 + name: gossip + command: ["/conf/update-node.sh", "redis-server", "/conf/redis.conf"] + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: conf + mountPath: /conf + readOnly: false + - name: data + mountPath: /data + readOnly: false + volumes: + - name: conf + configMap: + name: tb-redis + defaultMode: 0755 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 100Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: tb-redis + namespace: thingsboard +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: 6379 + name: client + - port: 16379 + targetPort: 16379 + name: gossip + selector: + app: tb-redis