From 73458484c2f4d54d537c3e195071bd627c165eca Mon Sep 17 00:00:00 2001 From: volodymyr-babak Date: Tue, 9 May 2017 23:38:42 +0300 Subject: [PATCH] TB-58: k8 base scripts --- .../cassandra}/cassandra.yaml | 0 .../thingsboard-schema.yaml | 5 +- .../thingsboard}/thingsboard.yaml | 69 +++++--- docker/zookeeper/Dockerfile | 55 +++++++ docker/zookeeper/Makefile | 12 ++ docker/zookeeper/zkGenConfig.sh | 151 ++++++++++++++++++ docker/zookeeper/zkOk.sh | 26 +++ .../zookeeper}/zookeeper.yaml | 10 +- 8 files changed, 299 insertions(+), 29 deletions(-) rename {kubernetes => docker/cassandra}/cassandra.yaml (100%) rename {kubernetes => docker/thingsboard-db-schema}/thingsboard-schema.yaml (84%) rename {kubernetes => docker/thingsboard}/thingsboard.yaml (62%) create mode 100644 docker/zookeeper/Dockerfile create mode 100644 docker/zookeeper/Makefile create mode 100755 docker/zookeeper/zkGenConfig.sh create mode 100755 docker/zookeeper/zkOk.sh rename {kubernetes => docker/zookeeper}/zookeeper.yaml (95%) diff --git a/kubernetes/cassandra.yaml b/docker/cassandra/cassandra.yaml similarity index 100% rename from kubernetes/cassandra.yaml rename to docker/cassandra/cassandra.yaml diff --git a/kubernetes/thingsboard-schema.yaml b/docker/thingsboard-db-schema/thingsboard-schema.yaml similarity index 84% rename from kubernetes/thingsboard-schema.yaml rename to docker/thingsboard-db-schema/thingsboard-schema.yaml index ca0f8b1759..141e1e3d41 100644 --- a/kubernetes/thingsboard-schema.yaml +++ b/docker/thingsboard-db-schema/thingsboard-schema.yaml @@ -1,10 +1,11 @@ apiVersion: v1 kind: Pod metadata: - name: tb-db-schema-deployment + name: tb-db-schema spec: containers: - - name: tb-db-schema-k8 + - name: tb-db-schema + imagePullPolicy: Always image: thingsboard/thingsboard-db-schema:k8test env: - name: SKIP_SCHEMA_CREATION diff --git a/kubernetes/thingsboard.yaml b/docker/thingsboard/thingsboard.yaml similarity index 62% rename from kubernetes/thingsboard.yaml rename to docker/thingsboard/thingsboard.yaml index dfe84b25fc..702a7215f5 100644 --- a/kubernetes/thingsboard.yaml +++ b/docker/thingsboard/thingsboard.yaml @@ -2,43 +2,53 @@ apiVersion: v1 kind: Service metadata: - name: thingsboard-external + name: tb-external-ip labels: - app: thingsboard-external + app: tb-external-ip spec: ports: - port: 8080 - name: ui + name: ui-port - port: 1883 - name: mqtt + name: mqtt-port - port: 5683 - name: coap + name: coap-port - port: 9001 - name: rpc + name: rpc-port selector: - app: thingsboard + app: tb type: LoadBalancer --- apiVersion: v1 kind: Service metadata: - name: thingsboard-headless + name: tb-headless labels: - app: thingsboard-headless + app: tb-headless spec: ports: - port: 8080 - name: ui + name: ui-port - port: 1883 - name: mqtt + name: mqtt-port - port: 5683 - name: coap + name: coap-port - port: 9001 - name: rpc + name: rpc-port selector: - app: thingsboard + app: tb clusterIP: None --- +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: tb-budget +spec: + selector: + matchLabels: + app: tb + minAvailable: 2 +--- apiVersion: v1 kind: ConfigMap metadata: @@ -53,12 +63,12 @@ kind: StatefulSet metadata: name: tb spec: - serviceName: "thingsboard-headless" + serviceName: "tb-headless" replicas: 2 template: metadata: labels: - app: thingsboard + app: tb spec: terminationGracePeriodSeconds: 10 affinity: @@ -69,20 +79,21 @@ spec: - key: "app" operator: In values: - - thingsboard-headless + - tb-headless topologyKey: "kubernetes.io/hostname" containers: - - name: thingsboard + - name: tb + imagePullPolicy: Always image: thingsboard/application:k8test ports: - containerPort: 8080 - name: ui + name: ui-port - containerPort: 1883 - name: mqtt + name: mqtt-port - containerPort: 5683 - name: coap + name: coap-port - containerPort: 9001 - name: rpc + name: rpc-port env: - name: ZOOKEEPER_ENABLED valueFrom: @@ -102,4 +113,16 @@ spec: command: - sh - -c - - ./run_thingsboard.sh \ No newline at end of file + - ./run_thingsboard.sh +# readinessProbe: +# httpGet: +# path: /login +# port: ui-port +# initialDelaySeconds: 30 +# periodSeconds: 10 + livenessProbe: + httpGet: + path: /login + port: ui-port + initialDelaySeconds: 30 + periodSeconds: 10 \ No newline at end of file diff --git a/docker/zookeeper/Dockerfile b/docker/zookeeper/Dockerfile new file mode 100644 index 0000000000..7752f93b3a --- /dev/null +++ b/docker/zookeeper/Dockerfile @@ -0,0 +1,55 @@ +FROM ubuntu:16.04 +ENV ZK_USER=zookeeper \ +ZK_DATA_DIR=/var/lib/zookeeper/data \ +ZK_DATA_LOG_DIR=/var/lib/zookeeper/log \ +ZK_LOG_DIR=/var/log/zookeeper \ +JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 + +ARG GPG_KEY=C823E3E5B12AF29C67F81976F5CECB3CB5E9BD2D +ARG ZK_DIST=zookeeper-3.4.9 +RUN set -x \ + && apt-get update \ + && apt-get install -y openjdk-8-jre-headless wget netcat-openbsd \ + && wget -q "http://www.apache.org/dist/zookeeper/$ZK_DIST/$ZK_DIST.tar.gz" \ + && wget -q "http://www.apache.org/dist/zookeeper/$ZK_DIST/$ZK_DIST.tar.gz.asc" \ + && export GNUPGHOME="$(mktemp -d)" \ + && gpg --keyserver ha.pool.sks-keyservers.net --recv-key "$GPG_KEY" \ + && gpg --batch --verify "$ZK_DIST.tar.gz.asc" "$ZK_DIST.tar.gz" \ + && tar -xzf "$ZK_DIST.tar.gz" -C /opt \ + && rm -r "$GNUPGHOME" "$ZK_DIST.tar.gz" "$ZK_DIST.tar.gz.asc" \ + && ln -s /opt/$ZK_DIST /opt/zookeeper \ + && rm -rf /opt/zookeeper/CHANGES.txt \ + /opt/zookeeper/README.txt \ + /opt/zookeeper/NOTICE.txt \ + /opt/zookeeper/CHANGES.txt \ + /opt/zookeeper/README_packaging.txt \ + /opt/zookeeper/build.xml \ + /opt/zookeeper/config \ + /opt/zookeeper/contrib \ + /opt/zookeeper/dist-maven \ + /opt/zookeeper/docs \ + /opt/zookeeper/ivy.xml \ + /opt/zookeeper/ivysettings.xml \ + /opt/zookeeper/recipes \ + /opt/zookeeper/src \ + /opt/zookeeper/$ZK_DIST.jar.asc \ + /opt/zookeeper/$ZK_DIST.jar.md5 \ + /opt/zookeeper/$ZK_DIST.jar.sha1 \ + && apt-get autoremove -y wget \ + && rm -rf /var/lib/apt/lists/* + +#Copy configuration generator script to bin +COPY zkGenConfig.sh zkOk.sh /opt/zookeeper/bin/ + +# Create a user for the zookeeper process and configure file system ownership +# for nessecary directories and symlink the distribution as a user executable +RUN set -x \ + && useradd $ZK_USER \ + && [ `id -u $ZK_USER` -eq 1000 ] \ + && [ `id -g $ZK_USER` -eq 1000 ] \ + && mkdir -p $ZK_DATA_DIR $ZK_DATA_LOG_DIR $ZK_LOG_DIR /usr/share/zookeeper /tmp/zookeeper /usr/etc/ \ + && chown -R "$ZK_USER:$ZK_USER" /opt/$ZK_DIST $ZK_DATA_DIR $ZK_LOG_DIR $ZK_DATA_LOG_DIR /tmp/zookeeper \ + && ln -s /opt/zookeeper/conf/ /usr/etc/zookeeper \ + && ln -s /opt/zookeeper/bin/* /usr/bin \ + && ln -s /opt/zookeeper/$ZK_DIST.jar /usr/share/zookeeper/ \ + && ln -s /opt/zookeeper/lib/* /usr/share/zookeeper diff --git a/docker/zookeeper/Makefile b/docker/zookeeper/Makefile new file mode 100644 index 0000000000..a28937f1d5 --- /dev/null +++ b/docker/zookeeper/Makefile @@ -0,0 +1,12 @@ +VERSION=k8test +PROJECT=thingsboard + +all: build + +build: + docker build --pull -t ${PROJECT}/k8szk:${VERSION} . + +push: build + docker push ${PROJECT}/k8szk:${VERSION} + +.PHONY: all build push diff --git a/docker/zookeeper/zkGenConfig.sh b/docker/zookeeper/zkGenConfig.sh new file mode 100755 index 0000000000..cef42f64f7 --- /dev/null +++ b/docker/zookeeper/zkGenConfig.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ZK_USER=${ZK_USER:-"zookeeper"} +ZK_LOG_LEVEL=${ZK_LOG_LEVEL:-"INFO"} +ZK_DATA_DIR=${ZK_DATA_DIR:-"/var/lib/zookeeper/data"} +ZK_DATA_LOG_DIR=${ZK_DATA_LOG_DIR:-"/var/lib/zookeeper/log"} +ZK_LOG_DIR=${ZK_LOG_DIR:-"var/log/zookeeper"} +ZK_CONF_DIR=${ZK_CONF_DIR:-"/opt/zookeeper/conf"} +ZK_CLIENT_PORT=${ZK_CLIENT_PORT:-2181} +ZK_SERVER_PORT=${ZK_SERVER_PORT:-2888} +ZK_ELECTION_PORT=${ZK_ELECTION_PORT:-3888} +ZK_TICK_TIME=${ZK_TICK_TIME:-2000} +ZK_INIT_LIMIT=${ZK_INIT_LIMIT:-10} +ZK_SYNC_LIMIT=${ZK_SYNC_LIMIT:-5} +ZK_HEAP_SIZE=${ZK_HEAP_SIZE:-2G} +ZK_MAX_CLIENT_CNXNS=${ZK_MAX_CLIENT_CNXNS:-60} +ZK_MIN_SESSION_TIMEOUT=${ZK_MIN_SESSION_TIMEOUT:- $((ZK_TICK_TIME*2))} +ZK_MAX_SESSION_TIMEOUT=${ZK_MAX_SESSION_TIMEOUT:- $((ZK_TICK_TIME*20))} +ZK_SNAP_RETAIN_COUNT=${ZK_SNAP_RETAIN_COUNT:-3} +ZK_PURGE_INTERVAL=${ZK_PURGE_INTERVAL:-0} +ID_FILE="$ZK_DATA_DIR/myid" +ZK_CONFIG_FILE="$ZK_CONF_DIR/zoo.cfg" +LOGGER_PROPS_FILE="$ZK_CONF_DIR/log4j.properties" +JAVA_ENV_FILE="$ZK_CONF_DIR/java.env" +HOST=`hostname -s` +DOMAIN=`hostname -d` + +function print_servers() { + for (( i=1; i<=$ZK_REPLICAS; i++ )) + do + echo "server.$i=$NAME-$((i-1)).$DOMAIN:$ZK_SERVER_PORT:$ZK_ELECTION_PORT" + done +} + +function validate_env() { + echo "Validating environment" + if [ -z $ZK_REPLICAS ]; then + echo "ZK_REPLICAS is a mandatory environment variable" + exit 1 + fi + + if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then + NAME=${BASH_REMATCH[1]} + ORD=${BASH_REMATCH[2]} + else + echo "Failed to extract ordinal from hostname $HOST" + exit 1 + fi + MY_ID=$((ORD+1)) + echo "ZK_REPLICAS=$ZK_REPLICAS" + echo "MY_ID=$MY_ID" + echo "ZK_LOG_LEVEL=$ZK_LOG_LEVEL" + echo "ZK_DATA_DIR=$ZK_DATA_DIR" + echo "ZK_DATA_LOG_DIR=$ZK_DATA_LOG_DIR" + echo "ZK_LOG_DIR=$ZK_LOG_DIR" + echo "ZK_CLIENT_PORT=$ZK_CLIENT_PORT" + echo "ZK_SERVER_PORT=$ZK_SERVER_PORT" + echo "ZK_ELECTION_PORT=$ZK_ELECTION_PORT" + echo "ZK_TICK_TIME=$ZK_TICK_TIME" + echo "ZK_INIT_LIMIT=$ZK_INIT_LIMIT" + echo "ZK_SYNC_LIMIT=$ZK_SYNC_LIMIT" + echo "ZK_MAX_CLIENT_CNXNS=$ZK_MAX_CLIENT_CNXNS" + echo "ZK_MIN_SESSION_TIMEOUT=$ZK_MIN_SESSION_TIMEOUT" + echo "ZK_MAX_SESSION_TIMEOUT=$ZK_MAX_SESSION_TIMEOUT" + echo "ZK_HEAP_SIZE=$ZK_HEAP_SIZE" + echo "ZK_SNAP_RETAIN_COUNT=$ZK_SNAP_RETAIN_COUNT" + echo "ZK_PURGE_INTERVAL=$ZK_PURGE_INTERVAL" + echo "ENSEMBLE" + print_servers + echo "Environment validation successful" +} + +function create_config() { + rm -f $ZK_CONFIG_FILE + echo "Creating ZooKeeper configuration" + echo "#This file was autogenerated by k8szk DO NOT EDIT" >> $ZK_CONFIG_FILE + echo "clientPort=$ZK_CLIENT_PORT" >> $ZK_CONFIG_FILE + echo "dataDir=$ZK_DATA_DIR" >> $ZK_CONFIG_FILE + echo "dataLogDir=$ZK_DATA_LOG_DIR" >> $ZK_CONFIG_FILE + echo "tickTime=$ZK_TICK_TIME" >> $ZK_CONFIG_FILE + echo "initLimit=$ZK_INIT_LIMIT" >> $ZK_CONFIG_FILE + echo "syncLimit=$ZK_SYNC_LIMIT" >> $ZK_CONFIG_FILE + echo "maxClientCnxns=$ZK_MAX_CLIENT_CNXNS" >> $ZK_CONFIG_FILE + echo "minSessionTimeout=$ZK_MIN_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE + echo "maxSessionTimeout=$ZK_MAX_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE + echo "autopurge.snapRetainCount=$ZK_SNAP_RETAIN_COUNT" >> $ZK_CONFIG_FILE + echo "autopurge.purgeInteval=$ZK_PURGE_INTERVAL" >> $ZK_CONFIG_FILE + + if [ $ZK_REPLICAS -gt 1 ]; then + print_servers >> $ZK_CONFIG_FILE + fi + echo "Wrote ZooKeeper configuration file to $ZK_CONFIG_FILE" +} + +function create_data_dirs() { + echo "Creating ZooKeeper data directories and setting permissions" + if [ ! -d $ZK_DATA_DIR ]; then + mkdir -p $ZK_DATA_DIR + chown -R $ZK_USER:$ZK_USER $ZK_DATA_DIR + fi + + if [ ! -d $ZK_DATA_LOG_DIR ]; then + mkdir -p $ZK_DATA_LOG_DIR + chown -R $ZK_USER:$ZK_USER $ZK_DATA_LOG_DIR + fi + + if [ ! -d $ZK_LOG_DIR ]; then + mkdir -p $ZK_LOG_DIR + chown -R $ZK_USER:$ZK_USER $ZK_LOG_DIR + fi + if [ ! -f $ID_FILE ]; then + echo $MY_ID >> $ID_FILE + fi + echo "Created ZooKeeper data directories and set permissions in $ZK_DATA_DIR" +} + +function create_log_props () { + rm -f $LOGGER_PROPS_FILE + echo "Creating ZooKeeper log4j configuration" + echo "zookeeper.root.logger=CONSOLE" >> $LOGGER_PROPS_FILE + echo "zookeeper.console.threshold="$ZK_LOG_LEVEL >> $LOGGER_PROPS_FILE + echo "log4j.rootLogger=\${zookeeper.root.logger}" >> $LOGGER_PROPS_FILE + echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender" >> $LOGGER_PROPS_FILE + echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}" >> $LOGGER_PROPS_FILE + echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout" >> $LOGGER_PROPS_FILE + echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n" >> $LOGGER_PROPS_FILE + echo "Wrote log4j configuration to $LOGGER_PROPS_FILE" +} + +function create_java_env() { + rm -f $JAVA_ENV_FILE + echo "Creating JVM configuration file" + echo "ZOO_LOG_DIR=$ZK_LOG_DIR" >> $JAVA_ENV_FILE + echo "JVMFLAGS=\"-Xmx$ZK_HEAP_SIZE -Xms$ZK_HEAP_SIZE\"" >> $JAVA_ENV_FILE + echo "Wrote JVM configuration to $JAVA_ENV_FILE" +} + +validate_env && create_config && create_log_props && create_data_dirs && create_java_env diff --git a/docker/zookeeper/zkOk.sh b/docker/zookeeper/zkOk.sh new file mode 100755 index 0000000000..dbe417682f --- /dev/null +++ b/docker/zookeeper/zkOk.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# zkOk.sh uses the ruok ZooKeeper four letter work to determine if the instance +# is health. The $? variable will be set to 0 if server responds that it is +# healthy, or 1 if the server fails to respond. + +ZK_CLIENT_PORT=${ZK_CLIENT_PORT:-2181} +OK=$(echo ruok | nc 127.0.0.1 $ZK_CLIENT_PORT) +if [ "$OK" == "imok" ]; then + exit 0 +else + exit 1 +fi \ No newline at end of file diff --git a/kubernetes/zookeeper.yaml b/docker/zookeeper/zookeeper.yaml similarity index 95% rename from kubernetes/zookeeper.yaml rename to docker/zookeeper/zookeeper.yaml index 7107e07ece..4184a3ab8e 100644 --- a/kubernetes/zookeeper.yaml +++ b/docker/zookeeper/zookeeper.yaml @@ -20,7 +20,7 @@ metadata: name: zk-config data: ensemble: "zk-0;zk-1;zk-2" - jvm.heap: "1G" + jvm.heap: "500m" tick: "2000" init: "10" sync: "5" @@ -65,11 +65,11 @@ spec: containers: - name: k8szk imagePullPolicy: Always - image: gcr.io/google_samples/k8szk:v1 + image: thingsboard/k8szk:k8test resources: requests: - memory: "0.5Gi" - cpu: "0.2" + memory: "500Mi" + cpu: "250m" ports: - containerPort: 2181 name: client @@ -83,6 +83,8 @@ spec: configMapKeyRef: name: zk-config key: ensemble + - name : ZK_REPLICAS + value: "3" - name : ZK_HEAP_SIZE valueFrom: configMapKeyRef: