TB-58: k8 base scripts

2017-05-09 23:38:42 +03:00 · 2017-05-09 23:38:42 +03:00 · 73458484c2
commit 73458484c2
parent b39416232d
8 changed files with 299 additions and 29 deletions
--- a/docker/cassandra/cassandra.yaml
+++ b/docker/cassandra/cassandra.yaml
--- a/docker/thingsboard-db-schema/thingsboard-schema.yaml
+++ b/docker/thingsboard-db-schema/thingsboard-schema.yaml
@ -1,10 +1,11 @@
 apiVersion: v1
 kind: Pod
 metadata:
-  name: tb-db-schema-deployment
+  name: tb-db-schema
 spec:
  containers:
-  - name: tb-db-schema-k8
+  - name: tb-db-schema
+    imagePullPolicy: Always
    image: thingsboard/thingsboard-db-schema:k8test
    env:
    - name: SKIP_SCHEMA_CREATION
--- a/docker/thingsboard/thingsboard.yaml
+++ b/docker/thingsboard/thingsboard.yaml
@ -2,43 +2,53 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: thingsboard-external
+  name: tb-external-ip
  labels:
-    app: thingsboard-external
+    app: tb-external-ip
 spec:
  ports:
  - port: 8080
-    name: ui
+    name: ui-port
  - port: 1883
-    name: mqtt
+    name: mqtt-port
  - port: 5683
-    name: coap
+    name: coap-port
  - port: 9001
-    name: rpc
+    name: rpc-port
  selector:
-    app: thingsboard
+    app: tb
  type: LoadBalancer
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: thingsboard-headless
+  name: tb-headless
  labels:
-    app: thingsboard-headless
+    app: tb-headless
 spec:
  ports:
  - port: 8080
-    name: ui
+    name: ui-port
  - port: 1883
-    name: mqtt
+    name: mqtt-port
  - port: 5683
-    name: coap
+    name: coap-port
  - port: 9001
-    name: rpc
+    name: rpc-port
  selector:
-    app: thingsboard
+    app: tb
  clusterIP: None
 ---
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  name: tb-budget
+spec:
+  selector:
+    matchLabels:
+      app: tb
+  minAvailable: 2
+---
 apiVersion: v1
 kind: ConfigMap
 metadata:
@ -53,12 +63,12 @@ kind: StatefulSet
 metadata:
  name: tb
 spec:
-  serviceName: "thingsboard-headless"
+  serviceName: "tb-headless"
  replicas: 2
  template:
    metadata:
      labels:
-        app: thingsboard
+        app: tb
    spec:
      terminationGracePeriodSeconds: 10
      affinity:
@ -69,20 +79,21 @@ spec:
                  - key: "app"
                    operator: In
                    values:
-                    - thingsboard-headless
+                    - tb-headless
              topologyKey: "kubernetes.io/hostname"
      containers:
-      - name: thingsboard
+      - name: tb
+        imagePullPolicy: Always
        image: thingsboard/application:k8test
        ports:
        - containerPort: 8080
-          name: ui
+          name: ui-port
        - containerPort: 1883
-          name: mqtt
+          name: mqtt-port
        - containerPort: 5683
-          name: coap
+          name: coap-port
        - containerPort: 9001
-          name: rpc
+          name: rpc-port
        env:
        - name: ZOOKEEPER_ENABLED
          valueFrom:
@ -102,4 +113,16 @@ spec:
        command:
        - sh
        - -c
-        - ./run_thingsboard.sh
+        - ./run_thingsboard.sh
+#        readinessProbe:
+#          httpGet:
+#            path: /login
+#            port: ui-port
+#          initialDelaySeconds: 30
+#          periodSeconds: 10
+        livenessProbe:
+          httpGet:
+            path: /login
+            port: ui-port
+          initialDelaySeconds: 30
+          periodSeconds: 10
--- a/docker/zookeeper/Dockerfile
+++ b/docker/zookeeper/Dockerfile
@ -0,0 +1,55 @@
+FROM ubuntu:16.04 
+ENV ZK_USER=zookeeper \
+ZK_DATA_DIR=/var/lib/zookeeper/data \
+ZK_DATA_LOG_DIR=/var/lib/zookeeper/log \
+ZK_LOG_DIR=/var/log/zookeeper \
+JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+
+ARG GPG_KEY=C823E3E5B12AF29C67F81976F5CECB3CB5E9BD2D
+ARG ZK_DIST=zookeeper-3.4.9
+RUN set -x \
+    && apt-get update \
+    && apt-get install -y openjdk-8-jre-headless wget netcat-openbsd \
+	&& wget -q "http://www.apache.org/dist/zookeeper/$ZK_DIST/$ZK_DIST.tar.gz" \
+    && wget -q "http://www.apache.org/dist/zookeeper/$ZK_DIST/$ZK_DIST.tar.gz.asc" \
+    && export GNUPGHOME="$(mktemp -d)" \
+    && gpg --keyserver ha.pool.sks-keyservers.net --recv-key "$GPG_KEY" \
+    && gpg --batch --verify "$ZK_DIST.tar.gz.asc" "$ZK_DIST.tar.gz" \
+    && tar -xzf "$ZK_DIST.tar.gz" -C /opt \
+    && rm -r "$GNUPGHOME" "$ZK_DIST.tar.gz" "$ZK_DIST.tar.gz.asc" \
+    && ln -s /opt/$ZK_DIST /opt/zookeeper \
+    && rm -rf /opt/zookeeper/CHANGES.txt \
+    /opt/zookeeper/README.txt \
+    /opt/zookeeper/NOTICE.txt \
+    /opt/zookeeper/CHANGES.txt \
+    /opt/zookeeper/README_packaging.txt \
+    /opt/zookeeper/build.xml \
+    /opt/zookeeper/config \
+    /opt/zookeeper/contrib \
+    /opt/zookeeper/dist-maven \
+    /opt/zookeeper/docs \
+    /opt/zookeeper/ivy.xml \
+    /opt/zookeeper/ivysettings.xml \
+    /opt/zookeeper/recipes \
+    /opt/zookeeper/src \
+    /opt/zookeeper/$ZK_DIST.jar.asc \
+    /opt/zookeeper/$ZK_DIST.jar.md5 \
+    /opt/zookeeper/$ZK_DIST.jar.sha1 \
+	&& apt-get autoremove -y wget \
+	&& rm -rf /var/lib/apt/lists/*
+
+#Copy configuration generator script to bin
+COPY zkGenConfig.sh zkOk.sh /opt/zookeeper/bin/
+
+# Create a user for the zookeeper process and configure file system ownership 
+# for nessecary directories and symlink the distribution as a user executable
+RUN set -x \
+	&& useradd $ZK_USER \
+    && [ `id -u $ZK_USER` -eq 1000 ] \
+    && [ `id -g $ZK_USER` -eq 1000 ] \
+    && mkdir -p $ZK_DATA_DIR $ZK_DATA_LOG_DIR $ZK_LOG_DIR /usr/share/zookeeper /tmp/zookeeper /usr/etc/ \
+	&& chown -R "$ZK_USER:$ZK_USER" /opt/$ZK_DIST $ZK_DATA_DIR $ZK_LOG_DIR $ZK_DATA_LOG_DIR /tmp/zookeeper \
+	&& ln -s /opt/zookeeper/conf/ /usr/etc/zookeeper \
+	&& ln -s /opt/zookeeper/bin/* /usr/bin \
+	&& ln -s /opt/zookeeper/$ZK_DIST.jar /usr/share/zookeeper/ \
+	&& ln -s /opt/zookeeper/lib/* /usr/share/zookeeper 
--- a/docker/zookeeper/Makefile
+++ b/docker/zookeeper/Makefile
@ -0,0 +1,12 @@
+VERSION=k8test
+PROJECT=thingsboard
+
+all: build
+
+build:
+	docker build --pull -t ${PROJECT}/k8szk:${VERSION} .
+
+push: build
+	docker push ${PROJECT}/k8szk:${VERSION}
+
+.PHONY: all build push
--- a/docker/zookeeper/zkGenConfig.sh
+++ b/docker/zookeeper/zkGenConfig.sh
@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ZK_USER=${ZK_USER:-"zookeeper"}
+ZK_LOG_LEVEL=${ZK_LOG_LEVEL:-"INFO"}
+ZK_DATA_DIR=${ZK_DATA_DIR:-"/var/lib/zookeeper/data"}
+ZK_DATA_LOG_DIR=${ZK_DATA_LOG_DIR:-"/var/lib/zookeeper/log"}
+ZK_LOG_DIR=${ZK_LOG_DIR:-"var/log/zookeeper"}
+ZK_CONF_DIR=${ZK_CONF_DIR:-"/opt/zookeeper/conf"}
+ZK_CLIENT_PORT=${ZK_CLIENT_PORT:-2181}
+ZK_SERVER_PORT=${ZK_SERVER_PORT:-2888}
+ZK_ELECTION_PORT=${ZK_ELECTION_PORT:-3888}
+ZK_TICK_TIME=${ZK_TICK_TIME:-2000}
+ZK_INIT_LIMIT=${ZK_INIT_LIMIT:-10}
+ZK_SYNC_LIMIT=${ZK_SYNC_LIMIT:-5}
+ZK_HEAP_SIZE=${ZK_HEAP_SIZE:-2G}
+ZK_MAX_CLIENT_CNXNS=${ZK_MAX_CLIENT_CNXNS:-60}
+ZK_MIN_SESSION_TIMEOUT=${ZK_MIN_SESSION_TIMEOUT:- $((ZK_TICK_TIME*2))}
+ZK_MAX_SESSION_TIMEOUT=${ZK_MAX_SESSION_TIMEOUT:- $((ZK_TICK_TIME*20))}
+ZK_SNAP_RETAIN_COUNT=${ZK_SNAP_RETAIN_COUNT:-3}
+ZK_PURGE_INTERVAL=${ZK_PURGE_INTERVAL:-0}
+ID_FILE="$ZK_DATA_DIR/myid"
+ZK_CONFIG_FILE="$ZK_CONF_DIR/zoo.cfg"
+LOGGER_PROPS_FILE="$ZK_CONF_DIR/log4j.properties"
+JAVA_ENV_FILE="$ZK_CONF_DIR/java.env"
+HOST=`hostname -s`
+DOMAIN=`hostname -d`
+
+function print_servers() {
+	 for (( i=1; i<=$ZK_REPLICAS; i++ ))
+	do
+		echo "server.$i=$NAME-$((i-1)).$DOMAIN:$ZK_SERVER_PORT:$ZK_ELECTION_PORT"
+	done
+}
+
+function validate_env() {
+    echo "Validating environment"
+	if [ -z $ZK_REPLICAS ]; then
+		echo "ZK_REPLICAS is a mandatory environment variable"
+		exit 1
+	fi
+
+	if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
+		NAME=${BASH_REMATCH[1]}
+		ORD=${BASH_REMATCH[2]}
+	else
+		echo "Failed to extract ordinal from hostname $HOST"
+		exit 1
+	fi
+	MY_ID=$((ORD+1))
+	echo "ZK_REPLICAS=$ZK_REPLICAS"
+    echo "MY_ID=$MY_ID"
+    echo "ZK_LOG_LEVEL=$ZK_LOG_LEVEL"
+    echo "ZK_DATA_DIR=$ZK_DATA_DIR"
+    echo "ZK_DATA_LOG_DIR=$ZK_DATA_LOG_DIR"
+    echo "ZK_LOG_DIR=$ZK_LOG_DIR"
+    echo "ZK_CLIENT_PORT=$ZK_CLIENT_PORT"
+    echo "ZK_SERVER_PORT=$ZK_SERVER_PORT"
+    echo "ZK_ELECTION_PORT=$ZK_ELECTION_PORT"
+    echo "ZK_TICK_TIME=$ZK_TICK_TIME"
+    echo "ZK_INIT_LIMIT=$ZK_INIT_LIMIT"
+    echo "ZK_SYNC_LIMIT=$ZK_SYNC_LIMIT"
+    echo "ZK_MAX_CLIENT_CNXNS=$ZK_MAX_CLIENT_CNXNS"
+    echo "ZK_MIN_SESSION_TIMEOUT=$ZK_MIN_SESSION_TIMEOUT"
+    echo "ZK_MAX_SESSION_TIMEOUT=$ZK_MAX_SESSION_TIMEOUT"
+    echo "ZK_HEAP_SIZE=$ZK_HEAP_SIZE"
+    echo "ZK_SNAP_RETAIN_COUNT=$ZK_SNAP_RETAIN_COUNT"
+    echo "ZK_PURGE_INTERVAL=$ZK_PURGE_INTERVAL"
+    echo "ENSEMBLE"
+    print_servers
+    echo "Environment validation successful"
+}
+
+function create_config() {
+	rm -f $ZK_CONFIG_FILE
+    echo "Creating ZooKeeper configuration"
+    echo "#This file was autogenerated by k8szk DO NOT EDIT" >> $ZK_CONFIG_FILE
+	echo "clientPort=$ZK_CLIENT_PORT" >> $ZK_CONFIG_FILE
+    echo "dataDir=$ZK_DATA_DIR" >> $ZK_CONFIG_FILE
+    echo "dataLogDir=$ZK_DATA_LOG_DIR" >> $ZK_CONFIG_FILE
+    echo "tickTime=$ZK_TICK_TIME" >> $ZK_CONFIG_FILE
+    echo "initLimit=$ZK_INIT_LIMIT" >> $ZK_CONFIG_FILE
+    echo "syncLimit=$ZK_SYNC_LIMIT" >> $ZK_CONFIG_FILE
+    echo "maxClientCnxns=$ZK_MAX_CLIENT_CNXNS" >> $ZK_CONFIG_FILE
+    echo "minSessionTimeout=$ZK_MIN_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE
+    echo "maxSessionTimeout=$ZK_MAX_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE
+    echo "autopurge.snapRetainCount=$ZK_SNAP_RETAIN_COUNT" >> $ZK_CONFIG_FILE
+    echo "autopurge.purgeInteval=$ZK_PURGE_INTERVAL" >> $ZK_CONFIG_FILE
+
+    if [ $ZK_REPLICAS -gt 1 ]; then
+    	print_servers >> $ZK_CONFIG_FILE
+    fi
+    echo "Wrote ZooKeeper configuration file to $ZK_CONFIG_FILE"
+}
+
+function create_data_dirs() {
+	echo "Creating ZooKeeper data directories and setting permissions"
+    if [ ! -d $ZK_DATA_DIR  ]; then
+        mkdir -p $ZK_DATA_DIR
+        chown -R $ZK_USER:$ZK_USER $ZK_DATA_DIR
+    fi
+
+    if [ ! -d $ZK_DATA_LOG_DIR  ]; then
+        mkdir -p $ZK_DATA_LOG_DIR
+        chown -R $ZK_USER:$ZK_USER $ZK_DATA_LOG_DIR
+    fi
+
+    if [ ! -d $ZK_LOG_DIR  ]; then
+        mkdir -p $ZK_LOG_DIR
+        chown -R $ZK_USER:$ZK_USER $ZK_LOG_DIR
+    fi
+    if [ ! -f $ID_FILE ]; then
+        echo $MY_ID >> $ID_FILE
+    fi
+    echo "Created ZooKeeper data directories and set permissions in $ZK_DATA_DIR"
+}
+
+function create_log_props () {
+	rm -f $LOGGER_PROPS_FILE
+    echo "Creating ZooKeeper log4j configuration"
+	echo "zookeeper.root.logger=CONSOLE" >> $LOGGER_PROPS_FILE
+	echo "zookeeper.console.threshold="$ZK_LOG_LEVEL >> $LOGGER_PROPS_FILE
+	echo "log4j.rootLogger=\${zookeeper.root.logger}" >> $LOGGER_PROPS_FILE
+	echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender" >> $LOGGER_PROPS_FILE
+	echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}" >> $LOGGER_PROPS_FILE
+	echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout" >> $LOGGER_PROPS_FILE
+	echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n" >> $LOGGER_PROPS_FILE
+	echo "Wrote log4j configuration to $LOGGER_PROPS_FILE"
+}
+
+function create_java_env() {
+    rm -f $JAVA_ENV_FILE
+    echo "Creating JVM configuration file"
+    echo "ZOO_LOG_DIR=$ZK_LOG_DIR" >> $JAVA_ENV_FILE
+    echo "JVMFLAGS=\"-Xmx$ZK_HEAP_SIZE -Xms$ZK_HEAP_SIZE\"" >> $JAVA_ENV_FILE
+    echo "Wrote JVM configuration to $JAVA_ENV_FILE"
+}
+
+validate_env && create_config && create_log_props && create_data_dirs && create_java_env
--- a/docker/zookeeper/zkOk.sh
+++ b/docker/zookeeper/zkOk.sh
@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# zkOk.sh uses the ruok ZooKeeper four letter work to determine if the instance
+# is health. The $? variable will be set to 0 if server responds that it is 
+# healthy, or 1 if the server fails to respond.
+
+ZK_CLIENT_PORT=${ZK_CLIENT_PORT:-2181}
+OK=$(echo ruok | nc 127.0.0.1 $ZK_CLIENT_PORT)
+if [ "$OK" == "imok" ]; then
+	exit 0
+else
+	exit 1
+fi
--- a/docker/zookeeper/zookeeper.yaml
+++ b/docker/zookeeper/zookeeper.yaml
@ -20,7 +20,7 @@ metadata:
  name: zk-config
 data:
  ensemble: "zk-0;zk-1;zk-2"
-  jvm.heap: "1G"
+  jvm.heap: "500m"
  tick: "2000"
  init: "10"
  sync: "5"
@ -65,11 +65,11 @@ spec:
      containers:
      - name: k8szk
        imagePullPolicy: Always
-        image: gcr.io/google_samples/k8szk:v1
+        image: thingsboard/k8szk:k8test
        resources:
          requests:
-            memory: "0.5Gi"
-            cpu: "0.2"
+            memory: "500Mi"
+            cpu: "250m"
        ports:
        - containerPort: 2181
          name: client
@ -83,6 +83,8 @@ spec:
            configMapKeyRef:
              name: zk-config
              key: ensemble
+        - name : ZK_REPLICAS
+          value: "3"
        - name : ZK_HEAP_SIZE
          valueFrom:
            configMapKeyRef: