From cd11bb6c053e2e6a94926ca3b0c9de05cbbd240e Mon Sep 17 00:00:00 2001 From: volodymyr-babak Date: Sun, 7 May 2017 19:54:50 +0300 Subject: [PATCH 01/12] Kubernetes, ZK --- kubernetes/zookeeper.yaml | 167 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 kubernetes/zookeeper.yaml diff --git a/kubernetes/zookeeper.yaml b/kubernetes/zookeeper.yaml new file mode 100644 index 0000000000..efe8d99c27 --- /dev/null +++ b/kubernetes/zookeeper.yaml @@ -0,0 +1,167 @@ +apiVersion: v1 +kind: Service +metadata: + name: zk-headless + labels: + app: zk-headless +spec: + ports: + - port: 2888 + name: server + - port: 3888 + name: leader-election + clusterIP: None + selector: + app: zk +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: zk-config +data: + ensemble: "zk-0;zk-1;zk-2" + jvm.heap: "2G" + tick: "2000" + init: "10" + sync: "5" + client.cnxns: "60" + snap.retain: "3" + purge.interval: "1" +--- +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: zk-budget +spec: + selector: + matchLabels: + app: zk + minAvailable: 2 +--- +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: zk +spec: + serviceName: zk-headless + replicas: 3 + template: + metadata: + labels: + app: zk + annotations: + pod.alpha.kubernetes.io/initialized: "true" + scheduler.alpha.kubernetes.io/affinity: > + { + "podAntiAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": [ + { + "labelSelector": { + "matchExpressions": [ + { + "key": "app", + "operator": "In", + "values": [ "zk-headless" ] + } + ] + }, + "topologyKey": "kubernetes.io/hostname" + } + ] + } + } + spec: + containers: + - name: k8szk + imagePullPolicy: Always + image: gcr.io/google_samples/k8szk:v1 + resources: + requests: + memory: "1Gi" + cpu: "0.5" + ports: + - containerPort: 2181 + name: client + - containerPort: 2888 + name: server + - containerPort: 3888 + name: leader-election + env: + - name : ZK_ENSEMBLE + valueFrom: + configMapKeyRef: + name: zk-config + key: ensemble + - name : ZK_HEAP_SIZE + valueFrom: + configMapKeyRef: + name: zk-config + key: jvm.heap + - name : ZK_TICK_TIME + valueFrom: + configMapKeyRef: + name: zk-config + key: tick + - name : ZK_INIT_LIMIT + valueFrom: + configMapKeyRef: + name: zk-config + key: init + - name : ZK_SYNC_LIMIT + valueFrom: + configMapKeyRef: + name: zk-config + key: tick + - name : ZK_MAX_CLIENT_CNXNS + valueFrom: + configMapKeyRef: + name: zk-config + key: client.cnxns + - name: ZK_SNAP_RETAIN_COUNT + valueFrom: + configMapKeyRef: + name: zk-config + key: snap.retain + - name: ZK_PURGE_INTERVAL + valueFrom: + configMapKeyRef: + name: zk-config + key: purge.interval + - name: ZK_CLIENT_PORT + value: "2181" + - name: ZK_SERVER_PORT + value: "2888" + - name: ZK_ELECTION_PORT + value: "3888" + command: + - sh + - -c + - zkGenConfig.sh && zkServer.sh start-foreground + readinessProbe: + exec: + command: + - "zkOk.sh" + initialDelaySeconds: 15 + timeoutSeconds: 5 + livenessProbe: + exec: + command: + - "zkOk.sh" + initialDelaySeconds: 15 + timeoutSeconds: 5 + volumeMounts: + - name: datadir + mountPath: /var/lib/zookeeper + securityContext: + runAsUser: 1000 + fsGroup: 1000 + volumeClaimTemplates: + - metadata: + name: datadir + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 20Gi \ No newline at end of file From b39416232d59f24d312ce0fb600b882138a8e228 Mon Sep 17 00:00:00 2001 From: volodymyr-babak Date: Tue, 9 May 2017 16:31:48 +0300 Subject: [PATCH 02/12] TB-58: k8 base scripts --- docker/thingsboard-db-schema.env | 3 +- .../thingsboard-db-schema/build_and_deploy.sh | 10 +- .../thingsboard-db-schema/install_schema.sh | 8 +- docker/thingsboard/build_and_deploy.sh | 8 +- docker/thingsboard/run_thingsboard.sh | 3 + kubernetes/cassandra.yaml | 134 ++++++++++++++++++ kubernetes/thingsboard-schema.yaml | 22 +++ kubernetes/thingsboard.yaml | 105 ++++++++++++++ kubernetes/zookeeper.yaml | 37 ++--- 9 files changed, 296 insertions(+), 34 deletions(-) create mode 100644 kubernetes/cassandra.yaml create mode 100644 kubernetes/thingsboard-schema.yaml create mode 100644 kubernetes/thingsboard.yaml diff --git a/docker/thingsboard-db-schema.env b/docker/thingsboard-db-schema.env index c8d2bd83d8..4b20f39a35 100644 --- a/docker/thingsboard-db-schema.env +++ b/docker/thingsboard-db-schema.env @@ -2,4 +2,5 @@ SKIP_SCHEMA_CREATION=false SKIP_SYSTEM_DATA=false -SKIP_DEMO_DATA=false \ No newline at end of file +SKIP_DEMO_DATA=false +CASSANDRA_URL=db \ No newline at end of file diff --git a/docker/thingsboard-db-schema/build_and_deploy.sh b/docker/thingsboard-db-schema/build_and_deploy.sh index d03413735d..428016c839 100755 --- a/docker/thingsboard-db-schema/build_and_deploy.sh +++ b/docker/thingsboard-db-schema/build_and_deploy.sh @@ -20,9 +20,13 @@ cp ../../dao/src/main/resources/schema.cql schema.cql cp ../../dao/src/main/resources/demo-data.cql demo-data.cql cp ../../dao/src/main/resources/system-data.cql system-data.cql -docker build -t thingsboard/thingsboard-db-schema:1.2.2 -t thingsboard/thingsboard-db-schema:latest . +docker build -t thingsboard/thingsboard-db-schema:k8test . docker login -docker push thingsboard/thingsboard-db-schema:1.2.2 -docker push thingsboard/thingsboard-db-schema:latest \ No newline at end of file +docker push thingsboard/thingsboard-db-schema:k8test + +# cleanup +rm schema.cql +rm demo-data.cql +rm system-data.cql \ No newline at end of file diff --git a/docker/thingsboard-db-schema/install_schema.sh b/docker/thingsboard-db-schema/install_schema.sh index f113b90a69..279711933c 100644 --- a/docker/thingsboard-db-schema/install_schema.sh +++ b/docker/thingsboard-db-schema/install_schema.sh @@ -16,7 +16,7 @@ # -until nmap db -p 9042 | grep "9042/tcp open" +until nmap $CASSANDRA_URL -p 9042 | grep "9042/tcp open" do echo "Wait for Cassandra..." sleep 10 @@ -24,7 +24,7 @@ done if [ "$SKIP_SCHEMA_CREATION" == "false" ]; then echo "Creating 'Thingsboard' keyspace..." - cqlsh db -f /root/schema.cql + cqlsh $CASSANDRA_URL -f /root/schema.cql if [ "$?" -eq 0 ]; then echo "'Thingsboard' keyspace was successfully created!" else @@ -34,7 +34,7 @@ fi if [ "$SKIP_SYSTEM_DATA" == "false" ]; then echo "Adding system data..." - cqlsh db -f /root/system-data.cql + cqlsh $CASSANDRA_URL -f /root/system-data.cql if [ "$?" -eq 0 ]; then echo "System data was successfully added!" else @@ -44,7 +44,7 @@ fi if [ "$SKIP_DEMO_DATA" == "false" ]; then echo "Adding demo data..." - cqlsh db -f /root/demo-data.cql + cqlsh $CASSANDRA_URL -f /root/demo-data.cql if [ "$?" -eq 0 ]; then echo "Demo data was successfully added!" else diff --git a/docker/thingsboard/build_and_deploy.sh b/docker/thingsboard/build_and_deploy.sh index 7c02de571c..d2d1e2cb76 100755 --- a/docker/thingsboard/build_and_deploy.sh +++ b/docker/thingsboard/build_and_deploy.sh @@ -18,9 +18,11 @@ cp ../../application/target/thingsboard.deb thingsboard.deb -docker build -t thingsboard/application:1.2.2 -t thingsboard/application:latest . +docker build -t thingsboard/application:k8test . docker login -docker push thingsboard/application:1.2.2 -docker push thingsboard/application:latest \ No newline at end of file +docker push thingsboard/application:k8test + +# cleanup +rm thingsboard.deb \ No newline at end of file diff --git a/docker/thingsboard/run_thingsboard.sh b/docker/thingsboard/run_thingsboard.sh index 877fed7fec..404cc8df9d 100755 --- a/docker/thingsboard/run_thingsboard.sh +++ b/docker/thingsboard/run_thingsboard.sh @@ -34,6 +34,9 @@ done # Copying env variables into conf files printenv | awk -F "=" '{print "export " $1 "='\''" $2 "'\''"}' >> /usr/share/thingsboard/conf/thingsboard.conf +# Set env variable for RPC address +echo "export RPC_HOST='"$(hostname -f)"'" | tee -a /usr/share/thingsboard/conf/thingsboard.conf + cat /usr/share/thingsboard/conf/thingsboard.conf echo "Starting 'Thingsboard' service..." diff --git a/kubernetes/cassandra.yaml b/kubernetes/cassandra.yaml new file mode 100644 index 0000000000..9333f4d939 --- /dev/null +++ b/kubernetes/cassandra.yaml @@ -0,0 +1,134 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: cassandra + name: cassandra +spec: + clusterIP: None + ports: + - port: 9042 + selector: + app: cassandra +--- +apiVersion: storage.k8s.io/v1beta1 +kind: StorageClass +metadata: + name: regular +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-standard +--- +apiVersion: "apps/v1beta1" +kind: StatefulSet +metadata: + name: cassandra +spec: + serviceName: cassandra + replicas: 3 + template: + metadata: + labels: + app: cassandra + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - cassandra + topologyKey: "kubernetes.io/hostname" + containers: + - name: cassandra + image: gcr.io/google-samples/cassandra:v12 + imagePullPolicy: Always + ports: + - containerPort: 7000 + name: intra-node + - containerPort: 7001 + name: tls-intra-node + - containerPort: 7199 + name: jmx + - containerPort: 9042 + name: cql + - containerPort: 9160 + name: thrift + resources: + limits: + cpu: "500m" + memory: 1Gi + requests: + cpu: "500m" + memory: 1Gi + securityContext: + capabilities: + add: + - IPC_LOCK + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "PID=$(pidof java) && kill $PID && while ps -p $PID > /dev/null; do sleep 1; done"] + env: + - name: MAX_HEAP_SIZE + value: 512M + - name: HEAP_NEWSIZE + value: 100M + - name: CASSANDRA_SEEDS + value: "cassandra-0.cassandra.default.svc.cluster.local" + - name: CASSANDRA_CLUSTER_NAME + value: "K8Demo" + - name: CASSANDRA_DC + value: "DC1-K8Demo" + - name: CASSANDRA_RACK + value: "Rack1-K8Demo" + - name: CASSANDRA_AUTO_BOOTSTRAP + value: "false" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + readinessProbe: + exec: + command: + - /bin/bash + - -c + - /ready-probe.sh + initialDelaySeconds: 15 + timeoutSeconds: 5 + # These volume mounts are persistent. They are like inline claims, + # but not exactly because the names need to match exactly one of + # the stateful pod volumes. + volumeMounts: + - name: cassandra-data + mountPath: /cassandra_data + - name: cassandra-commitlog + mountPath: /cassandra_commitlog + # These are converted to volume claims by the controller + # and mounted at the paths mentioned above. + # do not use these in production until ssd GCEPersistentDisk or other ssd pd + volumeClaimTemplates: + - metadata: + name: cassandra-data + annotations: + volume.beta.kubernetes.io/storage-class: regular + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 2Gi + - metadata: + name: cassandra-commitlog + annotations: + volume.beta.kubernetes.io/storage-class: regular + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/kubernetes/thingsboard-schema.yaml b/kubernetes/thingsboard-schema.yaml new file mode 100644 index 0000000000..ca0f8b1759 --- /dev/null +++ b/kubernetes/thingsboard-schema.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + name: tb-db-schema-deployment +spec: + containers: + - name: tb-db-schema-k8 + image: thingsboard/thingsboard-db-schema:k8test + env: + - name: SKIP_SCHEMA_CREATION + value: "false" + - name: SKIP_SYSTEM_DATA + value: "false" + - name : SKIP_DEMO_DATA + value: "false" + - name : CASSANDRA_URL + value: "cassandra" + command: + - sh + - -c + - ./install_schema.sh + restartPolicy: Never \ No newline at end of file diff --git a/kubernetes/thingsboard.yaml b/kubernetes/thingsboard.yaml new file mode 100644 index 0000000000..dfe84b25fc --- /dev/null +++ b/kubernetes/thingsboard.yaml @@ -0,0 +1,105 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: thingsboard-external + labels: + app: thingsboard-external +spec: + ports: + - port: 8080 + name: ui + - port: 1883 + name: mqtt + - port: 5683 + name: coap + - port: 9001 + name: rpc + selector: + app: thingsboard + type: LoadBalancer +--- +apiVersion: v1 +kind: Service +metadata: + name: thingsboard-headless + labels: + app: thingsboard-headless +spec: + ports: + - port: 8080 + name: ui + - port: 1883 + name: mqtt + - port: 5683 + name: coap + - port: 9001 + name: rpc + selector: + app: thingsboard + clusterIP: None +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: tb-config +data: + zookeeper.enabled: "true" + zookeeper.url: "zk-headless" + cassandra.url: "cassandra:9042" +--- +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: tb +spec: + serviceName: "thingsboard-headless" + replicas: 2 + template: + metadata: + labels: + app: thingsboard + spec: + terminationGracePeriodSeconds: 10 + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - thingsboard-headless + topologyKey: "kubernetes.io/hostname" + containers: + - name: thingsboard + image: thingsboard/application:k8test + ports: + - containerPort: 8080 + name: ui + - containerPort: 1883 + name: mqtt + - containerPort: 5683 + name: coap + - containerPort: 9001 + name: rpc + env: + - name: ZOOKEEPER_ENABLED + valueFrom: + configMapKeyRef: + name: tb-config + key: zookeeper.enabled + - name: ZOOKEEPER_URL + valueFrom: + configMapKeyRef: + name: tb-config + key: zookeeper.url + - name : CASSANDRA_URL + valueFrom: + configMapKeyRef: + name: tb-config + key: cassandra.url + command: + - sh + - -c + - ./run_thingsboard.sh \ No newline at end of file diff --git a/kubernetes/zookeeper.yaml b/kubernetes/zookeeper.yaml index efe8d99c27..7107e07ece 100644 --- a/kubernetes/zookeeper.yaml +++ b/kubernetes/zookeeper.yaml @@ -20,7 +20,7 @@ metadata: name: zk-config data: ensemble: "zk-0;zk-1;zk-2" - jvm.heap: "2G" + jvm.heap: "1G" tick: "2000" init: "10" sync: "5" @@ -51,34 +51,25 @@ spec: app: zk annotations: pod.alpha.kubernetes.io/initialized: "true" - scheduler.alpha.kubernetes.io/affinity: > - { - "podAntiAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": [ - { - "labelSelector": { - "matchExpressions": [ - { - "key": "app", - "operator": "In", - "values": [ "zk-headless" ] - } - ] - }, - "topologyKey": "kubernetes.io/hostname" - } - ] - } - } spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - zk-headless + topologyKey: "kubernetes.io/hostname" containers: - name: k8szk imagePullPolicy: Always image: gcr.io/google_samples/k8szk:v1 resources: requests: - memory: "1Gi" - cpu: "0.5" + memory: "0.5Gi" + cpu: "0.2" ports: - containerPort: 2181 name: client @@ -164,4 +155,4 @@ spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: 20Gi \ No newline at end of file + storage: 1Gi \ No newline at end of file From 73458484c2f4d54d537c3e195071bd627c165eca Mon Sep 17 00:00:00 2001 From: volodymyr-babak Date: Tue, 9 May 2017 23:38:42 +0300 Subject: [PATCH 03/12] TB-58: k8 base scripts --- .../cassandra}/cassandra.yaml | 0 .../thingsboard-schema.yaml | 5 +- .../thingsboard}/thingsboard.yaml | 69 +++++--- docker/zookeeper/Dockerfile | 55 +++++++ docker/zookeeper/Makefile | 12 ++ docker/zookeeper/zkGenConfig.sh | 151 ++++++++++++++++++ docker/zookeeper/zkOk.sh | 26 +++ .../zookeeper}/zookeeper.yaml | 10 +- 8 files changed, 299 insertions(+), 29 deletions(-) rename {kubernetes => docker/cassandra}/cassandra.yaml (100%) rename {kubernetes => docker/thingsboard-db-schema}/thingsboard-schema.yaml (84%) rename {kubernetes => docker/thingsboard}/thingsboard.yaml (62%) create mode 100644 docker/zookeeper/Dockerfile create mode 100644 docker/zookeeper/Makefile create mode 100755 docker/zookeeper/zkGenConfig.sh create mode 100755 docker/zookeeper/zkOk.sh rename {kubernetes => docker/zookeeper}/zookeeper.yaml (95%) diff --git a/kubernetes/cassandra.yaml b/docker/cassandra/cassandra.yaml similarity index 100% rename from kubernetes/cassandra.yaml rename to docker/cassandra/cassandra.yaml diff --git a/kubernetes/thingsboard-schema.yaml b/docker/thingsboard-db-schema/thingsboard-schema.yaml similarity index 84% rename from kubernetes/thingsboard-schema.yaml rename to docker/thingsboard-db-schema/thingsboard-schema.yaml index ca0f8b1759..141e1e3d41 100644 --- a/kubernetes/thingsboard-schema.yaml +++ b/docker/thingsboard-db-schema/thingsboard-schema.yaml @@ -1,10 +1,11 @@ apiVersion: v1 kind: Pod metadata: - name: tb-db-schema-deployment + name: tb-db-schema spec: containers: - - name: tb-db-schema-k8 + - name: tb-db-schema + imagePullPolicy: Always image: thingsboard/thingsboard-db-schema:k8test env: - name: SKIP_SCHEMA_CREATION diff --git a/kubernetes/thingsboard.yaml b/docker/thingsboard/thingsboard.yaml similarity index 62% rename from kubernetes/thingsboard.yaml rename to docker/thingsboard/thingsboard.yaml index dfe84b25fc..702a7215f5 100644 --- a/kubernetes/thingsboard.yaml +++ b/docker/thingsboard/thingsboard.yaml @@ -2,43 +2,53 @@ apiVersion: v1 kind: Service metadata: - name: thingsboard-external + name: tb-external-ip labels: - app: thingsboard-external + app: tb-external-ip spec: ports: - port: 8080 - name: ui + name: ui-port - port: 1883 - name: mqtt + name: mqtt-port - port: 5683 - name: coap + name: coap-port - port: 9001 - name: rpc + name: rpc-port selector: - app: thingsboard + app: tb type: LoadBalancer --- apiVersion: v1 kind: Service metadata: - name: thingsboard-headless + name: tb-headless labels: - app: thingsboard-headless + app: tb-headless spec: ports: - port: 8080 - name: ui + name: ui-port - port: 1883 - name: mqtt + name: mqtt-port - port: 5683 - name: coap + name: coap-port - port: 9001 - name: rpc + name: rpc-port selector: - app: thingsboard + app: tb clusterIP: None --- +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: tb-budget +spec: + selector: + matchLabels: + app: tb + minAvailable: 2 +--- apiVersion: v1 kind: ConfigMap metadata: @@ -53,12 +63,12 @@ kind: StatefulSet metadata: name: tb spec: - serviceName: "thingsboard-headless" + serviceName: "tb-headless" replicas: 2 template: metadata: labels: - app: thingsboard + app: tb spec: terminationGracePeriodSeconds: 10 affinity: @@ -69,20 +79,21 @@ spec: - key: "app" operator: In values: - - thingsboard-headless + - tb-headless topologyKey: "kubernetes.io/hostname" containers: - - name: thingsboard + - name: tb + imagePullPolicy: Always image: thingsboard/application:k8test ports: - containerPort: 8080 - name: ui + name: ui-port - containerPort: 1883 - name: mqtt + name: mqtt-port - containerPort: 5683 - name: coap + name: coap-port - containerPort: 9001 - name: rpc + name: rpc-port env: - name: ZOOKEEPER_ENABLED valueFrom: @@ -102,4 +113,16 @@ spec: command: - sh - -c - - ./run_thingsboard.sh \ No newline at end of file + - ./run_thingsboard.sh +# readinessProbe: +# httpGet: +# path: /login +# port: ui-port +# initialDelaySeconds: 30 +# periodSeconds: 10 + livenessProbe: + httpGet: + path: /login + port: ui-port + initialDelaySeconds: 30 + periodSeconds: 10 \ No newline at end of file diff --git a/docker/zookeeper/Dockerfile b/docker/zookeeper/Dockerfile new file mode 100644 index 0000000000..7752f93b3a --- /dev/null +++ b/docker/zookeeper/Dockerfile @@ -0,0 +1,55 @@ +FROM ubuntu:16.04 +ENV ZK_USER=zookeeper \ +ZK_DATA_DIR=/var/lib/zookeeper/data \ +ZK_DATA_LOG_DIR=/var/lib/zookeeper/log \ +ZK_LOG_DIR=/var/log/zookeeper \ +JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 + +ARG GPG_KEY=C823E3E5B12AF29C67F81976F5CECB3CB5E9BD2D +ARG ZK_DIST=zookeeper-3.4.9 +RUN set -x \ + && apt-get update \ + && apt-get install -y openjdk-8-jre-headless wget netcat-openbsd \ + && wget -q "http://www.apache.org/dist/zookeeper/$ZK_DIST/$ZK_DIST.tar.gz" \ + && wget -q "http://www.apache.org/dist/zookeeper/$ZK_DIST/$ZK_DIST.tar.gz.asc" \ + && export GNUPGHOME="$(mktemp -d)" \ + && gpg --keyserver ha.pool.sks-keyservers.net --recv-key "$GPG_KEY" \ + && gpg --batch --verify "$ZK_DIST.tar.gz.asc" "$ZK_DIST.tar.gz" \ + && tar -xzf "$ZK_DIST.tar.gz" -C /opt \ + && rm -r "$GNUPGHOME" "$ZK_DIST.tar.gz" "$ZK_DIST.tar.gz.asc" \ + && ln -s /opt/$ZK_DIST /opt/zookeeper \ + && rm -rf /opt/zookeeper/CHANGES.txt \ + /opt/zookeeper/README.txt \ + /opt/zookeeper/NOTICE.txt \ + /opt/zookeeper/CHANGES.txt \ + /opt/zookeeper/README_packaging.txt \ + /opt/zookeeper/build.xml \ + /opt/zookeeper/config \ + /opt/zookeeper/contrib \ + /opt/zookeeper/dist-maven \ + /opt/zookeeper/docs \ + /opt/zookeeper/ivy.xml \ + /opt/zookeeper/ivysettings.xml \ + /opt/zookeeper/recipes \ + /opt/zookeeper/src \ + /opt/zookeeper/$ZK_DIST.jar.asc \ + /opt/zookeeper/$ZK_DIST.jar.md5 \ + /opt/zookeeper/$ZK_DIST.jar.sha1 \ + && apt-get autoremove -y wget \ + && rm -rf /var/lib/apt/lists/* + +#Copy configuration generator script to bin +COPY zkGenConfig.sh zkOk.sh /opt/zookeeper/bin/ + +# Create a user for the zookeeper process and configure file system ownership +# for nessecary directories and symlink the distribution as a user executable +RUN set -x \ + && useradd $ZK_USER \ + && [ `id -u $ZK_USER` -eq 1000 ] \ + && [ `id -g $ZK_USER` -eq 1000 ] \ + && mkdir -p $ZK_DATA_DIR $ZK_DATA_LOG_DIR $ZK_LOG_DIR /usr/share/zookeeper /tmp/zookeeper /usr/etc/ \ + && chown -R "$ZK_USER:$ZK_USER" /opt/$ZK_DIST $ZK_DATA_DIR $ZK_LOG_DIR $ZK_DATA_LOG_DIR /tmp/zookeeper \ + && ln -s /opt/zookeeper/conf/ /usr/etc/zookeeper \ + && ln -s /opt/zookeeper/bin/* /usr/bin \ + && ln -s /opt/zookeeper/$ZK_DIST.jar /usr/share/zookeeper/ \ + && ln -s /opt/zookeeper/lib/* /usr/share/zookeeper diff --git a/docker/zookeeper/Makefile b/docker/zookeeper/Makefile new file mode 100644 index 0000000000..a28937f1d5 --- /dev/null +++ b/docker/zookeeper/Makefile @@ -0,0 +1,12 @@ +VERSION=k8test +PROJECT=thingsboard + +all: build + +build: + docker build --pull -t ${PROJECT}/k8szk:${VERSION} . + +push: build + docker push ${PROJECT}/k8szk:${VERSION} + +.PHONY: all build push diff --git a/docker/zookeeper/zkGenConfig.sh b/docker/zookeeper/zkGenConfig.sh new file mode 100755 index 0000000000..cef42f64f7 --- /dev/null +++ b/docker/zookeeper/zkGenConfig.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ZK_USER=${ZK_USER:-"zookeeper"} +ZK_LOG_LEVEL=${ZK_LOG_LEVEL:-"INFO"} +ZK_DATA_DIR=${ZK_DATA_DIR:-"/var/lib/zookeeper/data"} +ZK_DATA_LOG_DIR=${ZK_DATA_LOG_DIR:-"/var/lib/zookeeper/log"} +ZK_LOG_DIR=${ZK_LOG_DIR:-"var/log/zookeeper"} +ZK_CONF_DIR=${ZK_CONF_DIR:-"/opt/zookeeper/conf"} +ZK_CLIENT_PORT=${ZK_CLIENT_PORT:-2181} +ZK_SERVER_PORT=${ZK_SERVER_PORT:-2888} +ZK_ELECTION_PORT=${ZK_ELECTION_PORT:-3888} +ZK_TICK_TIME=${ZK_TICK_TIME:-2000} +ZK_INIT_LIMIT=${ZK_INIT_LIMIT:-10} +ZK_SYNC_LIMIT=${ZK_SYNC_LIMIT:-5} +ZK_HEAP_SIZE=${ZK_HEAP_SIZE:-2G} +ZK_MAX_CLIENT_CNXNS=${ZK_MAX_CLIENT_CNXNS:-60} +ZK_MIN_SESSION_TIMEOUT=${ZK_MIN_SESSION_TIMEOUT:- $((ZK_TICK_TIME*2))} +ZK_MAX_SESSION_TIMEOUT=${ZK_MAX_SESSION_TIMEOUT:- $((ZK_TICK_TIME*20))} +ZK_SNAP_RETAIN_COUNT=${ZK_SNAP_RETAIN_COUNT:-3} +ZK_PURGE_INTERVAL=${ZK_PURGE_INTERVAL:-0} +ID_FILE="$ZK_DATA_DIR/myid" +ZK_CONFIG_FILE="$ZK_CONF_DIR/zoo.cfg" +LOGGER_PROPS_FILE="$ZK_CONF_DIR/log4j.properties" +JAVA_ENV_FILE="$ZK_CONF_DIR/java.env" +HOST=`hostname -s` +DOMAIN=`hostname -d` + +function print_servers() { + for (( i=1; i<=$ZK_REPLICAS; i++ )) + do + echo "server.$i=$NAME-$((i-1)).$DOMAIN:$ZK_SERVER_PORT:$ZK_ELECTION_PORT" + done +} + +function validate_env() { + echo "Validating environment" + if [ -z $ZK_REPLICAS ]; then + echo "ZK_REPLICAS is a mandatory environment variable" + exit 1 + fi + + if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then + NAME=${BASH_REMATCH[1]} + ORD=${BASH_REMATCH[2]} + else + echo "Failed to extract ordinal from hostname $HOST" + exit 1 + fi + MY_ID=$((ORD+1)) + echo "ZK_REPLICAS=$ZK_REPLICAS" + echo "MY_ID=$MY_ID" + echo "ZK_LOG_LEVEL=$ZK_LOG_LEVEL" + echo "ZK_DATA_DIR=$ZK_DATA_DIR" + echo "ZK_DATA_LOG_DIR=$ZK_DATA_LOG_DIR" + echo "ZK_LOG_DIR=$ZK_LOG_DIR" + echo "ZK_CLIENT_PORT=$ZK_CLIENT_PORT" + echo "ZK_SERVER_PORT=$ZK_SERVER_PORT" + echo "ZK_ELECTION_PORT=$ZK_ELECTION_PORT" + echo "ZK_TICK_TIME=$ZK_TICK_TIME" + echo "ZK_INIT_LIMIT=$ZK_INIT_LIMIT" + echo "ZK_SYNC_LIMIT=$ZK_SYNC_LIMIT" + echo "ZK_MAX_CLIENT_CNXNS=$ZK_MAX_CLIENT_CNXNS" + echo "ZK_MIN_SESSION_TIMEOUT=$ZK_MIN_SESSION_TIMEOUT" + echo "ZK_MAX_SESSION_TIMEOUT=$ZK_MAX_SESSION_TIMEOUT" + echo "ZK_HEAP_SIZE=$ZK_HEAP_SIZE" + echo "ZK_SNAP_RETAIN_COUNT=$ZK_SNAP_RETAIN_COUNT" + echo "ZK_PURGE_INTERVAL=$ZK_PURGE_INTERVAL" + echo "ENSEMBLE" + print_servers + echo "Environment validation successful" +} + +function create_config() { + rm -f $ZK_CONFIG_FILE + echo "Creating ZooKeeper configuration" + echo "#This file was autogenerated by k8szk DO NOT EDIT" >> $ZK_CONFIG_FILE + echo "clientPort=$ZK_CLIENT_PORT" >> $ZK_CONFIG_FILE + echo "dataDir=$ZK_DATA_DIR" >> $ZK_CONFIG_FILE + echo "dataLogDir=$ZK_DATA_LOG_DIR" >> $ZK_CONFIG_FILE + echo "tickTime=$ZK_TICK_TIME" >> $ZK_CONFIG_FILE + echo "initLimit=$ZK_INIT_LIMIT" >> $ZK_CONFIG_FILE + echo "syncLimit=$ZK_SYNC_LIMIT" >> $ZK_CONFIG_FILE + echo "maxClientCnxns=$ZK_MAX_CLIENT_CNXNS" >> $ZK_CONFIG_FILE + echo "minSessionTimeout=$ZK_MIN_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE + echo "maxSessionTimeout=$ZK_MAX_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE + echo "autopurge.snapRetainCount=$ZK_SNAP_RETAIN_COUNT" >> $ZK_CONFIG_FILE + echo "autopurge.purgeInteval=$ZK_PURGE_INTERVAL" >> $ZK_CONFIG_FILE + + if [ $ZK_REPLICAS -gt 1 ]; then + print_servers >> $ZK_CONFIG_FILE + fi + echo "Wrote ZooKeeper configuration file to $ZK_CONFIG_FILE" +} + +function create_data_dirs() { + echo "Creating ZooKeeper data directories and setting permissions" + if [ ! -d $ZK_DATA_DIR ]; then + mkdir -p $ZK_DATA_DIR + chown -R $ZK_USER:$ZK_USER $ZK_DATA_DIR + fi + + if [ ! -d $ZK_DATA_LOG_DIR ]; then + mkdir -p $ZK_DATA_LOG_DIR + chown -R $ZK_USER:$ZK_USER $ZK_DATA_LOG_DIR + fi + + if [ ! -d $ZK_LOG_DIR ]; then + mkdir -p $ZK_LOG_DIR + chown -R $ZK_USER:$ZK_USER $ZK_LOG_DIR + fi + if [ ! -f $ID_FILE ]; then + echo $MY_ID >> $ID_FILE + fi + echo "Created ZooKeeper data directories and set permissions in $ZK_DATA_DIR" +} + +function create_log_props () { + rm -f $LOGGER_PROPS_FILE + echo "Creating ZooKeeper log4j configuration" + echo "zookeeper.root.logger=CONSOLE" >> $LOGGER_PROPS_FILE + echo "zookeeper.console.threshold="$ZK_LOG_LEVEL >> $LOGGER_PROPS_FILE + echo "log4j.rootLogger=\${zookeeper.root.logger}" >> $LOGGER_PROPS_FILE + echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender" >> $LOGGER_PROPS_FILE + echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}" >> $LOGGER_PROPS_FILE + echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout" >> $LOGGER_PROPS_FILE + echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n" >> $LOGGER_PROPS_FILE + echo "Wrote log4j configuration to $LOGGER_PROPS_FILE" +} + +function create_java_env() { + rm -f $JAVA_ENV_FILE + echo "Creating JVM configuration file" + echo "ZOO_LOG_DIR=$ZK_LOG_DIR" >> $JAVA_ENV_FILE + echo "JVMFLAGS=\"-Xmx$ZK_HEAP_SIZE -Xms$ZK_HEAP_SIZE\"" >> $JAVA_ENV_FILE + echo "Wrote JVM configuration to $JAVA_ENV_FILE" +} + +validate_env && create_config && create_log_props && create_data_dirs && create_java_env diff --git a/docker/zookeeper/zkOk.sh b/docker/zookeeper/zkOk.sh new file mode 100755 index 0000000000..dbe417682f --- /dev/null +++ b/docker/zookeeper/zkOk.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# zkOk.sh uses the ruok ZooKeeper four letter work to determine if the instance +# is health. The $? variable will be set to 0 if server responds that it is +# healthy, or 1 if the server fails to respond. + +ZK_CLIENT_PORT=${ZK_CLIENT_PORT:-2181} +OK=$(echo ruok | nc 127.0.0.1 $ZK_CLIENT_PORT) +if [ "$OK" == "imok" ]; then + exit 0 +else + exit 1 +fi \ No newline at end of file diff --git a/kubernetes/zookeeper.yaml b/docker/zookeeper/zookeeper.yaml similarity index 95% rename from kubernetes/zookeeper.yaml rename to docker/zookeeper/zookeeper.yaml index 7107e07ece..4184a3ab8e 100644 --- a/kubernetes/zookeeper.yaml +++ b/docker/zookeeper/zookeeper.yaml @@ -20,7 +20,7 @@ metadata: name: zk-config data: ensemble: "zk-0;zk-1;zk-2" - jvm.heap: "1G" + jvm.heap: "500m" tick: "2000" init: "10" sync: "5" @@ -65,11 +65,11 @@ spec: containers: - name: k8szk imagePullPolicy: Always - image: gcr.io/google_samples/k8szk:v1 + image: thingsboard/k8szk:k8test resources: requests: - memory: "0.5Gi" - cpu: "0.2" + memory: "500Mi" + cpu: "250m" ports: - containerPort: 2181 name: client @@ -83,6 +83,8 @@ spec: configMapKeyRef: name: zk-config key: ensemble + - name : ZK_REPLICAS + value: "3" - name : ZK_HEAP_SIZE valueFrom: configMapKeyRef: From 08e9e903f393cbef4c0ff154537bb276579117e6 Mon Sep 17 00:00:00 2001 From: volodymyr-babak Date: Wed, 10 May 2017 21:01:06 +0300 Subject: [PATCH 04/12] TB-58: k8s feature improvements --- docker/cassandra/Dockerfile | 5 ++ docker/cassandra/Makefile | 13 ++++ docker/cassandra/cassandra.yaml | 37 ++++------- .../ready-probe.sh} | 27 ++++---- docker/common/common.yaml | 9 +++ docker/deploy.sh | 31 ---------- docker/deploy_cassandra_zookeeper.sh | 31 ---------- docker/thingsboard-db-schema/Makefile | 13 ++++ .../thingsboard-db-schema/build_and_deploy.sh | 32 ---------- ...schema.yaml => thingsboard-db-schema.yaml} | 4 +- docker/thingsboard/Makefile | 13 ++++ docker/thingsboard/thingsboard.yaml | 35 +++++------ docker/zookeeper/Makefile | 7 ++- docker/zookeeper/zkGenConfig.sh | 2 +- docker/zookeeper/zookeeper.yaml | 62 ++++++++++++------- 15 files changed, 141 insertions(+), 180 deletions(-) create mode 100644 docker/cassandra/Dockerfile create mode 100644 docker/cassandra/Makefile rename docker/{thingsboard/build_and_deploy.sh => cassandra/ready-probe.sh} (69%) create mode 100644 docker/common/common.yaml delete mode 100755 docker/deploy.sh delete mode 100755 docker/deploy_cassandra_zookeeper.sh create mode 100644 docker/thingsboard-db-schema/Makefile delete mode 100755 docker/thingsboard-db-schema/build_and_deploy.sh rename docker/thingsboard-db-schema/{thingsboard-schema.yaml => thingsboard-db-schema.yaml} (81%) create mode 100644 docker/thingsboard/Makefile diff --git a/docker/cassandra/Dockerfile b/docker/cassandra/Dockerfile new file mode 100644 index 0000000000..72084cd0af --- /dev/null +++ b/docker/cassandra/Dockerfile @@ -0,0 +1,5 @@ +FROM cassandra:3.9 + +COPY ready-probe.sh / + +CMD ["cassandra", "-f"] \ No newline at end of file diff --git a/docker/cassandra/Makefile b/docker/cassandra/Makefile new file mode 100644 index 0000000000..ea71c6340c --- /dev/null +++ b/docker/cassandra/Makefile @@ -0,0 +1,13 @@ +VERSION=k8stest +PROJECT=thingsboard +APP=cassandra + +all: build + +build: + docker build --pull -t ${PROJECT}/${APP}:${VERSION} . + +push: build + docker push ${PROJECT}/${APP}:${VERSION} + +.PHONY: all build push \ No newline at end of file diff --git a/docker/cassandra/cassandra.yaml b/docker/cassandra/cassandra.yaml index 9333f4d939..c3772b15e1 100644 --- a/docker/cassandra/cassandra.yaml +++ b/docker/cassandra/cassandra.yaml @@ -1,30 +1,23 @@ apiVersion: v1 kind: Service metadata: + name: cassandra-headless labels: - app: cassandra - name: cassandra + app: cassandra-headless spec: - clusterIP: None ports: - port: 9042 + name: cql + clusterIP: None selector: app: cassandra --- -apiVersion: storage.k8s.io/v1beta1 -kind: StorageClass -metadata: - name: regular -provisioner: kubernetes.io/gce-pd -parameters: - type: pd-standard ---- apiVersion: "apps/v1beta1" kind: StatefulSet metadata: name: cassandra spec: - serviceName: cassandra + serviceName: cassandra-headless replicas: 3 template: metadata: @@ -39,11 +32,11 @@ spec: - key: "app" operator: In values: - - cassandra + - cassandra-headless topologyKey: "kubernetes.io/hostname" containers: - name: cassandra - image: gcr.io/google-samples/cassandra:v12 + image: thingsboard/cassandra:k8stest imagePullPolicy: Always ports: - containerPort: 7000 @@ -77,7 +70,7 @@ spec: - name: HEAP_NEWSIZE value: 100M - name: CASSANDRA_SEEDS - value: "cassandra-0.cassandra.default.svc.cluster.local" + value: "cassandra-0.cassandra-headless.default.svc.cluster.local" - name: CASSANDRA_CLUSTER_NAME value: "K8Demo" - name: CASSANDRA_DC @@ -102,22 +95,16 @@ spec: - /ready-probe.sh initialDelaySeconds: 15 timeoutSeconds: 5 - # These volume mounts are persistent. They are like inline claims, - # but not exactly because the names need to match exactly one of - # the stateful pod volumes. volumeMounts: - name: cassandra-data - mountPath: /cassandra_data + mountPath: /var/lib/cassandra/data - name: cassandra-commitlog - mountPath: /cassandra_commitlog - # These are converted to volume claims by the controller - # and mounted at the paths mentioned above. - # do not use these in production until ssd GCEPersistentDisk or other ssd pd + mountPath: /var/lib/cassandra/commitlog volumeClaimTemplates: - metadata: name: cassandra-data annotations: - volume.beta.kubernetes.io/storage-class: regular + volume.beta.kubernetes.io/storage-class: slow spec: accessModes: [ "ReadWriteOnce" ] resources: @@ -126,7 +113,7 @@ spec: - metadata: name: cassandra-commitlog annotations: - volume.beta.kubernetes.io/storage-class: regular + volume.beta.kubernetes.io/storage-class: slow spec: accessModes: [ "ReadWriteOnce" ] resources: diff --git a/docker/thingsboard/build_and_deploy.sh b/docker/cassandra/ready-probe.sh similarity index 69% rename from docker/thingsboard/build_and_deploy.sh rename to docker/cassandra/ready-probe.sh index d2d1e2cb76..989f4d90a1 100755 --- a/docker/thingsboard/build_and_deploy.sh +++ b/docker/cassandra/ready-probe.sh @@ -1,6 +1,6 @@ #!/bin/bash -# -# Copyright © 2016-2017 The Thingsboard Authors + +# Copyright 2016 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,16 +13,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# - -cp ../../application/target/thingsboard.deb thingsboard.deb - -docker build -t thingsboard/application:k8test . - -docker login - -docker push thingsboard/application:k8test - -# cleanup -rm thingsboard.deb \ No newline at end of file +if [[ $(nodetool status | grep $POD_IP) == *"UN"* ]]; then + if [[ $DEBUG ]]; then + echo "UN"; + fi + exit 0; +else + if [[ $DEBUG ]]; then + echo "Not Up"; + fi + exit 1; +fi \ No newline at end of file diff --git a/docker/common/common.yaml b/docker/common/common.yaml new file mode 100644 index 0000000000..610c027911 --- /dev/null +++ b/docker/common/common.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: storage.k8s.io/v1beta1 +kind: StorageClass +metadata: + name: slow +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-standard +--- \ No newline at end of file diff --git a/docker/deploy.sh b/docker/deploy.sh deleted file mode 100755 index ed69f7b0c7..0000000000 --- a/docker/deploy.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# -# Copyright © 2016-2017 The Thingsboard Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -command='docker-compose -f docker-compose.yml -f docker-compose.random.yml' - -echo "stopping images.." -$command stop - -echo "removing stopped images.." -$command rm -f - -echo "building images.." -$command build - -echo "starting images..." -$command up -d diff --git a/docker/deploy_cassandra_zookeeper.sh b/docker/deploy_cassandra_zookeeper.sh deleted file mode 100755 index a116e2130e..0000000000 --- a/docker/deploy_cassandra_zookeeper.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# -# Copyright © 2016-2017 The Thingsboard Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -command='docker-compose -f docker-compose.yml -f docker-compose.static.yml' - -echo "stopping images.." -$command stop - -echo "removing stopped images.." -$command rm -f - -echo "building images.." -$command build - -echo "starting cassandra, zookeeper, thingsboard-db-schema images..." -$command up -d db zk thingsboard-db-schema diff --git a/docker/thingsboard-db-schema/Makefile b/docker/thingsboard-db-schema/Makefile new file mode 100644 index 0000000000..653153940f --- /dev/null +++ b/docker/thingsboard-db-schema/Makefile @@ -0,0 +1,13 @@ +VERSION=k8stest +PROJECT=thingsboard +APP=thingsboard-db-schema + +all: build + +build: + docker build --pull -t ${PROJECT}/${APP}:${VERSION} . + +push: build + docker push ${PROJECT}/${APP}:${VERSION} + +.PHONY: all build push \ No newline at end of file diff --git a/docker/thingsboard-db-schema/build_and_deploy.sh b/docker/thingsboard-db-schema/build_and_deploy.sh deleted file mode 100755 index 428016c839..0000000000 --- a/docker/thingsboard-db-schema/build_and_deploy.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# -# Copyright © 2016-2017 The Thingsboard Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -cp ../../dao/src/main/resources/schema.cql schema.cql -cp ../../dao/src/main/resources/demo-data.cql demo-data.cql -cp ../../dao/src/main/resources/system-data.cql system-data.cql - -docker build -t thingsboard/thingsboard-db-schema:k8test . - -docker login - -docker push thingsboard/thingsboard-db-schema:k8test - -# cleanup -rm schema.cql -rm demo-data.cql -rm system-data.cql \ No newline at end of file diff --git a/docker/thingsboard-db-schema/thingsboard-schema.yaml b/docker/thingsboard-db-schema/thingsboard-db-schema.yaml similarity index 81% rename from docker/thingsboard-db-schema/thingsboard-schema.yaml rename to docker/thingsboard-db-schema/thingsboard-db-schema.yaml index 141e1e3d41..dbc4f587f2 100644 --- a/docker/thingsboard-db-schema/thingsboard-schema.yaml +++ b/docker/thingsboard-db-schema/thingsboard-db-schema.yaml @@ -6,7 +6,7 @@ spec: containers: - name: tb-db-schema imagePullPolicy: Always - image: thingsboard/thingsboard-db-schema:k8test + image: thingsboard/thingsboard-db-schema:k8stest env: - name: SKIP_SCHEMA_CREATION value: "false" @@ -15,7 +15,7 @@ spec: - name : SKIP_DEMO_DATA value: "false" - name : CASSANDRA_URL - value: "cassandra" + value: "cassandra-headless" command: - sh - -c diff --git a/docker/thingsboard/Makefile b/docker/thingsboard/Makefile new file mode 100644 index 0000000000..29203e30d3 --- /dev/null +++ b/docker/thingsboard/Makefile @@ -0,0 +1,13 @@ +VERSION=k8stest +PROJECT=thingsboard +APP=application + +all: build + +build: + docker build --pull -t ${PROJECT}/${APP}:${VERSION} . + +push: build + docker push ${PROJECT}/${APP}:${VERSION} + +.PHONY: all build push \ No newline at end of file diff --git a/docker/thingsboard/thingsboard.yaml b/docker/thingsboard/thingsboard.yaml index 702a7215f5..a9c1df37d8 100644 --- a/docker/thingsboard/thingsboard.yaml +++ b/docker/thingsboard/thingsboard.yaml @@ -8,13 +8,11 @@ metadata: spec: ports: - port: 8080 - name: ui-port + name: ui - port: 1883 - name: mqtt-port + name: mqtt - port: 5683 - name: coap-port - - port: 9001 - name: rpc-port + name: coap selector: app: tb type: LoadBalancer @@ -27,14 +25,8 @@ metadata: app: tb-headless spec: ports: - - port: 8080 - name: ui-port - - port: 1883 - name: mqtt-port - - port: 5683 - name: coap-port - port: 9001 - name: rpc-port + name: rpc selector: app: tb clusterIP: None @@ -56,7 +48,7 @@ metadata: data: zookeeper.enabled: "true" zookeeper.url: "zk-headless" - cassandra.url: "cassandra:9042" + cassandra.url: "cassandra-headless:9042" --- apiVersion: apps/v1beta1 kind: StatefulSet @@ -84,16 +76,23 @@ spec: containers: - name: tb imagePullPolicy: Always - image: thingsboard/application:k8test + image: thingsboard/application:k8stest ports: - containerPort: 8080 - name: ui-port + name: ui - containerPort: 1883 - name: mqtt-port + name: mqtt - containerPort: 5683 - name: coap-port + name: coap - containerPort: 9001 - name: rpc-port + name: rpc + resources: + limits: + cpu: "250m" + memory: "500Mi" + requests: + cpu: "250m" + memory: "500Mi" env: - name: ZOOKEEPER_ENABLED valueFrom: diff --git a/docker/zookeeper/Makefile b/docker/zookeeper/Makefile index a28937f1d5..4c7a8c4007 100644 --- a/docker/zookeeper/Makefile +++ b/docker/zookeeper/Makefile @@ -1,12 +1,13 @@ -VERSION=k8test +VERSION=k8stest PROJECT=thingsboard +APP=zk all: build build: - docker build --pull -t ${PROJECT}/k8szk:${VERSION} . + docker build --pull -t ${PROJECT}/${APP}:${VERSION} . push: build - docker push ${PROJECT}/k8szk:${VERSION} + docker push ${PROJECT}/${APP}:${VERSION} .PHONY: all build push diff --git a/docker/zookeeper/zkGenConfig.sh b/docker/zookeeper/zkGenConfig.sh index cef42f64f7..0b3365bcac 100755 --- a/docker/zookeeper/zkGenConfig.sh +++ b/docker/zookeeper/zkGenConfig.sh @@ -86,7 +86,7 @@ function validate_env() { function create_config() { rm -f $ZK_CONFIG_FILE echo "Creating ZooKeeper configuration" - echo "#This file was autogenerated by k8szk DO NOT EDIT" >> $ZK_CONFIG_FILE + echo "#This file was autogenerated by zk DO NOT EDIT" >> $ZK_CONFIG_FILE echo "clientPort=$ZK_CLIENT_PORT" >> $ZK_CONFIG_FILE echo "dataDir=$ZK_DATA_DIR" >> $ZK_CONFIG_FILE echo "dataLogDir=$ZK_DATA_LOG_DIR" >> $ZK_CONFIG_FILE diff --git a/docker/zookeeper/zookeeper.yaml b/docker/zookeeper/zookeeper.yaml index 4184a3ab8e..00bca9c902 100644 --- a/docker/zookeeper/zookeeper.yaml +++ b/docker/zookeeper/zookeeper.yaml @@ -20,6 +20,7 @@ metadata: name: zk-config data: ensemble: "zk-0;zk-1;zk-2" + replicas: "3" jvm.heap: "500m" tick: "2000" init: "10" @@ -27,6 +28,9 @@ data: client.cnxns: "60" snap.retain: "3" purge.interval: "1" + client.port: "2181" + server.port: "2888" + election.port: "3888" --- apiVersion: policy/v1beta1 kind: PodDisruptionBudget @@ -63,9 +67,9 @@ spec: - zk-headless topologyKey: "kubernetes.io/hostname" containers: - - name: k8szk + - name: zk imagePullPolicy: Always - image: thingsboard/k8szk:k8test + image: thingsboard/zk:k8stest resources: requests: memory: "500Mi" @@ -84,48 +88,60 @@ spec: name: zk-config key: ensemble - name : ZK_REPLICAS - value: "3" + valueFrom: + configMapKeyRef: + name: zk-config + key: replicas - name : ZK_HEAP_SIZE valueFrom: configMapKeyRef: - name: zk-config - key: jvm.heap + name: zk-config + key: jvm.heap - name : ZK_TICK_TIME valueFrom: configMapKeyRef: - name: zk-config - key: tick + name: zk-config + key: tick - name : ZK_INIT_LIMIT valueFrom: configMapKeyRef: - name: zk-config - key: init + name: zk-config + key: init - name : ZK_SYNC_LIMIT valueFrom: configMapKeyRef: - name: zk-config - key: tick + name: zk-config + key: tick - name : ZK_MAX_CLIENT_CNXNS valueFrom: configMapKeyRef: - name: zk-config - key: client.cnxns + name: zk-config + key: client.cnxns - name: ZK_SNAP_RETAIN_COUNT valueFrom: configMapKeyRef: - name: zk-config - key: snap.retain + name: zk-config + key: snap.retain - name: ZK_PURGE_INTERVAL valueFrom: configMapKeyRef: - name: zk-config - key: purge.interval + name: zk-config + key: purge.interval - name: ZK_CLIENT_PORT - value: "2181" + valueFrom: + configMapKeyRef: + name: zk-config + key: client.port - name: ZK_SERVER_PORT - value: "2888" + valueFrom: + configMapKeyRef: + name: zk-config + key: server.port - name: ZK_ELECTION_PORT - value: "3888" + valueFrom: + configMapKeyRef: + name: zk-config + key: election.port command: - sh - -c @@ -143,16 +159,16 @@ spec: initialDelaySeconds: 15 timeoutSeconds: 5 volumeMounts: - - name: datadir + - name: zkdatadir mountPath: /var/lib/zookeeper securityContext: runAsUser: 1000 fsGroup: 1000 volumeClaimTemplates: - metadata: - name: datadir + name: zkdatadir annotations: - volume.alpha.kubernetes.io/storage-class: anything + volume.beta.kubernetes.io/storage-class: slow spec: accessModes: [ "ReadWriteOnce" ] resources: From 3a1eef449b024747dc271b361d8af5a31d89ff3b Mon Sep 17 00:00:00 2001 From: volodymyr-babak Date: Wed, 10 May 2017 21:34:49 +0300 Subject: [PATCH 05/12] TB-58: k8s feature improvements --- docker/thingsboard/run_thingsboard.sh | 3 --- docker/thingsboard/thingsboard.yaml | 37 ++++++++------------------- 2 files changed, 10 insertions(+), 30 deletions(-) diff --git a/docker/thingsboard/run_thingsboard.sh b/docker/thingsboard/run_thingsboard.sh index 404cc8df9d..877fed7fec 100755 --- a/docker/thingsboard/run_thingsboard.sh +++ b/docker/thingsboard/run_thingsboard.sh @@ -34,9 +34,6 @@ done # Copying env variables into conf files printenv | awk -F "=" '{print "export " $1 "='\''" $2 "'\''"}' >> /usr/share/thingsboard/conf/thingsboard.conf -# Set env variable for RPC address -echo "export RPC_HOST='"$(hostname -f)"'" | tee -a /usr/share/thingsboard/conf/thingsboard.conf - cat /usr/share/thingsboard/conf/thingsboard.conf echo "Starting 'Thingsboard' service..." diff --git a/docker/thingsboard/thingsboard.yaml b/docker/thingsboard/thingsboard.yaml index a9c1df37d8..02326c1671 100644 --- a/docker/thingsboard/thingsboard.yaml +++ b/docker/thingsboard/thingsboard.yaml @@ -2,9 +2,9 @@ apiVersion: v1 kind: Service metadata: - name: tb-external-ip + name: tb-service labels: - app: tb-external-ip + app: tb-service spec: ports: - port: 8080 @@ -17,20 +17,6 @@ spec: app: tb type: LoadBalancer --- -apiVersion: v1 -kind: Service -metadata: - name: tb-headless - labels: - app: tb-headless -spec: - ports: - - port: 9001 - name: rpc - selector: - app: tb - clusterIP: None ---- apiVersion: policy/v1beta1 kind: PodDisruptionBudget metadata: @@ -55,14 +41,13 @@ kind: StatefulSet metadata: name: tb spec: - serviceName: "tb-headless" + serviceName: "tb-service" replicas: 2 template: metadata: labels: app: tb spec: - terminationGracePeriodSeconds: 10 affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -71,7 +56,7 @@ spec: - key: "app" operator: In values: - - tb-headless + - tb-service topologyKey: "kubernetes.io/hostname" containers: - name: tb @@ -109,19 +94,17 @@ spec: configMapKeyRef: name: tb-config key: cassandra.url + - name : RPC_HOST + valueFrom: + fieldRef: + fieldPath: status.podIP command: - sh - -c - ./run_thingsboard.sh -# readinessProbe: -# httpGet: -# path: /login -# port: ui-port -# initialDelaySeconds: 30 -# periodSeconds: 10 livenessProbe: httpGet: path: /login port: ui-port - initialDelaySeconds: 30 - periodSeconds: 10 \ No newline at end of file + initialDelaySeconds: 120 + timeoutSeconds: 10 \ No newline at end of file From 2f5c9ecba5963e0199f061fa4cfb07d7e0147390 Mon Sep 17 00:00:00 2001 From: volodymyr-babak Date: Wed, 17 May 2017 12:08:10 +0300 Subject: [PATCH 06/12] improved docker makefiles --- docker/cassandra/Makefile | 6 +----- docker/thingsboard-db-schema/Makefile | 8 ++++---- docker/thingsboard/Makefile | 6 ++---- docker/zookeeper/Makefile | 4 ---- 4 files changed, 7 insertions(+), 17 deletions(-) diff --git a/docker/cassandra/Makefile b/docker/cassandra/Makefile index ea71c6340c..cb577cb502 100644 --- a/docker/cassandra/Makefile +++ b/docker/cassandra/Makefile @@ -2,12 +2,8 @@ VERSION=k8stest PROJECT=thingsboard APP=cassandra -all: build - build: docker build --pull -t ${PROJECT}/${APP}:${VERSION} . push: build - docker push ${PROJECT}/${APP}:${VERSION} - -.PHONY: all build push \ No newline at end of file + docker push ${PROJECT}/${APP}:${VERSION} \ No newline at end of file diff --git a/docker/thingsboard-db-schema/Makefile b/docker/thingsboard-db-schema/Makefile index 653153940f..ef526e1a6a 100644 --- a/docker/thingsboard-db-schema/Makefile +++ b/docker/thingsboard-db-schema/Makefile @@ -2,12 +2,12 @@ VERSION=k8stest PROJECT=thingsboard APP=thingsboard-db-schema -all: build - build: + cp ../../dao/src/main/resources/schema.cql . + cp ../../dao/src/main/resources/demo-data.cql . + cp ../../dao/src/main/resources/system-data.cql . docker build --pull -t ${PROJECT}/${APP}:${VERSION} . push: build docker push ${PROJECT}/${APP}:${VERSION} - -.PHONY: all build push \ No newline at end of file + rm schema.cql demo-data.cql system-data.cql \ No newline at end of file diff --git a/docker/thingsboard/Makefile b/docker/thingsboard/Makefile index 29203e30d3..72ada4d0f7 100644 --- a/docker/thingsboard/Makefile +++ b/docker/thingsboard/Makefile @@ -2,12 +2,10 @@ VERSION=k8stest PROJECT=thingsboard APP=application -all: build - build: + cp ../../application/target/thingsboard.deb . docker build --pull -t ${PROJECT}/${APP}:${VERSION} . push: build docker push ${PROJECT}/${APP}:${VERSION} - -.PHONY: all build push \ No newline at end of file + rm thingsboard.deb \ No newline at end of file diff --git a/docker/zookeeper/Makefile b/docker/zookeeper/Makefile index 4c7a8c4007..b179f07291 100644 --- a/docker/zookeeper/Makefile +++ b/docker/zookeeper/Makefile @@ -2,12 +2,8 @@ VERSION=k8stest PROJECT=thingsboard APP=zk -all: build - build: docker build --pull -t ${PROJECT}/${APP}:${VERSION} . push: build docker push ${PROJECT}/${APP}:${VERSION} - -.PHONY: all build push From aaf3c5d4390fefd11d5ab69cc05d156eee6f7df2 Mon Sep 17 00:00:00 2001 From: volodymyr-babak Date: Wed, 17 May 2017 13:17:36 +0300 Subject: [PATCH 07/12] small docker makefiles fixes --- docker/thingsboard-db-schema/Makefile | 4 ++-- docker/thingsboard/Makefile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/thingsboard-db-schema/Makefile b/docker/thingsboard-db-schema/Makefile index ef526e1a6a..604520e90d 100644 --- a/docker/thingsboard-db-schema/Makefile +++ b/docker/thingsboard-db-schema/Makefile @@ -7,7 +7,7 @@ build: cp ../../dao/src/main/resources/demo-data.cql . cp ../../dao/src/main/resources/system-data.cql . docker build --pull -t ${PROJECT}/${APP}:${VERSION} . + rm schema.cql demo-data.cql system-data.cql push: build - docker push ${PROJECT}/${APP}:${VERSION} - rm schema.cql demo-data.cql system-data.cql \ No newline at end of file + docker push ${PROJECT}/${APP}:${VERSION} \ No newline at end of file diff --git a/docker/thingsboard/Makefile b/docker/thingsboard/Makefile index 72ada4d0f7..000968e447 100644 --- a/docker/thingsboard/Makefile +++ b/docker/thingsboard/Makefile @@ -5,7 +5,7 @@ APP=application build: cp ../../application/target/thingsboard.deb . docker build --pull -t ${PROJECT}/${APP}:${VERSION} . + rm thingsboard.deb push: build - docker push ${PROJECT}/${APP}:${VERSION} - rm thingsboard.deb \ No newline at end of file + docker push ${PROJECT}/${APP}:${VERSION} \ No newline at end of file From 7120d61f59eb1e72ab16c643ca95003e3ae93a2f Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Thu, 18 May 2017 21:09:54 +0300 Subject: [PATCH 08/12] Misc fixes after performance testing --- docker/cassandra/Dockerfile | 16 +++++++++++ docker/cassandra/cassandra.yaml | 27 ++++++++++++------- docker/cassandra/ready-probe.sh | 5 ++-- docker/common/common.yaml | 24 +++++++++++++++++ .../thingsboard-db-schema.yaml | 16 +++++++++++ docker/thingsboard/thingsboard.yaml | 23 +++++++++++----- docker/zookeeper/Dockerfile | 16 +++++++++++ docker/zookeeper/zkGenConfig.sh | 4 ++- docker/zookeeper/zkOk.sh | 4 ++- docker/zookeeper/zookeeper.yaml | 20 +++++++++++--- 10 files changed, 131 insertions(+), 24 deletions(-) diff --git a/docker/cassandra/Dockerfile b/docker/cassandra/Dockerfile index 72084cd0af..ec8117358f 100644 --- a/docker/cassandra/Dockerfile +++ b/docker/cassandra/Dockerfile @@ -1,3 +1,19 @@ +# +# Copyright © 2016-2017 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + FROM cassandra:3.9 COPY ready-probe.sh / diff --git a/docker/cassandra/cassandra.yaml b/docker/cassandra/cassandra.yaml index c3772b15e1..6da5532c3f 100644 --- a/docker/cassandra/cassandra.yaml +++ b/docker/cassandra/cassandra.yaml @@ -1,3 +1,19 @@ +# +# Copyright © 2016-2017 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + apiVersion: v1 kind: Service metadata: @@ -49,13 +65,6 @@ spec: name: cql - containerPort: 9160 name: thrift - resources: - limits: - cpu: "500m" - memory: 1Gi - requests: - cpu: "500m" - memory: 1Gi securityContext: capabilities: add: @@ -104,7 +113,7 @@ spec: - metadata: name: cassandra-data annotations: - volume.beta.kubernetes.io/storage-class: slow + volume.beta.kubernetes.io/storage-class: fast spec: accessModes: [ "ReadWriteOnce" ] resources: @@ -113,7 +122,7 @@ spec: - metadata: name: cassandra-commitlog annotations: - volume.beta.kubernetes.io/storage-class: slow + volume.beta.kubernetes.io/storage-class: fast spec: accessModes: [ "ReadWriteOnce" ] resources: diff --git a/docker/cassandra/ready-probe.sh b/docker/cassandra/ready-probe.sh index 989f4d90a1..68faf927fe 100755 --- a/docker/cassandra/ready-probe.sh +++ b/docker/cassandra/ready-probe.sh @@ -1,6 +1,6 @@ #!/bin/bash - -# Copyright 2016 The Kubernetes Authors. +# +# Copyright © 2016-2017 The Thingsboard Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# if [[ $(nodetool status | grep $POD_IP) == *"UN"* ]]; then if [[ $DEBUG ]]; then diff --git a/docker/common/common.yaml b/docker/common/common.yaml index 610c027911..f650118efd 100644 --- a/docker/common/common.yaml +++ b/docker/common/common.yaml @@ -1,3 +1,19 @@ +# +# Copyright © 2016-2017 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + --- apiVersion: storage.k8s.io/v1beta1 kind: StorageClass @@ -6,4 +22,12 @@ metadata: provisioner: kubernetes.io/gce-pd parameters: type: pd-standard +--- +apiVersion: storage.k8s.io/v1beta1 +kind: StorageClass +metadata: + name: fast +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-ssd --- \ No newline at end of file diff --git a/docker/thingsboard-db-schema/thingsboard-db-schema.yaml b/docker/thingsboard-db-schema/thingsboard-db-schema.yaml index dbc4f587f2..6d6040f2eb 100644 --- a/docker/thingsboard-db-schema/thingsboard-db-schema.yaml +++ b/docker/thingsboard-db-schema/thingsboard-db-schema.yaml @@ -1,3 +1,19 @@ +# +# Copyright © 2016-2017 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + apiVersion: v1 kind: Pod metadata: diff --git a/docker/thingsboard/thingsboard.yaml b/docker/thingsboard/thingsboard.yaml index 02326c1671..6ef5ab16b4 100644 --- a/docker/thingsboard/thingsboard.yaml +++ b/docker/thingsboard/thingsboard.yaml @@ -1,3 +1,19 @@ +# +# Copyright © 2016-2017 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + --- apiVersion: v1 kind: Service @@ -71,13 +87,6 @@ spec: name: coap - containerPort: 9001 name: rpc - resources: - limits: - cpu: "250m" - memory: "500Mi" - requests: - cpu: "250m" - memory: "500Mi" env: - name: ZOOKEEPER_ENABLED valueFrom: diff --git a/docker/zookeeper/Dockerfile b/docker/zookeeper/Dockerfile index 7752f93b3a..7d3480e825 100644 --- a/docker/zookeeper/Dockerfile +++ b/docker/zookeeper/Dockerfile @@ -1,3 +1,19 @@ +# +# Copyright © 2016-2017 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + FROM ubuntu:16.04 ENV ZK_USER=zookeeper \ ZK_DATA_DIR=/var/lib/zookeeper/data \ diff --git a/docker/zookeeper/zkGenConfig.sh b/docker/zookeeper/zkGenConfig.sh index 0b3365bcac..02fde70fa1 100755 --- a/docker/zookeeper/zkGenConfig.sh +++ b/docker/zookeeper/zkGenConfig.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Copyright 2016 The Kubernetes Authors. +# +# Copyright © 2016-2017 The Thingsboard Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,6 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# ZK_USER=${ZK_USER:-"zookeeper"} ZK_LOG_LEVEL=${ZK_LOG_LEVEL:-"INFO"} diff --git a/docker/zookeeper/zkOk.sh b/docker/zookeeper/zkOk.sh index dbe417682f..6dd38ca030 100755 --- a/docker/zookeeper/zkOk.sh +++ b/docker/zookeeper/zkOk.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Copyright 2016 The Kubernetes Authors. +# +# Copyright © 2016-2017 The Thingsboard Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,6 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# # zkOk.sh uses the ruok ZooKeeper four letter work to determine if the instance # is health. The $? variable will be set to 0 if server responds that it is diff --git a/docker/zookeeper/zookeeper.yaml b/docker/zookeeper/zookeeper.yaml index 00bca9c902..ed7a9f6ba2 100644 --- a/docker/zookeeper/zookeeper.yaml +++ b/docker/zookeeper/zookeeper.yaml @@ -1,3 +1,19 @@ +# +# Copyright © 2016-2017 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + apiVersion: v1 kind: Service metadata: @@ -70,10 +86,6 @@ spec: - name: zk imagePullPolicy: Always image: thingsboard/zk:k8stest - resources: - requests: - memory: "500Mi" - cpu: "250m" ports: - containerPort: 2181 name: client From 2b18ed7a9294d72ecfcf24534dc2b70cb8592fbf Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 19 May 2017 11:46:15 +0300 Subject: [PATCH 09/12] Misc improvements --- docker/cassandra/cassandra.yaml | 10 ++++++---- docker/thingsboard/thingsboard.yaml | 6 ++++-- docker/zookeeper/zookeeper.yaml | 6 ++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docker/cassandra/cassandra.yaml b/docker/cassandra/cassandra.yaml index 6da5532c3f..24312ae562 100644 --- a/docker/cassandra/cassandra.yaml +++ b/docker/cassandra/cassandra.yaml @@ -34,12 +34,14 @@ metadata: name: cassandra spec: serviceName: cassandra-headless - replicas: 3 + replicas: 2 template: metadata: labels: app: cassandra spec: + nodeSelector: + machinetype: other affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -75,7 +77,7 @@ spec: command: ["/bin/sh", "-c", "PID=$(pidof java) && kill $PID && while ps -p $PID > /dev/null; do sleep 1; done"] env: - name: MAX_HEAP_SIZE - value: 512M + value: 2048M - name: HEAP_NEWSIZE value: 100M - name: CASSANDRA_SEEDS @@ -118,7 +120,7 @@ spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: 2Gi + storage: 3Gi - metadata: name: cassandra-commitlog annotations: @@ -127,4 +129,4 @@ spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: 1Gi \ No newline at end of file + storage: 2Gi \ No newline at end of file diff --git a/docker/thingsboard/thingsboard.yaml b/docker/thingsboard/thingsboard.yaml index 6ef5ab16b4..f03f49bcf1 100644 --- a/docker/thingsboard/thingsboard.yaml +++ b/docker/thingsboard/thingsboard.yaml @@ -41,7 +41,7 @@ spec: selector: matchLabels: app: tb - minAvailable: 2 + minAvailable: 3 --- apiVersion: v1 kind: ConfigMap @@ -58,12 +58,14 @@ metadata: name: tb spec: serviceName: "tb-service" - replicas: 2 + replicas: 3 template: metadata: labels: app: tb spec: + nodeSelector: + machinetype: tb affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: diff --git a/docker/zookeeper/zookeeper.yaml b/docker/zookeeper/zookeeper.yaml index ed7a9f6ba2..062d8cc4ee 100644 --- a/docker/zookeeper/zookeeper.yaml +++ b/docker/zookeeper/zookeeper.yaml @@ -35,7 +35,7 @@ kind: ConfigMap metadata: name: zk-config data: - ensemble: "zk-0;zk-1;zk-2" + ensemble: "zk-0;zk-1" replicas: "3" jvm.heap: "500m" tick: "2000" @@ -64,7 +64,7 @@ metadata: name: zk spec: serviceName: zk-headless - replicas: 3 + replicas: 2 template: metadata: labels: @@ -72,6 +72,8 @@ spec: annotations: pod.alpha.kubernetes.io/initialized: "true" spec: + nodeSelector: + machinetype: other affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: From 8c50c63ca970ae36d2bd1b86b3ea998e39efad7e Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 23 May 2017 09:25:31 +0300 Subject: [PATCH 10/12] Docker SSL support --- .../src/main/resources/thingsboard.yml | 46 ++++++++++++------- .../dao/cache/ServiceCacheConfiguration.java | 1 - docker/docker-compose.yml | 8 ++-- docker/thingsboard.env | 6 ++- docker/thingsboard/build_and_deploy.sh | 7 +-- .../mqtt/MqttSslHandlerProvider.java | 2 +- 6 files changed, 41 insertions(+), 29 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index e64e9cd95c..f015aa156e 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -19,12 +19,18 @@ server: address: "${HTTP_BIND_ADDRESS:0.0.0.0}" # Server bind port port: "${HTTP_BIND_PORT:8080}" -# Uncomment the following section to enable ssl -# ssl: -# key-store: classpath:keystore/keystore.p12 -# key-store-password: thingsboard -# keyStoreType: PKCS12 -# keyAlias: tomcat + # Server SSL configuration + ssl: + # Enable/disable SSL support + enabled: "${SSL_ENABLED:false}" + # Path to the key store that holds the SSL certificate + key-store: "${SSL_KEY_STORE:classpath:keystore/keystore.p12}" + # Password used to access the key store + key-store-password: "${SSL_KEY_STORE_PASSWORD:thingsboard}" + # Type of the key store + key-store-type: "${SSL_KEY_STORE_TYPE:PKCS12}" + # Alias that identifies the key in the key store + key-alias: "${SSL_KEY_ALIAS:tomcat}" # Zookeeper connection parameters. Used for service discovery. zk: @@ -60,10 +66,10 @@ plugins: # JWT Token parameters security.jwt: - tokenExpirationTime: "${JWT_TOKEN_EXPIRATION_TIME:900}" # Number of seconds (15 mins) - refreshTokenExpTime: "${JWT_REFRESH_TOKEN_EXPIRATION_TIME:3600}" # Seconds (1 hour) - tokenIssuer: "${JWT_TOKEN_ISSUER:thingsboard.io}" - tokenSigningKey: "${JWT_TOKEN_SIGNING_KEY:thingsboardDefaultSigningKey}" + token-expiration-time: "${JWT_TOKEN_EXPIRATION_TIME:900}" # Number of seconds (15 mins) + refresh-token-exp-time: "${JWT_REFRESH_TOKEN_EXPIRATION_TIME:3600}" # Seconds (1 hour) + token-issuer: "${JWT_TOKEN_ISSUER:thingsboard.io}" + token-signing-key: "${JWT_TOKEN_SIGNING_KEY:thingsboardDefaultSigningKey}" # Device communication protocol parameters http: @@ -72,19 +78,25 @@ http: # MQTT server parameters mqtt: bind_address: "${MQTT_BIND_ADDRESS:0.0.0.0}" - bind_port: "${MQTT_BIND_PORT:1883}" + bind_port: "${MQTT_BIND_PORT:1884}" adaptor: "${MQTT_ADAPTOR_NAME:JsonMqttAdaptor}" timeout: "${MQTT_TIMEOUT:10000}" netty: leak_detector_level: "${NETTY_LEASK_DETECTOR_LVL:DISABLED}" boss_group_thread_count: "${NETTY_BOSS_GROUP_THREADS:1}" worker_group_thread_count: "${NETTY_WORKER_GROUP_THREADS:12}" -# Uncomment the following lines to enable ssl for MQTT -# ssl: -# key_store: mqttserver.jks -# key_store_password: server_ks_password -# key_password: server_key_password -# key_store_type: JKS + # MQTT SSL configuration + ssl: + # Enable/disable SSL support + enabled: "${MQTT_SSL_ENABLED:false}" + # Path to the key store that holds the SSL certificate + key_store: "${MQTT_SSL_KEY_STORE:mqttserver.jks}" + # Password used to access the key store + key_store_password: "${MQTT_SSL_KEY_STORE_PASSWORD:server_ks_password}" + # Password used to access the key + key_password: "${MQTT_SSL_KEY_PASSWORD:server_key_password}" + # Type of the key store + key_store_type: "${MQTT_SSL_KEY_STORE_TYPE:JKS}" # CoAP server parameters coap: diff --git a/dao/src/main/java/org/thingsboard/server/dao/cache/ServiceCacheConfiguration.java b/dao/src/main/java/org/thingsboard/server/dao/cache/ServiceCacheConfiguration.java index b4fbc65507..7c435bf779 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cache/ServiceCacheConfiguration.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cache/ServiceCacheConfiguration.java @@ -45,7 +45,6 @@ public class ServiceCacheConfiguration { @Value("${cache.device_credentials.time_to_live}") private Integer cacheDeviceCredentialsTTL; - @Value("${zk.enabled}") private boolean zkEnabled; @Value("${zk.url}") diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d19332b8de..b3b182364c 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -18,13 +18,15 @@ version: '2' services: thingsboard: - image: "thingsboard/application:1.2.2" + image: "thingsboard/application:1.2.3-test" ports: - - "8080:8080" - - "1883:1883" + - "8443:8443" + - "1884:1883" - "5683:5683/udp" env_file: - thingsboard.env + volumes: + - "~/keystore.p12:/root/keystore.p12" entrypoint: ./run_thingsboard.sh thingsboard-db-schema: image: "thingsboard/thingsboard-db-schema:1.2.2" diff --git a/docker/thingsboard.env b/docker/thingsboard.env index 2325790cd6..5d9ca6ddf3 100644 --- a/docker/thingsboard.env +++ b/docker/thingsboard.env @@ -5,4 +5,8 @@ ZOOKEEPER_URL=zk:2181 MQTT_BIND_ADDRESS=0.0.0.0 MQTT_BIND_PORT=1883 COAP_BIND_ADDRESS=0.0.0.0 -COAP_BIND_PORT=5683 \ No newline at end of file +COAP_BIND_PORT=5683 + +SSL_ENABLED=true +SSL_KEY_STORE=/root/keystore.p12 +HTTP_BIND_PORT=8443 \ No newline at end of file diff --git a/docker/thingsboard/build_and_deploy.sh b/docker/thingsboard/build_and_deploy.sh index 7c02de571c..58d7cd65e3 100755 --- a/docker/thingsboard/build_and_deploy.sh +++ b/docker/thingsboard/build_and_deploy.sh @@ -18,9 +18,4 @@ cp ../../application/target/thingsboard.deb thingsboard.deb -docker build -t thingsboard/application:1.2.2 -t thingsboard/application:latest . - -docker login - -docker push thingsboard/application:1.2.2 -docker push thingsboard/application:latest \ No newline at end of file +docker build -t thingsboard/application:1.2.3-test . \ No newline at end of file diff --git a/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttSslHandlerProvider.java b/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttSslHandlerProvider.java index 80e4e011fa..aed9a0c4d4 100644 --- a/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttSslHandlerProvider.java +++ b/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttSslHandlerProvider.java @@ -41,7 +41,7 @@ import java.security.cert.X509Certificate; */ @Slf4j @Component("MqttSslHandlerProvider") -@ConditionalOnProperty(prefix = "mqtt.ssl", value = "key-store", havingValue = "", matchIfMissing = false) +@ConditionalOnProperty(prefix = "mqtt.ssl", value = "enabled", havingValue = "true", matchIfMissing = false) public class MqttSslHandlerProvider { public static final String TLS = "TLS"; From 5da5a1a1347540c0a2091f4ea37353d4814aaddc Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 24 May 2017 15:24:48 +0300 Subject: [PATCH 11/12] removed unnecessary config options --- docker/docker-compose.yml | 2 -- docker/thingsboard.env | 6 +----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index e5015f789f..2617133206 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -25,8 +25,6 @@ services: - "5683:5683/udp" env_file: - thingsboard.env - volumes: - - "~/keystore.p12:/root/keystore.p12" entrypoint: ./run_thingsboard.sh thingsboard-db-schema: image: "thingsboard/thingsboard-db-schema:1.2.3" diff --git a/docker/thingsboard.env b/docker/thingsboard.env index 5d9ca6ddf3..2325790cd6 100644 --- a/docker/thingsboard.env +++ b/docker/thingsboard.env @@ -5,8 +5,4 @@ ZOOKEEPER_URL=zk:2181 MQTT_BIND_ADDRESS=0.0.0.0 MQTT_BIND_PORT=1883 COAP_BIND_ADDRESS=0.0.0.0 -COAP_BIND_PORT=5683 - -SSL_ENABLED=true -SSL_KEY_STORE=/root/keystore.p12 -HTTP_BIND_PORT=8443 \ No newline at end of file +COAP_BIND_PORT=5683 \ No newline at end of file From 9c29deba7327f0759fff94dd87f6249308f59adf Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 24 May 2017 15:31:37 +0300 Subject: [PATCH 12/12] Revert to default values --- application/src/main/resources/thingsboard.yml | 2 +- docker/docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index ce6ecb1ac2..24cea0e4af 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -78,7 +78,7 @@ http: # MQTT server parameters mqtt: bind_address: "${MQTT_BIND_ADDRESS:0.0.0.0}" - bind_port: "${MQTT_BIND_PORT:1884}" + bind_port: "${MQTT_BIND_PORT:1883}" adaptor: "${MQTT_ADAPTOR_NAME:JsonMqttAdaptor}" timeout: "${MQTT_TIMEOUT:10000}" netty: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2617133206..97873e82ce 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -20,8 +20,8 @@ services: thingsboard: image: "thingsboard/application:1.2.3" ports: - - "8443:8443" - - "1884:1883" + - "8080:8080" + - "1883:1883" - "5683:5683/udp" env_file: - thingsboard.env