From 8b829498018cbcb23d32706d7553009b9959f9c9 Mon Sep 17 00:00:00 2001 From: Lin Ru Date: Wed, 20 May 2020 12:43:25 +0800 Subject: [PATCH 1/4] update scripts runbroker & runserver, add default variables settings --- image-build/scripts/runbroker-customize.sh | 10 +++++----- image-build/scripts/runserver-customize.sh | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/image-build/scripts/runbroker-customize.sh b/image-build/scripts/runbroker-customize.sh index e1edad0..82ca6ef 100755 --- a/image-build/scripts/runbroker-customize.sh +++ b/image-build/scripts/runbroker-customize.sh @@ -125,10 +125,10 @@ calculate_heap_sizes() calculate_heap_sizes # Dynamically calculate parameters, for reference. -Xms=$MAX_HEAP_SIZE -Xmx=$MAX_HEAP_SIZE -Xmn=$HEAP_NEWSIZE -MaxDirectMemorySize=$MAX_HEAP_SIZE +Xms=${JVM_MIN_HEAP_SIZE:-$MAX_HEAP_SIZE} +Xmx=${JVM_MAX_HEAP_SIZE:-$MAX_HEAP_SIZE} +Xmn=${JVM_NEW_HEAP_SIZE:-$HEAP_NEWSIZE} +MaxDirectMemorySize=${Xmx:-$MAX_HEAP_SIZE} # Set for `JAVA_OPT`. JAVA_OPT="${JAVA_OPT} -server -Xms${Xms} -Xmx${Xmx} -Xmn${Xmn}" JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8" @@ -153,4 +153,4 @@ then fi else $JAVA ${JAVA_OPT} $@ -fi +fi \ No newline at end of file diff --git a/image-build/scripts/runserver-customize.sh b/image-build/scripts/runserver-customize.sh index c05e77e..bc3b352 100755 --- a/image-build/scripts/runserver-customize.sh +++ b/image-build/scripts/runserver-customize.sh @@ -125,9 +125,9 @@ calculate_heap_sizes() calculate_heap_sizes # Dynamically calculate parameters, for reference. -Xms=$MAX_HEAP_SIZE -Xmx=$MAX_HEAP_SIZE -Xmn=$HEAP_NEWSIZE +Xms=${JVM_MIN_HEAP_SIZE:-$MAX_HEAP_SIZE} +Xmx=${JVM_MAX_HEAP_SIZE:-$MAX_HEAP_SIZE} +Xmn=${JVM_NEW_HEAP_SIZE:-$HEAP_NEWSIZE} # Set for `JAVA_OPT`. JAVA_OPT="${JAVA_OPT} -server -Xms${Xms} -Xmx${Xmx} -Xmn${Xmn}" JAVA_OPT="${JAVA_OPT} -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:-UseParNewGC" @@ -139,4 +139,4 @@ JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${JAVA_HOME}/jre/lib/ext:${BASE_DIR}/lib" JAVA_OPT="${JAVA_OPT} ${JAVA_OPT_EXT}" JAVA_OPT="${JAVA_OPT} -cp ${CLASSPATH}" -$JAVA ${JAVA_OPT} $@ +$JAVA ${JAVA_OPT} $@ \ No newline at end of file From 2ed264e05578dc47d29282cd99a33275936107f8 Mon Sep 17 00:00:00 2001 From: Lin Ru Date: Wed, 20 May 2020 13:05:59 +0800 Subject: [PATCH 2/4] Update Dockerfile-alpine, remove unuse files such as 'bin/*.cmd' and 'bin/mqbroker.numanode*', set 644 to bin/*.xml --- image-build/Dockerfile-alpine | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/image-build/Dockerfile-alpine b/image-build/Dockerfile-alpine index 90bbda5..70c2b33 100644 --- a/image-build/Dockerfile-alpine +++ b/image-build/Dockerfile-alpine @@ -47,7 +47,6 @@ RUN set -eux; \ curl -L https://archive.apache.org/dist/rocketmq/${ROCKETMQ_VERSION}/rocketmq-all-${ROCKETMQ_VERSION}-bin-release.zip.asc -o rocketmq.zip.asc; \ #https://www.apache.org/dist/rocketmq/KEYS curl -L https://www.apache.org/dist/rocketmq/KEYS -o KEYS; \ - \ gpg --import KEYS; \ gpg --batch --verify rocketmq.zip.asc rocketmq.zip; \ unzip rocketmq.zip; \ @@ -56,6 +55,7 @@ RUN set -eux; \ rm rocketmq.zip rocketmq.zip.asc KEYS; \ apk del .build-deps ; \ rm -rf /var/cache/apk/* ; \ + rm -rf bin/*.cmd bin/mqbroker.numanode* ; \ rm -rf /tmp/* # Copy customized scripts @@ -69,7 +69,8 @@ EXPOSE 9876 # Override customized scripts for namesrv RUN mv ${ROCKETMQ_HOME}/bin/runserver-customize.sh ${ROCKETMQ_HOME}/bin/runserver.sh \ && chmod a+x ${ROCKETMQ_HOME}/bin/runserver.sh \ - && chmod a+x ${ROCKETMQ_HOME}/bin/mqnamesrv + && chmod a+x ${ROCKETMQ_HOME}/bin/mqnamesrv \ + && chmod a-x ${ROCKETMQ_HOME}/bin/*.xml # Expose broker ports EXPOSE 10909 10911 10912 From d356f8648dd822a548f0c2aeab45d3475be4787c Mon Sep 17 00:00:00 2001 From: Lin Ru Date: Wed, 20 May 2020 15:14:43 +0800 Subject: [PATCH 3/4] Add StatefulSet templates --- README.md | 9 ++ .../statefulset/rocketmq-broker.yaml | 116 ++++++++++++++++++ .../statefulset/rocketmq-configmap.yaml | 57 +++++++++ .../statefulset/rocketmq-nameserver.yaml | 74 +++++++++++ templates/play-kubernetes-statefulset.sh | 23 ++++ 5 files changed, 279 insertions(+) create mode 100644 templates/kubernetes/statefulset/rocketmq-broker.yaml create mode 100644 templates/kubernetes/statefulset/rocketmq-configmap.yaml create mode 100644 templates/kubernetes/statefulset/rocketmq-nameserver.yaml create mode 100755 templates/play-kubernetes-statefulset.sh diff --git a/README.md b/README.md index 2e8a341..dc36307 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ cd stages/4.5.0 #### 3. Cluster on Kubernetes +##### Deployment Run: ``` @@ -81,6 +82,14 @@ cd stages/4.5.0 ``` +##### StatefulSet +Run: + +``` +cd templates +./play-kubernetes-statefulset.sh +``` + #### 4. Cluster of Dledger storage Run: (Note: This feature needs RMQ version is 4.4.0 or above) diff --git a/templates/kubernetes/statefulset/rocketmq-broker.yaml b/templates/kubernetes/statefulset/rocketmq-broker.yaml new file mode 100644 index 0000000..c79304b --- /dev/null +++ b/templates/kubernetes/statefulset/rocketmq-broker.yaml @@ -0,0 +1,116 @@ +# RocketMQ run.sh for Kubernetes +# +# Created by Lin.Ru@DaoCloud.io, 2020-05-20 + +apiVersion: v1 +kind: Service +metadata: + name: rocketmq-broker-svc + labels: + app: rocketmq-broker-svc + service: rocketmq +spec: + ports: + - port: 10909 + name: vipchannel + - port: 10911 + name: server + - port: 10912 + name: ha + clusterIP: None + selector: + app: rocketmq-broker +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: rocketmq-broker +spec: + serviceName: rocketmq-broker-svc + replicas: 4 + selector: + matchLabels: + app: rocketmq-broker + template: + metadata: + labels: + app: rocketmq-broker + service: rocketmq + spec: + initContainers: + - name: init-dir + image: busybox + command: + - /bin/sh + - -c + - mkdir -p /data/rocketmq/store && chown -v 3000. /data/rocketmq/store + volumeMounts: + - name: store + mountPath: /data/rocketmq + containers: + - name: rocketmq + image: apacherocketmq/rocketmq:4.4.0-alpine + imagePullPolicy: Always + command: + - /bin/sh + - -c + - /run.sh + env: + - name: JVM_MIN_HEAP_SIZE + value: 512M + - name: JVM_MAX_HEAP_SIZE + value: 512M + - name: JVM_NEW_HEAP_SIZE + value: 256M + - name: RMQ_RUN_MODE + value: 2m-2s-sync + - name: RMQ_NAMESERVERS + value: rocketmq-ns-0.rocketmq-ns-svc:9876;rocketmq-ns-1.rocketmq-ns-svc:9876;rocketmq-ns-2.rocketmq-ns-svc:9876 + securityContext: + runAsUser: 3000 + resources: + requests: + memory: 1Gi + cpu: 300m + limits: + memory: 1Gi + cpu: 300m + ports: + - containerPort: 10909 + name: vipchannel + - containerPort: 10911 + name: server + - containerPort: 10912 + name: ha + livenessProbe: + tcpSocket: + port: 10911 + initialDelaySeconds: 15 + timeoutSeconds: 5 + periodSeconds: 20 + readinessProbe: + tcpSocket: + port: 10911 + initialDelaySeconds: 15 + timeoutSeconds: 5 + periodSeconds: 20 + volumeMounts: + - name: store + mountPath: /data/rocketmq + - name: runsh + mountPath: /run.sh + subPath: run.sh + volumes: + - name: runsh + configMap: + defaultMode: 0755 + name: rocketmq-run + volumeClaimTemplates: + - metadata: + name: store + spec: + storageClassName: data + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 50Gi \ No newline at end of file diff --git a/templates/kubernetes/statefulset/rocketmq-configmap.yaml b/templates/kubernetes/statefulset/rocketmq-configmap.yaml new file mode 100644 index 0000000..b297645 --- /dev/null +++ b/templates/kubernetes/statefulset/rocketmq-configmap.yaml @@ -0,0 +1,57 @@ +# RocketMQ run.sh for Kubernetes +# +# Created by Lin.Ru@DaoCloud.io, 2020-05-20 + +apiVersion: v1 +kind: ConfigMap +metadata: + name: rocketmq-run + labels: + app: rocketmq-broker +data: + run.sh: | + #! /bin/sh + + # RocketMQ run.sh for Kubernetes + # + # Created by Lin.Ru@DaoCloud.io, 2020-05-20 + + config=$ROCKETMQ_HOME/conf/broker.properties + + GenProperties(){ + host_id=${HOSTNAME##*-} + result=$(( $host_id % 2 )) + + case $result in + 0) # MASTER + # Set brokerRole for '2m-2s-async' & '2m-noslave' default to 'ASYNC_MASTER' + test "$RMQ_RUN_MODE" = "2m-2s-sync" && RMQ_BROKER_ROLE="SYNC_MASTER" + RMQ_BROKER_NAME=broker-$host_id + ;; + 1) # SLAVE + RMQ_BROKER_ROLE=SLAVE + RMQ_BROKER_NAME=broker-$(($host_id - 1)) + ;; + esac + + RMQ_BROKER_ID=$host_id + + echo "Generate Broker Properties" + { + echo "brokerClusterName=${RMQ_CLUSTER_NAME:-DefaultCluster}" + echo "brokerName=${RMQ_BROKER_NAME:-broker-a}" + echo "brokerId=${RMQ_BROKER_ID:-0}" + echo "deleteWhen=${RMQ_DELETE_WHEN:-40}" + echo "fileReservedTime=${RMQ_FILE_REV_TIME:-48}" + echo "brokerRole=${RMQ_BROKER_ROLE:-ASYNC_MASTER}" + echo "flushDiskType=${RMQ_FLUSH_DISKTYPE:-ASYNC_FLUSH}" + echo "storePathRootDir=/data/rocketmq/store" + } > $config + } + + # Generate Config + GenProperties + + # Startup + echo "Start Broker with id $RMQ_BROKER_ID" + sh mqbroker -n $RMQ_NAMESERVERS -c $config \ No newline at end of file diff --git a/templates/kubernetes/statefulset/rocketmq-nameserver.yaml b/templates/kubernetes/statefulset/rocketmq-nameserver.yaml new file mode 100644 index 0000000..2df6351 --- /dev/null +++ b/templates/kubernetes/statefulset/rocketmq-nameserver.yaml @@ -0,0 +1,74 @@ +# RocketMQ run.sh for Kubernetes +# +# Created by Lin.Ru@DaoCloud.io, 2020-05-20 + +apiVersion: v1 +kind: Service +metadata: + name: rocketmq-ns-svc + labels: + app: rocketmq-ns-svc + service: rocketmq +spec: + ports: + - port: 9876 + name: server + clusterIP: None + selector: + app: rocketmq-ns +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: rocketmq-ns +spec: + serviceName: rocketmq-ns-svc + replicas: 3 + selector: + matchLabels: + app: rocketmq-ns + template: + metadata: + labels: + app: rocketmq-ns + service: rocketmq + spec: + containers: + - name: rocketmq + image: apacherocketmq/rocketmq:4.4.0-alpine + imagePullPolicy: Always + command: + - /bin/sh + - -c + - mqnamesrv + env: + - name: RMQ_MIN_HEAP_SIZE + value: 256M + - name: RMQ_MAX_HEAP_SIZE + value: 256M + - name: RMQ_NEW_HEAP_SIZE + value: 256M + securityContext: + runAsUser: 3000 + resources: + requests: + memory: 512Mi + cpu: 200m + limits: + memory: 512Mi + cpu: 200m + ports: + - containerPort: 9876 + name: server + livenessProbe: + tcpSocket: + port: 9876 + initialDelaySeconds: 15 + timeoutSeconds: 5 + periodSeconds: 20 + readinessProbe: + tcpSocket: + port: 9876 + initialDelaySeconds: 15 + timeoutSeconds: 5 + periodSeconds: 20 \ No newline at end of file diff --git a/templates/play-kubernetes-statefulset.sh b/templates/play-kubernetes-statefulset.sh new file mode 100755 index 0000000..cc1cbfd --- /dev/null +++ b/templates/play-kubernetes-statefulset.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run nameserver and broker on your Kubernetes cluster +kubectl apply -f kubernetes/statefulset/rocketmq-nameserver.yaml + +kubectl apply -f kubernetes/statefulset/rocketmq-configmap.yaml + +kubectl apply -f kubernetes/statefulset/rocketmq-broker.yaml \ No newline at end of file From 2b1d7386b33b84c3ca73ac7ccc2d83374a858e1a Mon Sep 17 00:00:00 2001 From: Lin Ru Date: Wed, 20 May 2020 16:54:57 +0800 Subject: [PATCH 4/4] delete chmod for *.xml --- image-build/Dockerfile-alpine | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/image-build/Dockerfile-alpine b/image-build/Dockerfile-alpine index 70c2b33..7a1ecdd 100644 --- a/image-build/Dockerfile-alpine +++ b/image-build/Dockerfile-alpine @@ -69,8 +69,7 @@ EXPOSE 9876 # Override customized scripts for namesrv RUN mv ${ROCKETMQ_HOME}/bin/runserver-customize.sh ${ROCKETMQ_HOME}/bin/runserver.sh \ && chmod a+x ${ROCKETMQ_HOME}/bin/runserver.sh \ - && chmod a+x ${ROCKETMQ_HOME}/bin/mqnamesrv \ - && chmod a-x ${ROCKETMQ_HOME}/bin/*.xml + && chmod a+x ${ROCKETMQ_HOME}/bin/mqnamesrv # Expose broker ports EXPOSE 10909 10911 10912