From 0f94391beb69fe1c0ede02e957c7f2704860d1b8 Mon Sep 17 00:00:00 2001 From: Dioni Date: Wed, 4 Dec 2019 12:31:28 -0300 Subject: [PATCH 01/13] Added script to get harness pod ip --- Dockerfile | 5 ++++- setHarnessHost.sh | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 setHarnessHost.sh diff --git a/Dockerfile b/Dockerfile index eca935b..d776979 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ ENV SPARK_HOME=/spark \ SPARK_PGP_KEYS="6EC5F1052DF08FF4 DCE4BFD807461E96" RUN adduser -Ds /bin/bash -h ${SPARK_HOME} spark && \ - apk add --no-cache bash tini libc6-compat linux-pam krb5 krb5-libs && \ + apk add --no-cache jq curl-dev bash tini libc6-compat linux-pam krb5 krb5-libs && \ # download dist apk add --virtual .deps --no-cache curl tar gnupg && \ cd /tmp && export GNUPGHOME=/tmp && \ @@ -38,8 +38,11 @@ RUN adduser -Ds /bin/bash -h ${SPARK_HOME} spark && \ apk --no-cache del .deps && ls -A | xargs rm -rf COPY entrypoint.sh / +COPY setHarnessHost.sh / COPY spark-env.sh ${SPARK_HOME}/conf/ +RUN chmod +x setHarnessHost.sh && echo "* * * * * root /bin/bash /setHarnessHost.sh" >> /etc/crontabs/root + WORKDIR ${SPARK_HOME}/work ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/setHarnessHost.sh b/setHarnessHost.sh new file mode 100644 index 0000000..0f227a8 --- /dev/null +++ b/setHarnessHost.sh @@ -0,0 +1,10 @@ +#!/bin/bash + HARNESS_HOST=`curl -s https://$KUBERNETES_SERVICE_HOST/api/v1/namespaces/default/pods/harness-0 --header "Authorization: Bearer $KUBE_TOKEN" --insecure | jq -r .status.podIP` + HARNESS=`cat /etc/hosts | grep harness-0` + + if [ "$HARNESS" != ""] + then + sed -i "s|.*.harness-0|$HARNESS_HOST \tharness-0|g" /etc/hosts + else + echo "$HARNESS_HOST harness-0" >> /etc/hosts + fi \ No newline at end of file From a0745eae855824c08c6d27c1d4c38b3febfed9b6 Mon Sep 17 00:00:00 2001 From: Dioni Date: Wed, 4 Dec 2019 12:49:04 -0300 Subject: [PATCH 02/13] Updated build --- .circleci/config.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 646ccfc..e4c0900 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,12 +44,11 @@ jobs: case "${CIRCLE_BRANCH}" in master|develop|devops) - docker image build -f Dockerfile \ + docker build -t actionml/spark:latest \ --build-arg release=${CIRCLE_TAG:-build} \ --build-arg version=${VERSION%%-*} \ --build-arg GIT_HASH=$SHORT_GIT_HASH --build-arg DATE_BUILD=$DATE_BUILD \ - --build-arg BRANCH=${CIRCLE_BRANCH} \ - -t actionml/spark:latest ./ + --build-arg BRANCH=${CIRCLE_BRANCH} . ;; *) ;; From d5ff16d189fa0e029e52ecf340c569a07b5addeb Mon Sep 17 00:00:00 2001 From: Dioni Date: Wed, 4 Dec 2019 13:11:34 -0300 Subject: [PATCH 03/13] Updated --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e4c0900..8351cdb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,7 +44,7 @@ jobs: case "${CIRCLE_BRANCH}" in master|develop|devops) - docker build -t actionml/spark:latest \ + docker build -t actionml/spark:latest --no-cache\ --build-arg release=${CIRCLE_TAG:-build} \ --build-arg version=${VERSION%%-*} \ --build-arg GIT_HASH=$SHORT_GIT_HASH --build-arg DATE_BUILD=$DATE_BUILD \ From 12c11125e4198dee0c687e0f5fa022582c00f2a5 Mon Sep 17 00:00:00 2001 From: Dioni Date: Wed, 4 Dec 2019 14:20:27 -0300 Subject: [PATCH 04/13] Debug --- .circleci/config.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8351cdb..aeb7187 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,6 +41,18 @@ jobs: command: | SHORT_GIT_HASH=$(echo $CIRCLE_SHA1 | cut -c -7) DATE_BUILD=$(date +'%Y-%m-%d') + + RELEASE=${CIRCLE_TAG:-build} + VERSION_TEST=${VERSION%%-*} + GIT_HASH_TEST=$SHORT_GIT_HASH + DATE_BUILD_TEST=$DATE_BUILD + + echo "RELEASE - $RELEASE" + echo "VERSION_TEST - $VERSION_TEST" + echo "GIT_HASH_TEST - $GIT_HASH_TEST" + echo "DATE_BUILD_TEST - $DATE_BUILD_TEST" + echo "CIRCLE_BRANCH - ${CIRCLE_BRANCH}" + case "${CIRCLE_BRANCH}" in master|develop|devops) From d29d9af27a5566e81f07629c1016489b66c2e051 Mon Sep 17 00:00:00 2001 From: Dioni Date: Wed, 4 Dec 2019 14:58:01 -0300 Subject: [PATCH 05/13] Updated --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index aeb7187..1c8bab7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,8 @@ jobs: name: Setup dependencies command: | latests=$(wget -qO - https://raw.githubusercontent.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/master/.latests) - version=${CIRCLE_TAG#v} + version=$(cat .latests) + # version=${CIRCLE_TAG#v} [ -n "$version" ] || version=$(echo "${latests}" | cut -f1 -d' ') printf 'export LATESTS=%s\n' ${latests} >> $BASH_ENV printf 'export VERSION=%s\n' ${version} >> $BASH_ENV From 602fad1c5858fbc694e7e70e6ce8a550f342f228 Mon Sep 17 00:00:00 2001 From: Dioni Date: Wed, 4 Dec 2019 15:02:11 -0300 Subject: [PATCH 06/13] update --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1c8bab7..c1406ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,7 @@ jobs: name: Setup dependencies command: | latests=$(wget -qO - https://raw.githubusercontent.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/master/.latests) - version=$(cat .latests) + version=$(cat latests) # version=${CIRCLE_TAG#v} [ -n "$version" ] || version=$(echo "${latests}" | cut -f1 -d' ') printf 'export LATESTS=%s\n' ${latests} >> $BASH_ENV From e131b0b2969330a3d6cef7c924941dd4d7a91870 Mon Sep 17 00:00:00 2001 From: Dioni Date: Wed, 4 Dec 2019 15:03:47 -0300 Subject: [PATCH 07/13] Update version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c1406ec..2032df2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,7 @@ jobs: name: Setup dependencies command: | latests=$(wget -qO - https://raw.githubusercontent.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/master/.latests) - version=$(cat latests) + version=2.3.3 # version=${CIRCLE_TAG#v} [ -n "$version" ] || version=$(echo "${latests}" | cut -f1 -d' ') printf 'export LATESTS=%s\n' ${latests} >> $BASH_ENV From 25d9de016067fbae5af20011279929a6008f4885 Mon Sep 17 00:00:00 2001 From: Dioni Date: Wed, 4 Dec 2019 20:14:18 -0300 Subject: [PATCH 08/13] Fix cron starting --- Dockerfile | 10 ++++++---- setHarnessHost.sh | 20 ++++++++++++++------ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index d776979..c01c57d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,9 +19,11 @@ ENV DATE_BUILD=${DATE_BUILD} ENV SPARK_HOME=/spark \ SPARK_PGP_KEYS="6EC5F1052DF08FF4 DCE4BFD807461E96" +RUN apk update +RUN apk add --no-cache curl jq RUN adduser -Ds /bin/bash -h ${SPARK_HOME} spark && \ - apk add --no-cache jq curl-dev bash tini libc6-compat linux-pam krb5 krb5-libs && \ + apk add --no-cache bash tini libc6-compat linux-pam krb5 krb5-libs && \ # download dist apk add --virtual .deps --no-cache curl tar gnupg && \ cd /tmp && export GNUPGHOME=/tmp && \ @@ -38,13 +40,13 @@ RUN adduser -Ds /bin/bash -h ${SPARK_HOME} spark && \ apk --no-cache del .deps && ls -A | xargs rm -rf COPY entrypoint.sh / -COPY setHarnessHost.sh / +COPY setHarnessHost.sh /harnesstool/ COPY spark-env.sh ${SPARK_HOME}/conf/ -RUN chmod +x setHarnessHost.sh && echo "* * * * * root /bin/bash /setHarnessHost.sh" >> /etc/crontabs/root +RUN chmod +x /harnesstool/setHarnessHost.sh && echo "* * * * * /bin/bash /harnesstool/setHarnessHost.sh" >> /etc/crontabs/root WORKDIR ${SPARK_HOME}/work ENTRYPOINT [ "/entrypoint.sh" ] # Specify the User that the actual main process will run as -USER spark:spark +# USER spark:spark diff --git a/setHarnessHost.sh b/setHarnessHost.sh index 0f227a8..56c1171 100644 --- a/setHarnessHost.sh +++ b/setHarnessHost.sh @@ -1,10 +1,18 @@ #!/bin/bash - HARNESS_HOST=`curl -s https://$KUBERNETES_SERVICE_HOST/api/v1/namespaces/default/pods/harness-0 --header "Authorization: Bearer $KUBE_TOKEN" --insecure | jq -r .status.podIP` - HARNESS=`cat /etc/hosts | grep harness-0` + + POD_IP=`curl -s https://$KUBERNETES_SERVICE_HOST/api/v1/namespaces/default/pods/harness-0 --header "Authorization: Bearer $KUBE_TOKEN" --insecure | jq -r .status.podIP` - if [ "$HARNESS" != ""] + TEMP_HOSTS=/harnesstool/newhosts + echo "$(cat /etc/hosts)" > $TEMP_HOSTS + HOSTS=`cat $TEMP_HOSTS | grep harness-0` + + if [ "$HOSTS" != "" ] then - sed -i "s|.*.harness-0|$HARNESS_HOST \tharness-0|g" /etc/hosts + sed -i "s|.*.harness-0|$POD_IP \tharness-0|g" $TEMP_HOSTS else - echo "$HARNESS_HOST harness-0" >> /etc/hosts - fi \ No newline at end of file + echo "$POD_IP harness-0" >> $TEMP_HOSTS + fi + + echo "$(cat $TEMP_HOSTS)" > /etc/hosts + + echo "$(date '+%Y-%m-%d %T') Haness-0 IP: $POD_IP" >> /harnesstool/harness-host.log \ No newline at end of file From bce6296627213b21e758d75849e6c38e93fc0625 Mon Sep 17 00:00:00 2001 From: Dioni Date: Wed, 4 Dec 2019 20:20:47 -0300 Subject: [PATCH 09/13] Fix cron starting --- Dockerfile | 11 +++++++---- setHarnessHost.sh | 20 ++++++++++++++------ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index d776979..2f1165e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,9 +19,11 @@ ENV DATE_BUILD=${DATE_BUILD} ENV SPARK_HOME=/spark \ SPARK_PGP_KEYS="6EC5F1052DF08FF4 DCE4BFD807461E96" +RUN apk update && apk add --no-cache curl jq RUN adduser -Ds /bin/bash -h ${SPARK_HOME} spark && \ - apk add --no-cache jq curl-dev bash tini libc6-compat linux-pam krb5 krb5-libs && \ + apk add --no-cache bash tini libc6-compat linux-pam krb5 krb5-libs && \ + # download dist apk add --virtual .deps --no-cache curl tar gnupg && \ cd /tmp && export GNUPGHOME=/tmp && \ @@ -30,6 +32,7 @@ RUN adduser -Ds /bin/bash -h ${SPARK_HOME} spark && \ https://archive.apache.org/dist/spark/spark-${version}/{${file},${file}.asc} && \ gpg --keyserver hkp://pool.sks-keyservers.net --recv-keys ${SPARK_PGP_KEYS} && \ gpg --batch --verify ${file}.asc ${file} && \ + # create spark directories mkdir -p ${SPARK_HOME}/work ${SPARK_HOME}/conf && chown spark:spark ${SPARK_HOME}/work && \ tar -xzf ${file} --no-same-owner --strip-components 1 && \ @@ -38,13 +41,13 @@ RUN adduser -Ds /bin/bash -h ${SPARK_HOME} spark && \ apk --no-cache del .deps && ls -A | xargs rm -rf COPY entrypoint.sh / -COPY setHarnessHost.sh / +COPY setHarnessHost.sh /harnesstool/ COPY spark-env.sh ${SPARK_HOME}/conf/ -RUN chmod +x setHarnessHost.sh && echo "* * * * * root /bin/bash /setHarnessHost.sh" >> /etc/crontabs/root +RUN chmod +x /harnesstool/setHarnessHost.sh && echo "* * * * * /bin/bash /harnesstool/setHarnessHost.sh" >> /etc/crontabs/root WORKDIR ${SPARK_HOME}/work ENTRYPOINT [ "/entrypoint.sh" ] # Specify the User that the actual main process will run as -USER spark:spark +# USER spark:spark diff --git a/setHarnessHost.sh b/setHarnessHost.sh index 0f227a8..56c1171 100644 --- a/setHarnessHost.sh +++ b/setHarnessHost.sh @@ -1,10 +1,18 @@ #!/bin/bash - HARNESS_HOST=`curl -s https://$KUBERNETES_SERVICE_HOST/api/v1/namespaces/default/pods/harness-0 --header "Authorization: Bearer $KUBE_TOKEN" --insecure | jq -r .status.podIP` - HARNESS=`cat /etc/hosts | grep harness-0` + + POD_IP=`curl -s https://$KUBERNETES_SERVICE_HOST/api/v1/namespaces/default/pods/harness-0 --header "Authorization: Bearer $KUBE_TOKEN" --insecure | jq -r .status.podIP` - if [ "$HARNESS" != ""] + TEMP_HOSTS=/harnesstool/newhosts + echo "$(cat /etc/hosts)" > $TEMP_HOSTS + HOSTS=`cat $TEMP_HOSTS | grep harness-0` + + if [ "$HOSTS" != "" ] then - sed -i "s|.*.harness-0|$HARNESS_HOST \tharness-0|g" /etc/hosts + sed -i "s|.*.harness-0|$POD_IP \tharness-0|g" $TEMP_HOSTS else - echo "$HARNESS_HOST harness-0" >> /etc/hosts - fi \ No newline at end of file + echo "$POD_IP harness-0" >> $TEMP_HOSTS + fi + + echo "$(cat $TEMP_HOSTS)" > /etc/hosts + + echo "$(date '+%Y-%m-%d %T') Haness-0 IP: $POD_IP" >> /harnesstool/harness-host.log \ No newline at end of file From 1b520f763e9f1bcfa477983ef4fd6e89536e817c Mon Sep 17 00:00:00 2001 From: Dioni Date: Thu, 5 Dec 2019 12:04:02 -0300 Subject: [PATCH 10/13] Cleannup --- .circleci/config.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2032df2..99e3374 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,13 +48,6 @@ jobs: GIT_HASH_TEST=$SHORT_GIT_HASH DATE_BUILD_TEST=$DATE_BUILD - echo "RELEASE - $RELEASE" - echo "VERSION_TEST - $VERSION_TEST" - echo "GIT_HASH_TEST - $GIT_HASH_TEST" - echo "DATE_BUILD_TEST - $DATE_BUILD_TEST" - echo "CIRCLE_BRANCH - ${CIRCLE_BRANCH}" - - case "${CIRCLE_BRANCH}" in master|develop|devops) docker build -t actionml/spark:latest --no-cache\ From 3362b190fec70056a9285fee65dae141dde4b530 Mon Sep 17 00:00:00 2001 From: Dioni Date: Thu, 5 Dec 2019 12:06:59 -0300 Subject: [PATCH 11/13] Cleannup --- .circleci/config.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2032df2..d0db1a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,17 +43,6 @@ jobs: SHORT_GIT_HASH=$(echo $CIRCLE_SHA1 | cut -c -7) DATE_BUILD=$(date +'%Y-%m-%d') - RELEASE=${CIRCLE_TAG:-build} - VERSION_TEST=${VERSION%%-*} - GIT_HASH_TEST=$SHORT_GIT_HASH - DATE_BUILD_TEST=$DATE_BUILD - - echo "RELEASE - $RELEASE" - echo "VERSION_TEST - $VERSION_TEST" - echo "GIT_HASH_TEST - $GIT_HASH_TEST" - echo "DATE_BUILD_TEST - $DATE_BUILD_TEST" - echo "CIRCLE_BRANCH - ${CIRCLE_BRANCH}" - case "${CIRCLE_BRANCH}" in master|develop|devops) From ab222e67fb6209d035fe67ee02a113b9208b06cf Mon Sep 17 00:00:00 2001 From: Dioni Date: Thu, 5 Dec 2019 12:10:09 -0300 Subject: [PATCH 12/13] Cleanup --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 43c837b..8857591 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,5 +48,3 @@ RUN chmod +x /harnesstool/setHarnessHost.sh && echo "* * * * * /bin/bash /harnes WORKDIR ${SPARK_HOME}/work ENTRYPOINT [ "/entrypoint.sh" ] -# Specify the User that the actual main process will run as -# USER spark:spark From fd5ea0ea9eeb673764d61f2010a2f718297c1d69 Mon Sep 17 00:00:00 2001 From: Dioni Date: Fri, 21 Feb 2020 10:52:44 -0300 Subject: [PATCH 13/13] Added jmx exporter agent --- Dockerfile | 3 +++ config.yaml | 7 +++++++ 2 files changed, 10 insertions(+) create mode 100644 config.yaml diff --git a/Dockerfile b/Dockerfile index 8857591..afaae21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,9 @@ COPY spark-env.sh ${SPARK_HOME}/conf/ RUN chmod +x /harnesstool/setHarnessHost.sh && echo "* * * * * /bin/bash /harnesstool/setHarnessHost.sh" >> /etc/crontabs/root +RUN curl https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.12.0/jmx_prometheus_javaagent-0.12.0.jar --create-dirs -o ${SPARK_HOME}/prometheus/jmx_prometheus_javaagent-0.12.0.jar +COPY config.yaml ${SPARK_HOME}/prometheus/conf/config.yaml + WORKDIR ${SPARK_HOME}/work ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..c7f99ec --- /dev/null +++ b/config.yaml @@ -0,0 +1,7 @@ +--- +startDelaySeconds: 0 +ssl: false +lowercaseOutputName: false +lowercaseOutputLabelNames: false +rules: + - pattern: ".*" \ No newline at end of file