From e7108a6f7623f9ff8d0826baa2833a37ba18c1a7 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Fri, 14 May 2021 14:55:56 -0700 Subject: [PATCH 1/8] make chart work with pre-existing data-volume (pvc) --- helm/templates/_helpers.tpl | 4 +++- helm/values.yaml | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 56dcb4ad..30a5623b 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -32,9 +32,11 @@ The data volume which is used in both the Collection Manager and the Granule Ing server: {{ .Values.ingestion.granules.nfsServer }} path: {{ .Values.ingestion.granules.path }} {{- else }} + {{- if .Values.ingestion.granules.path -}} hostPath: path: {{ .Values.ingestion.granules.path }} -{{- end }} + {{- end -}} + {{- end -}} {{- end -}} {{/* diff --git a/helm/values.yaml b/helm/values.yaml index a2cba8c0..70c8bba7 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -29,7 +29,7 @@ ingestion: granuleIngester: replicas: 2 - image: nexusjpl/granule-ingester:0.1.5 + image: nexusjpl/granule-ingester:0.1.6 ## cpu refers to both request and limit cpu: 1 @@ -38,7 +38,7 @@ ingestion: memory: 1Gi collectionManager: - image: nexusjpl/collection-manager:0.1.5 + image: nexusjpl/collection-manager:0.1.6 ## cpu refers to both request and limit cpu: 0.5 From 7026407cb3dfc966f9f5f901df6c03295857a511 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Wed, 9 Jun 2021 10:40:31 -0700 Subject: [PATCH 2/8] updates for MOC deployment --- helm/templates/config-operator-rbac.yml | 24 ----------------- helm/templates/config-operator.yml | 25 ------------------ helm/templates/gitbasedconfig-crd.yml | 35 ------------------------- helm/templates/role.yaml | 21 +++++++++++++++ helm/templates/rolebinding.yaml | 17 ++++++++++++ helm/templates/spark-sa.yaml | 8 ++++++ helm/templates/spark-serviceaccount.yml | 17 ------------ helm/templates/webapp.yml | 4 +-- 8 files changed, 48 insertions(+), 103 deletions(-) delete mode 100644 helm/templates/config-operator-rbac.yml delete mode 100644 helm/templates/config-operator.yml delete mode 100644 helm/templates/gitbasedconfig-crd.yml create mode 100644 helm/templates/role.yaml create mode 100644 helm/templates/rolebinding.yaml create mode 100644 helm/templates/spark-sa.yaml delete mode 100644 helm/templates/spark-serviceaccount.yml diff --git a/helm/templates/config-operator-rbac.yml b/helm/templates/config-operator-rbac.yml deleted file mode 100644 index b2954309..00000000 --- a/helm/templates/config-operator-rbac.yml +++ /dev/null @@ -1,24 +0,0 @@ -{{ if .Values.ingestion.enabled }} -{{ if not .Values.ingestion.collections.configMap }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: config-operator - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: config-operator-role-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: - - kind: ServiceAccount - name: config-operator - namespace: {{ .Release.Namespace }} -{{ end }} -{{ end }} - diff --git a/helm/templates/config-operator.yml b/helm/templates/config-operator.yml deleted file mode 100644 index 298095e3..00000000 --- a/helm/templates/config-operator.yml +++ /dev/null @@ -1,25 +0,0 @@ -{{ if .Values.ingestion.enabled }} -{{ if not .Values.ingestion.collections.configMap }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: config-operator - labels: - app: config-operator -spec: - replicas: 1 - selector: - matchLabels: - app: config-operator - template: - metadata: - labels: - app: config-operator - spec: - serviceAccountName: config-operator - containers: - - name: config-operator - image: {{ .Values.ingestion.configOperator.image }} - imagePullPolicy: Always -{{ end }} -{{ end }} diff --git a/helm/templates/gitbasedconfig-crd.yml b/helm/templates/gitbasedconfig-crd.yml deleted file mode 100644 index 8c1dd4cf..00000000 --- a/helm/templates/gitbasedconfig-crd.yml +++ /dev/null @@ -1,35 +0,0 @@ -{{ if .Values.ingestion.collections.createCrd }} -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: gitbasedconfigs.sdap.apache.org -spec: - group: sdap.apache.org - versions: - - name: v1 - served: true - storage: true - scope: Namespaced - names: - plural: gitbasedconfigs - singular: gitbasedconfig - kind: GitBasedConfig - shortNames: - - gitcfg - validation: - openAPIV3Schema: - type: object - properties: - git-url: - type: string - git-branch: - type: string - git-token: - type: string - update-every-seconds: - type: int - local-dir: - type: string - config-map: - type: string -{{ end }} diff --git a/helm/templates/role.yaml b/helm/templates/role.yaml new file mode 100644 index 00000000..152bb23e --- /dev/null +++ b/helm/templates/role.yaml @@ -0,0 +1,21 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/instance: spark-operator + app.kubernetes.io/name: spark-operator + app.kubernetes.io/version: v1beta2-1.2.3-3.1.1 + name: sdap-spark-role +rules: + - verbs: + - '*' + apiGroups: + - '' + resources: + - pods + - verbs: + - '*' + apiGroups: + - '' + resources: + - services diff --git a/helm/templates/rolebinding.yaml b/helm/templates/rolebinding.yaml new file mode 100644 index 00000000..34ee0103 --- /dev/null +++ b/helm/templates/rolebinding.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: spark-operator + app.kubernetes.io/name: spark-operator + app.kubernetes.io/version: v1beta2-1.2.3-3.1.1 + name: spark +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: sdap-spark-role +subjects: +- kind: ServiceAccount + name: sdap-spark + namespace: $(namespace) + diff --git a/helm/templates/spark-sa.yaml b/helm/templates/spark-sa.yaml new file mode 100644 index 00000000..a977729e --- /dev/null +++ b/helm/templates/spark-sa.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sdap-spark + labels: + app.kubernetes.io/instance: spark-operator + app.kubernetes.io/name: spark-operator + app.kubernetes.io/version: v1beta2-1.2.3-3.1.1 diff --git a/helm/templates/spark-serviceaccount.yml b/helm/templates/spark-serviceaccount.yml deleted file mode 100644 index 76a47c7f..00000000 --- a/helm/templates/spark-serviceaccount.yml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: spark-serviceaccount -automountServiceAccountToken: true ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: spark-serviceaccount-role-binding -subjects: -- kind: ServiceAccount - name: spark-serviceaccount -roleRef: - kind: ClusterRole - name: admin - apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/helm/templates/webapp.yml b/helm/templates/webapp.yml index d14f8777..319a66a8 100644 --- a/helm/templates/webapp.yml +++ b/helm/templates/webapp.yml @@ -27,7 +27,7 @@ spec: {{ .Values.webapp.distributed.driver | toYaml | indent 4 }} labels: version: 2.4.4 - serviceAccount: spark-serviceaccount + serviceAccount: sdap-spark executor: {{ .Values.webapp.distributed.executor| toYaml | indent 4 }} labels: @@ -50,4 +50,4 @@ spec: port: 8083 targetPort: 8083 -{{ end }} \ No newline at end of file +{{ end }} From 3e8ebd5d6d2781988d9c696cfe1aa3b9e3c4f2d8 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Wed, 16 Jun 2021 10:00:01 -0700 Subject: [PATCH 3/8] refine spark-operator role verbs --- helm/templates/role.yaml | 19 +++++++------------ helm/templates/root-webpage.yml | 7 ++++++- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/helm/templates/role.yaml b/helm/templates/role.yaml index 152bb23e..6ba78be4 100644 --- a/helm/templates/role.yaml +++ b/helm/templates/role.yaml @@ -7,15 +7,10 @@ metadata: app.kubernetes.io/version: v1beta2-1.2.3-3.1.1 name: sdap-spark-role rules: - - verbs: - - '*' - apiGroups: - - '' - resources: - - pods - - verbs: - - '*' - apiGroups: - - '' - resources: - - services + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "list", "delete"] + - apiGroups: [""] + resources: ["services"] + verbs: ["create"] + diff --git a/helm/templates/root-webpage.yml b/helm/templates/root-webpage.yml index c55b313f..baca3a12 100644 --- a/helm/templates/root-webpage.yml +++ b/helm/templates/root-webpage.yml @@ -41,5 +41,10 @@ spec: { echo -e 'HTTP/1.1 200 OK\r\n'; } | nc -l -p 8080 done name: root-webpage + resources: + limits: + cpu: 0.5 + requests: + cpu: 0.3 restartPolicy: Always -{{ end }} \ No newline at end of file +{{ end }} From d59254e116fcde7de559bd28d9427fc39757536a Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Tue, 22 Jun 2021 11:38:17 -0700 Subject: [PATCH 4/8] wip: upgrade to run on openshift --- helm/requirements.yaml | 8 ++++---- helm/templates/_helpers.tpl | 2 +- helm/templates/webapp.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/helm/requirements.yaml b/helm/requirements.yaml index b2f68d45..0b8516fd 100644 --- a/helm/requirements.yaml +++ b/helm/requirements.yaml @@ -4,15 +4,15 @@ dependencies: repository: https://charts.helm.sh/stable condition: nginx-ingress.enabled - name: rabbitmq - version: 8.0.1 + version: 8.15.3 repository: https://charts.bitnami.com/bitnami condition: rabbitmq.enabled - name: solr - version: 1.5.2 - repository: https://charts.helm.sh/incubator + version: 0.3.3 + repository: https://charts.bitnami.com/bitnami condition: solr.enabled - name: cassandra - version: 5.5.3 + version: 7.6.0 repository: https://charts.bitnami.com/bitnami condition: cassandra.enabled diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 30a5623b..3084fa9d 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -48,7 +48,7 @@ The data volume mount which is used in both the Collection Manager and the Granu {{- end -}} {{- define "nexus.urls.solr" -}} -{{ .Values.external.solrHostAndPort | default (print "http://" .Release.Name "-solr-svc:8983") }} +{{ .Values.external.solrHostAndPort | default (print "http://" .Release.Name "-solr:8983") }} {{- end -}} {{- define "nexus.urls.zookeeper" -}} diff --git a/helm/templates/webapp.yml b/helm/templates/webapp.yml index 319a66a8..88a024a5 100644 --- a/helm/templates/webapp.yml +++ b/helm/templates/webapp.yml @@ -6,7 +6,7 @@ metadata: name: nexus-webapp spec: type: Python - pythonVersion: "2" + pythonVersion: "3" mode: cluster image: {{ .Values.webapp.distributed.image }} imagePullPolicy: Always From 431eb1dc962129c8758497f37254488259689d67 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Tue, 22 Jun 2021 11:39:33 -0700 Subject: [PATCH 5/8] wip: upgrade to run on openshift --- helm/templates/spark-serviceaccount.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 helm/templates/spark-serviceaccount.yml diff --git a/helm/templates/spark-serviceaccount.yml b/helm/templates/spark-serviceaccount.yml new file mode 100644 index 00000000..76a47c7f --- /dev/null +++ b/helm/templates/spark-serviceaccount.yml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: spark-serviceaccount +automountServiceAccountToken: true +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: spark-serviceaccount-role-binding +subjects: +- kind: ServiceAccount + name: spark-serviceaccount +roleRef: + kind: ClusterRole + name: admin + apiGroup: rbac.authorization.k8s.io \ No newline at end of file From ed654f83188bb3430caedf0d71c01c0cea1a6a96 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Wed, 7 Jul 2021 11:04:47 -0700 Subject: [PATCH 6/8] make deployment run without root-containers, upgrade chart for spark 3.1.1 --- docker/nexus-webapp/Dockerfile | 66 +++++++++------------------------ helm/templates/_helpers.tpl | 3 ++ helm/templates/rolebinding.yaml | 5 ++- helm/templates/webapp.yml | 6 +-- 4 files changed, 27 insertions(+), 53 deletions(-) diff --git a/docker/nexus-webapp/Dockerfile b/docker/nexus-webapp/Dockerfile index 68f7d873..6a0cdcc5 100644 --- a/docker/nexus-webapp/Dockerfile +++ b/docker/nexus-webapp/Dockerfile @@ -13,9 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM nexusjpl/alpine-pyspark:3.1.1 - - +#FROM nexusjpl/alpine-pyspark:3.1.1 +FROM tloubrieu/openshift-pyshark MAINTAINER Apache SDAP "dev@sdap.apache.org" @@ -23,53 +22,23 @@ ARG CONDA_VERSION="4.7.12.1" ARG CONDA_MD5="81c773ff87af5cfac79ab862942ab6b3" ARG CONDA_DIR="/opt/conda" -ENV \ - PYTHONPATH=/opt/conda/share/py4j/py4j0.10.9.2.jar \ - NEXUS_SRC=/tmp/incubator-sdap-nexus \ - PROJ_LIB=/opt/conda/lib/python3.8/site-packages/pyproj/data \ - PATH="$CONDA_DIR/bin:$PATH" \ - PYTHONDONTWRITEBYTECODE=1 \ - SPARK_HOME=/opt/spark \ - PYSPARK_DRIVER_PYTHON=/opt/conda/lib/python3.8 \ - PYSPARK_PYTHON=/opt/conda/lib/python3.8 \ - LD_LIBRARY_PATH=/usr/lib - -RUN apk add --update --no-cache \ - bzip2 \ - gcc \ - git \ - mesa-gl \ - wget \ - curl \ - which \ - python3 \ - bash==4.4.19-r1 \ - libc-dev \ - libressl2.7-libcrypto - -RUN apk upgrade musl +ENV PATH="/opt/conda/bin:${PATH}" +ARG PATH="/opt/conda/bin:${PATH}" + +RUN apt-get update && apt-get install -y wget && rm -rf /var/lib/apt/lists/* WORKDIR /tmp -RUN apk del libc6-compat -RUN apk --no-cache add wget zlib && \ - wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub && \ - wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.30-r0/glibc-2.30-r0.apk && \ - apk add glibc-2.30-r0.apk && \ - ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ - ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib && \ - ln -s /usr/lib/libgcc_s.so.1 /usr/glibc-compat/lib +RUN wget \ + https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && mkdir /root/.conda \ + && bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda \ + && rm -f Miniconda3-latest-Linux-x86_64.sh + + -COPY docker/nexus-webapp/install_conda.sh ./install_conda.sh -RUN /tmp/install_conda.sh RUN conda install python=3.8 -RUN cd /usr/lib && ln -s libcom_err.so.2 libcom_err.so.3 && \ - cd /opt/conda/lib && \ - ln -s libnetcdf.so.11 libnetcdf.so.7 && \ - ln -s libkea.so.1.4.6 libkea.so.1.4.5 && \ - ln -s libhdf5_cpp.so.12 libhdf5_cpp.so.10 && \ - ln -s libjpeg.so.9 libjpeg.so.8 # Change REBUILD_CODE if you want tell Docker not to use cached layers from this line on ARG REBUILD_CODE=0 @@ -77,6 +46,7 @@ ARG REBUILD_CODE=0 ARG APACHE_NEXUSPROTO=https://github.com/apache/incubator-sdap-nexusproto.git ARG APACHE_NEXUSPROTO_BRANCH=master +RUN apt-get update && apt-get install -y git COPY docker/nexus-webapp/install_nexusproto.sh ./install_nexusproto.sh RUN /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH @@ -97,8 +67,8 @@ RUN pip3 install -r requirements.txt RUN pip3 install cython RUN rm requirements.txt -WORKDIR /incubator-sdap-nexus +RUN chmod -R a+rwX /opt/conda/ -# Upgrade kubernetes client jar from the default version -RUN rm /opt/spark/jars/kubernetes-client-4.12.0.jar -ADD https://repo1.maven.org/maven2/io/fabric8/kubernetes-client/4.12.0/kubernetes-client-4.12.0.jar /opt/spark/jars +ENV PROJ_LIB=/opt/conda/lib/python3.8/site-packages/pyproj/data + +WORKDIR /incubator-sdap-nexus diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 3084fa9d..f1cf360b 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -35,6 +35,9 @@ The data volume which is used in both the Collection Manager and the Granule Ing {{- if .Values.ingestion.granules.path -}} hostPath: path: {{ .Values.ingestion.granules.path }} + {{- else }} + persistentVolumeClaim: + claimName: {{ .Values.ingestion.granules.pvc }} {{- end -}} {{- end -}} {{- end -}} diff --git a/helm/templates/rolebinding.yaml b/helm/templates/rolebinding.yaml index 34ee0103..9366fa48 100644 --- a/helm/templates/rolebinding.yaml +++ b/helm/templates/rolebinding.yaml @@ -5,7 +5,8 @@ metadata: app.kubernetes.io/instance: spark-operator app.kubernetes.io/name: spark-operator app.kubernetes.io/version: v1beta2-1.2.3-3.1.1 - name: spark + name: sdap-spark-rolebinding + namespace: {{ .Release.Namespace }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -13,5 +14,5 @@ roleRef: subjects: - kind: ServiceAccount name: sdap-spark - namespace: $(namespace) + namespace: {{ .Release.Namespace }} diff --git a/helm/templates/webapp.yml b/helm/templates/webapp.yml index 88a024a5..f9214072 100644 --- a/helm/templates/webapp.yml +++ b/helm/templates/webapp.yml @@ -16,7 +16,7 @@ spec: - --cassandra-username={{ include "nexus.credentials.cassandra.username" . }} - --cassandra-password={{ include "nexus.credentials.cassandra.password" . }} - --solr-host={{ include "nexus.urls.solr" . }} - sparkVersion: "2.4.4" + sparkVersion: "3.1.1" restartPolicy: type: OnFailure onFailureRetries: 10 @@ -26,12 +26,12 @@ spec: driver: {{ .Values.webapp.distributed.driver | toYaml | indent 4 }} labels: - version: 2.4.4 + version: 3.1.1 serviceAccount: sdap-spark executor: {{ .Values.webapp.distributed.executor| toYaml | indent 4 }} labels: - version: 2.4.4 + version: 3.1.1 --- From 7a698e7b7d4ff3b234e81b4310fd78a98009583a Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Sun, 11 Jul 2021 11:45:47 -0700 Subject: [PATCH 7/8] adjust resource settings --- helm/templates/_helpers.tpl | 4 ++-- helm/templates/root-webpage.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index f1cf360b..b0b69799 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -63,9 +63,9 @@ The data volume mount which is used in both the Collection Manager and the Granu {{- end -}} {{- define "nexus.credentials.cassandra.username" -}} -{{ .Values.external.cassandraUsername | default "cassandra" }} +{{ .Values.external.cassandraUsername | default (.Values.cassandra.dbUser.user) }} {{- end -}} {{- define "nexus.credentials.cassandra.password" -}} -{{ .Values.external.cassandraPassword | default "cassandra" }} +{{ .Values.external.cassandraPassword | default (.Values.cassandra.dbUser.password) }} {{- end -}} diff --git a/helm/templates/root-webpage.yml b/helm/templates/root-webpage.yml index baca3a12..f761d8af 100644 --- a/helm/templates/root-webpage.yml +++ b/helm/templates/root-webpage.yml @@ -43,8 +43,8 @@ spec: name: root-webpage resources: limits: - cpu: 0.5 + cpu: 0.1 requests: - cpu: 0.3 + cpu: 0.1 restartPolicy: Always {{ end }} From d41d5c264144cdae407d4fd8bd3cc5653e97b1ab Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Wed, 21 Jul 2021 14:43:14 -0700 Subject: [PATCH 8/8] make max thread work in granule ingester chart --- helm/templates/granule-ingester.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/templates/granule-ingester.yml b/helm/templates/granule-ingester.yml index 405edb85..9cfc7c70 100644 --- a/helm/templates/granule-ingester.yml +++ b/helm/templates/granule-ingester.yml @@ -36,7 +36,7 @@ spec: - name: ZK_HOST_AND_PORT value: {{ include "nexus.urls.zookeeper" . }} {{ if .Values.ingestion.granuleIngester.maxConcurrency }} - - name: MAX_CONCURRENCY + - name: MAX_THREADS value: "{{ .Values.ingestion.granuleIngester.maxConcurrency }}" {{ end }} {{- range $name, $value := .Values.ingestion.granules.s3.awsCredsEnvs }}