Moc deployment

docker/nexus-webapp/Dockerfile

@@ -13,70 +13,40 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM nexusjpl/alpine-pyspark:3.1.1
-
-
+#FROM nexusjpl/alpine-pyspark:3.1.1
+FROM tloubrieu/openshift-pyshark
 MAINTAINER Apache SDAP "dev@sdap.apache.org"
 
 
 ARG CONDA_VERSION="4.7.12.1"
 ARG CONDA_MD5="81c773ff87af5cfac79ab862942ab6b3"
 ARG CONDA_DIR="/opt/conda"
 
-ENV  \
-    PYTHONPATH=/opt/conda/share/py4j/py4j0.10.9.2.jar \
-    NEXUS_SRC=/tmp/incubator-sdap-nexus \
-    PROJ_LIB=/opt/conda/lib/python3.8/site-packages/pyproj/data	\
-    PATH="$CONDA_DIR/bin:$PATH" \
-    PYTHONDONTWRITEBYTECODE=1 \
-    SPARK_HOME=/opt/spark \
-    PYSPARK_DRIVER_PYTHON=/opt/conda/lib/python3.8 \
-    PYSPARK_PYTHON=/opt/conda/lib/python3.8 \
-    LD_LIBRARY_PATH=/usr/lib
-
-RUN apk add --update --no-cache \
-    bzip2 \
-    gcc \
-    git \
-    mesa-gl \
-    wget \
-    curl \
-    which \
-    python3 \
-    bash==4.4.19-r1 \
-    libc-dev \
-    libressl2.7-libcrypto
-
-RUN  apk upgrade musl
+ENV PATH="/opt/conda/bin:${PATH}"
+ARG PATH="/opt/conda/bin:${PATH}"
+
+RUN apt-get update && apt-get install -y wget && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /tmp
 
-RUN apk del libc6-compat
-RUN apk --no-cache add wget zlib && \
-    wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub && \
-    wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.30-r0/glibc-2.30-r0.apk && \
-    apk add glibc-2.30-r0.apk && \
-    ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \
-    ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib && \
-    ln -s /usr/lib/libgcc_s.so.1 /usr/glibc-compat/lib
+RUN wget \
+    https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+    && mkdir /root/.conda \
+    && bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda \
+    && rm -f Miniconda3-latest-Linux-x86_64.sh
+
+
 
-COPY docker/nexus-webapp/install_conda.sh ./install_conda.sh
-RUN /tmp/install_conda.sh 
 
 RUN conda install python=3.8
-RUN cd /usr/lib && ln -s libcom_err.so.2 libcom_err.so.3 && \ 
-    cd /opt/conda/lib && \
-    ln -s libnetcdf.so.11 libnetcdf.so.7 && \
-    ln -s libkea.so.1.4.6 libkea.so.1.4.5 && \
-    ln -s libhdf5_cpp.so.12 libhdf5_cpp.so.10 && \
-    ln -s libjpeg.so.9 libjpeg.so.8
 
 # Change REBUILD_CODE if you want tell Docker not to use cached layers from this line on
 ARG REBUILD_CODE=0
 
 ARG APACHE_NEXUSPROTO=https://github.com/apache/incubator-sdap-nexusproto.git
 ARG APACHE_NEXUSPROTO_BRANCH=master
 
+RUN apt-get update && apt-get install -y git
 COPY docker/nexus-webapp/install_nexusproto.sh ./install_nexusproto.sh
 RUN /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH
 
@@ -97,8 +67,8 @@ RUN pip3 install -r requirements.txt
 RUN pip3 install cython
 RUN rm requirements.txt
 
-WORKDIR /incubator-sdap-nexus
+RUN chmod -R a+rwX /opt/conda/
 
-# Upgrade kubernetes client jar from the default version
-RUN rm /opt/spark/jars/kubernetes-client-4.12.0.jar
-ADD https://repo1.maven.org/maven2/io/fabric8/kubernetes-client/4.12.0/kubernetes-client-4.12.0.jar /opt/spark/jars
+ENV PROJ_LIB=/opt/conda/lib/python3.8/site-packages/pyproj/data
+
+WORKDIR /incubator-sdap-nexus

helm/requirements.yaml

@@ -4,15 +4,15 @@ dependencies:
     repository: https://charts.helm.sh/stable
     condition: nginx-ingress.enabled
   - name: rabbitmq
-    version: 8.0.1
+    version: 8.15.3
     repository: https://charts.bitnami.com/bitnami
     condition: rabbitmq.enabled
   - name: solr
-    version: 1.5.2
-    repository: https://charts.helm.sh/incubator
+    version: 0.3.3
+    repository: https://charts.bitnami.com/bitnami
     condition: solr.enabled
   - name: cassandra
-    version: 5.5.3
+    version: 7.6.0
     repository: https://charts.bitnami.com/bitnami
     condition: cassandra.enabled
 

helm/templates/_helpers.tpl

@@ -32,9 +32,14 @@ The data volume which is used in both the Collection Manager and the Granule Ing
     server: {{ .Values.ingestion.granules.nfsServer }}
     path: {{ .Values.ingestion.granules.path }}
   {{- else }}
+  {{- if .Values.ingestion.granules.path -}}
   hostPath:
     path: {{ .Values.ingestion.granules.path }}
-{{- end }}
+  {{- else }}
+  persistentVolumeClaim:
+    claimName: {{ .Values.ingestion.granules.pvc }}
+  {{- end -}}
+  {{- end -}}
 {{- end -}}
 
 {{/*
@@ -46,7 +51,7 @@ The data volume mount which is used in both the Collection Manager and the Granu
 {{- end -}}
 
 {{- define "nexus.urls.solr" -}}
-{{ .Values.external.solrHostAndPort | default (print "http://" .Release.Name "-solr-svc:8983") }}
+{{ .Values.external.solrHostAndPort | default (print "http://" .Release.Name "-solr:8983") }}
 {{- end -}}
 
 {{- define "nexus.urls.zookeeper" -}}
@@ -58,9 +63,9 @@ The data volume mount which is used in both the Collection Manager and the Granu
 {{- end -}}
 
 {{- define "nexus.credentials.cassandra.username" -}}
-{{ .Values.external.cassandraUsername | default "cassandra" }}
+{{ .Values.external.cassandraUsername | default (.Values.cassandra.dbUser.user) }}
 {{- end -}}
 
 {{- define "nexus.credentials.cassandra.password" -}}
-{{ .Values.external.cassandraPassword | default "cassandra" }}
+{{ .Values.external.cassandraPassword | default (.Values.cassandra.dbUser.password)  }}
 {{- end -}}

helm/templates/granule-ingester.yml

@@ -36,7 +36,7 @@ spec:
             - name: ZK_HOST_AND_PORT
               value: {{ include "nexus.urls.zookeeper" . }}
             {{ if .Values.ingestion.granuleIngester.maxConcurrency }}
-            - name: MAX_CONCURRENCY
+            - name: MAX_THREADS
               value: "{{ .Values.ingestion.granuleIngester.maxConcurrency }}"
             {{ end }}
             {{- range $name, $value := .Values.ingestion.granules.s3.awsCredsEnvs }}

helm/templates/role.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/instance: spark-operator
+    app.kubernetes.io/name: spark-operator
+    app.kubernetes.io/version: v1beta2-1.2.3-3.1.1
+  name: sdap-spark-role
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["create", "get", "list", "delete"]
+  - apiGroups: [""]
+    resources: ["services"]
+    verbs: ["create"]
+

helm/templates/rolebinding.yaml

@@ -0,0 +1,18 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/instance: spark-operator
+    app.kubernetes.io/name: spark-operator
+    app.kubernetes.io/version: v1beta2-1.2.3-3.1.1
+  name: sdap-spark-rolebinding
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: sdap-spark-role
+subjects:
+- kind: ServiceAccount
+  name: sdap-spark
+  namespace: {{ .Release.Namespace }}
+

helm/templates/root-webpage.yml

@@ -41,5 +41,10 @@ spec:
               { echo -e 'HTTP/1.1 200 OK\r\n'; } | nc -l -p  8080
             done
         name: root-webpage
+        resources:
+          limits:
+            cpu: 0.1
+          requests:
+            cpu: 0.1
       restartPolicy: Always
-{{ end }}
+{{ end }}

helm/templates/spark-sa.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: sdap-spark
+  labels:
+    app.kubernetes.io/instance: spark-operator
+    app.kubernetes.io/name: spark-operator
+    app.kubernetes.io/version: v1beta2-1.2.3-3.1.1

helm/templates/webapp.yml

@@ -6,7 +6,7 @@ metadata:
   name: nexus-webapp
 spec:
   type: Python
-  pythonVersion: "2"
+  pythonVersion: "3"
   mode: cluster
   image: {{ .Values.webapp.distributed.image }}
   imagePullPolicy: Always
@@ -16,7 +16,7 @@ spec:
     - --cassandra-username={{ include "nexus.credentials.cassandra.username" . }}
     - --cassandra-password={{ include "nexus.credentials.cassandra.password" . }}
     - --solr-host={{ include "nexus.urls.solr" . }}
-  sparkVersion: "2.4.4"
+  sparkVersion: "3.1.1"
   restartPolicy:
     type: OnFailure
     onFailureRetries: 10
@@ -26,12 +26,12 @@ spec:
   driver:
 {{ .Values.webapp.distributed.driver | toYaml | indent 4 }}
     labels:
-      version: 2.4.4
-    serviceAccount: spark-serviceaccount
+      version: 3.1.1
+    serviceAccount: sdap-spark
   executor:
 {{ .Values.webapp.distributed.executor| toYaml | indent 4 }}
     labels:
-      version: 2.4.4
+      version: 3.1.1
 
 ---
 
@@ -50,4 +50,4 @@ spec:
     port: 8083
     targetPort: 8083
 
-{{ end }}
+{{ end }}

helm/values.yaml

@@ -29,7 +29,7 @@ ingestion:
 
   granuleIngester:
     replicas: 2
-    image: nexusjpl/granule-ingester:0.1.5
+    image: nexusjpl/granule-ingester:0.1.6
 
     ## cpu refers to both request and limit
     cpu: 1
@@ -38,7 +38,7 @@ ingestion:
     memory: 1Gi
 
   collectionManager:
-    image: nexusjpl/collection-manager:0.1.5
+    image: nexusjpl/collection-manager:0.1.6
 
     ## cpu refers to both request and limit
     cpu: 0.5