diff --git a/docker/nexus-webapp/Dockerfile b/docker/nexus-webapp/Dockerfile index 68f7d873..6a0cdcc5 100644 --- a/docker/nexus-webapp/Dockerfile +++ b/docker/nexus-webapp/Dockerfile @@ -13,9 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM nexusjpl/alpine-pyspark:3.1.1 - - +#FROM nexusjpl/alpine-pyspark:3.1.1 +FROM tloubrieu/openshift-pyshark MAINTAINER Apache SDAP "dev@sdap.apache.org" @@ -23,53 +22,23 @@ ARG CONDA_VERSION="4.7.12.1" ARG CONDA_MD5="81c773ff87af5cfac79ab862942ab6b3" ARG CONDA_DIR="/opt/conda" -ENV \ - PYTHONPATH=/opt/conda/share/py4j/py4j0.10.9.2.jar \ - NEXUS_SRC=/tmp/incubator-sdap-nexus \ - PROJ_LIB=/opt/conda/lib/python3.8/site-packages/pyproj/data \ - PATH="$CONDA_DIR/bin:$PATH" \ - PYTHONDONTWRITEBYTECODE=1 \ - SPARK_HOME=/opt/spark \ - PYSPARK_DRIVER_PYTHON=/opt/conda/lib/python3.8 \ - PYSPARK_PYTHON=/opt/conda/lib/python3.8 \ - LD_LIBRARY_PATH=/usr/lib - -RUN apk add --update --no-cache \ - bzip2 \ - gcc \ - git \ - mesa-gl \ - wget \ - curl \ - which \ - python3 \ - bash==4.4.19-r1 \ - libc-dev \ - libressl2.7-libcrypto - -RUN apk upgrade musl +ENV PATH="/opt/conda/bin:${PATH}" +ARG PATH="/opt/conda/bin:${PATH}" + +RUN apt-get update && apt-get install -y wget && rm -rf /var/lib/apt/lists/* WORKDIR /tmp -RUN apk del libc6-compat -RUN apk --no-cache add wget zlib && \ - wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub && \ - wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.30-r0/glibc-2.30-r0.apk && \ - apk add glibc-2.30-r0.apk && \ - ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ - ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib && \ - ln -s /usr/lib/libgcc_s.so.1 /usr/glibc-compat/lib +RUN wget \ + https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && mkdir /root/.conda \ + && bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda \ + && rm -f Miniconda3-latest-Linux-x86_64.sh + + -COPY docker/nexus-webapp/install_conda.sh ./install_conda.sh -RUN /tmp/install_conda.sh RUN conda install python=3.8 -RUN cd /usr/lib && ln -s libcom_err.so.2 libcom_err.so.3 && \ - cd /opt/conda/lib && \ - ln -s libnetcdf.so.11 libnetcdf.so.7 && \ - ln -s libkea.so.1.4.6 libkea.so.1.4.5 && \ - ln -s libhdf5_cpp.so.12 libhdf5_cpp.so.10 && \ - ln -s libjpeg.so.9 libjpeg.so.8 # Change REBUILD_CODE if you want tell Docker not to use cached layers from this line on ARG REBUILD_CODE=0 @@ -77,6 +46,7 @@ ARG REBUILD_CODE=0 ARG APACHE_NEXUSPROTO=https://github.com/apache/incubator-sdap-nexusproto.git ARG APACHE_NEXUSPROTO_BRANCH=master +RUN apt-get update && apt-get install -y git COPY docker/nexus-webapp/install_nexusproto.sh ./install_nexusproto.sh RUN /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH @@ -97,8 +67,8 @@ RUN pip3 install -r requirements.txt RUN pip3 install cython RUN rm requirements.txt -WORKDIR /incubator-sdap-nexus +RUN chmod -R a+rwX /opt/conda/ -# Upgrade kubernetes client jar from the default version -RUN rm /opt/spark/jars/kubernetes-client-4.12.0.jar -ADD https://repo1.maven.org/maven2/io/fabric8/kubernetes-client/4.12.0/kubernetes-client-4.12.0.jar /opt/spark/jars +ENV PROJ_LIB=/opt/conda/lib/python3.8/site-packages/pyproj/data + +WORKDIR /incubator-sdap-nexus diff --git a/helm/requirements.yaml b/helm/requirements.yaml index b2f68d45..0b8516fd 100644 --- a/helm/requirements.yaml +++ b/helm/requirements.yaml @@ -4,15 +4,15 @@ dependencies: repository: https://charts.helm.sh/stable condition: nginx-ingress.enabled - name: rabbitmq - version: 8.0.1 + version: 8.15.3 repository: https://charts.bitnami.com/bitnami condition: rabbitmq.enabled - name: solr - version: 1.5.2 - repository: https://charts.helm.sh/incubator + version: 0.3.3 + repository: https://charts.bitnami.com/bitnami condition: solr.enabled - name: cassandra - version: 5.5.3 + version: 7.6.0 repository: https://charts.bitnami.com/bitnami condition: cassandra.enabled diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 56dcb4ad..b0b69799 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -32,9 +32,14 @@ The data volume which is used in both the Collection Manager and the Granule Ing server: {{ .Values.ingestion.granules.nfsServer }} path: {{ .Values.ingestion.granules.path }} {{- else }} + {{- if .Values.ingestion.granules.path -}} hostPath: path: {{ .Values.ingestion.granules.path }} -{{- end }} + {{- else }} + persistentVolumeClaim: + claimName: {{ .Values.ingestion.granules.pvc }} + {{- end -}} + {{- end -}} {{- end -}} {{/* @@ -46,7 +51,7 @@ The data volume mount which is used in both the Collection Manager and the Granu {{- end -}} {{- define "nexus.urls.solr" -}} -{{ .Values.external.solrHostAndPort | default (print "http://" .Release.Name "-solr-svc:8983") }} +{{ .Values.external.solrHostAndPort | default (print "http://" .Release.Name "-solr:8983") }} {{- end -}} {{- define "nexus.urls.zookeeper" -}} @@ -58,9 +63,9 @@ The data volume mount which is used in both the Collection Manager and the Granu {{- end -}} {{- define "nexus.credentials.cassandra.username" -}} -{{ .Values.external.cassandraUsername | default "cassandra" }} +{{ .Values.external.cassandraUsername | default (.Values.cassandra.dbUser.user) }} {{- end -}} {{- define "nexus.credentials.cassandra.password" -}} -{{ .Values.external.cassandraPassword | default "cassandra" }} +{{ .Values.external.cassandraPassword | default (.Values.cassandra.dbUser.password) }} {{- end -}} diff --git a/helm/templates/config-operator-rbac.yml b/helm/templates/config-operator-rbac.yml deleted file mode 100644 index b2954309..00000000 --- a/helm/templates/config-operator-rbac.yml +++ /dev/null @@ -1,24 +0,0 @@ -{{ if .Values.ingestion.enabled }} -{{ if not .Values.ingestion.collections.configMap }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: config-operator - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: config-operator-role-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: - - kind: ServiceAccount - name: config-operator - namespace: {{ .Release.Namespace }} -{{ end }} -{{ end }} - diff --git a/helm/templates/config-operator.yml b/helm/templates/config-operator.yml deleted file mode 100644 index 298095e3..00000000 --- a/helm/templates/config-operator.yml +++ /dev/null @@ -1,25 +0,0 @@ -{{ if .Values.ingestion.enabled }} -{{ if not .Values.ingestion.collections.configMap }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: config-operator - labels: - app: config-operator -spec: - replicas: 1 - selector: - matchLabels: - app: config-operator - template: - metadata: - labels: - app: config-operator - spec: - serviceAccountName: config-operator - containers: - - name: config-operator - image: {{ .Values.ingestion.configOperator.image }} - imagePullPolicy: Always -{{ end }} -{{ end }} diff --git a/helm/templates/gitbasedconfig-crd.yml b/helm/templates/gitbasedconfig-crd.yml deleted file mode 100644 index 8c1dd4cf..00000000 --- a/helm/templates/gitbasedconfig-crd.yml +++ /dev/null @@ -1,35 +0,0 @@ -{{ if .Values.ingestion.collections.createCrd }} -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: gitbasedconfigs.sdap.apache.org -spec: - group: sdap.apache.org - versions: - - name: v1 - served: true - storage: true - scope: Namespaced - names: - plural: gitbasedconfigs - singular: gitbasedconfig - kind: GitBasedConfig - shortNames: - - gitcfg - validation: - openAPIV3Schema: - type: object - properties: - git-url: - type: string - git-branch: - type: string - git-token: - type: string - update-every-seconds: - type: int - local-dir: - type: string - config-map: - type: string -{{ end }} diff --git a/helm/templates/granule-ingester.yml b/helm/templates/granule-ingester.yml index 405edb85..9cfc7c70 100644 --- a/helm/templates/granule-ingester.yml +++ b/helm/templates/granule-ingester.yml @@ -36,7 +36,7 @@ spec: - name: ZK_HOST_AND_PORT value: {{ include "nexus.urls.zookeeper" . }} {{ if .Values.ingestion.granuleIngester.maxConcurrency }} - - name: MAX_CONCURRENCY + - name: MAX_THREADS value: "{{ .Values.ingestion.granuleIngester.maxConcurrency }}" {{ end }} {{- range $name, $value := .Values.ingestion.granules.s3.awsCredsEnvs }} diff --git a/helm/templates/role.yaml b/helm/templates/role.yaml new file mode 100644 index 00000000..6ba78be4 --- /dev/null +++ b/helm/templates/role.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/instance: spark-operator + app.kubernetes.io/name: spark-operator + app.kubernetes.io/version: v1beta2-1.2.3-3.1.1 + name: sdap-spark-role +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "list", "delete"] + - apiGroups: [""] + resources: ["services"] + verbs: ["create"] + diff --git a/helm/templates/rolebinding.yaml b/helm/templates/rolebinding.yaml new file mode 100644 index 00000000..9366fa48 --- /dev/null +++ b/helm/templates/rolebinding.yaml @@ -0,0 +1,18 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: spark-operator + app.kubernetes.io/name: spark-operator + app.kubernetes.io/version: v1beta2-1.2.3-3.1.1 + name: sdap-spark-rolebinding + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: sdap-spark-role +subjects: +- kind: ServiceAccount + name: sdap-spark + namespace: {{ .Release.Namespace }} + diff --git a/helm/templates/root-webpage.yml b/helm/templates/root-webpage.yml index c55b313f..f761d8af 100644 --- a/helm/templates/root-webpage.yml +++ b/helm/templates/root-webpage.yml @@ -41,5 +41,10 @@ spec: { echo -e 'HTTP/1.1 200 OK\r\n'; } | nc -l -p 8080 done name: root-webpage + resources: + limits: + cpu: 0.1 + requests: + cpu: 0.1 restartPolicy: Always -{{ end }} \ No newline at end of file +{{ end }} diff --git a/helm/templates/spark-sa.yaml b/helm/templates/spark-sa.yaml new file mode 100644 index 00000000..a977729e --- /dev/null +++ b/helm/templates/spark-sa.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sdap-spark + labels: + app.kubernetes.io/instance: spark-operator + app.kubernetes.io/name: spark-operator + app.kubernetes.io/version: v1beta2-1.2.3-3.1.1 diff --git a/helm/templates/webapp.yml b/helm/templates/webapp.yml index d14f8777..f9214072 100644 --- a/helm/templates/webapp.yml +++ b/helm/templates/webapp.yml @@ -6,7 +6,7 @@ metadata: name: nexus-webapp spec: type: Python - pythonVersion: "2" + pythonVersion: "3" mode: cluster image: {{ .Values.webapp.distributed.image }} imagePullPolicy: Always @@ -16,7 +16,7 @@ spec: - --cassandra-username={{ include "nexus.credentials.cassandra.username" . }} - --cassandra-password={{ include "nexus.credentials.cassandra.password" . }} - --solr-host={{ include "nexus.urls.solr" . }} - sparkVersion: "2.4.4" + sparkVersion: "3.1.1" restartPolicy: type: OnFailure onFailureRetries: 10 @@ -26,12 +26,12 @@ spec: driver: {{ .Values.webapp.distributed.driver | toYaml | indent 4 }} labels: - version: 2.4.4 - serviceAccount: spark-serviceaccount + version: 3.1.1 + serviceAccount: sdap-spark executor: {{ .Values.webapp.distributed.executor| toYaml | indent 4 }} labels: - version: 2.4.4 + version: 3.1.1 --- @@ -50,4 +50,4 @@ spec: port: 8083 targetPort: 8083 -{{ end }} \ No newline at end of file +{{ end }} diff --git a/helm/values.yaml b/helm/values.yaml index a2cba8c0..70c8bba7 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -29,7 +29,7 @@ ingestion: granuleIngester: replicas: 2 - image: nexusjpl/granule-ingester:0.1.5 + image: nexusjpl/granule-ingester:0.1.6 ## cpu refers to both request and limit cpu: 1 @@ -38,7 +38,7 @@ ingestion: memory: 1Gi collectionManager: - image: nexusjpl/collection-manager:0.1.5 + image: nexusjpl/collection-manager:0.1.6 ## cpu refers to both request and limit cpu: 0.5