diff --git a/core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/runtimeRoutesModels.scala b/core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/runtimeRoutesModels.scala index 7ff4db118f..b802407de2 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/runtimeRoutesModels.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/runtimeRoutesModels.scala @@ -114,7 +114,6 @@ final case class GetRuntimeResponse(id: Long, defaultClientId: Option[String], clusterImages: Set[RuntimeImage], scopes: Set[String], - welderEnabled: Boolean, patchInProgress: Boolean, customClusterEnvironmentVariables: Map[String, String], diskConfig: Option[DiskConfig] @@ -142,7 +141,6 @@ object GetRuntimeResponse { runtime.defaultClientId, runtime.runtimeImages, runtime.scopes, - runtime.welderEnabled, runtime.patchInProgress, runtime.customEnvironmentVariables, diskConfig diff --git a/core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/runtimeModels.scala b/core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/runtimeModels.scala index adde889fbe..de680816ba 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/runtimeModels.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/runtimeModels.scala @@ -41,7 +41,6 @@ final case class Runtime(id: Long, allowStop: Boolean, runtimeImages: Set[RuntimeImage], scopes: Set[String], - welderEnabled: Boolean, customEnvironmentVariables: Map[String, String], runtimeConfigId: RuntimeConfigId, patchInProgress: Boolean diff --git a/core/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/RuntimeRoutesTestJsonCodec.scala b/core/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/RuntimeRoutesTestJsonCodec.scala index 19b4b63624..1f3fe3e3a3 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/RuntimeRoutesTestJsonCodec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/RuntimeRoutesTestJsonCodec.scala @@ -197,7 +197,6 @@ object RuntimeRoutesTestJsonCodec { defaultClientId, clusterImages, scopes, - true, false, Map.empty, None diff --git a/http/src/main/resources/init-resources/gce-init.sh b/http/src/main/resources/init-resources/gce-init.sh index 50196db598..142d975e7b 100644 --- a/http/src/main/resources/init-resources/gce-init.sh +++ b/http/src/main/resources/init-resources/gce-init.sh @@ -31,7 +31,6 @@ export MEM_LIMIT=$(memLimit) export SHM_SIZE=$(shmSize) export WELDER_MEM_LIMIT=$(welderMemLimit) export PROXY_SERVER_HOST_NAME=$(proxyServerHostName) -export WELDER_ENABLED=$(welderEnabled) export NOTEBOOKS_DIR=$(notebooksDir) START_USER_SCRIPT_URI=$(startUserScriptUri) @@ -40,7 +39,8 @@ START_USER_SCRIPT_OUTPUT_URI=$(startUserScriptOutputUri) IS_GCE_FORMATTED=$(isGceFormatted) # Needs to be in sync with terra-docker container JUPYTER_HOME=/etc/jupyter -JUPYTER_SCRIPTS=$JUPYTER_HOME/scripts +JUPYTER_EXTENSIONS=$JUPYTER_HOME/extensions +JUPYTER_SCRIPTS=$JUPYTER_EXTENSIONS/scripts JUPYTER_USER_HOME=$(jupyterHomeDirectory) RSTUDIO_SCRIPTS=/etc/rstudio/scripts SERVER_CRT=$(proxyServerCrt) @@ -311,7 +311,7 @@ log 'Starting up the Jupyter...' # and wouldn't be remedied by retrying COMPOSE_FILES=(-f ${DOCKER_COMPOSE_FILES_DIRECTORY}/`basename ${PROXY_DOCKER_COMPOSE}`) cat ${DOCKER_COMPOSE_FILES_DIRECTORY}/`basename ${PROXY_DOCKER_COMPOSE}` -if [ ! -z "$WELDER_DOCKER_IMAGE" ] && [ "$WELDER_ENABLED" == "true" ] ; then +if [ ! -z "$WELDER_DOCKER_IMAGE" ]; then COMPOSE_FILES+=(-f ${DOCKER_COMPOSE_FILES_DIRECTORY}/`basename ${WELDER_DOCKER_COMPOSE}`) cat ${DOCKER_COMPOSE_FILES_DIRECTORY}/`basename ${WELDER_DOCKER_COMPOSE}` fi @@ -351,7 +351,6 @@ JUPYTER_DOCKER_IMAGE=${JUPYTER_DOCKER_IMAGE} NOTEBOOKS_DIR=${NOTEBOOKS_DIR} OWNER_EMAIL=${OWNER_EMAIL} PET_SA_EMAIL=${PET_SA_EMAIL} -WELDER_ENABLED=${WELDER_ENABLED} MEM_LIMIT=${MEM_LIMIT} SHM_SIZE=${SHM_SIZE} WELDER_SERVER_NAME=${WELDER_SERVER_NAME} @@ -412,11 +411,9 @@ if [ ! -z "$JUPYTER_DOCKER_IMAGE" ] ; then # See: https://github.com/DataBiosphere/leonardo/pull/4465/files if [ ! "$JUPYTER_USER_HOME" = "/home/jupyter" ] ; then # TODO: Remove once we stop supporting non AI notebooks based images - log 'Installing Jupyter kernelspecs...(Remove once we stop supporting non AI notebooks based images)' - KERNELSPEC_HOME=/usr/local/share/jupyter/kernels - + log 'Installing Jupyter kernelspecs' # Install kernelspecs inside the Jupyter container - retry 3 docker exec -u root ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/kernel/kernelspec.sh ${JUPYTER_SCRIPTS}/kernel ${KERNELSPEC_HOME} + retry 3 docker exec -u root ${JUPYTER_SERVER_NAME} ${JUPYTER_HOME}/kernel/kernelspec.sh ${JUPYTER_SCRIPTS}/kernel ${KERNELSPEC_HOME} fi # Install notebook.json which is used to populate Jupyter.notebook.config in JavaScript extensions. @@ -447,14 +444,14 @@ if [ ! -z "$JUPYTER_DOCKER_IMAGE" ] ; then $GSUTIL_CMD cp $ext /var JUPYTER_EXTENSION_ARCHIVE=`basename $ext` docker cp /var/${JUPYTER_EXTENSION_ARCHIVE} ${JUPYTER_SERVER_NAME}:${JUPYTER_HOME}/${JUPYTER_EXTENSION_ARCHIVE} - retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/extension/jupyter_install_notebook_extension.sh ${JUPYTER_HOME}/${JUPYTER_EXTENSION_ARCHIVE} + retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/jupyter_install_notebook_extension.sh ${JUPYTER_HOME}/${JUPYTER_EXTENSION_ARCHIVE} elif [[ $ext == 'http://'* || $ext == 'https://'* ]]; then JUPYTER_EXTENSION_FILE=`basename $ext` curl $ext -o /var/${JUPYTER_EXTENSION_FILE} docker cp /var/${JUPYTER_EXTENSION_FILE} ${JUPYTER_SERVER_NAME}:${JUPYTER_HOME}/${JUPYTER_EXTENSION_FILE} - retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/extension/jupyter_install_notebook_extension.sh ${JUPYTER_HOME}/${JUPYTER_EXTENSION_FILE} + retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/jupyter_install_notebook_extension.sh ${JUPYTER_HOME}/${JUPYTER_EXTENSION_FILE} else - retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/extension/jupyter_pip_install_notebook_extension.sh $ext + retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/jupyter_pip_install_notebook_extension.sh $ext fi done fi @@ -468,9 +465,9 @@ if [ ! -z "$JUPYTER_DOCKER_IMAGE" ] ; then $GSUTIL_CMD cp $ext /var JUPYTER_EXTENSION_ARCHIVE=`basename $ext` docker cp /var/${JUPYTER_EXTENSION_ARCHIVE} ${JUPYTER_SERVER_NAME}:${JUPYTER_HOME}/${JUPYTER_EXTENSION_ARCHIVE} - retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/extension/jupyter_install_server_extension.sh ${JUPYTER_HOME}/${JUPYTER_EXTENSION_ARCHIVE} + retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/jupyter_install_server_extension.sh ${JUPYTER_HOME}/${JUPYTER_EXTENSION_ARCHIVE} else - retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/extension/jupyter_pip_install_server_extension.sh $ext + retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/jupyter_pip_install_server_extension.sh $ext fi done fi @@ -485,9 +482,9 @@ if [ ! -z "$JUPYTER_DOCKER_IMAGE" ] ; then $GSUTIL_CMD cp $ext /var JUPYTER_EXTENSION_ARCHIVE=`basename $ext` docker cp /var/${JUPYTER_EXTENSION_ARCHIVE} ${JUPYTER_SERVER_NAME}:${JUPYTER_HOME}/${JUPYTER_EXTENSION_ARCHIVE} - retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/extension/jupyter_install_combined_extension.sh ${JUPYTER_EXTENSION_ARCHIVE} + retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/jupyter_install_combined_extension.sh ${JUPYTER_EXTENSION_ARCHIVE} else - retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/extension/jupyter_pip_install_combined_extension.sh $ext + retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/jupyter_pip_install_combined_extension.sh $ext fi done fi @@ -503,14 +500,14 @@ if [ ! -z "$JUPYTER_DOCKER_IMAGE" ] ; then $GSUTIL_CMD cp -r $ext /var JUPYTER_EXTENSION_ARCHIVE=`basename $ext` docker cp /var/${JUPYTER_EXTENSION_ARCHIVE} ${JUPYTER_SERVER_NAME}:${JUPYTER_HOME}/${JUPYTER_EXTENSION_ARCHIVE} - retry 3 docker exec -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/extension/jupyter_install_lab_extension.sh ${JUPYTER_HOME}/${JUPYTER_EXTENSION_ARCHIVE} + retry 3 docker exec -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/jupyter_install_lab_extension.sh ${JUPYTER_HOME}/${JUPYTER_EXTENSION_ARCHIVE} elif [[ $ext == 'http://'* || $ext == 'https://'* ]]; then JUPYTER_EXTENSION_FILE=`basename $ext` curl $ext -o /var/${JUPYTER_EXTENSION_FILE} docker cp /var/${JUPYTER_EXTENSION_FILE} ${JUPYTER_SERVER_NAME}:${JUPYTER_HOME}/${JUPYTER_EXTENSION_FILE} - retry 3 docker exec -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/extension/jupyter_install_lab_extension.sh ${JUPYTER_HOME}/${JUPYTER_EXTENSION_FILE} + retry 3 docker exec -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/jupyter_install_lab_extension.sh ${JUPYTER_HOME}/${JUPYTER_EXTENSION_FILE} else - retry 3 docker exec -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/extension/jupyter_install_lab_extension.sh $ext + retry 3 docker exec -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/jupyter_install_lab_extension.sh $ext fi done fi @@ -524,24 +521,26 @@ if [ ! -z "$JUPYTER_DOCKER_IMAGE" ] ; then docker exec $JUPYTER_SERVER_NAME /bin/bash -c "R -e '1+1'" || true # For older jupyter images, jupyter_delocalize.py is using 127.0.0.1 as welder's url, which won't work now that we're no longer using `network_mode: host` for GCE VMs - docker exec $JUPYTER_SERVER_NAME /bin/bash -c "sed -i 's/127.0.0.1/welder/g' /etc/jupyter/custom/jupyter_delocalize.py" + docker exec $JUPYTER_SERVER_NAME /bin/bash -c "sed -i 's/127.0.0.1/welder/g' $JUPYTER_EXTENSIONS/jupyter_delocalize.py" + + log 'Wget the gitignore_global file, set gitignore in Git Config' # Copy gitignore into jupyter container (ask AOU?) - docker exec $JUPYTER_SERVER_NAME /bin/bash -c "wget -N https://raw.githubusercontent.com/DataBiosphere/terra-docker/045a139dbac19fbf2b8c4080b8bc7fff7fc8b177/terra-jupyter-aou/gitignore_global" + docker exec $JUPYTER_SERVER_NAME /bin/bash -c "wget -N https://raw.githubusercontent.com/DataBiosphere/terra-docker/045a139dbac19fbf2b8c4080b8bc7fff7fc8b177/terra-jupyter-aou/gitignore_global \ + && git config --global core.excludesfile $JUPYTER_USER_HOME/gitignore_global" + + docker exec $JUPYTER_SERVER_NAME /bin/bash -c "whoami" - # Install nbstripout and set gitignore in Git Config (ask AOU?) - docker exec $JUPYTER_SERVER_NAME /bin/bash -c "pip install nbstripout \ - && nbstripout --install --global \ - && git config --global core.excludesfile $JUPYTER_USER_HOME/gitignore_global" + docker exec $JUPYTER_SERVER_NAME /bin/bash -c "ls -l $JUPYTER_EXTENSIONS" # Starts the locking logic (used for AOU). google_sign_in.js is likely not used anymore - docker exec -u 0 $JUPYTER_SERVER_NAME /bin/bash -c "$JUPYTER_HOME/scripts/extension/install_jupyter_contrib_nbextensions.sh \ + docker exec -u 0 $JUPYTER_SERVER_NAME /bin/bash -c "$JUPYTER_SCRIPTS/install_jupyter_contrib_nbextensions.sh \ && mkdir -p $JUPYTER_USER_HOME/.jupyter/custom/ \ - && cp $JUPYTER_HOME/custom/google_sign_in.js $JUPYTER_USER_HOME/.jupyter/custom/ \ - && ls -la $JUPYTER_HOME/custom/extension_entry_jupyter.js \ - && cp $JUPYTER_HOME/custom/extension_entry_jupyter.js $JUPYTER_USER_HOME/.jupyter/custom/custom.js \ - && cp $JUPYTER_HOME/custom/safe-mode.js $JUPYTER_USER_HOME/.jupyter/custom/ \ - && cp $JUPYTER_HOME/custom/edit-mode.js $JUPYTER_USER_HOME/.jupyter/custom/ \ + && cp $JUPYTER_EXTENSIONS/google_sign_in.js $JUPYTER_USER_HOME/.jupyter/custom/ \ + && ls -la $JUPYTER_EXTENSIONS/extension_entry_jupyter.js \ + && cp $JUPYTER_EXTENSIONS/extension_entry_jupyter.js $JUPYTER_USER_HOME/.jupyter/custom/custom.js \ + && cp $JUPYTER_EXTENSIONS/safe-mode.js $JUPYTER_USER_HOME/.jupyter/custom/ \ + && cp $JUPYTER_EXTENSIONS/edit-mode.js $JUPYTER_USER_HOME/.jupyter/custom/ \ && mkdir -p $JUPYTER_HOME/nbconfig" # In new jupyter images, we should update jupyter_notebook_config.py in terra-docker. @@ -566,7 +565,7 @@ if [ ! -z "$JUPYTER_DOCKER_IMAGE" ] ; then STEP_TIMINGS+=($(date +%s)) log 'Starting Jupyter Notebook...' - retry 3 docker exec -d $JUPYTER_SERVER_NAME /bin/bash -c "${JUPYTER_SCRIPTS}/run-jupyter.sh ${NOTEBOOKS_DIR}" + retry 3 docker exec -d $JUPYTER_SERVER_NAME /bin/bash -c "${JUPYTER_HOME}/run-jupyter.sh ${NOTEBOOKS_DIR}" # done start Jupyter STEP_TIMINGS+=($(date +%s)) diff --git a/http/src/main/resources/init-resources/init-actions.sh b/http/src/main/resources/init-resources/init-actions.sh index a48d0f2d99..e9337fb38a 100644 --- a/http/src/main/resources/init-resources/init-actions.sh +++ b/http/src/main/resources/init-resources/init-actions.sh @@ -159,7 +159,8 @@ if [[ "${ROLE}" == 'Master' ]]; then bash add-google-cloud-ops-agent-repo.sh --also-install JUPYTER_HOME=/etc/jupyter - JUPYTER_SCRIPTS=${JUPYTER_HOME}/scripts + JUPYTER_EXTENSIONS=$JUPYTER_HOME/extensions + JUPYTER_SCRIPTS=$JUPYTER_EXTENSIONS/scripts KERNELSPEC_HOME=/usr/local/share/jupyter/kernels # Set variables @@ -185,7 +186,6 @@ if [[ "${ROLE}" == 'Master' ]]; then export PROXY_DOCKER_IMAGE=$(proxyDockerImage) export WELDER_DOCKER_IMAGE=$(welderDockerImage) export CRYPTO_DETECTOR_DOCKER_IMAGE=$(cryptoDetectorDockerImage) - export WELDER_ENABLED=$(welderEnabled) export NOTEBOOKS_DIR=$(notebooksDir) export MEM_LIMIT=$(memLimit) export SHM_SIZE=$(shmSize) @@ -291,7 +291,7 @@ EOF cat /etc/`basename ${PROXY_DOCKER_COMPOSE}` - if [ ! -z ${WELDER_DOCKER_IMAGE} ] && [ "${WELDER_ENABLED}" == "true" ] ; then + if [ ! -z ${WELDER_DOCKER_IMAGE} ]; then COMPOSE_FILES+=(-f /etc/`basename ${WELDER_DOCKER_COMPOSE}`) cat /etc/`basename ${WELDER_DOCKER_COMPOSE}` fi @@ -360,7 +360,7 @@ EOF gsutil cp $ext /etc JUPYTER_EXTENSION_ARCHIVE=`basename $ext` docker cp /etc/${JUPYTER_EXTENSION_ARCHIVE} ${JUPYTER_SERVER_NAME}:${JUPYTER_HOME}/${JUPYTER_EXTENSION_ARCHIVE} - retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_SCRIPTS}/extension/jupyter_install_notebook_extension.sh ${JUPYTER_HOME}/${JUPYTER_EXTENSION_ARCHIVE} + retry 3 docker exec -u root -e PIP_USER=false ${JUPYTER_SERVER_NAME} ${JUPYTER_EXTENSIONS}/jupyter_install_notebook_extension.sh ${JUPYTER_HOME}/${JUPYTER_EXTENSION_ARCHIVE} elif [[ $ext == 'http://'* || $ext == 'https://'* ]]; then JUPYTER_EXTENSION_FILE=`basename $ext` curl $ext -o /etc/${JUPYTER_EXTENSION_FILE} @@ -459,7 +459,7 @@ EOF # jupyter_delocalize.py now assumes welder's url is `http://welder:8080`, but on dataproc, we're still using host network # A better to do this might be to take welder host as an argument to the script - docker exec $JUPYTER_SERVER_NAME /bin/bash -c "sed -i 's/http:\/\/welder/http:\/\/127.0.0.1/g' /etc/jupyter/custom/jupyter_delocalize.py" + docker exec $JUPYTER_SERVER_NAME /bin/bash -c "sed -i 's/http:\/\/welder/http:\/\/127.0.0.1/g' ${JUPYTER_EXTENSIONS}/jupyter_delocalize.py" # In new jupyter images, we should update jupyter_notebook_config.py in terra-docker. # This is to make it so that older images will still work after we change notebooks location to home dir @@ -474,17 +474,17 @@ EOF && git config --global core.excludesfile $JUPYTER_USER_HOME/gitignore_global" # Install the custom jupyter extensions needed to lock notebooks into edit or safe modes (required by AOU) - docker exec -u 0 $JUPYTER_SERVER_NAME /bin/bash -c "$JUPYTER_HOME/scripts/extension/install_jupyter_contrib_nbextensions.sh \ - && mkdir -p $JUPYTER_USER_HOME/.jupyter/custom/ \ - && cp $JUPYTER_HOME/custom/google_sign_in.js $JUPYTER_USER_HOME/.jupyter/custom/ \ - && ls -la $JUPYTER_HOME/custom/extension_entry_jupyter.js \ - && cp $JUPYTER_HOME/custom/extension_entry_jupyter.js $JUPYTER_USER_HOME/.jupyter/custom/custom.js \ - && cp $JUPYTER_HOME/custom/safe-mode.js $JUPYTER_USER_HOME/.jupyter/custom/ \ - && cp $JUPYTER_HOME/custom/edit-mode.js $JUPYTER_USER_HOME/.jupyter/custom/ \ - && mkdir -p $JUPYTER_HOME/nbconfig" +# docker exec -u 0 $JUPYTER_SERVER_NAME /bin/bash -c "$JUPYTER_SCRIPTS/install_jupyter_contrib_nbextensions.sh \ +# && mkdir -p $JUPYTER_USER_HOME/.jupyter/custom/ \ +# && cp $JUPYTER_EXTENSIONS/google_sign_in.js $JUPYTER_USER_HOME/.jupyter/custom/ \ +# && ls -la $JUPYTER_EXTENSIONS/extension_entry_jupyter.js \ +# && cp $JUPYTER_EXTENSIONS/extension_entry_jupyter.js $JUPYTER_USER_HOME/.jupyter/custom/custom.js \ +# && cp $JUPYTER_EXTENSIONS/safe-mode.js $JUPYTER_USER_HOME/.jupyter/custom/ \ +# && cp $JUPYTER_EXTENSIONS/edit-mode.js $JUPYTER_USER_HOME/.jupyter/custom/ \ +# && mkdir -p $JUPYTER_HOME/nbconfig" log 'Starting Jupyter Notebook...' - retry 3 docker exec -d ${JUPYTER_SERVER_NAME} /bin/bash -c "${JUPYTER_SCRIPTS}/run-jupyter.sh ${NOTEBOOKS_DIR}" + retry 3 docker exec -d ${JUPYTER_SERVER_NAME} /bin/bash -c "${JUPYTER_HOME}/custom/run-jupyter.sh ${NOTEBOOKS_DIR}" STEP_TIMINGS+=($(date +%s)) fi diff --git a/http/src/main/resources/init-resources/jupyter-docker-compose-gce.yaml b/http/src/main/resources/init-resources/jupyter-docker-compose-gce.yaml index 375e4209ee..2d8ee7fa0f 100644 --- a/http/src/main/resources/init-resources/jupyter-docker-compose-gce.yaml +++ b/http/src/main/resources/init-resources/jupyter-docker-compose-gce.yaml @@ -27,14 +27,13 @@ services: OWNER_EMAIL: "${OWNER_EMAIL}" PET_SA_EMAIL: "${PET_SA_EMAIL}" # Value must be the string "true" to enable. - WELDER_ENABLED: "${WELDER_ENABLED}" NOTEBOOKS_DIR: "${NOTEBOOKS_DIR}" PIP_USER: "true" R_LIBS: "${NOTEBOOKS_DIR}/packages" # The next two lines aren't great. But they're for updating PYTHONPATH, PATH in older than (inclusive) us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.0.2 # We should remove the two lines once we no longer support older images. In the meantime, we need to be careful updating Jupyter base images. PYTHONPATH: "/etc/jupyter/custom:/usr/lib/spark/python:${NOTEBOOKS_DIR}/packages" - PATH: "/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${NOTEBOOKS_DIR}/.local/bin:${NOTEBOOKS_DIR}/packages/bin" + PATH: "/opt/conda/bin:/opt/uv/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${NOTEBOOKS_DIR}/.local/bin:${NOTEBOOKS_DIR}/packages/bin" env_file: - /var/custom_env_vars.env # See https://docs.docker.com/engine/reference/run/#user-memory-constraints diff --git a/http/src/main/resources/init-resources/jupyter-docker-compose.yaml b/http/src/main/resources/init-resources/jupyter-docker-compose.yaml index 34f538e706..9a18ada045 100644 --- a/http/src/main/resources/init-resources/jupyter-docker-compose.yaml +++ b/http/src/main/resources/init-resources/jupyter-docker-compose.yaml @@ -44,7 +44,6 @@ services: OWNER_EMAIL: "${OWNER_EMAIL}" PET_SA_EMAIL: "${PET_SA_EMAIL}" # Value must be the string "true" to enable. - WELDER_ENABLED: "${WELDER_ENABLED}" NOTEBOOKS_DIR: "${NOTEBOOKS_DIR}" MEM_LIMIT: "${MEM_LIMIT}" # (1/6/2022) When it's a year from now, consider removing the next two lines. @@ -52,7 +51,7 @@ services: # We should remove the two lines once we no longer support older images. # When we update base image in terra-docker next time, we should verify the paths are still valid PYTHONPATH: "/etc/jupyter/custom:/usr/lib/spark/python:${NOTEBOOKS_DIR}/packages" - PATH: "/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${HOME}/.local/bin:${NOTEBOOKS_DIR}/packages/bin" + PATH: "/opt/conda/bin:/opt/uv/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${HOME}/.local/bin:${NOTEBOOKS_DIR}/packages/bin" env_file: - /var/custom_env_vars.env # See https://docs.docker.com/engine/reference/run/#user-memory-constraints diff --git a/http/src/main/resources/init-resources/notebook.json b/http/src/main/resources/init-resources/notebook.json index d6e21fc722..69382f8a07 100644 --- a/http/src/main/resources/init-resources/notebook.json +++ b/http/src/main/resources/init-resources/notebook.json @@ -3,5 +3,4 @@ "clusterName": $(clusterName), "loginHint": $(loginHint), "googleClientId": $(googleClientId), - "welderEnabled": $(welderEnabled) } \ No newline at end of file diff --git a/http/src/main/resources/init-resources/startup.sh b/http/src/main/resources/init-resources/startup.sh index 7e90819ac2..26cf77300d 100644 --- a/http/src/main/resources/init-resources/startup.sh +++ b/http/src/main/resources/init-resources/startup.sh @@ -54,7 +54,6 @@ COMPLETE_JUPYTER_DOCKER_COMPOSE="-f $JUPYTER_DOCKER_COMPOSE" RSTUDIO_DOCKER_COMPOSE=$(ls ${DOCKER_COMPOSE_FILES_DIRECTORY}/rstudio-docker*) COMPLETE_RSTUDIO_DOCKER_COMPOSE="-f $RSTUDIO_DOCKER_COMPOSE" export CRYPTO_DETECTOR_DOCKER_IMAGE=$(cryptoDetectorDockerImage) -export WELDER_ENABLED=$(welderEnabled) export UPDATE_WELDER=$(updateWelder) export WELDER_DOCKER_IMAGE=$(welderDockerImage) export DISABLE_DELOCALIZATION=$(disableDelocalization) @@ -133,7 +132,7 @@ function validateCert() { IMAGES_TO_RESTART=(-f /var/docker-compose-files/proxy-docker-compose-gce.yaml) DATAPROC_IMAGES_TO_RESTART=(-f /etc/proxy-docker-compose.yaml) - if [ ! -z ${WELDER_DOCKER_IMAGE} ] && [ "${WELDER_ENABLED}" == "true" ]; then + if [ ! -z ${WELDER_DOCKER_IMAGE} ] ]; then IMAGES_TO_RESTART+=(-f /var/docker-compose-files/welder-docker-compose-gce.yaml) DATAPROC_IMAGES_TO_RESTART+=(-f /etc/welder-docker-compose.yaml) fi @@ -209,7 +208,6 @@ GOOGLE_PROJECT=${GOOGLE_PROJECT} RUNTIME_NAME=${RUNTIME_NAME} OWNER_EMAIL=${OWNER_EMAIL} PET_SA_EMAIL=${PET_SA_EMAIL} -WELDER_ENABLED=${WELDER_ENABLED} WELDER_SERVER_NAME=${WELDER_SERVER_NAME} WELDER_DOCKER_IMAGE=${WELDER_DOCKER_IMAGE} STAGING_BUCKET=${STAGING_BUCKET} @@ -242,7 +240,6 @@ GOOGLE_PROJECT=${GOOGLE_PROJECT} RUNTIME_NAME=${RUNTIME_NAME} OWNER_EMAIL=${OWNER_EMAIL} PET_SA_EMAIL=${PET_SA_EMAIL} -WELDER_ENABLED=${WELDER_ENABLED} SHM_SIZE=${SHM_SIZE} END @@ -278,7 +275,6 @@ GOOGLE_PROJECT=${GOOGLE_PROJECT} RUNTIME_NAME=${RUNTIME_NAME} OWNER_EMAIL=${OWNER_EMAIL} PET_SA_EMAIL=${PET_SA_EMAIL} -WELDER_ENABLED=${WELDER_ENABLED} SHM_SIZE=${SHM_SIZE} END @@ -312,7 +308,7 @@ else # jupyter_delocalize.py now assumes welder's url is `http://welder:8080`, but on dataproc, we're still using host network # A better to do this might be to take welder host as an argument to the script - docker exec $JUPYTER_SERVER_NAME /bin/bash -c "sed -i 's/http:\/\/welder/http:\/\/127.0.0.1/g' /etc/jupyter/custom/jupyter_delocalize.py" + docker exec $JUPYTER_SERVER_NAME /bin/bash -c "sed -i 's/http:\/\/welder/http:\/\/127.0.0.1/g' /etc/jupyter/custom/extensions/jupyter_delocalize.py" fi fi @@ -366,7 +362,7 @@ if [ ! -z "$JUPYTER_DOCKER_IMAGE" ] ; then docker exec ${JUPYTER_SERVER_NAME} sed -i '/^# to mount there as it effectively deletes existing files on the image/,+5d' ${JUPYTER_HOME}/jupyter_notebook_config.py # Start Jupyter server - docker exec -d $JUPYTER_SERVER_NAME /bin/bash -c "export WELDER_ENABLED=$WELDER_ENABLED && export NOTEBOOKS_DIR=$NOTEBOOKS_DIR && (/etc/jupyter/scripts/run-jupyter.sh $NOTEBOOKS_DIR || /opt/conda/bin/jupyter notebook)" + docker exec -d $JUPYTER_SERVER_NAME /bin/bash -c "export NOTEBOOKS_DIR=$NOTEBOOKS_DIR && (/etc/jupyter/scripts/run-jupyter.sh $NOTEBOOKS_DIR || /etc/jupyter/bin/jupyter notebook)" fi # Configuring RStudio, if enabled diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpDockerDAO.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpDockerDAO.scala index cb6bd33d82..05c9a24ea1 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpDockerDAO.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpDockerDAO.scala @@ -149,6 +149,10 @@ class HttpDockerDAO[F[_]] private (httpClient: Client[F])(implicit logger: Logge tokenOpt.fold(Headers.empty)(t => Headers(Authorization(Credentials.Token(AuthScheme.Bearer, t.token)))) private[dao] def parseImage(image: ContainerImage)(implicit ev: Ask[F, TraceId]): F[ParsedImage] = + // traceId <- ev.ask + // _ <- logger.info(s"Parsing image ${image.imageUrl}, $traceId") + // img <- F.pure(ParsedImage(GCR, Uri.unsafeFromString(s"https://us.gcr.io/v2"), "terra-docker-base", Tag("latest"))) + // } yield img image.imageUrl match { case GCR.regex(registry, imageName, tagOpt, shaOpt) => val version = Option(tagOpt) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/ClusterComponent.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/ClusterComponent.scala index 2aedd1e062..2f6a36d64d 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/ClusterComponent.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/ClusterComponent.scala @@ -410,7 +410,6 @@ object clusterQuery extends TableQuery(new ClusterTable(_)) { rec._1._1.userScriptUri, rec._1._1.startUserScriptUri, rec._1._1.defaultClientId, - rec._1._1.welderEnabled, rec._1._1.customClusterEnvironmentVariables ) } @@ -757,7 +756,7 @@ object clusterQuery extends TableQuery(new ClusterTable(_)) { runtime.asyncRuntimeFields.map(_.stagingBucket.value), runtime.autopauseThreshold, runtime.defaultClientId, - runtime.welderEnabled, + true, runtime.customEnvironmentVariables, runtime.runtimeConfigId, None, @@ -912,7 +911,6 @@ object clusterQuery extends TableQuery(new ClusterTable(_)) { allowStop = false, runtimeImages = clusterImages, scopes = scopeQuery.unmarshallScopes(scopes), - welderEnabled = clusterRecord.welderEnabled, customEnvironmentVariables = clusterRecord.customClusterEnvironmentVariables, runtimeConfigId = clusterRecord.runtimeConfigId.value, patchInProgress = patchInProgress, diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/RuntimeServiceDbQueries.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/RuntimeServiceDbQueries.scala index 35f5110dc6..94e9f31a6b 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/RuntimeServiceDbQueries.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/RuntimeServiceDbQueries.scala @@ -281,7 +281,6 @@ object RuntimeServiceDbQueries { clusterRecord.defaultClientId, clusterImages, scopeQuery.unmarshallScopes(scopes.toList), - clusterRecord.welderEnabled, patchInProgress, clusterRecord.customClusterEnvironmentVariables, diskConfig diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/RuntimeServiceInterp.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/RuntimeServiceInterp.scala index 619c3b76c7..2de7907395 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/RuntimeServiceInterp.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/RuntimeServiceInterp.scala @@ -913,7 +913,6 @@ object RuntimeServiceInterp { allowStop = false, runtimeImages = clusterImages, scopes = clusterScopes, - welderEnabled = true, customEnvironmentVariables = req.customEnvironmentVariables, runtimeConfigId = RuntimeConfigId(-1), patchInProgress = false diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/BaseCloudServiceRuntimeMonitor.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/BaseCloudServiceRuntimeMonitor.scala index 2c6c73b41c..d4b0e1d4d1 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/BaseCloudServiceRuntimeMonitor.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/BaseCloudServiceRuntimeMonitor.scala @@ -595,7 +595,7 @@ abstract class BaseCloudServiceRuntimeMonitor[F[_]] { Some("tool_start_up") ), mainDataprocInstance, - deleteRuntimeOnFail + false ) } } yield r diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/GceRuntimeMonitor.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/GceRuntimeMonitor.scala index 6050664b7a..58e274ecf2 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/GceRuntimeMonitor.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/GceRuntimeMonitor.scala @@ -179,7 +179,8 @@ class GceRuntimeMonitor[F[_]: Parallel]( "Creation may have failed due to temporary resource unavailability in Google Cloud Platform (`ZONE_RESOURCE_POOL_EXHAUSTED` error). Please try again later or refer to http://broad.io/different-zone for creating a cloud environment in a different zone.", shortMessage = Some("fail_to_create") ), - None + None, + false ) else checkAgain(monitorContext, runtimeAndRuntimeConfig, None, Some(s"Can't retrieve instance yet")) @@ -247,7 +248,8 @@ class GceRuntimeMonitor[F[_]: Parallel]( RuntimeErrorDetails(s"unexpected GCE instance status ${ss} when trying to creating an instance", shortMessage = Some("unexpected_status") ), - None + None, + false ) } } yield r diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/MonitorAtBoot.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/MonitorAtBoot.scala index ba58f747b6..ee14c067c3 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/MonitorAtBoot.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/MonitorAtBoot.scala @@ -298,7 +298,6 @@ class MonitorAtBoot[F[_]](publisherQueue: Queue[F, LeoPubsubMessage], runtime.defaultClientId, extra.runtimeImages, extra.scopes, - runtime.welderEnabled, runtime.customEnvironmentVariables, rtConfigInMessage, Some(traceId), @@ -343,7 +342,6 @@ final case class RuntimeToMonitor( userScriptUri: Option[UserScriptPath], startUserScriptUri: Option[UserScriptPath], defaultClientId: Option[String], - welderEnabled: Boolean, customEnvironmentVariables: Map[String, String] ) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/leoPubsubMessageSubscriberModels.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/leoPubsubMessageSubscriberModels.scala index c2eba6b90e..453df39869 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/leoPubsubMessageSubscriberModels.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/leoPubsubMessageSubscriberModels.scala @@ -178,7 +178,6 @@ object LeoPubsubMessage { defaultClientId: Option[String], runtimeImages: Set[RuntimeImage], scopes: Set[String], - welderEnabled: Boolean, customEnvironmentVariables: Map[String, String], runtimeConfig: RuntimeConfigInCreateRuntimeMessage, traceId: Option[TraceId], @@ -205,7 +204,6 @@ object LeoPubsubMessage { runtime.defaultClientId, runtime.runtimeImages, runtime.scopes, - runtime.welderEnabled, runtime.customEnvironmentVariables, runtimeConfig, traceId, @@ -566,7 +564,7 @@ object LeoPubsubCodec { } implicit val createRuntimeMessageEncoder: Encoder[CreateRuntimeMessage] = - Encoder.forProduct17( + Encoder.forProduct16( "messageType", "id", "clusterProjectAndName", @@ -579,7 +577,6 @@ object LeoPubsubCodec { "defaultClientId", "clusterImages", "scopes", - "welderEnabled", "customClusterEnvironmentVariables", "runtimeConfig", "checkToolsInterruptAfter", @@ -597,7 +594,6 @@ object LeoPubsubCodec { x.defaultClientId, x.runtimeImages, x.scopes, - x.welderEnabled, x.customEnvironmentVariables, x.runtimeConfig, x.checkToolsInterruptAfter, @@ -670,7 +666,7 @@ object LeoPubsubCodec { } implicit val createRuntimeMessageDecoder: Decoder[CreateRuntimeMessage] = - Decoder.forProduct16( + Decoder.forProduct15( "id", "clusterProjectAndName", "serviceAccountInfo", @@ -682,7 +678,6 @@ object LeoPubsubCodec { "defaultClientId", "clusterImages", "scopes", - "welderEnabled", "customClusterEnvironmentVariables", "runtimeConfig", "traceId", diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/BaseRuntimeInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/BaseRuntimeInterpreter.scala index 6f4e028ecb..26db2eafe6 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/BaseRuntimeInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/BaseRuntimeInterpreter.scala @@ -66,8 +66,6 @@ abstract private[util] class BaseRuntimeInterpreter[F[_]]( s"Failed to flush welder cache for ${params.runtimeAndRuntimeConfig.runtime.projectNameString}" ) ) - .whenA(params.runtimeAndRuntimeConfig.runtime.welderEnabled) - _ <- logger.info( s"StopRuntimeMessage timing: Updating the hostIp, [runtime = ${runtimeName}, traceId = ${ctx.traceId.asString},time = ${(now.toEpochMilli - ctx.now.toEpochMilli).toString}]" ) @@ -116,27 +114,19 @@ abstract private[util] class BaseRuntimeInterpreter[F[_]]( } yield res } - private def getWelderAction(runtime: Runtime): Option[WelderAction] = - if (runtime.welderEnabled) { - // Welder is already enabled; do we need to update it? - val labelFound = config.welderConfig.updateWelderLabel.exists(runtime.labels.contains) - - val imageChanged = runtime.runtimeImages.find(_.imageType == Welder) match { - case Some(welderImage) if welderImage.hash != Some(config.imageConfig.welderHash) => true - case _ => false - } + private def getWelderAction(runtime: Runtime): Option[WelderAction] = { + // Welder is already enabled; do we need to update it? + val labelFound = config.welderConfig.updateWelderLabel.exists(runtime.labels.contains) - if (labelFound && imageChanged) Some(UpdateWelder) - else None - } else { - // Welder is not enabled; do we need to deploy it? - val labelFound = config.welderConfig.deployWelderLabel.exists(runtime.labels.contains) - if (labelFound) { - if (isClusterBeforeCutoffDate(runtime)) Some(DisableDelocalization) - else None - } else None + val imageChanged = runtime.runtimeImages.find(_.imageType == Welder) match { + case Some(welderImage) if welderImage.hash != Some(config.imageConfig.welderHash) => true + case _ => false } + if (labelFound && imageChanged) Some(UpdateWelder) + else None + } + private def isClusterBeforeCutoffDate(runtime: Runtime): Boolean = (for { dateStr <- config.welderConfig.deployWelderCutoffDate @@ -172,9 +162,7 @@ abstract private[util] class BaseRuntimeInterpreter[F[_]]( clusterQuery.updateWelder(runtime.id, RuntimeImage(Welder, newWelderImageUrl, None, now), now) } - newRuntime = runtime.copy(runtimeImages = runtime.runtimeImages.filterNot(_.imageType == Welder) + welderImage, - welderEnabled = true - ) + newRuntime = runtime.copy(runtimeImages = runtime.runtimeImages.filterNot(_.imageType == Welder) + welderImage) } yield newRuntime override def updateMachineType(params: UpdateMachineTypeParams)(implicit ev: Ask[F, AppContext]): F[Unit] = diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/RuntimeAlgebra.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/RuntimeAlgebra.scala index 52b8dce68d..3a29e4d763 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/RuntimeAlgebra.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/RuntimeAlgebra.scala @@ -50,7 +50,6 @@ final case class CreateRuntimeParams(id: Long, defaultClientId: Option[String], runtimeImages: Set[RuntimeImage], scopes: Set[String], - welderEnabled: Boolean, customEnvironmentVariables: Map[String, String], runtimeConfig: RuntimeConfigInCreateRuntimeMessage ) @@ -68,7 +67,6 @@ object CreateRuntimeParams { message.defaultClientId, message.runtimeImages, message.scopes, - message.welderEnabled, message.customEnvironmentVariables, message.runtimeConfig ) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/RuntimeTemplateValues.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/RuntimeTemplateValues.scala index 61a38764f1..0297413d8d 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/RuntimeTemplateValues.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/RuntimeTemplateValues.scala @@ -49,7 +49,6 @@ case class RuntimeTemplateValues private (googleProject: String, jupyterNotebookConfigUri: String, jupyterNotebookFrontendConfigUri: String, googleClientId: String, - welderEnabled: String, notebooksDir: String, customEnvVarsConfigUri: String, memLimit: String, @@ -85,7 +84,6 @@ case class RuntimeTemplateValuesConfig private (runtimeProjectAndName: RuntimePr petSaEmail: WorkbenchEmail, userJupyterExtensionConfig: Option[UserJupyterExtensionConfig], defaultClientId: Option[String], - welderEnabled: Boolean, auditInfo: AuditInfo, imageConfig: ImageConfig, welderConfig: WelderConfig, @@ -128,7 +126,6 @@ object RuntimeTemplateValuesConfig { params.serviceAccountInfo, params.userJupyterExtensionConfig, params.defaultClientId, - params.welderEnabled, params.auditInfo, imageConfig, welderConfig, @@ -173,7 +170,6 @@ object RuntimeTemplateValuesConfig { runtime.serviceAccount, runtime.userJupyterExtensionConfig, runtime.defaultClientId, - runtime.welderEnabled, runtime.auditInfo, imageConfig, welderConfig, @@ -273,7 +269,6 @@ object RuntimeTemplateValues { ) .getOrElse(""), config.defaultClientId.getOrElse(""), - config.welderEnabled.toString, // TODO: remove this and conditional below when welder is rolled out to all clusters notebooksDir = s"${jupyterUserhome}", config.initBucketName .map(n => GcsPath(n, GcsObjectName(config.clusterResourcesConfig.customEnvVarsConfigUri.asString)).toUri) diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/CommonTestData.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/CommonTestData.scala index 8618cf12d3..26da4e394f 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/CommonTestData.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/CommonTestData.scala @@ -330,7 +330,6 @@ object CommonTestData { allowStop = false, runtimeImages = Set(jupyterImage, welderImage), scopes = defaultScopes, - welderEnabled = false, customEnvironmentVariables = Map.empty, runtimeConfigId = RuntimeConfigId(-1), patchInProgress = false @@ -366,7 +365,6 @@ object CommonTestData { allowStop = false, runtimeImages = Set(jupyterImage, welderImage, proxyImage, cryptoDetectorImage), scopes = defaultScopes, - welderEnabled = true, customEnvironmentVariables = Map.empty, runtimeConfigId = RuntimeConfigId(-1), patchInProgress = false diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/LeoPublisherSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/LeoPublisherSpec.scala index 56c3b0d8e5..75438a8d46 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/LeoPublisherSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/LeoPublisherSpec.scala @@ -133,7 +133,6 @@ class LeoPublisherSpec extends AnyFlatSpecLike with MockitoSugar with Matchers w None, Set.empty, Set.empty, - false, Map.empty, RuntimeConfigInCreateRuntimeMessage.GceConfig(MachineTypeName("n1-standard-4"), DiskSize(50), diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/api/RuntimeRoutesSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/api/RuntimeRoutesSpec.scala index d82e9b73a8..8afff9da05 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/api/RuntimeRoutesSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/api/RuntimeRoutesSpec.scala @@ -406,7 +406,6 @@ class RuntimeRoutesSpec extends AnyFlatSpec with Matchers with LeonardoTestSuite Set(jupyterImage, welderImage, proxyImage, cryptoDetectorImage).map(_.copy(timestamp = date)), defaultScopes, true, - true, Map("ev1" -> "a", "ev2" -> "b"), Some(DiskConfig(DiskName("disk"), DiskSize(100), DiskType.Standard, BlockSize(1024))) ) diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubCodecSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubCodecSpec.scala index 918707dc8e..7c5e565d75 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubCodecSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubCodecSpec.scala @@ -34,7 +34,6 @@ class LeoPubsubCodecSpec extends AnyFlatSpec with Matchers { None, Set.empty, Set.empty, - false, Map.empty, RuntimeConfigInCreateRuntimeMessage.GceConfig(MachineTypeName("n1-standard-4"), DiskSize(50), @@ -65,7 +64,6 @@ class LeoPubsubCodecSpec extends AnyFlatSpec with Matchers { None, Set.empty, Set.empty, - false, Map.empty, RuntimeConfigInCreateRuntimeMessage.GceWithPdConfig(MachineTypeName("n1-standard-4"), DiskId(2), diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/util/GceInterpreterSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/util/GceInterpreterSpec.scala index 3067e389f0..82ee9059d9 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/util/GceInterpreterSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/util/GceInterpreterSpec.scala @@ -97,7 +97,6 @@ class GceInterpreterSpec extends AnyFlatSpecLike with TestComponent with Leonard None, runtime.runtimeImages, Set.empty, - true, Map.empty, runtimeConfig ) diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/util/RuntimeTemplateValuesSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/util/RuntimeTemplateValuesSpec.scala index c60b854486..91f2bb60b9 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/util/RuntimeTemplateValuesSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/util/RuntimeTemplateValuesSpec.scala @@ -88,7 +88,6 @@ class RuntimeTemplateValuesSpec extends LeonardoTestSuite with AnyFlatSpecLike { GcsObjectName("test-welder-docker-compose.yaml") ).toUri result.welderDockerImage shouldBe CommonTestData.welderImage.imageUrl - result.welderEnabled shouldBe "true" result.welderMemLimit shouldBe "805306368b" // 768 MB result.welderServerName shouldBe "welder-server" result.shouldDeleteJupyterDir shouldBe "false" diff --git a/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/provider/RuntimeStateManager.scala b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/provider/RuntimeStateManager.scala index 9413bfef2c..4e581c46dc 100644 --- a/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/provider/RuntimeStateManager.scala +++ b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/provider/RuntimeStateManager.scala @@ -53,7 +53,6 @@ object RuntimeStateManager { Some("clientId"), Set(jupyterImage, welderImage, proxyImage, cryptoDetectorImage).map(_.copy(timestamp = date)), defaultScopes, - welderEnabled = true, patchInProgress = true, Map("ev1" -> "a", "ev2" -> "b"), Some(DiskConfig(DiskName("disk"), DiskSize(100), DiskType.Standard, BlockSize(1024)))