From 5fa22b46bdfb8afa473cbf6c622321fe818ab332 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 12 Jul 2023 01:37:33 +0200 Subject: [PATCH 1/2] =?UTF-8?q?add=20deps-conda=20for=20system=20dependenc?= =?UTF-8?q?ies=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - factor `get-conda` out of `deps-cuda` so it can run independently - add `deps-conda` as alternative to `deps-ubuntu`: installing system dependencies (including Python) for core - when doing `make deps-cuda` or `make deps-conda` and there is not `conda` already installed, run `make get-conda` --- Makefile | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 6df7d5fd0c..e80d3c9f92 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,8 @@ help: @echo "" @echo " Targets" @echo "" + @echo " get-conda Install Conda distribution" + @echo " deps-conda Dependencies for deployment via Conda" @echo " deps-cuda Dependencies for deployment with GPU support via Conda" @echo " deps-ubuntu Dependencies for deployment in an Ubuntu/Debian Linux" @echo " deps-test Install test python deps via pip" @@ -52,16 +54,32 @@ help: # pip install command. Default: $(PIP_INSTALL) PIP_INSTALL ?= $(PIP) install -deps-cuda: CONDA_EXE ?= /usr/local/bin/conda -deps-cuda: export CONDA_PREFIX ?= /conda -deps-cuda: PYTHON_PREFIX != $(PYTHON) -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])' -deps-cuda: - curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba +.PHONY: get-conda deps-cuda deps-conda deps-ubuntu + +ifeq ($(shell command -v conda),) +# no prior Conda distribution, so install Mamba now +# (see https://mamba.readthedocs.io/en/latest/installation.html) +get-conda: CONDA_EXE ?= /usr/local/bin/conda +get-conda: export CONDA_PREFIX ?= /conda +# first part of recipe: see micro.mamba.pm/install.sh +get-conda: OS != uname +get-conda: PLATFORM = $(subst Darwin,osx,$(subst Linux,linux,$(OS))) +get-conda: ARCH != uname -m +get-conda: MACHINE = $(or $(filter aarch64 arm64 ppc64le, $(ARCH)), 64) +get-conda: URL = https://micro.mamba.pm/api/micromamba/$(PLATFORM)-$(MACHINE)/latest +get-conda: + curl -Ls $(URL) | tar -xvj bin/micromamba mv bin/micromamba $(CONDA_EXE) # Install Conda system-wide (for interactive / login shells) echo 'export MAMBA_EXE=$(CONDA_EXE) MAMBA_ROOT_PREFIX=$(CONDA_PREFIX) CONDA_PREFIX=$(CONDA_PREFIX) PATH=$(CONDA_PREFIX)/bin:$$PATH' >> /etc/profile.d/98-conda.sh mkdir -p $(CONDA_PREFIX)/lib $(CONDA_PREFIX)/include echo $(CONDA_PREFIX)/lib >> /etc/ld.so.conf.d/conda.conf +else +get-conda: ; +endif + +deps-cuda: PYTHON_PREFIX != $(PYTHON) -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])' +deps-cuda: get-conda # Get CUDA toolkit, including compiler and libraries with dev, # however, the Nvidia channels do not provide (recent) cudnn (needed for Torch, TF etc): #MAMBA_ROOT_PREFIX=$(CONDA_PREFIX) \ @@ -70,7 +88,6 @@ deps-cuda: # The conda-forge channel has cudnn and cudatoolkit but no cudatoolkit-dev anymore (and we need both!), # so let's combine nvidia and conda-forge (will be same lib versions, no waste of space), # but omitting cuda-cudart-dev and cuda-libraries-dev (as these will be pulled by pip for torch anyway): - MAMBA_ROOT_PREFIX=$(CONDA_PREFIX) \ conda install -c nvidia/label/cuda-11.8.0 \ cuda-nvcc \ cuda-cccl \ @@ -106,10 +123,16 @@ deps-cuda: && ldconfig # gputil/nvidia-smi would be nice, too – but that drags in Python as a conda dependency... -# Dependencies for deployment in an ubuntu/debian linux +# Dependencies for deployment via Conda +deps-conda: get-conda + conda install -c conda-forge python==3.8.* imagemagick geos pkgconfig + +# Dependencies for deployment in an Ubuntu/Debian Linux deps-ubuntu: apt-get install -y python3 imagemagick libgeos-dev +.PHONY: deps-test install install-dev uninstall + # Install test python deps via pip deps-test: $(PIP) install -U pip From 619e1a3284648c4e3b4db108840b638f5823d4b8 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 12 Jul 2023 01:38:37 +0200 Subject: [PATCH 2/2] rebase Docker image on conda for system dependencies, including Python --- Dockerfile | 17 +++++++++++------ Dockerfile.cuda | 9 +-------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index e814420eae..5784c4c6f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,13 @@ ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 ENV PIP=pip +ENV MAMBA_EXE=/usr/local/bin/conda +ENV MAMBA_ROOT_PREFIX=/conda +ENV PATH=$MAMBA_ROOT_PREFIX/bin:$PATH +ENV CONDA_EXE=$MAMBA_EXE +ENV CONDA_PREFIX=$MAMBA_ROOT_PREFIX +ENV CONDA_SHLVL='1' + WORKDIR /build-ocrd COPY ocrd ./ocrd COPY ocrd_modelfactory ./ocrd_modelfactory/ @@ -20,11 +27,8 @@ COPY Makefile . COPY README.md . COPY LICENSE . RUN echo 'APT::Install-Recommends "0"; APT::Install-Suggests "0";' >/etc/apt/apt.conf.d/ocr-d.conf -RUN apt-get update && apt-get -y install software-properties-common \ - && apt-get update && apt-get -y install \ +RUN apt-get update && apt-get -y install \ ca-certificates \ - python3-dev \ - python3-venv \ gcc \ make \ wget \ @@ -32,7 +36,8 @@ RUN apt-get update && apt-get -y install software-properties-common \ curl \ sudo \ git \ - && make deps-ubuntu \ + && make get-conda \ + && make deps-conda \ && python3 -m venv /usr/local \ && hash -r \ && pip install --upgrade pip setuptools wheel \ @@ -42,4 +47,4 @@ RUN apt-get update && apt-get -y install software-properties-common \ WORKDIR /data -CMD ["/usr/local/bin/ocrd", "--help"] +CMD ["/conda/bin/ocrd", "--help"] diff --git a/Dockerfile.cuda b/Dockerfile.cuda index 52d7a27619..c19058b200 100644 --- a/Dockerfile.cuda +++ b/Dockerfile.cuda @@ -1,13 +1,6 @@ ARG BASE_IMAGE FROM $BASE_IMAGE -ENV MAMBA_EXE=/usr/local/bin/conda -ENV MAMBA_ROOT_PREFIX=/conda -ENV PATH=$MAMBA_ROOT_PREFIX/bin:$PATH -ENV CONDA_EXE=$MAMBA_EXE -ENV CONDA_PREFIX=$MAMBA_ROOT_PREFIX -ENV CONDA_SHLVL='1' - WORKDIR /build COPY Makefile . @@ -18,5 +11,5 @@ WORKDIR /data RUN rm -fr /build -CMD ["/usr/local/bin/ocrd", "--help"] +CMD ["/conda/bin/ocrd", "--help"]