From edcf85555f851586e7ba832eb36d49e708ebf206 Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Wed, 15 Mar 2023 20:30:03 +0100 Subject: [PATCH 01/35] Dockerfile: Use virtual environment python3-pip can now be removed, and running pip instead of pip3 is sufficient. This also avoids the installation of python3-setuptools and python3-wheel (requirements of python3-pip). Signed-off-by: Stefan Weil --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6b23842ee4..a6008feb02 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ENV DEBIAN_FRONTEND noninteractive ENV PYTHONIOENCODING utf8 ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 -ENV PIP=pip3 +ENV PIP=pip WORKDIR /build-ocrd COPY ocrd ./ocrd @@ -23,7 +23,6 @@ RUN apt-get update && apt-get -y install software-properties-common \ && apt-get update && apt-get -y install \ ca-certificates \ python3-dev \ - python3-pip \ python3-venv \ gcc \ make \ @@ -33,7 +32,8 @@ RUN apt-get update && apt-get -y install software-properties-common \ sudo \ git \ && make deps-ubuntu \ - && pip3 install --upgrade pip setuptools \ + && python3 -m venv /usr/local \ + && pip install --upgrade pip setuptools \ && make install \ && apt-get remove -y gcc \ && apt-get autoremove -y \ From 5c003fa3dca79b2c0de1e65da47d4bac5c8870eb Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Sat, 1 Apr 2023 11:10:37 +0200 Subject: [PATCH 02/35] give up workaround for shapely-CUDA issue --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 72a9bb7652..4d43130d5a 100644 --- a/Makefile +++ b/Makefile @@ -72,8 +72,6 @@ install: @# speedup for end-of-life builds if $(PYTHON) -V | fgrep -e 3.5 -e 3.6; then $(PIP) install --prefer-binary opencv-python-headless numpy; fi for mod in $(BUILD_ORDER);do (cd $$mod ; $(PIP_INSTALL) .);done - @# workaround for shapely#1598 - $(PIP) install --no-binary shapely --force-reinstall shapely # Install with pip install -e install-dev: uninstall From 2250550d85b626da18ca8050f0ed8997eba6a8fd Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Sat, 1 Apr 2023 11:12:00 +0200 Subject: [PATCH 03/35] rehash after pip upgrade --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 6b23842ee4..f425bb82b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,7 @@ RUN apt-get update && apt-get -y install software-properties-common \ git \ && make deps-ubuntu \ && pip3 install --upgrade pip setuptools \ + && hash -r \ && make install \ && apt-get remove -y gcc \ && apt-get autoremove -y \ From 00a0f6f6f72817239f6f6214e72c9a79e1d29478 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Sat, 1 Apr 2023 11:12:36 +0200 Subject: [PATCH 04/35] keep gcc, no autoremove --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f425bb82b1..720c4d3b6a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,8 +36,6 @@ RUN apt-get update && apt-get -y install software-properties-common \ && pip3 install --upgrade pip setuptools \ && hash -r \ && make install \ - && apt-get remove -y gcc \ - && apt-get autoremove -y \ && $FIXUP \ && rm -rf /build-ocrd From 410783f4767882d3b73574dfe6866f788360ada3 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Sat, 1 Apr 2023 11:13:37 +0200 Subject: [PATCH 05/35] docker-cuda: change base image, no multi-CUDA runtimes --- Dockerfile | 4 ++++ Makefile | 35 ++++++----------------------------- 2 files changed, 10 insertions(+), 29 deletions(-) diff --git a/Dockerfile b/Dockerfile index 720c4d3b6a..52f8eb4a2b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,4 +41,8 @@ RUN apt-get update && apt-get -y install software-properties-common \ WORKDIR /data +# remove any entry points from base image +RUN rm -fr /opt/nvidia +ENTRYPOINT [] + CMD ["/usr/local/bin/ocrd", "--help"] diff --git a/Makefile b/Makefile index 4d43130d5a..b907993be3 100644 --- a/Makefile +++ b/Makefile @@ -214,38 +214,15 @@ pyclean: # Build docker image docker docker-cuda: - docker build -t $(DOCKER_TAG) --build-arg BASE_IMAGE=$(DOCKER_BASE_IMAGE) $(DOCKER_ARGS) . + docker build --progress=plain -t $(DOCKER_TAG) --build-arg BASE_IMAGE=$(DOCKER_BASE_IMAGE) $(DOCKER_ARGS) . # Build docker GPU / CUDA image -docker-cuda: DOCKER_BASE_IMAGE = nvidia/cuda:11.3.1-cudnn8-runtime-ubuntu20.04 +# (we need *devel* for compilation via nvcc etc. of our dependencies inside the build, +# no good *cudnn* because it will not be used by pip installs, +# and cudnn-devel does not contain nvcc etc.) +# cf. https://github.com/NVIDIA/nvidia-docker/wiki/CUDA#description +docker-cuda: DOCKER_BASE_IMAGE = nvidia/cuda:11.3.1-devel-ubuntu20.04 docker-cuda: DOCKER_TAG = ocrd/core-cuda -docker-cuda: DOCKER_ARGS += --build-arg FIXUP="make cuda-ubuntu cuda-ldconfig" - -# -# CUDA -# - -.PHONY: cuda-ubuntu cuda-ldconfig - -# Install native CUDA toolkit in different versions -cuda-ubuntu: cuda-ldconfig - apt-get -y install --no-install-recommends cuda-runtime-11-0 cuda-runtime-11-1 cuda-runtime-11-3 cuda-runtime-11-7 cuda-runtime-12-1 - -cuda-ldconfig: /etc/ld.so.conf.d/cuda.conf - ldconfig - -/etc/ld.so.conf.d/cuda.conf: - @echo > $@ - @echo /usr/local/cuda-11.0/lib64 >> $@ - @echo /usr/local/cuda-11.0/targets/x86_64-linux/lib >> $@ - @echo /usr/local/cuda-11.1/lib64 >> $@ - @echo /usr/local/cuda-11.1/targets/x86_64-linux/lib >> $@ - @echo /usr/local/cuda-11.3/lib64 >> $@ - @echo /usr/local/cuda-11.3/targets/x86_64-linux/lib >> $@ - @echo /usr/local/cuda-11.7/lib64 >> $@ - @echo /usr/local/cuda-11.7/targets/x86_64-linux/lib >> $@ - @echo /usr/local/cuda-12.1/lib64 >> $@ - @echo /usr/local/cuda-12.1/targets/x86_64-linux/lib >> $@ # Build wheels and source dist and twine upload them pypi: uninstall install From de86e0fc380c7786222f92f38b971824bb84af00 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 12 Apr 2023 17:24:26 +0200 Subject: [PATCH 06/35] reinstate workaround for shapely, but more robust --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index b907993be3..ecaa44e4fb 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,8 @@ install: @# speedup for end-of-life builds if $(PYTHON) -V | fgrep -e 3.5 -e 3.6; then $(PIP) install --prefer-binary opencv-python-headless numpy; fi for mod in $(BUILD_ORDER);do (cd $$mod ; $(PIP_INSTALL) .);done + @# workaround for shapely#1598 + $(PIP) config set global.no-binary shapely # Install with pip install -e install-dev: uninstall @@ -223,6 +225,7 @@ docker docker-cuda: # cf. https://github.com/NVIDIA/nvidia-docker/wiki/CUDA#description docker-cuda: DOCKER_BASE_IMAGE = nvidia/cuda:11.3.1-devel-ubuntu20.04 docker-cuda: DOCKER_TAG = ocrd/core-cuda +docker-cuda: DOCKER_ARGS += --build-arg FIXUP="apt-get install libcudnn8" # Build wheels and source dist and twine upload them pypi: uninstall install From c1178f9076cbbcc8b45ce487bc39eb8739b82ed7 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Sat, 15 Apr 2023 23:11:56 +0200 Subject: [PATCH 07/35] core-cuda: use CUDA 11.8, install cuDNN via pip and make available system-wide via ld.so.conf --- Dockerfile | 2 +- Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 52f8eb4a2b..b8b98c15ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,7 @@ RUN apt-get update && apt-get -y install software-properties-common \ && pip3 install --upgrade pip setuptools \ && hash -r \ && make install \ - && $FIXUP \ + && eval $FIXUP \ && rm -rf /build-ocrd WORKDIR /data diff --git a/Makefile b/Makefile index ecaa44e4fb..28f43fca70 100644 --- a/Makefile +++ b/Makefile @@ -223,9 +223,9 @@ docker docker-cuda: # no good *cudnn* because it will not be used by pip installs, # and cudnn-devel does not contain nvcc etc.) # cf. https://github.com/NVIDIA/nvidia-docker/wiki/CUDA#description -docker-cuda: DOCKER_BASE_IMAGE = nvidia/cuda:11.3.1-devel-ubuntu20.04 +docker-cuda: DOCKER_BASE_IMAGE = nvidia/cuda:11.8.0-devel-ubuntu20.04 docker-cuda: DOCKER_TAG = ocrd/core-cuda -docker-cuda: DOCKER_ARGS += --build-arg FIXUP="apt-get install libcudnn8" +docker-cuda: DOCKER_ARGS += --build-arg FIXUP="pip3 install nvidia-pyindex && pip3 install nvidia-cudnn-cu11==8.6.0.163 && echo /usr/local/lib/python3.8/dist-packages/nvidia/cudnn/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && echo /usr/local/lib/python3.8/dist-packages/nvidia/cublas/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && ldconfig" # Build wheels and source dist and twine upload them pypi: uninstall install From d3d54bff959d0f812b651049bcb433eaeee3f333 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 21 Apr 2023 01:38:07 +0200 Subject: [PATCH 08/35] core-cuda: install more CUDA libs via pip and ld.so.conf, simplify Dockerfile for that --- Dockerfile | 20 ++++++++++++++++++++ Makefile | 1 - 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b8b98c15ba..5c2954bd8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ ARG BASE_IMAGE FROM $BASE_IMAGE +ARG BASE_IMAGE ARG FIXUP=echo MAINTAINER OCR-D ENV DEBIAN_FRONTEND noninteractive @@ -38,6 +39,25 @@ RUN apt-get update && apt-get -y install software-properties-common \ && make install \ && eval $FIXUP \ && rm -rf /build-ocrd +RUN if echo $BASE_IMAGE | fgrep -q cuda; then \ + pip3 install nvidia-pyindex && \ + pip3 install nvidia-cudnn-cu11==8.6.0.163 && \ + pip3 install nvidia-cublas-cu11 && \ + pip3 install nvidia-cusparse-cu11 && \ + pip3 install nvidia-cusolver-cu11 && \ + pip3 install nvidia-curand-cu11 && \ + pip3 install nvidia-cufft-cu11 && \ + pip3 install nvidia-cuda-runtime-cu11 && \ + pip3 install nvidia-cuda-nvrtc-cu11 && \ + echo /usr/local/lib/python3.8/dist-packages/nvidia/cudnn/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ + echo /usr/local/lib/python3.8/dist-packages/nvidia/cublas/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ + echo /usr/local/lib/python3.8/dist-packages/nvidia/cusparse/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ + echo /usr/local/lib/python3.8/dist-packages/nvidia/cusolver/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ + echo /usr/local/lib/python3.8/dist-packages/nvidia/curand/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ + echo /usr/local/lib/python3.8/dist-packages/nvidia/cufft/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ + echo /usr/local/lib/python3.8/dist-packages/nvidia/cuda_runtime/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ + echo /usr/local/lib/python3.8/dist-packages/nvidia/cuda_nvrtc/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ + ldconfig; fi WORKDIR /data diff --git a/Makefile b/Makefile index 28f43fca70..91807cb025 100644 --- a/Makefile +++ b/Makefile @@ -225,7 +225,6 @@ docker docker-cuda: # cf. https://github.com/NVIDIA/nvidia-docker/wiki/CUDA#description docker-cuda: DOCKER_BASE_IMAGE = nvidia/cuda:11.8.0-devel-ubuntu20.04 docker-cuda: DOCKER_TAG = ocrd/core-cuda -docker-cuda: DOCKER_ARGS += --build-arg FIXUP="pip3 install nvidia-pyindex && pip3 install nvidia-cudnn-cu11==8.6.0.163 && echo /usr/local/lib/python3.8/dist-packages/nvidia/cudnn/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && echo /usr/local/lib/python3.8/dist-packages/nvidia/cublas/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && ldconfig" # Build wheels and source dist and twine upload them pypi: uninstall install From 357e72984e80d84b6bcfd0bfa85c9e78f9b08c7e Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 21 Apr 2023 01:39:23 +0200 Subject: [PATCH 09/35] make install on py36: prefer binary OpenCV/Numpy via pip config instead of preinstall --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 91807cb025..bdf79eeee3 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ deps-test: install: $(PIP) install -U pip wheel setuptools fastentrypoints @# speedup for end-of-life builds - if $(PYTHON) -V | fgrep -e 3.5 -e 3.6; then $(PIP) install --prefer-binary opencv-python-headless numpy; fi + if $(PYTHON) -V | fgrep -e 3.5 -e 3.6; then $(PIP) config set global.prefer-binary "opencv-python-headless numpy"; fi for mod in $(BUILD_ORDER);do (cd $$mod ; $(PIP_INSTALL) .);done @# workaround for shapely#1598 $(PIP) config set global.no-binary shapely From 2640b711c98ca5ae21ad9a3af463602ed26da116 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 21 Apr 2023 17:22:54 +0200 Subject: [PATCH 10/35] make install on py36: fix prefer-binary syntax --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index bdf79eeee3..8f7431a9fe 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ deps-test: install: $(PIP) install -U pip wheel setuptools fastentrypoints @# speedup for end-of-life builds - if $(PYTHON) -V | fgrep -e 3.5 -e 3.6; then $(PIP) config set global.prefer-binary "opencv-python-headless numpy"; fi + if $(PYTHON) -V | fgrep -e 3.5 -e 3.6; then $(PIP) config set global.prefer-binary "opencv-python-headless=1 numpy=1"; fi for mod in $(BUILD_ORDER);do (cd $$mod ; $(PIP_INSTALL) .);done @# workaround for shapely#1598 $(PIP) config set global.no-binary shapely @@ -215,7 +215,7 @@ pyclean: .PHONY: docker docker-cuda # Build docker image -docker docker-cuda: +docker docker-cuda: docker build --progress=plain -t $(DOCKER_TAG) --build-arg BASE_IMAGE=$(DOCKER_BASE_IMAGE) $(DOCKER_ARGS) . # Build docker GPU / CUDA image From b3618a98b770da287be55bf23d006787362e7782 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 21 Apr 2023 19:09:58 +0200 Subject: [PATCH 11/35] make install on py36: revert to prefer-binary via install --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8f7431a9fe..35914b626f 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,8 @@ deps-test: install: $(PIP) install -U pip wheel setuptools fastentrypoints @# speedup for end-of-life builds - if $(PYTHON) -V | fgrep -e 3.5 -e 3.6; then $(PIP) config set global.prefer-binary "opencv-python-headless=1 numpy=1"; fi + @# we cannot use pip config here due to pip#11988 + if $(PYTHON) -V | fgrep -e 3.5 -e 3.6; then $(PIP) install --prefer-binary opencv-python-headless numpy; fi for mod in $(BUILD_ORDER);do (cd $$mod ; $(PIP_INSTALL) .);done @# workaround for shapely#1598 $(PIP) config set global.no-binary shapely From b7130307ce68c5f074c1ceea24b66ae9ee9ef289 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Mon, 24 Apr 2023 13:39:15 +0200 Subject: [PATCH 12/35] :package: v2.50.0 --- CHANGELOG.md | 4 ++++ ocrd_utils/setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82323b0675..cb6e6611f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ Versioned according to [Semantic Versioning](http://semver.org/). ## Unreleased +## [2.50.0] - 2023-04-24 + Added: * :fire: `ocrd_network`: Components related to OCR-D Web API, #974 @@ -1729,6 +1731,8 @@ Fixed Initial Release +[2.50.0]: ../../compare/v2.50.0..v2.49.0 +[2.49.0]: ../../compare/v2.49.0..v2.48.1 [2.48.1]: ../../compare/v2.48.1..v2.48.0 [2.48.0]: ../../compare/v2.48.0..v2.47.4 [2.47.4]: ../../compare/v2.47.4..v2.47.3 diff --git a/ocrd_utils/setup.py b/ocrd_utils/setup.py index 870eede6ee..6b0b8690d7 100644 --- a/ocrd_utils/setup.py +++ b/ocrd_utils/setup.py @@ -5,7 +5,7 @@ setup( name='ocrd_utils', - version='2.49.0', + version='2.50.0', description='OCR-D framework - shared code, helpers, constants', long_description=open('README.md').read(), long_description_content_type='text/markdown', From a6cf5ff2ff62a421b0f764f3a00388edead14e6c Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 27 Apr 2023 15:11:08 +0200 Subject: [PATCH 13/35] core-cuda: use same CUDA libs as needed for Torch anyway --- Dockerfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 069cb0e0ae..506bfa0143 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,13 +43,13 @@ RUN apt-get update && apt-get -y install software-properties-common \ RUN if echo $BASE_IMAGE | fgrep -q cuda; then \ pip3 install nvidia-pyindex && \ pip3 install nvidia-cudnn-cu11==8.6.0.163 && \ - pip3 install nvidia-cublas-cu11 && \ - pip3 install nvidia-cusparse-cu11 && \ - pip3 install nvidia-cusolver-cu11 && \ - pip3 install nvidia-curand-cu11 && \ - pip3 install nvidia-cufft-cu11 && \ - pip3 install nvidia-cuda-runtime-cu11 && \ - pip3 install nvidia-cuda-nvrtc-cu11 && \ + pip3 install nvidia-cublas-cu117 && \ + pip3 install nvidia-cusparse-cu117 && \ + pip3 install nvidia-cusolver-cu117 && \ + pip3 install nvidia-curand-cu117 && \ + pip3 install nvidia-cufft-cu117 && \ + pip3 install nvidia-cuda-runtime-cu117 && \ + pip3 install nvidia-cuda-nvrtc-cu117 && \ echo /usr/local/lib/python3.8/site-packages/nvidia/cudnn/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ echo /usr/local/lib/python3.8/site-packages/nvidia/cublas/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ echo /usr/local/lib/python3.8/site-packages/nvidia/cusparse/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ From 1da66f4300677f8b2109666d6d498e46c002549f Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 10 May 2023 11:44:16 +0200 Subject: [PATCH 14/35] Fix mongodb credentials usage Previously mongodb credentials from config were accidentially ignored and it was always deployed without credentials --- ocrd_network/ocrd_network/deployer.py | 11 ++++++++++- ocrd_network/ocrd_network/runtime_data.py | 12 +++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index 491e6a632d..f83a19b3e5 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -305,11 +305,20 @@ def deploy_mongodb( ports_mapping = { 27017: self.data_mongo.port } + if self.data_mongo.username: + environment = [ + f'MONGO_INITDB_ROOT_USERNAME={self.data_mongo.username}', + f'MONGO_INITDB_ROOT_PASSWORD={self.data_mongo.password}' + ] + else: + environment = [] + res = client.containers.run( image=image, detach=detach, remove=remove, - ports=ports_mapping + ports=ports_mapping, + environment=environment ) if not res or not res.id: raise RuntimeError('Failed to start MongoDB docker container on host: ' diff --git a/ocrd_network/ocrd_network/runtime_data.py b/ocrd_network/ocrd_network/runtime_data.py index 8ab9a2896a..254652f1cf 100644 --- a/ocrd_network/ocrd_network/runtime_data.py +++ b/ocrd_network/ocrd_network/runtime_data.py @@ -101,9 +101,15 @@ def __init__(self, config: Dict) -> None: self.ssh_username = config['ssh']['username'] self.ssh_keypath = config['ssh'].get('path_to_privkey', None) self.ssh_password = config['ssh'].get('password', None) - self.username = config['credentials']['username'] - self.password = config['credentials']['password'] - self.url = f'mongodb://{self.address}:{self.port}' + + if 'credentials' in config: + self.username = config['credentials']['username'] + self.password = config['credentials']['password'] + self.url = f'mongodb://{self.username}:{self.password}@{self.address}:{self.port}' + else: + self.username = None + self.password = None + self.url = f'mongodb://{self.address}:{self.port}' # Assigned when deployed self.pid = None From 8530bd9006c9bfbf91ef0e9cade582103a2dc04e Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 10 May 2023 12:01:47 +0200 Subject: [PATCH 15/35] Add skip_deployment flag for queue and database --- ocrd_network/ocrd_network/runtime_data.py | 25 ++++++++++++++----- .../processing_server_config.schema.yml | 6 +++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/ocrd_network/ocrd_network/runtime_data.py b/ocrd_network/ocrd_network/runtime_data.py index 254652f1cf..59c658ada1 100644 --- a/ocrd_network/ocrd_network/runtime_data.py +++ b/ocrd_network/ocrd_network/runtime_data.py @@ -98,9 +98,14 @@ class DataMongoDB: def __init__(self, config: Dict) -> None: self.address = config['address'] self.port = int(config['port']) - self.ssh_username = config['ssh']['username'] - self.ssh_keypath = config['ssh'].get('path_to_privkey', None) - self.ssh_password = config['ssh'].get('password', None) + if 'ssh' in config: + self.ssh_username = config['ssh']['username'] + self.ssh_keypath = config['ssh'].get('path_to_privkey', None) + self.ssh_password = config['ssh'].get('password', None) + else: + self.ssh_username = None + self.ssh_keypath = None + self.ssh_password = None if 'credentials' in config: self.username = config['credentials']['username'] @@ -110,6 +115,7 @@ def __init__(self, config: Dict) -> None: self.username = None self.password = None self.url = f'mongodb://{self.address}:{self.port}' + self.skip_deployment = config.get('skip_deployment', False) # Assigned when deployed self.pid = None @@ -118,12 +124,19 @@ class DataRabbitMQ: def __init__(self, config: Dict) -> None: self.address = config['address'] self.port = int(config['port']) - self.ssh_username = config['ssh']['username'] - self.ssh_keypath = config['ssh'].get('path_to_privkey', None) - self.ssh_password = config['ssh'].get('password', None) + if 'ssh' in config: + self.ssh_username = config['ssh']['username'] + self.ssh_keypath = config['ssh'].get('path_to_privkey', None) + self.ssh_password = config['ssh'].get('password', None) + else: + self.ssh_username = None + self.ssh_keypath = None + self.ssh_password = None + self.vhost = '/' self.username = config['credentials']['username'] self.password = config['credentials']['password'] self.url = f'amqp://{self.username}:{self.password}@{self.address}:{self.port}{self.vhost}' + self.skip_deployment = config.get('skip_deployment', False) # Assigned when deployed self.pid = None diff --git a/ocrd_validators/ocrd_validators/processing_server_config.schema.yml b/ocrd_validators/ocrd_validators/processing_server_config.schema.yml index 4039e4917f..6cbe8d41c3 100644 --- a/ocrd_validators/ocrd_validators/processing_server_config.schema.yml +++ b/ocrd_validators/ocrd_validators/processing_server_config.schema.yml @@ -26,6 +26,9 @@ properties: ssh: description: Information required for an SSH connection $ref: "#/$defs/ssh" + skip_deployment: + description: set to true to deploy queue yourself + type: boolean database: description: Information about the MongoDB type: object @@ -46,6 +49,9 @@ properties: ssh: description: Information required for an SSH connection $ref: "#/$defs/ssh" + skip_deployment: + description: set to true to deploy database yourself + type: boolean hosts: description: A list of hosts where Processing Servers will be deployed type: array From b4e65762b2f724642bf590461aa74132231d30f7 Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 10 May 2023 12:10:36 +0200 Subject: [PATCH 16/35] Make deployer optionally skip mongo and rabbitmq --- ocrd_network/ocrd_network/deployer.py | 34 +++++++++++++++++-- ocrd_network/ocrd_network/deployment_utils.py | 20 +++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index f83a19b3e5..7d67a7e79d 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -17,7 +17,9 @@ from .deployment_utils import ( create_docker_client, DeployType, - wait_for_rabbitmq_availability + wait_for_rabbitmq_availability, + verify_mongodb_available, + verify_rabbitmq_available, ) from .runtime_data import ( @@ -229,6 +231,23 @@ def deploy_rabbitmq( remove: bool, ports_mapping: Union[Dict, None] = None ) -> str: + if self.data_queue.skip_deployment: + self.log.debug(f"RabbitMQ is externaly managed. Skipping deployment") + verify_rabbitmq_available( + self.data_queue.address, + self.data_queue.port, + self.data_queue.vhost, + self.data_queue.username, + self.data_queue.password + ) + wait_for_rabbitmq_availability( + host=self.data_queue.address, + port=int(self.data_queue.port), + vhost='/', + username=self.data_queue.username, + password=self.data_queue.password + ) + return self.data_queue.url self.log.debug(f"Trying to deploy '{image}', with modes: " f"detach='{detach}', remove='{remove}'") @@ -289,6 +308,11 @@ def deploy_mongodb( remove: bool, ports_mapping: Union[Dict, None] = None ) -> str: + if self.data_mongo.skip_deployment: + self.log.debug('MongoDb is externaly managed. Skipping deployment') + verify_mongodb_available(self.data_mongo.url); + return self.data_mongo.url + self.log.debug(f"Trying to deploy '{image}', with modes: " f"detach='{detach}', remove='{remove}'") @@ -331,7 +355,9 @@ def deploy_mongodb( return self.data_mongo.url def kill_rabbitmq(self) -> None: - if not self.data_queue.pid: + if self.data_queue.skip_deployment: + return + elif not self.data_queue.pid: self.log.warning('No running RabbitMQ instance found') return client = create_docker_client( @@ -346,7 +372,9 @@ def kill_rabbitmq(self) -> None: self.log.info('The RabbitMQ is stopped') def kill_mongodb(self) -> None: - if not self.data_mongo.pid: + if self.data_mongo.skip_deployment: + return + elif not self.data_mongo.pid: self.log.warning('No running MongoDB instance found') return client = create_docker_client( diff --git a/ocrd_network/ocrd_network/deployment_utils.py b/ocrd_network/ocrd_network/deployment_utils.py index 9be063cb2c..937f2efdb6 100644 --- a/ocrd_network/ocrd_network/deployment_utils.py +++ b/ocrd_network/ocrd_network/deployment_utils.py @@ -4,8 +4,10 @@ from docker.transport import SSHHTTPAdapter from paramiko import AutoAddPolicy, SSHClient from time import sleep +import re from .rabbitmq_utils import RMQPublisher +from pymongo import MongoClient __all__ = [ 'create_docker_client', @@ -104,6 +106,24 @@ def wait_for_rabbitmq_availability( raise RuntimeError('Error waiting for queue startup: timeout exceeded') +def verify_rabbitmq_available(host: str, port: int, vhost: str, username: str, + password: str) -> None: + try: + dummy_publisher = RMQPublisher(host=host, port=port, vhost=vhost) + dummy_publisher.authenticate_and_connect(username=username, password=password) + except Exception: + raise Exception(f'Cannot connet to Rabbitmq host: {host}, port: {port}, ' + f'vhost: {vhost}, username: {username}') + + +def verify_mongodb_available(mongo_url: str) -> None: + try: + client = MongoClient(mongo_url, serverSelectionTimeoutMS=1000.0) + client.admin.command("ismaster") + except Exception: + raise Exception(f'Cannot connet to MongoDb: {re.sub(r":[^@]+@", ":****@", mongo_url)}') + + class DeployType(Enum): """ Deploy-Type of the processing worker/processor server. """ From 8da78680ae792cf2674d1a29b42780a4e3f5ce1e Mon Sep 17 00:00:00 2001 From: joschrew Date: Thu, 11 May 2023 09:34:44 +0200 Subject: [PATCH 17/35] Remove redundant rabbitmq availability check --- ocrd_network/ocrd_network/deployer.py | 10 +--------- ocrd_network/ocrd_network/deployment_utils.py | 17 ++++------------- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index 7d67a7e79d..b6318d1a8c 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -17,7 +17,6 @@ from .deployment_utils import ( create_docker_client, DeployType, - wait_for_rabbitmq_availability, verify_mongodb_available, verify_rabbitmq_available, ) @@ -240,13 +239,6 @@ def deploy_rabbitmq( self.data_queue.username, self.data_queue.password ) - wait_for_rabbitmq_availability( - host=self.data_queue.address, - port=int(self.data_queue.port), - vhost='/', - username=self.data_queue.username, - password=self.data_queue.password - ) return self.data_queue.url self.log.debug(f"Trying to deploy '{image}', with modes: " f"detach='{detach}', remove='{remove}'") @@ -290,7 +282,7 @@ def deploy_rabbitmq( rmq_port = int(self.data_queue.port) rmq_vhost = '/' - wait_for_rabbitmq_availability( + verify_rabbitmq_available( host=rmq_host, port=rmq_port, vhost=rmq_vhost, diff --git a/ocrd_network/ocrd_network/deployment_utils.py b/ocrd_network/ocrd_network/deployment_utils.py index 937f2efdb6..b02d5d2778 100644 --- a/ocrd_network/ocrd_network/deployment_utils.py +++ b/ocrd_network/ocrd_network/deployment_utils.py @@ -85,7 +85,7 @@ def _create_paramiko_client(self, base_url: str) -> None: self.ssh_client.set_missing_host_key_policy(AutoAddPolicy) -def wait_for_rabbitmq_availability( +def verify_rabbitmq_available( host: str, port: int, vhost: str, @@ -103,17 +103,8 @@ def wait_for_rabbitmq_availability( else: # TODO: Disconnect the dummy_publisher here before returning... return - raise RuntimeError('Error waiting for queue startup: timeout exceeded') - - -def verify_rabbitmq_available(host: str, port: int, vhost: str, username: str, - password: str) -> None: - try: - dummy_publisher = RMQPublisher(host=host, port=port, vhost=vhost) - dummy_publisher.authenticate_and_connect(username=username, password=password) - except Exception: - raise Exception(f'Cannot connet to Rabbitmq host: {host}, port: {port}, ' - f'vhost: {vhost}, username: {username}') + raise RuntimeError(f'Cannot connet to Rabbitmq host: {host}, port: {port}, ' + f'vhost: {vhost}, username: {username}') def verify_mongodb_available(mongo_url: str) -> None: @@ -121,7 +112,7 @@ def verify_mongodb_available(mongo_url: str) -> None: client = MongoClient(mongo_url, serverSelectionTimeoutMS=1000.0) client.admin.command("ismaster") except Exception: - raise Exception(f'Cannot connet to MongoDb: {re.sub(r":[^@]+@", ":****@", mongo_url)}') + raise RuntimeError(f'Cannot connet to MongoDb: {re.sub(r":[^@]+@", ":****@", mongo_url)}') class DeployType(Enum): From f749c2426abfc39f4f980e3f896a78d4da1960dd Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 15 May 2023 14:40:29 +0200 Subject: [PATCH 18/35] Correct spelling --- ocrd_network/ocrd_network/deployer.py | 2 +- ocrd_network/ocrd_network/deployment_utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index b6318d1a8c..f5b7f045d5 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -301,7 +301,7 @@ def deploy_mongodb( ports_mapping: Union[Dict, None] = None ) -> str: if self.data_mongo.skip_deployment: - self.log.debug('MongoDb is externaly managed. Skipping deployment') + self.log.debug('MongoDB is externaly managed. Skipping deployment') verify_mongodb_available(self.data_mongo.url); return self.data_mongo.url diff --git a/ocrd_network/ocrd_network/deployment_utils.py b/ocrd_network/ocrd_network/deployment_utils.py index b02d5d2778..a5c01de6ec 100644 --- a/ocrd_network/ocrd_network/deployment_utils.py +++ b/ocrd_network/ocrd_network/deployment_utils.py @@ -103,7 +103,7 @@ def verify_rabbitmq_available( else: # TODO: Disconnect the dummy_publisher here before returning... return - raise RuntimeError(f'Cannot connet to Rabbitmq host: {host}, port: {port}, ' + raise RuntimeError(f'Cannot connect to RabbitMQ host: {host}, port: {port}, ' f'vhost: {vhost}, username: {username}') @@ -112,7 +112,7 @@ def verify_mongodb_available(mongo_url: str) -> None: client = MongoClient(mongo_url, serverSelectionTimeoutMS=1000.0) client.admin.command("ismaster") except Exception: - raise RuntimeError(f'Cannot connet to MongoDb: {re.sub(r":[^@]+@", ":****@", mongo_url)}') + raise RuntimeError(f'Cannot connect to MongoDB: {re.sub(r":[^@]+@", ":****@", mongo_url)}') class DeployType(Enum): From f5e212afbfd0c7043d8da91af196782b05dc407d Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 1 Jun 2023 20:14:07 +0200 Subject: [PATCH 19/35] =?UTF-8?q?docker-cuda:=20rewrite=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - extra Dockerfile.cuda instead of guessing from base image name - re-use non-CUDA ocrd/core instead of nvidia/cuda base image - install CUDA toolkit and libraries via (micro)mamba instead of nvidia-pyindex CUDA libraries made available system-wide - get cuDNN and CUDA libs from conda-forge and nvidia channels --- Dockerfile | 24 ------------------------ Dockerfile.cuda | 38 ++++++++++++++++++++++++++++++++++++++ Makefile | 33 ++++++++++++++------------------- 3 files changed, 52 insertions(+), 43 deletions(-) create mode 100644 Dockerfile.cuda diff --git a/Dockerfile b/Dockerfile index 506bfa0143..04df1e3564 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ ARG BASE_IMAGE FROM $BASE_IMAGE -ARG BASE_IMAGE ARG FIXUP=echo MAINTAINER OCR-D ENV DEBIAN_FRONTEND noninteractive @@ -40,30 +39,7 @@ RUN apt-get update && apt-get -y install software-properties-common \ && make install \ && eval $FIXUP \ && rm -rf /build-ocrd -RUN if echo $BASE_IMAGE | fgrep -q cuda; then \ - pip3 install nvidia-pyindex && \ - pip3 install nvidia-cudnn-cu11==8.6.0.163 && \ - pip3 install nvidia-cublas-cu117 && \ - pip3 install nvidia-cusparse-cu117 && \ - pip3 install nvidia-cusolver-cu117 && \ - pip3 install nvidia-curand-cu117 && \ - pip3 install nvidia-cufft-cu117 && \ - pip3 install nvidia-cuda-runtime-cu117 && \ - pip3 install nvidia-cuda-nvrtc-cu117 && \ - echo /usr/local/lib/python3.8/site-packages/nvidia/cudnn/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ - echo /usr/local/lib/python3.8/site-packages/nvidia/cublas/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ - echo /usr/local/lib/python3.8/site-packages/nvidia/cusparse/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ - echo /usr/local/lib/python3.8/site-packages/nvidia/cusolver/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ - echo /usr/local/lib/python3.8/site-packages/nvidia/curand/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ - echo /usr/local/lib/python3.8/site-packages/nvidia/cufft/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ - echo /usr/local/lib/python3.8/site-packages/nvidia/cuda_runtime/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ - echo /usr/local/lib/python3.8/site-packages/nvidia/cuda_nvrtc/lib/ >> /etc/ld.so.conf.d/000_cuda.conf && \ - ldconfig; fi WORKDIR /data -# remove any entry points from base image -RUN rm -fr /opt/nvidia -ENTRYPOINT [] - CMD ["/usr/local/bin/ocrd", "--help"] diff --git a/Dockerfile.cuda b/Dockerfile.cuda new file mode 100644 index 0000000000..a6c51a71f1 --- /dev/null +++ b/Dockerfile.cuda @@ -0,0 +1,38 @@ +ARG BASE_IMAGE +FROM $BASE_IMAGE + +ENV MAMBA_EXE=/usr/local/bin/conda +ENV MAMBA_ROOT_PREFIX=/conda +ENV PATH=$MAMBA_ROOT_PREFIX/bin:$PATH +ENV CONDA_PREFIX=$MAMBA_ROOT_PREFIX +ENV CONDA_SHLVL='1' + +WORKDIR $MAMBA_ROOT_PREFIX + +RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba +RUN mv bin/micromamba $MAMBA_EXE +RUN hash -r +RUN echo $CONDA_PREFIX/lib >> /etc/ld.so.conf.d/conda.conf +# get CUDA toolkit, including compiler and libraries with dev +# nvidia channels do not provide (recent) cudnn (needed for Torch, TF etc): +#RUN conda install -c nvidia/label/cuda-11.8.0 cuda && conda clean -a +# conda-forge has cudnn but no cudatoolkit-dev anymore +# so let's combine nvidia and conda-forge (will be same lib versions, no waste of space): +RUN conda install -c conda-forge \ + cudatoolkit=11.8.0 \ + cudnn=8.8.* && \ + conda clean -a && \ + find $CONDA_PREFIX -name "*_static.a" -delete +RUN conda install -c nvidia/label/cuda-11.8.0 \ + cuda-nvcc \ + cuda-cudart-dev \ + cuda-libraries-dev && \ + conda clean -a && \ + find $CONDA_PREFIX -name "*_static.a" -delete +# gputil/nvidia-smi would be nice, too – but that drags in Python as a conda dependency... +RUN ldconfig + +WORKDIR /data + +CMD ["/usr/local/bin/ocrd", "--help"] + diff --git a/Makefile b/Makefile index d87ccb8d9f..ff28d1fc5e 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,6 @@ help: @echo " docs-coverage Calculate docstring coverage" @echo " docker Build docker image" @echo " docker-cuda Build docker GPU / CUDA image" - @echo " cuda-ubuntu Install native CUDA toolkit in different versions" @echo " pypi Build wheels and source dist and twine upload them" @echo "" @echo " Variables" @@ -45,15 +44,6 @@ help: # END-EVAL -# Docker tag. Default: '$(DOCKER_TAG)'. -DOCKER_TAG = ocrd/core - -# Docker base image. Default: '$(DOCKER_BASE_IMAGE)'. -DOCKER_BASE_IMAGE = ubuntu:20.04 - -# Additional arguments to docker build. Default: '$(DOCKER_ARGS)' -DOCKER_ARGS = - # pip install command. Default: $(PIP_INSTALL) PIP_INSTALL = $(PIP) install @@ -215,17 +205,22 @@ pyclean: .PHONY: docker docker-cuda +# Additional arguments to docker build. Default: '$(DOCKER_ARGS)' +DOCKER_ARGS = + # Build docker image -docker docker-cuda: - docker build --progress=plain -t $(DOCKER_TAG) --build-arg BASE_IMAGE=$(DOCKER_BASE_IMAGE) $(DOCKER_ARGS) . - -# Build docker GPU / CUDA image -# (we need *devel* for compilation via nvcc etc. of our dependencies inside the build, -# no good *cudnn* because it will not be used by pip installs, -# and cudnn-devel does not contain nvcc etc.) -# cf. https://github.com/NVIDIA/nvidia-docker/wiki/CUDA#description -docker-cuda: DOCKER_BASE_IMAGE = nvidia/cuda:11.8.0-devel-ubuntu20.04 +docker: DOCKER_BASE_IMAGE = ubuntu:20.04 +docker: DOCKER_TAG = ocrd/core +docker: DOCKER_FILE = Dockerfile + +docker-cuda: DOCKER_BASE_IMAGE = ocrd/core docker-cuda: DOCKER_TAG = ocrd/core-cuda +docker-cuda: DOCKER_FILE = Dockerfile.cuda + +docker-cuda: docker + +docker docker-cuda: + docker build --progress=plain -f $(DOCKER_FILE) -t $(DOCKER_TAG) --build-arg BASE_IMAGE=$(DOCKER_BASE_IMAGE) $(DOCKER_ARGS) . # Build wheels and source dist and twine upload them pypi: uninstall install From 85a5d16a3e37ae9ac33cd2193d62c4e32eaf4eb2 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 2 Jun 2023 02:36:50 +0200 Subject: [PATCH 20/35] =?UTF-8?q?docker-cuda:=20improve=20(reduce=20size)?= =?UTF-8?q?=20again=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - install CUDA libraries via nvidia-pyindex again (but not nvcc) - ensure they can be compiled/linked against system-wide (with nvcc) --- Dockerfile.cuda | 51 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/Dockerfile.cuda b/Dockerfile.cuda index a6c51a71f1..c968ae9f48 100644 --- a/Dockerfile.cuda +++ b/Dockerfile.cuda @@ -12,26 +12,51 @@ WORKDIR $MAMBA_ROOT_PREFIX RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba RUN mv bin/micromamba $MAMBA_EXE RUN hash -r +RUN mkdir -p $CONDA_PREFIX/lib $CONDA_PREFIX/include RUN echo $CONDA_PREFIX/lib >> /etc/ld.so.conf.d/conda.conf -# get CUDA toolkit, including compiler and libraries with dev -# nvidia channels do not provide (recent) cudnn (needed for Torch, TF etc): +# Get CUDA toolkit, including compiler and libraries with dev. +# The nvidia channels do not provide (recent) cudnn (needed for Torch, TF etc): #RUN conda install -c nvidia/label/cuda-11.8.0 cuda && conda clean -a -# conda-forge has cudnn but no cudatoolkit-dev anymore +# The conda-forge channel has cudnn but no cudatoolkit-dev anymore, # so let's combine nvidia and conda-forge (will be same lib versions, no waste of space): -RUN conda install -c conda-forge \ - cudatoolkit=11.8.0 \ - cudnn=8.8.* && \ - conda clean -a && \ - find $CONDA_PREFIX -name "*_static.a" -delete RUN conda install -c nvidia/label/cuda-11.8.0 \ - cuda-nvcc \ - cuda-cudart-dev \ - cuda-libraries-dev && \ - conda clean -a && \ - find $CONDA_PREFIX -name "*_static.a" -delete + cuda-nvcc \ + cuda-cccl \ + && conda clean -a \ + && find $CONDA_PREFIX -name "*_static.a" -delete + # cuda-cudart-dev \ + # cuda-libraries-dev \ +#RUN conda install -c conda-forge \ +# cudatoolkit=11.8.0 \ +# cudnn=8.8.* && \ +# conda clean -a && \ +# find $CONDA_PREFIX -name "*_static.a" -delete +# Since Torch will pull in the CUDA libraries (as Python pkgs) anyway, +# let's jump the shark and pull these via NGC index directly, +# but then share them with the rest of the system so native compilation/linking +# works, too: +RUN pip3 install nvidia-pyindex \ + && pip3 install nvidia-cudnn-cu11==8.6.0.163 \ + nvidia-cublas-cu11 \ + nvidia-cusparse-cu11 \ + nvidia-cusolver-cu11 \ + nvidia-curand-cu11 \ + nvidia-cufft-cu11 \ + nvidia-cuda-runtime-cu11 \ + nvidia-cuda-nvrtc-cu11 \ + && for pkg in cudnn cublas cusparse cusolver curand cufft cuda_runtime cuda_nvrtc; do \ + for lib in /usr/local/lib/python3.8/site-packages/nvidia/$pkg/lib/lib*.so.*; do \ + base=$(basename $lib); \ + ln -s $lib $CONDA_PREFIX/lib/$base.so; \ + ln -s $lib $CONDA_PREFIX/lib/${base%.so.*}.so; \ + done \ + && ln -s /usr/local/lib/python3.8/site-packages/nvidia/$pkg/include/* $CONDA_PREFIX/include/; \ + done \ + && ldconfig # gputil/nvidia-smi would be nice, too – but that drags in Python as a conda dependency... RUN ldconfig + WORKDIR /data CMD ["/usr/local/bin/ocrd", "--help"] From 47eff22f7a18928ff1bb56f734f63a3a033f6772 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 2 Jun 2023 03:58:58 +0200 Subject: [PATCH 21/35] remove out-dated processor resources --- ocrd/ocrd/resource_list.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/ocrd/ocrd/resource_list.yml b/ocrd/ocrd/resource_list.yml index fcedf85913..5a047daad8 100644 --- a/ocrd/ocrd/resource_list.yml +++ b/ocrd/ocrd/resource_list.yml @@ -59,19 +59,3 @@ ocrd-sbb-binarize: type: archive path_in_archive: models size: 1654623597 -ocrd-sbb-textline-detector: - - url: https://qurator-data.de/sbb_textline_detector/models.tar.gz - description: default models provided by github.com/qurator-spk - name: default - type: archive - size: 1194551551 -ocrd-kraken-segment: - - url: https://github.com/mittagessen/kraken/raw/master/kraken/blla.mlmodel - description: Pretrained baseline segmentation model - name: blla.mlmodel - size: 5046835 -ocrd-kraken-recognize: - - url: https://zenodo.org/record/2577813/files/en_best.mlmodel?download=1 - name: en_best.mlmodel - description: This model has been trained on a large corpus of modern printed English text\naugmented with ~10000 lines of historical pages - size: 2930723 From 12e781c67812e7cda97659786d5b79ec2b3aa31a Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 2 Jun 2023 04:13:54 +0200 Subject: [PATCH 22/35] Revert "Merge remote-tracking branch 'hnesk/no-more-pkg_resources' into release-2.36.0" This reverts commit 9a54ef6ae95c84fcf9d0513afa732c4abc33e8eb, reversing changes made to a78d4c50e92b2acc8fcb0f18fdb23da366631b41. --- Makefile | 2 +- ocrd/ocrd/constants.py | 2 +- .../builtin/{ => dummy}/ocrd-tool.json | 0 .../ocrd/processor/builtin/dummy_processor.py | 4 +- ocrd/ocrd/workspace_bagger.py | 2 +- ocrd/setup.py | 1 - ocrd_models/ocrd_models/constants.py | 2 +- ocrd_utils/ocrd_utils/constants.py | 3 +- ocrd_utils/ocrd_utils/package_resources.py | 49 ------------------- ocrd_utils/requirements.txt | 2 - ocrd_validators/ocrd_validators/constants.py | 2 +- 11 files changed, 8 insertions(+), 61 deletions(-) rename ocrd/ocrd/processor/builtin/{ => dummy}/ocrd-tool.json (100%) delete mode 100644 ocrd_utils/ocrd_utils/package_resources.py diff --git a/Makefile b/Makefile index ff28d1fc5e..7a66526d1e 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ deps-test: # (Re)install the tool install: - $(PIP) install -U pip wheel setuptools fastentrypoints + $(PIP) install -U pip wheel setuptools @# speedup for end-of-life builds @# we cannot use pip config here due to pip#11988 if $(PYTHON) -V | fgrep -e 3.5 -e 3.6; then $(PIP) install --prefer-binary opencv-python-headless numpy; fi diff --git a/ocrd/ocrd/constants.py b/ocrd/ocrd/constants.py index 2e9c17c649..1d436a7fa9 100644 --- a/ocrd/ocrd/constants.py +++ b/ocrd/ocrd/constants.py @@ -1,7 +1,7 @@ """ Constants for ocrd. """ -from ocrd_utils.package_resources import resource_filename +from pkg_resources import resource_filename __all__ = [ 'TMP_PREFIX', diff --git a/ocrd/ocrd/processor/builtin/ocrd-tool.json b/ocrd/ocrd/processor/builtin/dummy/ocrd-tool.json similarity index 100% rename from ocrd/ocrd/processor/builtin/ocrd-tool.json rename to ocrd/ocrd/processor/builtin/dummy/ocrd-tool.json diff --git a/ocrd/ocrd/processor/builtin/dummy_processor.py b/ocrd/ocrd/processor/builtin/dummy_processor.py index 0fb07fc6d7..c0371e2d0e 100644 --- a/ocrd/ocrd/processor/builtin/dummy_processor.py +++ b/ocrd/ocrd/processor/builtin/dummy_processor.py @@ -1,6 +1,6 @@ # pylint: disable=missing-module-docstring,invalid-name from os.path import join, basename -from ocrd_utils.package_resources import resource_string +from pkg_resources import resource_string import click @@ -17,7 +17,7 @@ ) from ocrd_modelfactory import page_from_file -OCRD_TOOL = parse_json_string_with_comments(resource_string(__name__, 'ocrd-tool.json').decode('utf8')) +OCRD_TOOL = parse_json_string_with_comments(resource_string(__name__, 'dummy/ocrd-tool.json').decode('utf8')) class DummyProcessor(Processor): """ diff --git a/ocrd/ocrd/workspace_bagger.py b/ocrd/ocrd/workspace_bagger.py index 97d4527180..447f33d139 100644 --- a/ocrd/ocrd/workspace_bagger.py +++ b/ocrd/ocrd/workspace_bagger.py @@ -9,6 +9,7 @@ import sys from bagit import Bag, make_manifests, _load_tag_file, _make_tag_file, _make_tagmanifest_file # pylint: disable=no-name-in-module from distutils.dir_util import copy_tree +from pkg_resources import get_distribution from ocrd_utils import ( pushd_popd, @@ -22,7 +23,6 @@ from ocrd_validators.constants import BAGIT_TXT, TMP_BAGIT_PREFIX, OCRD_BAGIT_PROFILE_URL from ocrd_modelfactory import page_from_file from ocrd_models.ocrd_page import to_xml -from ocrd_utils.package_resources import get_distribution from .workspace import Workspace diff --git a/ocrd/setup.py b/ocrd/setup.py index be28ba0d6b..654b362378 100644 --- a/ocrd/setup.py +++ b/ocrd/setup.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import fastentrypoints from setuptools import setup, find_packages from ocrd_utils import VERSION diff --git a/ocrd_models/ocrd_models/constants.py b/ocrd_models/ocrd_models/constants.py index b3fe89a4c9..6c8b0e1017 100644 --- a/ocrd_models/ocrd_models/constants.py +++ b/ocrd_models/ocrd_models/constants.py @@ -1,7 +1,7 @@ """ Constants for ocrd_models. """ -from ocrd_utils.package_resources import resource_string +from pkg_resources import resource_string import re __all__ = [ diff --git a/ocrd_utils/ocrd_utils/constants.py b/ocrd_utils/ocrd_utils/constants.py index cfe5b20e09..3b1449e30c 100644 --- a/ocrd_utils/ocrd_utils/constants.py +++ b/ocrd_utils/ocrd_utils/constants.py @@ -1,12 +1,11 @@ """ Constants for ocrd_utils. """ +from pkg_resources import get_distribution from re import compile as regex_compile from os import environ from os.path import join, expanduser -from ocrd_utils.package_resources import get_distribution - __all__ = [ 'EXT_TO_MIME', 'LOG_FORMAT', diff --git a/ocrd_utils/ocrd_utils/package_resources.py b/ocrd_utils/ocrd_utils/package_resources.py deleted file mode 100644 index c142f0e119..0000000000 --- a/ocrd_utils/ocrd_utils/package_resources.py +++ /dev/null @@ -1,49 +0,0 @@ -import atexit -from contextlib import ExitStack -from pathlib import Path - -try: - from importlib.resources import as_file, files -except ImportError: - from importlib_resources import as_file, files # type: ignore - -try: - from importlib.metadata import distribution as get_distribution -except ImportError: - from importlib_metadata import distribution as get_distribution - -# See https://importlib-resources.readthedocs.io/en/latest/migration.html#pkg-resources-resource-filename -_file_manager = ExitStack() -atexit.register(_file_manager.close) - - -def resource_filename(package: str, resource: str) -> Path: - """ - Reimplementation of the function with the same name from pkg_resources - - Using importlib for better performance - - package : str - The package from where to start looking for resource (often __name__) - resource : str - The resource to look up - """ - parent_package = package.rsplit('.',1)[0] - return _file_manager.enter_context(as_file(files(parent_package).joinpath(resource))) - - -def resource_string(package: str, resource: str) -> bytes: - """ - Reimplementation of the function with the same name from pkg_resources - - Using importlib for better performance - - package : str - The package from where to start looking for resource (often __name__) - resource : str - The resource to look up - """ - parent_package = package.rsplit('.',1)[0] - return files(parent_package).joinpath(resource).read_bytes() - -__all__ = ['resource_filename', 'resource_string', 'get_distribution'] diff --git a/ocrd_utils/requirements.txt b/ocrd_utils/requirements.txt index 12f36abec3..b4dccec0ad 100644 --- a/ocrd_utils/requirements.txt +++ b/ocrd_utils/requirements.txt @@ -3,6 +3,4 @@ Pillow >= 7.2.0 # tensorflow versions might require different versions numpy atomicwrites >= 1.3.0 -importlib_metadata;python_version<'3.8' -importlib_resources;python_version<'3.9' frozendict>=2.3.4 diff --git a/ocrd_validators/ocrd_validators/constants.py b/ocrd_validators/ocrd_validators/constants.py index b3834f7eb0..fc1ff445ae 100644 --- a/ocrd_validators/ocrd_validators/constants.py +++ b/ocrd_validators/ocrd_validators/constants.py @@ -2,7 +2,7 @@ Constants for ocrd_validators. """ import yaml -from ocrd_utils.package_resources import resource_string, resource_filename +from pkg_resources import resource_string, resource_filename __all__ = [ 'PROCESSING_SERVER_CONFIG_SCHEMA', From bac1a45b1393028cebf85371a4e0225de00936cb Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Sat, 3 Jun 2023 00:17:29 +0200 Subject: [PATCH 23/35] make help: improve description --- Dockerfile.cuda | 2 -- Makefile | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Dockerfile.cuda b/Dockerfile.cuda index c968ae9f48..0e470a4744 100644 --- a/Dockerfile.cuda +++ b/Dockerfile.cuda @@ -54,8 +54,6 @@ RUN pip3 install nvidia-pyindex \ done \ && ldconfig # gputil/nvidia-smi would be nice, too – but that drags in Python as a conda dependency... -RUN ldconfig - WORKDIR /data diff --git a/Makefile b/Makefile index 7a66526d1e..f6537b69e6 100644 --- a/Makefile +++ b/Makefile @@ -32,13 +32,13 @@ help: @echo " docs-clean Clean docs" @echo " docs-coverage Calculate docstring coverage" @echo " docker Build docker image" - @echo " docker-cuda Build docker GPU / CUDA image" + @echo " docker-cuda Build docker image for GPU / CUDA" @echo " pypi Build wheels and source dist and twine upload them" @echo "" @echo " Variables" @echo "" - @echo " DOCKER_TAG Docker tag. Default: '$(DOCKER_TAG)'." - @echo " DOCKER_BASE_IMAGE Docker base image. Default: '$(DOCKER_BASE_IMAGE)'." + @echo " DOCKER_TAG Docker target image tag. Default: '$(DOCKER_TAG)'." + @echo " DOCKER_BASE_IMAGE Docker source image tag. Default: '$(DOCKER_BASE_IMAGE)'." @echo " DOCKER_ARGS Additional arguments to docker build. Default: '$(DOCKER_ARGS)'" @echo " PIP_INSTALL pip install command. Default: $(PIP_INSTALL)" From 95062b06064d687bc7d1eb226860544f85fe56cb Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Wed, 7 Jun 2023 13:25:21 +0200 Subject: [PATCH 24/35] :memo: changelog --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb6e6611f5..ce40cb2a6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ Versioned according to [Semantic Versioning](http://semver.org/). ## Unreleased +Changed: + + * `core cuda` Docker: CUDA base image working again, based on `ocrd/core` not `nvidia/cuda` in a separate `Dockerfile.cuda`, #1041 + * `core-cuda` Docker: adopt #1008 (venv under /usr/local, as in ocrd_all, instead of dist-packages), #1041 + * `core-cuda` Docker: use conda ([micromamba](https://mamba.readthedocs.io/en/latest/user_guide/micromamba.html)) for CUDA toolkit, and [nvidia-pyindex](https://pypi.org/project/nvidia-pyindex/) for CUDA libs – instead of [nvidia/cuda](https://hub.docker.com/r/nvidia/cuda) base image, #1041 + * more robust workaround for shapely#1598, #1041 + +Removed: + + * Revert #882 (fastentrypoints) as it enforces deps versions at runtime + * Drop `ocrd_utils.package_resources` and use `pkg_resources.*` directly, #1041 + * `ocrd resmgr`: Drop redundant (processor-provided) entries in the central `resource_list.yml`. + ## [2.50.0] - 2023-04-24 Added: From c636f2c060c91d42ec1eb1d5b847a7a7aef38d2e Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Wed, 7 Jun 2023 13:27:03 +0200 Subject: [PATCH 25/35] :package: v2.51.0 --- CHANGELOG.md | 3 +++ ocrd_utils/setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce40cb2a6f..d156af5d80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ Versioned according to [Semantic Versioning](http://semver.org/). ## Unreleased +## [2.51.0] - 2023-06-07 + Changed: * `core cuda` Docker: CUDA base image working again, based on `ocrd/core` not `nvidia/cuda` in a separate `Dockerfile.cuda`, #1041 @@ -1744,6 +1746,7 @@ Fixed Initial Release +[2.51.0]: ../../compare/v2.51.0..v2.50.0 [2.50.0]: ../../compare/v2.50.0..v2.49.0 [2.49.0]: ../../compare/v2.49.0..v2.48.1 [2.48.1]: ../../compare/v2.48.1..v2.48.0 diff --git a/ocrd_utils/setup.py b/ocrd_utils/setup.py index 6b0b8690d7..1ab0b56d17 100644 --- a/ocrd_utils/setup.py +++ b/ocrd_utils/setup.py @@ -5,7 +5,7 @@ setup( name='ocrd_utils', - version='2.50.0', + version='2.51.0', description='OCR-D framework - shared code, helpers, constants', long_description=open('README.md').read(), long_description_content_type='text/markdown', From 8c7b761046a09c0733d2b1dfabeb2a6cc85cd2b1 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Wed, 7 Jun 2023 14:53:45 +0200 Subject: [PATCH 26/35] docker-image: reuse local ghcr.io image instead of docker.io (only cosmetic) --- .github/workflows/docker-image.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 453cae65b6..25999a199e 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -23,9 +23,11 @@ jobs: name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Build the Docker image + # default tag uses docker.io, so override on command-line run: make docker DOCKER_TAG=${{ env.DOCKER_TAG }} - name: Build the Docker image with GPU support - run: make docker-cuda DOCKER_TAG=${{ env.DOCKER_TAG }}-cuda + # default tag uses docker.io, so override on command-line + run: make docker-cuda DOCKER_TAG=${{ env.DOCKER_TAG }}-cuda DOCKER_BASE_IMAGE=${{ env.DOCKER_TAG }} - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: From 4bfac5e9d870c7ad56335ae441830da3213be511 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Wed, 7 Jun 2023 18:27:31 +0200 Subject: [PATCH 27/35] disable logging tests until properly fixed --- Makefile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f6537b69e6..8197b07d08 100644 --- a/Makefile +++ b/Makefile @@ -140,9 +140,16 @@ assets: repo/assets .PHONY: test # Run all unit tests test: assets + $(PYTHON) -m pytest --continue-on-collection-errors --durations=10\ + --ignore=$(TESTDIR)/test_logging.py \ + --ignore=$(TESTDIR)/test_logging_conf.py \ + --ignore-glob="$(TESTDIR)/**/*bench*.py" \ + $(TESTDIR) + #$(MAKE) test-logging + +test-logging: HOME=$(CURDIR)/ocrd_utils $(PYTHON) -m pytest --continue-on-collection-errors -k TestLogging $(TESTDIR) HOME=$(CURDIR) $(PYTHON) -m pytest --continue-on-collection-errors -k TestLogging $(TESTDIR) - $(PYTHON) -m pytest --continue-on-collection-errors --durations=10 --ignore=$(TESTDIR)/test_logging.py --ignore-glob="$(TESTDIR)/**/*bench*.py" $(TESTDIR) benchmark: $(PYTHON) -m pytest $(TESTDIR)/model/test_ocrd_mets_bench.py From ca5f3421c85ced15705cb449e3a8b32b5bedee64 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Wed, 7 Jun 2023 18:28:13 +0200 Subject: [PATCH 28/35] test_workspace_bagger: use ocr-d.de instead of google.com for testing --- tests/validator/test_workspace_bagger.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/validator/test_workspace_bagger.py b/tests/validator/test_workspace_bagger.py index fac3cd1084..fc3b035039 100644 --- a/tests/validator/test_workspace_bagger.py +++ b/tests/validator/test_workspace_bagger.py @@ -33,7 +33,7 @@ def tearDown(self): def test_bag_zip_and_spill(self): self.workspace.mets.find_all_files(ID='INPUT_0017')[0].url = 'bad-scheme://foo' - self.workspace.mets.find_all_files(ID='INPUT_0020')[0].url = 'http://google.com' + self.workspace.mets.find_all_files(ID='INPUT_0020')[0].url = 'http://ocr-d.de' self.bagger.bag(self.workspace, 'kant_aufklaerung_1784', skip_zip=False, dest=join(self.tempdir, 'out.ocrd.zip')) self.bagger.spill(join(self.tempdir, 'out.ocrd.zip'), join(self.tempdir, 'out')) @@ -52,7 +52,7 @@ def test_bag_wo_dest_zip(self): def test_bag_partial_http_nostrict(self): self.bagger.strict = False makedirs(BACKUPDIR) - self.workspace.mets.find_all_files(ID='INPUT_0020')[0].url = 'http://google.com' + self.workspace.mets.find_all_files(ID='INPUT_0020')[0].url = 'http://ocr-d.de' self.bagger.bag(self.workspace, 'kant_aufklaerung_1784') def test_bag_full(self): From 3f057457e18faa39e68f9de32500e63095411fc0 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Thu, 8 Jun 2023 14:22:40 +0200 Subject: [PATCH 29/35] test bashlib: /usr/bin/env bash instead of /bin/bash --- .circleci/config.yml | 4 +++- tests/cli/test_bashlib.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3fc9dcf157..f4e4436527 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,9 @@ jobs: steps: - checkout - run: HOMEBREW_NO_AUTO_UPDATE=1 brew install imagemagick geos bash - - run: make install + - run: which bash + - run: bash --version + - run: make install - run: make deps-test test benchmark test-python37: diff --git a/tests/cli/test_bashlib.py b/tests/cli/test_bashlib.py index e6db8c6f7a..3d178dcdd6 100644 --- a/tests/cli/test_bashlib.py +++ b/tests/cli/test_bashlib.py @@ -128,7 +128,7 @@ def test_bashlib_cp_processor(self): } } } - script = """#!/bin/bash + script = """#!/usr/bin/env bash set -eu set -o pipefail MIMETYPE_PAGE=$(ocrd bashlib constants MIMETYPE_PAGE) From af38a2cf01d703c3e3a9fecf3afd4db3343bf8c9 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Thu, 8 Jun 2023 15:20:08 +0200 Subject: [PATCH 30/35] debug gh actions --- .github/workflows/docker-image.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 25999a199e..fa81751440 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -2,7 +2,7 @@ name: Docker Image CI on: push: - branches: [ "master" ] + branches: [ "update-gh-actions" ] workflow_dispatch: # run manually env: @@ -22,20 +22,22 @@ jobs: - # Activate cache export feature to reduce build time of images name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Setup upterm session + uses: lhotari/action-upterm@v1 - name: Build the Docker image # default tag uses docker.io, so override on command-line run: make docker DOCKER_TAG=${{ env.DOCKER_TAG }} - name: Build the Docker image with GPU support # default tag uses docker.io, so override on command-line run: make docker-cuda DOCKER_TAG=${{ env.DOCKER_TAG }}-cuda DOCKER_BASE_IMAGE=${{ env.DOCKER_TAG }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Push images to Github Container Registry - run: | - docker push ${{ env.DOCKER_TAG }}:latest - docker push ${{ env.DOCKER_TAG }}-cuda:latest + #- name: Login to GitHub Container Registry + # uses: docker/login-action@v2 + # with: + # registry: ghcr.io + # username: ${{ github.actor }} + # password: ${{ secrets.GITHUB_TOKEN }} + #- name: Push images to Github Container Registry + # run: | + # docker push ${{ env.DOCKER_TAG }}:latest + # docker push ${{ env.DOCKER_TAG }}-cuda:latest From c63ab4c11d80a76a960dd60d4a232068751e19e6 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Thu, 8 Jun 2023 16:15:01 +0200 Subject: [PATCH 31/35] readme: remove dockerhub/travis badge, add GH actions badge --- .github/workflows/docker-image.yml | 22 +++++++++++----------- README.md | 3 +-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index fa81751440..0c48f6c88b 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -2,7 +2,7 @@ name: Docker Image CI on: push: - branches: [ "update-gh-actions" ] + branches: [ "master" ] workflow_dispatch: # run manually env: @@ -30,14 +30,14 @@ jobs: - name: Build the Docker image with GPU support # default tag uses docker.io, so override on command-line run: make docker-cuda DOCKER_TAG=${{ env.DOCKER_TAG }}-cuda DOCKER_BASE_IMAGE=${{ env.DOCKER_TAG }} - #- name: Login to GitHub Container Registry - # uses: docker/login-action@v2 - # with: - # registry: ghcr.io - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} - #- name: Push images to Github Container Registry - # run: | - # docker push ${{ env.DOCKER_TAG }}:latest - # docker push ${{ env.DOCKER_TAG }}-cuda:latest + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Push images to Github Container Registry + run: | + docker push ${{ env.DOCKER_TAG }}:latest + docker push ${{ env.DOCKER_TAG }}-cuda:latest diff --git a/README.md b/README.md index d2b34344b8..4572d7e32e 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,9 @@ > Python modules implementing [OCR-D specs](https://github.com/OCR-D/spec) and related tools [![image](https://img.shields.io/pypi/v/ocrd.svg)](https://pypi.org/project/ocrd/) -[![image](https://travis-ci.org/OCR-D/core.svg?branch=master)](https://travis-ci.org/OCR-D/core) [![image](https://circleci.com/gh/OCR-D/core.svg?style=svg)](https://circleci.com/gh/OCR-D/core) +[![Docker Image CI](https://github.com/OCR-D/core/actions/workflows/docker-image.yml/badge.svg)](https://github.com/OCR-D/core/actions/workflows/docker-image.yml) [![image](https://scrutinizer-ci.com/g/OCR-D/core/badges/build.png?b=master)](https://scrutinizer-ci.com/g/OCR-D/core) -[![Docker Automated build](https://img.shields.io/docker/automated/ocrd/core.svg)](https://hub.docker.com/r/ocrd/core/tags/) [![image](https://codecov.io/gh/OCR-D/core/branch/master/graph/badge.svg)](https://codecov.io/gh/OCR-D/core) [![image](https://scrutinizer-ci.com/g/OCR-D/core/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/OCR-D/core) From 555dece1614271508a21484baa19ce206ec093b6 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Thu, 8 Jun 2023 16:21:12 +0200 Subject: [PATCH 32/35] ci: disable upterm for gh actions --- .github/workflows/docker-image.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 0c48f6c88b..25999a199e 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -22,8 +22,6 @@ jobs: - # Activate cache export feature to reduce build time of images name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Setup upterm session - uses: lhotari/action-upterm@v1 - name: Build the Docker image # default tag uses docker.io, so override on command-line run: make docker DOCKER_TAG=${{ env.DOCKER_TAG }} From d76409edfca5955d60a10714d2cce1e405be2124 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 8 Jun 2023 16:39:18 +0200 Subject: [PATCH 33/35] docker-cuda: move recipe to reusable makefile target deps-cuda --- Dockerfile.cuda | 55 +++++++------------------------------------------ Makefile | 55 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 48 deletions(-) diff --git a/Dockerfile.cuda b/Dockerfile.cuda index 0e470a4744..52d7a27619 100644 --- a/Dockerfile.cuda +++ b/Dockerfile.cuda @@ -4,58 +4,19 @@ FROM $BASE_IMAGE ENV MAMBA_EXE=/usr/local/bin/conda ENV MAMBA_ROOT_PREFIX=/conda ENV PATH=$MAMBA_ROOT_PREFIX/bin:$PATH +ENV CONDA_EXE=$MAMBA_EXE ENV CONDA_PREFIX=$MAMBA_ROOT_PREFIX ENV CONDA_SHLVL='1' -WORKDIR $MAMBA_ROOT_PREFIX - -RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba -RUN mv bin/micromamba $MAMBA_EXE -RUN hash -r -RUN mkdir -p $CONDA_PREFIX/lib $CONDA_PREFIX/include -RUN echo $CONDA_PREFIX/lib >> /etc/ld.so.conf.d/conda.conf -# Get CUDA toolkit, including compiler and libraries with dev. -# The nvidia channels do not provide (recent) cudnn (needed for Torch, TF etc): -#RUN conda install -c nvidia/label/cuda-11.8.0 cuda && conda clean -a -# The conda-forge channel has cudnn but no cudatoolkit-dev anymore, -# so let's combine nvidia and conda-forge (will be same lib versions, no waste of space): -RUN conda install -c nvidia/label/cuda-11.8.0 \ - cuda-nvcc \ - cuda-cccl \ - && conda clean -a \ - && find $CONDA_PREFIX -name "*_static.a" -delete - # cuda-cudart-dev \ - # cuda-libraries-dev \ -#RUN conda install -c conda-forge \ -# cudatoolkit=11.8.0 \ -# cudnn=8.8.* && \ -# conda clean -a && \ -# find $CONDA_PREFIX -name "*_static.a" -delete -# Since Torch will pull in the CUDA libraries (as Python pkgs) anyway, -# let's jump the shark and pull these via NGC index directly, -# but then share them with the rest of the system so native compilation/linking -# works, too: -RUN pip3 install nvidia-pyindex \ - && pip3 install nvidia-cudnn-cu11==8.6.0.163 \ - nvidia-cublas-cu11 \ - nvidia-cusparse-cu11 \ - nvidia-cusolver-cu11 \ - nvidia-curand-cu11 \ - nvidia-cufft-cu11 \ - nvidia-cuda-runtime-cu11 \ - nvidia-cuda-nvrtc-cu11 \ - && for pkg in cudnn cublas cusparse cusolver curand cufft cuda_runtime cuda_nvrtc; do \ - for lib in /usr/local/lib/python3.8/site-packages/nvidia/$pkg/lib/lib*.so.*; do \ - base=$(basename $lib); \ - ln -s $lib $CONDA_PREFIX/lib/$base.so; \ - ln -s $lib $CONDA_PREFIX/lib/${base%.so.*}.so; \ - done \ - && ln -s /usr/local/lib/python3.8/site-packages/nvidia/$pkg/include/* $CONDA_PREFIX/include/; \ - done \ - && ldconfig -# gputil/nvidia-smi would be nice, too – but that drags in Python as a conda dependency... +WORKDIR /build + +COPY Makefile . + +RUN make deps-cuda WORKDIR /data +RUN rm -fr /build + CMD ["/usr/local/bin/ocrd", "--help"] diff --git a/Makefile b/Makefile index 8197b07d08..81ffe811b5 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,8 @@ help: @echo "" @echo " Targets" @echo "" - @echo " deps-ubuntu Dependencies for deployment in an ubuntu/debian linux" + @echo " deps-cuda Dependencies for deployment with GPU support via Conda" + @echo " deps-ubuntu Dependencies for deployment in an Ubuntu/Debian Linux" @echo " deps-test Install test python deps via pip" @echo " install (Re)install the tool" @echo " install-dev Install with pip install -e" @@ -47,6 +48,58 @@ help: # pip install command. Default: $(PIP_INSTALL) PIP_INSTALL = $(PIP) install +deps-cuda: CONDA_EXE ?= /usr/local/bin/conda +deps-cuda: export CONDA_PREFIX ?= /conda +deps-cuda: PYTHON_PREFIX != $(PYTHON) -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])' +deps-cuda: + curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba + mv bin/micromamba $(CONDA_EXE) +# Install Conda system-wide (for interactive / login shells) + echo 'export MAMBA_EXE=$(CONDA_EXE) MAMBA_ROOT_PREFIX=$(CONDA_PREFIX) CONDA_PREFIX=$(CONDA_PREFIX) PATH=$(CONDA_PREFIX)/bin:$$PATH' >> /etc/profile.d/98-conda.sh + mkdir -p $(CONDA_PREFIX)/lib $(CONDA_PREFIX)/include + echo $(CONDA_PREFIX)/lib >> /etc/ld.so.conf.d/conda.conf +# Get CUDA toolkit, including compiler and libraries with dev, +# however, the Nvidia channels do not provide (recent) cudnn (needed for Torch, TF etc): +#conda install -c nvidia/label/cuda-11.8.0 cuda && conda clean -a +# +# The conda-forge channel has cudnn and cudatoolkit but no cudatoolkit-dev anymore (and we need both!), +# so let's combine nvidia and conda-forge (will be same lib versions, no waste of space), +# but omitting cuda-cudart-dev and cuda-libraries-dev (as these will be pulled by pip for torch anyway): + conda install -c nvidia/label/cuda-11.8.0 \ + cuda-nvcc \ + cuda-cccl \ + && conda clean -a \ + && find $(CONDA_PREFIX) -name "*_static.a" -delete +#conda install -c conda-forge \ +# cudatoolkit=11.8.0 \ +# cudnn=8.8.* && \ +#conda clean -a && \ +#find $(CONDA_PREFIX) -name "*_static.a" -delete +# +# Since Torch will pull in the CUDA libraries (as Python pkgs) anyway, +# let's jump the shark and pull these via NGC index directly, +# but then share them with the rest of the system so native compilation/linking +# works, too: + $(PIP) install nvidia-pyindex \ + && $(PIP) install nvidia-cudnn-cu11==8.6.0.163 \ + nvidia-cublas-cu11 \ + nvidia-cusparse-cu11 \ + nvidia-cusolver-cu11 \ + nvidia-curand-cu11 \ + nvidia-cufft-cu11 \ + nvidia-cuda-runtime-cu11 \ + nvidia-cuda-nvrtc-cu11 \ + && for pkg in cudnn cublas cusparse cusolver curand cufft cuda_runtime cuda_nvrtc; do \ + for lib in $(PYTHON_PREFIX)/nvidia/$$pkg/lib/lib*.so.*; do \ + base=`basename $$lib`; \ + ln -s $$lib $(CONDA_PREFIX)/lib/$$base.so; \ + ln -s $$lib $(CONDA_PREFIX)/lib/$${base%.so.*}.so; \ + done \ + && ln -s $(PYTHON_PREFIX)/nvidia/$$pkg/include/* $(CONDA_PREFIX)/include/; \ + done \ + && ldconfig +# gputil/nvidia-smi would be nice, too – but that drags in Python as a conda dependency... + # Dependencies for deployment in an ubuntu/debian linux deps-ubuntu: apt-get install -y python3 imagemagick libgeos-dev From 2538979349a9bdded374af15638b57056cf4ba18 Mon Sep 17 00:00:00 2001 From: kba Date: Fri, 9 Jun 2023 17:49:08 +0000 Subject: [PATCH 34/35] make deps-cuda: Set MAMBA_ROOT_PREFIX to CONDA_PREFIX --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 81ffe811b5..f13549333d 100644 --- a/Makefile +++ b/Makefile @@ -60,11 +60,13 @@ deps-cuda: echo $(CONDA_PREFIX)/lib >> /etc/ld.so.conf.d/conda.conf # Get CUDA toolkit, including compiler and libraries with dev, # however, the Nvidia channels do not provide (recent) cudnn (needed for Torch, TF etc): +#MAMBA_ROOT_PREFIX=$(CONDA_PREFIX) \ #conda install -c nvidia/label/cuda-11.8.0 cuda && conda clean -a # # The conda-forge channel has cudnn and cudatoolkit but no cudatoolkit-dev anymore (and we need both!), # so let's combine nvidia and conda-forge (will be same lib versions, no waste of space), # but omitting cuda-cudart-dev and cuda-libraries-dev (as these will be pulled by pip for torch anyway): + MAMBA_ROOT_PREFIX=$(CONDA_PREFIX) \ conda install -c nvidia/label/cuda-11.8.0 \ cuda-nvcc \ cuda-cccl \ From 79ad301907fb5a41f85a822257cf814143cab595 Mon Sep 17 00:00:00 2001 From: kba Date: Fri, 9 Jun 2023 17:59:16 +0000 Subject: [PATCH 35/35] :memo: changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d156af5d80..2406864752 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Versioned according to [Semantic Versioning](http://semver.org/). ## Unreleased +Added: + + * `make deps-cuda`: Makefile target to set up a working CUDA installation, both for native and Dockerfile.cuda, #1055 + ## [2.51.0] - 2023-06-07 Changed: