diff --git a/docker/jupyter/.gitignore b/docker/jupyter/.gitignore new file mode 100644 index 0000000..5ceb386 --- /dev/null +++ b/docker/jupyter/.gitignore @@ -0,0 +1 @@ +venv diff --git a/docker/jupyter/Dockerfile b/docker/jupyter/Dockerfile new file mode 100644 index 0000000..be82297 --- /dev/null +++ b/docker/jupyter/Dockerfile @@ -0,0 +1,18 @@ +# Start from a core stack version +FROM jupyter/scipy-notebook:latest +# Install from requirements.txt file +COPY --chown=${NB_UID}:${NB_GID} requirements.txt /tmp/ +RUN pip install --quiet --no-cache-dir --requirement /tmp/requirements.txt && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +COPY start-wrapper.sh /usr/local/bin/ + +# Custom fonts config +RUN mkdir -p /home/${NB_USER}/.config/fontconfig +COPY etc/fonts.conf /home/${NB_USER}/.config/fontconfig/fonts.conf +COPY etc/fonts /home/${NB_USER}/.fonts +RUN rm -rf /home/${NB_USER}/.cache/matplotlib +RUN fc-cache --force +RUN python3 -c 'import matplotlib.font_manager; matplotlib.font_manager._load_fontmanager(try_read_cache=False)' +CMD ["start-wrapper.sh", "start-notebook.sh"] diff --git a/docker/jupyter/README.md b/docker/jupyter/README.md new file mode 100644 index 0000000..c1303ef --- /dev/null +++ b/docker/jupyter/README.md @@ -0,0 +1,37 @@ +# JupyterLab with jupytext and other custom extensions + +## Build & run + + + docker build -t imkteksim/dtool-jupyter . + docker run -it -p 8888:8888 -v ${HOME}/.config/dtool:/home/jovyan/.config/dtool -v ${HOME}:/home/jovyan/work imkteksim/dtool-jupyter + +## Creation protocol + +Describes how to extract imports from python scripts and dump as pip-processible `requirements.txt`. + +```console +python3 -m venv venv +source venv/bin/activate + +pip install --upgrade pip +pip install wheel +pip install pip-tools + +pip freeze > initial.txt +cat initial.txt | sed 's/==.*//g' > initial.in + +# extract requirements from some scripts with pipreqs +cat pipreqs_requirements.txt | sed 's/==.*//g' > pipreqs_requirements.in + +cp pipreqs_requirements.in edited_requirements.in # and edit manually + +cat initial.in edited_requirements.in + +cat initial.in edited_requirements.in > requirements.in + +# remove pkg_resources from requirements.in, +# manually add packages, i.e. jupytext, openpyxl, ... + +pip-compile requirements.in +``` diff --git a/docker/jupyter/edited_requirements.in b/docker/jupyter/edited_requirements.in new file mode 100644 index 0000000..8bebfaf --- /dev/null +++ b/docker/jupyter/edited_requirements.in @@ -0,0 +1,26 @@ +ase +asgiref +cycler +dtool +dtool_lookup_api +dtoolcore +dtool-s3 +gromacs +GromacsWrapper +imteksimfw +ipython +ipywidgets +Jinja2 +matplotlib +MDAnalysis +miniball +nglview +numpy +ovito +pandas +panedr +ParmEd +pymongo +PySide2 +PyYAML +scipy diff --git a/docker/jupyter/etc/.gitignore b/docker/jupyter/etc/.gitignore new file mode 100644 index 0000000..ef8212d --- /dev/null +++ b/docker/jupyter/etc/.gitignore @@ -0,0 +1 @@ +fonts/* diff --git a/docker/jupyter/etc/fonts.conf b/docker/jupyter/etc/fonts.conf new file mode 100644 index 0000000..f8b8752 --- /dev/null +++ b/docker/jupyter/etc/fonts.conf @@ -0,0 +1,6 @@ + + + + +~/.fonts + diff --git a/docker/jupyter/initial.in b/docker/jupyter/initial.in new file mode 100644 index 0000000..cf29bf7 --- /dev/null +++ b/docker/jupyter/initial.in @@ -0,0 +1,5 @@ +click +pep517 +pip-tools +pkg_resources +tomli diff --git a/docker/jupyter/initial.txt b/docker/jupyter/initial.txt new file mode 100644 index 0000000..287f678 --- /dev/null +++ b/docker/jupyter/initial.txt @@ -0,0 +1,5 @@ +click==8.0.4 +pep517==0.12.0 +pip-tools==6.5.1 +pkg_resources==0.0.0 +tomli==2.0.1 diff --git a/docker/jupyter/pipreqs_requirements.in b/docker/jupyter/pipreqs_requirements.in new file mode 100644 index 0000000..a993c32 --- /dev/null +++ b/docker/jupyter/pipreqs_requirements.in @@ -0,0 +1,29 @@ +ase +asgiref +cycler +dtool_lookup_api +dtoolcore +fireworks +FireWorks.egg +gromacs +GromacsWrapper +imteksimfw +imteksimfw.egg +ipython +ipywidgets +Jinja2 +matplotlib +MDAnalysis +miniball +nglview +numpy +ovito +pandas +panedr +ParmEd +pymongo +PySide2 +PyYAML +scikit_learn +scipy +SurfaceTopography diff --git a/docker/jupyter/pipreqs_requirements.txt b/docker/jupyter/pipreqs_requirements.txt new file mode 100644 index 0000000..c133907 --- /dev/null +++ b/docker/jupyter/pipreqs_requirements.txt @@ -0,0 +1,29 @@ +ase==3.21.1 +asgiref==3.3.1 +cycler==0.10.0 +dtool_lookup_api==0.5.0 +dtoolcore==3.18.0 +fireworks==2.0.2 +FireWorks.egg==info +gromacs==0.0.0 +GromacsWrapper==0.8.0 +imteksimfw==0.5.1 +imteksimfw.egg==info +ipython==8.1.1 +ipywidgets==7.6.3 +Jinja2==2.11.2 +matplotlib==3.3.3 +MDAnalysis==2.1.0 +miniball==1.0.4 +nglview==2.7.7 +numpy==1.19.5 +ovito==3.6.0 +pandas==1.2.1 +panedr==0.5.2 +ParmEd==3.2.0 +pymongo==3.11.2 +PySide2==5.15.2.1 +PyYAML==6.0 +scikit_learn==1.0.2 +scipy==1.6.0 +SurfaceTopography==0.101.3 diff --git a/docker/jupyter/requirements.in b/docker/jupyter/requirements.in new file mode 100644 index 0000000..0b1322f --- /dev/null +++ b/docker/jupyter/requirements.in @@ -0,0 +1,34 @@ +click +pep517 +pip-tools +tomli +ase +asgiref +cycler +dtool +dtool_lookup_api +dtoolcore +dtool-s3 +gromacs +GromacsWrapper +igraph +imteksimfw +ipython +ipywidgets +Jinja2 +jupyterlab_autorun_cells +jupytext +matplotlib +MDAnalysis +miniball +nglview +numpy +openpyxl +ovito +pandas +panedr +ParmEd +pymongo +PySide2 +PyYAML +scipy diff --git a/docker/jupyter/requirements.txt b/docker/jupyter/requirements.txt new file mode 100644 index 0000000..3fd06cc --- /dev/null +++ b/docker/jupyter/requirements.txt @@ -0,0 +1,552 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# pip-compile requirements.in +# +aiohttp==3.8.1 + # via dtool-lookup-api +aiosignal==1.2.0 + # via aiohttp +anyio==3.5.0 + # via jupyter-server +argon2-cffi==21.3.0 + # via + # jupyter-server + # notebook +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +arrow==1.2.2 + # via jinja2-time +ase==3.22.1 + # via -r requirements.in +asgiref==3.5.0 + # via + # -r requirements.in + # dtool-lookup-api +asttokens==2.0.5 + # via stack-data +async-timeout==4.0.2 + # via aiohttp +attrs==21.4.0 + # via + # aiohttp + # jsonschema + # markdown-it-py +backcall==0.2.0 + # via ipython +bcrypt==3.2.0 + # via paramiko +beautifulsoup4==4.10.0 + # via nbconvert +biopython==1.79 + # via mdanalysis +bleach==4.1.0 + # via nbconvert +boto3==1.21.19 + # via dtool-s3 +botocore==1.24.19 + # via + # boto3 + # s3transfer +certifi==2021.10.8 + # via requests +cffi==1.15.0 + # via + # argon2-cffi-bindings + # bcrypt + # cryptography + # pynacl +charset-normalizer==2.0.12 + # via + # aiohttp + # requests +click==8.0.4 + # via + # -r requirements.in + # click-plugins + # dtool-annotation + # dtool-cli + # dtool-config + # dtool-create + # dtool-info + # dtool-overlay + # dtool-s3 + # dtool-tag + # flask + # pip-tools +click-plugins==1.1.1 + # via dtool-cli +cryptography==36.0.1 + # via paramiko +cycler==0.11.0 + # via + # -r requirements.in + # matplotlib +debugpy==1.5.1 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +dill==0.3.4 + # via imteksimfw +dtool==3.26.1 + # via -r requirements.in +dtool-annotation==0.1.1 + # via dtool +dtool-cli==0.7.1 + # via + # dtool + # dtool-annotation + # dtool-create + # dtool-info + # dtool-overlay + # dtool-s3 + # dtool-tag +dtool-config==0.4.1 + # via dtool +dtool-create==0.23.4 + # via + # dtool + # imteksimfw +dtool-http==0.5.1 + # via + # dtool + # dtool-create +dtool-info==0.16.2 + # via dtool +dtool-lookup-api==0.5.0 + # via + # -r requirements.in + # imteksimfw +dtool-overlay==0.3.1 + # via dtool +dtool-s3==0.14.0 + # via -r requirements.in +dtool-symlink==0.3.1 + # via + # dtool + # dtool-create +dtool-tag==0.1.1 + # via dtool +dtoolcore==3.18.0 + # via + # -r requirements.in + # dtool + # dtool-annotation + # dtool-cli + # dtool-config + # dtool-create + # dtool-http + # dtool-info + # dtool-lookup-api + # dtool-overlay + # dtool-s3 + # dtool-symlink + # dtool-tag +entrypoints==0.4 + # via + # jupyter-client + # nbconvert +et-xmlfile==1.1.0 + # via openpyxl +executing==0.8.3 + # via stack-data +fireworks==2.0.2 + # via imteksimfw +flask==2.0.3 + # via + # fireworks + # flask-paginate +flask-paginate==2022.1.8 + # via fireworks +fonttools==4.30.0 + # via matplotlib +frozenlist==1.3.0 + # via + # aiohttp + # aiosignal +griddataformats==0.7.0 + # via mdanalysis +gromacs==0.0.0 + # via -r requirements.in +gromacswrapper==0.8.2 + # via -r requirements.in +gsd==2.5.1 + # via mdanalysis +gunicorn==20.1.0 + # via fireworks +idna==3.3 + # via + # anyio + # requests + # yarl +igraph==0.9.11 + # via -r requirements.in +importlib-resources==5.4.0 + # via jsonschema +imteksimfw==0.5.1 + # via -r requirements.in +ipykernel==6.9.2 + # via + # ipywidgets + # notebook +ipython==8.1.1 + # via + # -r requirements.in + # ipykernel + # ipywidgets +ipython-genutils==0.2.0 + # via + # ipywidgets + # notebook +ipywidgets==7.6.5 + # via + # -r requirements.in + # nglview +itsdangerous==2.1.1 + # via flask +jedi==0.18.1 + # via ipython +jinja2==3.0.3 + # via + # -r requirements.in + # dtool-info + # fireworks + # flask + # imteksimfw + # jinja2-time + # jupyter-server + # nbconvert + # notebook +jinja2-time==0.2.0 + # via imteksimfw +jmespath==0.10.0 + # via + # boto3 + # botocore +joblib==1.1.0 + # via mdanalysis +jsonschema==4.4.0 + # via nbformat +jupyter-client==7.1.2 + # via + # ipykernel + # jupyter-server + # nbclient + # notebook +jupyter-core==4.9.2 + # via + # jupyter-client + # jupyter-server + # nbconvert + # nbformat + # notebook +jupyter-server==1.16.0 + # via jupyterlab-autorun-cells +jupyterlab-autorun-cells==1.0.0 + # via -r requirements.in +jupyterlab-pygments==0.1.2 + # via nbconvert +jupyterlab-widgets==1.0.2 + # via + # ipywidgets + # nglview +jupytext==1.13.7 + # via -r requirements.in +kiwisolver==1.4.0 + # via matplotlib +markdown-it-py==1.1.0 + # via + # jupytext + # mdit-py-plugins +markupsafe==2.1.1 + # via jinja2 +matplotlib==3.5.1 + # via + # -r requirements.in + # ase + # gromacswrapper + # mdanalysis +matplotlib-inline==0.1.3 + # via + # ipykernel + # ipython +mdanalysis==2.1.0 + # via -r requirements.in +mdit-py-plugins==0.3.0 + # via jupytext +miniball==1.1.0 + # via -r requirements.in +mistune==0.8.4 + # via nbconvert +mmtf-python==1.1.2 + # via mdanalysis +monty==2022.3.12 + # via + # fireworks + # imteksimfw +mrcfile==1.3.0 + # via griddataformats +msgpack==1.0.3 + # via mmtf-python +multidict==6.0.2 + # via + # aiohttp + # yarl +nbclient==0.5.13 + # via nbconvert +nbconvert==6.4.4 + # via + # jupyter-server + # notebook +nbformat==5.2.0 + # via + # ipywidgets + # jupyter-server + # jupytext + # nbclient + # nbconvert + # notebook +nest-asyncio==1.5.4 + # via + # ipykernel + # jupyter-client + # nbclient + # notebook +networkx==2.7.1 + # via mdanalysis +nglview==3.0.3 + # via -r requirements.in +notebook==6.4.9 + # via widgetsnbextension +numkit==1.2.2 + # via gromacswrapper +numpy==1.22.3 + # via + # -r requirements.in + # ase + # biopython + # griddataformats + # gromacswrapper + # gsd + # matplotlib + # mdanalysis + # miniball + # mrcfile + # nglview + # numkit + # ovito + # pandas + # scipy +openpyxl==3.0.9 + # via -r requirements.in +ovito==3.7.2 + # via -r requirements.in +packaging==21.3 + # via + # bleach + # jupyter-server + # matplotlib + # mdanalysis +pandas==1.4.1 + # via + # -r requirements.in + # panedr +pandocfilters==1.5.0 + # via nbconvert +panedr==0.5.2 + # via -r requirements.in +paramiko==2.10.2 + # via imteksimfw +parmed==3.4.3 + # via -r requirements.in +parse==1.19.0 + # via dtool-overlay +parso==0.8.3 + # via jedi +pbr==5.8.1 + # via panedr +pep517==0.12.0 + # via + # -r requirements.in + # pip-tools +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython +pillow==9.0.1 + # via matplotlib +pip-tools==6.5.1 + # via -r requirements.in +prometheus-client==0.13.1 + # via + # jupyter-server + # notebook +prompt-toolkit==3.0.28 + # via ipython +psutil==5.9.0 + # via ipykernel +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pure-eval==0.2.2 + # via stack-data +pycparser==2.21 + # via cffi +pygments==2.11.2 + # via + # dtool-info + # ipython + # jupyterlab-pygments + # nbconvert +pymongo==4.0.2 + # via + # -r requirements.in + # fireworks +pynacl==1.5.0 + # via paramiko +pyparsing==3.0.7 + # via + # matplotlib + # packaging +pyrsistent==0.18.1 + # via jsonschema +pyside2==5.15.2.1 + # via -r requirements.in +pyside6==6.2.3 + # via ovito +python-dateutil==2.8.2 + # via + # arrow + # botocore + # fireworks + # jupyter-client + # matplotlib + # pandas +pytz==2021.3 + # via pandas +pyyaml==6.0 + # via + # -r requirements.in + # dtool-lookup-api + # jupytext +pyzmq==22.3.0 + # via + # jupyter-client + # jupyter-server + # notebook +requests==2.27.1 + # via dtool-http +ruamel-yaml==0.17.21 + # via + # dtool-create + # fireworks + # imteksimfw +ruamel-yaml-clib==0.2.6 + # via ruamel-yaml +s3transfer==0.5.2 + # via boto3 +scipy==1.8.0 + # via + # -r requirements.in + # ase + # griddataformats + # mdanalysis + # numkit +send2trash==1.8.0 + # via + # jupyter-server + # notebook +shiboken2==5.15.2.1 + # via pyside2 +shiboken6==6.2.3 + # via pyside6 +six==1.16.0 + # via + # asttokens + # bcrypt + # bleach + # fireworks + # griddataformats + # gromacswrapper + # imteksimfw + # numkit + # paramiko + # python-dateutil +sniffio==1.2.0 + # via anyio +soupsieve==2.3.1 + # via beautifulsoup4 +stack-data==0.2.0 + # via ipython +tabulate==0.8.9 + # via fireworks +terminado==0.13.3 + # via + # jupyter-server + # notebook +testpath==0.6.0 + # via nbconvert +texttable==1.6.4 + # via igraph +threadpoolctl==3.1.0 + # via mdanalysis +toml==0.10.2 + # via jupytext +tomli==2.0.1 + # via + # -r requirements.in + # pep517 +tornado==6.1 + # via + # ipykernel + # jupyter-client + # jupyter-server + # notebook + # terminado +tqdm==4.63.0 + # via + # fireworks + # mdanalysis +traitlets==5.1.1 + # via + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # jupyter-server + # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # notebook +traits==6.3.2 + # via ovito +urllib3==1.26.8 + # via + # botocore + # requests +wcwidth==0.2.5 + # via prompt-toolkit +webencodings==0.5.1 + # via bleach +websocket-client==1.3.2 + # via jupyter-server +werkzeug==2.0.3 + # via flask +wheel==0.37.1 + # via pip-tools +widgetsnbextension==3.5.2 + # via ipywidgets +yarl==1.7.2 + # via aiohttp +zipp==3.7.0 + # via importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/docker/jupyter/start-wrapper.sh b/docker/jupyter/start-wrapper.sh new file mode 100755 index 0000000..16f745c --- /dev/null +++ b/docker/jupyter/start-wrapper.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +fc-cache --force +echo +echo "matplotlib cache dir:" +echo +python3 -c "import matplotlib; print(matplotlib.get_cachedir())" +echo +echo "matplotlib fonts:" +echo +python3 -c "import matplotlib.font_manager ; [print(f) for f in matplotlib.font_manager.findSystemFonts(fontpaths=None)]" +echo +exec "$@" diff --git a/dtool/remove_datasets/remove_dataset_from_s3.sh b/dtool/remove_datasets/remove_dataset_from_s3.sh new file mode 100644 index 0000000..a0784ce --- /dev/null +++ b/dtool/remove_datasets/remove_dataset_from_s3.sh @@ -0,0 +1,37 @@ +#! /bin/bash + +UUID="$1" +ENDPOINT="${2:-https://s3.bwsfs.uni-freiburg.de:8082}" +BUCKET="${3:-frct-simdata}" +AWS_OPTS="${4}" # specify "--profile name-of-profile" to select credentials other than default +DRY_RUN=True + +echo "Removing dataset ${UUID} from bucket ${BUCKET} on endpoint ${ENDPOINT}..." + +# get prefix from registration key +aws s3 ${AWS_OPTS} --endpoint=${ENDPOINT} cp s3://${BUCKET}/dtool-${UUID} . +PREFIX=`cat dtool-${UUID}` +# rm -f dtool-${UUID} + +# remove dataset objects +for fn in `aws s3 ${AWS_OPTS} --endpoint=${ENDPOINT} ls --recursive s3://${BUCKET}/${PREFIX}${UUID}/ | awk '{ print $4 }'`; do + if [ -n "${DRY_RUN}" ]; then + echo "Dry run, would remove s3://${BUCKET}/${fn}" + else + echo "Remove s3://${BUCKET}/${fn}" + if ! aws ${AWS_OPTS} s3 --endpoint=${ENDPOINT} rm s3://${BUCKET}/${fn}; then + echo "Error removing dataset object s3://${BUCKET}/${fn}. Do you have the correct write permissions?" + exit 1 + fi + fi +done + +# remove registration key +if [ -n "${DRY_RUN}" ]; then + echo "Dry run, would remove s3://${BUCKET}/dtool-${UUID}" +else + echo "Remove s3://${BUCKET}/dtool-${UUID}" + if ! aws ${AWS_OPTS} s3 --endpoint=${ENDPOINT} rm s3://${BUCKET}/dtool-${UUID}; then + echo "Error removing registration key s3://${BUCKET}/dtool-${UUID}. Do you have the correct write permissions?" + fi +fi