Added UV support in pip-containerize

README.md

@@ -152,6 +152,7 @@ on the sqfs image itself. This behavior can be disabled by setting `CW_NO_FIX_PE
 - `pip-containerize`
  - Wrap new venv installation or edit existing
  - Will by default use currently available python
+ - Option to use uv to manage python and venv
  - Option to also use slim container image  (will then not mount full host)
 - `wrap-container`
  - Generate wrappers for existing container. Mainly

frontends/conda-containerize.py

@@ -17,8 +17,17 @@
 parser_new=add_new_pars(subparsers)
 parser_new.add_argument("env_file",help="conda env file")
 parser_new.add_argument("--mamba",help="use mamba for installation",action="store_true")
+parser_new.add_argument(
+    "--uv", help="use uv for pip dependencies (only with mamba)", action="store_true"
+)
+parser_new.add_argument(
+    "--nocache", help="Do not use pip/uv cache", action="store_true"
+)
 parser_upd=add_upd_pars(subparsers)
 add_adv_pars(subparsers)
+parser_upd.add_argument(
+    "--nocache", help="Do not use pip/uv cache", action="store_true"
+)
 
 ps=[parser_new,parser_upd]
 for p in ps:
@@ -31,17 +40,26 @@
 args = parser.parse_args()
 conf={}
 conf["add_ld"]="no"
+conf["use_uv"] = "no"
+conf["mode"] = "conda"
+conf["template_script"] = "conda.sh"
 if args.command == "new":
     conf["env_file"]=args.env_file
+    conf["update_installation"] = "no"
+    conf["installation_file_paths"] = [conf["env_file"]]
     if args.prefix:
         conf["installation_prefix"]=args.prefix
     conf["mode"]="conda"
     if args.mamba: 
         conf["mamba"]="yes" 
     else:
         conf["mamba"]="no"
+    if args.uv and args.mamba:
+        conf["use_uv"] = "yes"
+    elif args.uv and not args.mamba:
+        print_warn("Using --uv without --mamba does not have an effect")
 elif args.command == "update":
-    conf["mode"]="conda_modify"
+    conf["update_installation"] = "yes"
     get_old_conf(args.dir,conf)
 else:
     with open(args.yaml,'r') as y:
@@ -69,22 +87,18 @@
     global_conf=yaml.safe_load(g)
 
 parse_wrapper(conf,global_conf,args,False)
-if conf["mode"] == "conda":
-    conf["update_installation"]="no"
-    conf["template_script"]="conda.sh"
-    conf["installation_file_paths"]=[conf["env_file"]]
-elif conf["mode"]=="conda_modify":
-    conf["update_installation"]="yes"
-    conf["template_script"]="conda_modify.sh"
-else:
-    print_err("No or incorrent mode set, [conda,conda_modify]")
-    sys.exit(1)
+
 if "requirements_file" in conf:
     if "installation_file_paths" in conf:
         conf["installation_file_paths"].append(conf["requirements_file"])
     else:
         conf["installation_file_paths"]=conf["requirements_file"]
 
+if "pipcache" not in conf:
+    conf["pipcache"] = True
+
+if args.nocache is not None:
+    conf["pipcache"] = not args.nocache
 
 with open(os.getenv("_usr_yaml"),'a+') as f:
     yaml.dump(conf,f)

frontends/pip-containerize.py

@@ -18,48 +18,60 @@
 parser_new.add_argument("requirements_file", type=lambda x: is_valid_file(parser, x),help="requirements file for pip")
 parser_upd=add_upd_pars(subparsers)
 parser_upd.add_argument("-r","--requirements-file", type=lambda x: is_valid_file(parser, x),help="requirements file for pip")
+parser_upd.add_argument(
+    "--nocache", help="Do not use pip/uv cache", action="store_true"
+)
 add_adv_pars(subparsers)
-parser_new.add_argument("--slim",action='store_true',help="Use minimal base python container")
-parser_new.add_argument("--pyver",help="Docker tag to use for the slim python verison, e.g 3.12.9-bookworm")
+python_mode = parser_new.add_mutually_exclusive_group()
+python_mode.add_argument("--uv", action="store_true", help="Use uv")
+python_mode.add_argument(
+    "--slim", action="store_true", help="Use minimal base python container"
+)
+parser_new.add_argument("--pyver", help="Python version to use for slim or uv modes")
 parser_new.add_argument("--system-site-packages",action='store_true',help="Enable system and user site packages for the created installation")
+parser_new.add_argument("--nocache", help="Do not use pip/uv cache", action="store_true")
 
 ps=[parser_new,parser_upd]
 for p in ps:
     add_base_pars(p)
 
-
-
-
 if len(sys.argv) < 2:
     parser.print_help()
     sys.exit(0)
 args = parser.parse_args()
-conf={}
-pyver="3.12.9-slim-bookworm"
-
-
+conf = {}
 
 if args.requirements_file:
     conf["requirements_file"]=args.requirements_file
     conf["installation_file_paths"]=[conf["requirements_file"]]
 
 if args.command == "new":
+    conf["mode"] = "venv"
+    conf["use_uv"] = "no"
+    conf["update_installation"] = "no"
     if args.system_site_packages:
         conf["enable_site_packages"]="yes"
     if args.prefix:
         conf["installation_prefix"]=args.prefix
-    conf["mode"]="venv"
     if args.slim:
+        conf["pyver"] = "slim"
         if args.pyver:
-            pyver=args.pyver
-        conf["container_src"]="docker://python:{}".format(pyver)
-        conf["isolate"]="yes"
+            if "-slim" in args.pyver or args.pyver == "slim":
+                conf["pyver"] = args.pyver
+            else:
+                conf["pyver"] = args.pyver + "-slim"
+
+        conf["container_src"] = "docker://python:{}".format(conf["pyver"])
+        conf["isolate"] = "yes"
+    elif args.uv:
+        conf["use_uv"] = "yes"
+        conf["pyver"] = args.pyver if args.pyver else "3"
     else:
         if args.pyver:
-            print_warn("Using --pyver without --slim does not have an effect")
+            print_warn("Using --pyver without --slim or --uv does not have an effect")
 
 elif args.command == "update":
-    conf["mode"]="venv_modify"
+    conf["update_installation"] = "yes"
     get_old_conf(args.dir,conf)
 else:
     with open(args.yaml,'r') as y:
@@ -81,12 +93,14 @@
     global_conf=yaml.safe_load(g)
 
 parse_wrapper(conf,global_conf,args,False)
-if conf["mode"] == "venv":
-    conf["update_installation"]="no"
-    conf["template_script"]="venv.sh"
-else:
-    conf["update_installation"]="yes"
-    conf["template_script"]="venv_modify.sh"
+
+conf["template_script"] = "venv.sh"
+
+if "pipcache" not in conf:
+    conf["pipcache"] = True
+
+if args.nocache is not None:
+    conf["pipcache"] = not args.nocache
 
 with open(os.getenv("_usr_yaml"),'a+') as f:
     yaml.dump(conf,f)

frontends/script_shared.py

@@ -93,5 +93,12 @@ def get_old_conf(d,conf):
     conf["sqfs_image"]=old_conf["sqfs_image"]
     conf["container_image"]=old_conf["container_image"]
     conf["isolate"]=old_conf["isolate"]
+    conf["mode"] = old_conf["mode"]
+    if "mamba" in old_conf:
+        conf["mamba"] = old_conf["mamba"]
+    if "use_uv" in old_conf:
+        conf["use_uv"] = old_conf["use_uv"]
+    if "pipcache" in old_conf:
+        conf["pipcache"] = old_conf["pipcache"]
     if "wrapper_paths" in old_conf:
         conf["wrapper_paths"] = old_conf["wrapper_paths"]

generate_wrappers.sh

@@ -242,7 +242,7 @@ fi" >> _deploy/bin/$target
             if [[ ! -z "$($_CONTAINER_EXEC ls $wrapper_path/../pyvenv.cfg 2>/dev/null )" ]]; then
                 print_info "Target is a venv" 2
                 $_CONTAINER_EXEC cat $wrapper_path/../pyvenv.cfg > _deploy/pyvenv.cfg
-                _pyd=$($_CONTAINER_EXEC ls $wrapper_path/../lib)
+                _pyd=$(basename "$($_CONTAINER_EXEC find $wrapper_path/../lib -maxdepth 1 -type d -regex '.*/python[0-9]+\.[0-9]+$')")
                 mkdir -p _deploy/lib/$_pyd/
                 (cd _deploy/lib/$_pyd && ln -s $wrapper_path/../lib/$_pyd/site-packages site-packages )
                 (cd _deploy && ln -s lib lib64 )

templates/conda.sh

@@ -9,17 +9,25 @@ export env_root=$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/
 
 cd $CW_INSTALLATION_PATH
 
-[ "$CW_CONDA_VERSION" = "latest" ] && CW_CONDA_VERSION=$(curl -s https://api.github.com/repos/conda-forge/miniforge/releases/latest | grep "tag_name" | cut -d: -f2 | tr -d \" | tr -d , | tr -d " ")
+if [[ ! -e $CW_INSTALLATION_PATH/miniforge/ ]]; then
+    if [[ -e $CW_INSTALLATION_PATH/miniconda/ ]]; then
+        print_info "Updating older installation which is using miniconda and not miniforge\nCreating symlink miniforge -> miniconda" 1
+        ln -s "$CW_INSTALLATION_PATH/miniconda" "$CW_INSTALLATION_PATH/miniforge"
+    else
+        [ "$CW_CONDA_VERSION" = "latest" ] && CW_CONDA_VERSION=$(curl -s https://api.github.com/repos/conda-forge/miniforge/releases/latest | grep "tag_name" | cut -d: -f2 | tr -d \" | tr -d , | tr -d " ")
 
-print_info "Using miniforge version Miniforge3-$CW_CONDA_VERSION-$CW_CONDA_ARCH" 1
-print_info "Downloading miniforge " 2
-curl -sL https://github.com/conda-forge/miniforge/releases/download/$CW_CONDA_VERSION/Miniforge3-$CW_CONDA_VERSION-$CW_CONDA_ARCH.sh --output Miniforge_inst.sh &>/dev/null
-print_info "Installing miniforge " 1
-bash Miniforge_inst.sh -b -p $CW_INSTALLATION_PATH/miniforge > $CW_BUILD_TMPDIR/_inst_miniforge.log &
-inst_pid=$!
+        print_info "Using miniforge version Miniforge3-$CW_CONDA_VERSION-$CW_CONDA_ARCH" 1
+        print_info "Downloading miniforge " 2
+        curl -sL "https://github.com/conda-forge/miniforge/releases/download/$CW_CONDA_VERSION/Miniforge3-$CW_CONDA_VERSION-$CW_CONDA_ARCH.sh" --output Miniforge_inst.sh &>/dev/null
+        print_info "Installing miniforge " 1
+        bash Miniforge_inst.sh -b -p "$CW_INSTALLATION_PATH/miniforge" > "$CW_BUILD_TMPDIR/_inst_miniforge.log" &
+        inst_pid=$!
+
+        follow_log $inst_pid "$CW_BUILD_TMPDIR/_inst_miniforge.log" 20
+        rm Miniforge_inst.sh
+    fi
+fi
 
-follow_log $inst_pid $CW_BUILD_TMPDIR/_inst_miniforge.log 20
-rm Miniforge_inst.sh
 eval "$($CW_INSTALLATION_PATH/miniforge/bin/conda shell.bash hook)"
 
 cd $CW_WORKDIR
@@ -31,21 +39,44 @@ else
     _FF="--file"
 fi
 cd $CW_INSTALLATION_PATH
-print_info "Creating env, full log in $CW_BUILD_TMPDIR/build.log" 1
 
-if [[ ${CW_MAMBA} == "yes" ]] ;then
-    print_info "Using mamba to install packages" 1 
-    mamba $_EC create --name $CW_ENV_NAME $_FF $( basename $CW_ENV_FILE ) &>> $CW_BUILD_TMPDIR/build.log &
-else
-    conda $_EC create --name $CW_ENV_NAME $_FF $( basename $CW_ENV_FILE ) &>> $CW_BUILD_TMPDIR/build.log &
+if [[ "$CW_PIPCACHE" != "yes" ]]; then
+    export PIP_NO_CACHE_DIR=1
+    export UV_NO_CACHE=1
+fi
+
+_UV=""
+if [[ ${CW_USE_UV} == "yes" ]] ; then
+    export UV_LINK_MODE=copy
+    _UV="--use-uv"
 fi
 
-inst_pid=$!
-follow_log $inst_pid $CW_BUILD_TMPDIR/build.log 20 
-wait $inst_pid
-conda activate $CW_ENV_NAME
+if [[ ! -e "$env_root" ]]; then
+    print_info "Creating env, full log in $CW_BUILD_TMPDIR/build.log" 1
+
+    if [[ ${CW_MAMBA} == "yes" ]] ; then
+        print_info "Using mamba to install packages" 1
+        mamba $_EC create -y $_UV --name "$CW_ENV_NAME" $_FF "$( basename "$CW_ENV_FILE" )" &>> "$CW_BUILD_TMPDIR/build.log" &
+    else
+        conda $_EC create -y --name "$CW_ENV_NAME" $_FF "$( basename "$CW_ENV_FILE" )" &>> "$CW_BUILD_TMPDIR/build.log" &
+    fi
+    inst_pid=$!
+    follow_log $inst_pid "$CW_BUILD_TMPDIR/build.log" 20
+    wait $inst_pid
+fi
+
+conda activate "$CW_ENV_NAME"
+
 if [[ ${CW_REQUIREMENTS_FILE+defined}  ]];then
-    pip install -r $( basename "$CW_REQUIREMENTS_FILE" )
+    print_info "Installing requirements file" 1
+    if [[ ${CW_USE_UV} == "yes" ]] ; then
+        uv pip install -r "$( basename "$CW_REQUIREMENTS_FILE" )" > "$CW_BUILD_TMPDIR/_pip.log" &
+    else
+        pip install -r "$( basename "$CW_REQUIREMENTS_FILE" )"> "$CW_BUILD_TMPDIR/_pip.log" &
+    fi
+    bg_pid=$!
+    wait $bg_pid
+    follow_log $bg_pid "$CW_BUILD_TMPDIR/_pip.log" 20
 fi
 cd $CW_WORKDIR
 print_info "Running user supplied commands" 1

templates/venv.sh

@@ -9,24 +9,50 @@ export env_root=$CW_INSTALLATION_PATH/$CW_ENV_NAME/
 
 cd $CW_INSTALLATION_PATH
 
-
 cd $CW_WORKDIR
 source $CW_INSTALLATION_PATH/_pre_install.sh
 cd $CW_INSTALLATION_PATH
 
-if [[ ${CW_ENABLE_SITE_PACKAGES+defined} ]];then
-    print_info "Enabling system and user site packages" 1
-    _SP="--system-site-packages"
-else
-    print_info "Not enabling system and user site packages" 1
-    _SP=""
+
+_NC=""
+if [[ "$CW_PIPCACHE" != "yes" ]]; then
+    _NC="-n"
+fi
+
+if [[ "$CW_USE_UV" == "yes" ]]; then
+    if [[ ! -f "$CW_INSTALLATION_PATH/uv/bin/uv" ]]; then
+        print_info "Installing uv package manager" 1
+        curl -LsSf https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL="$CW_INSTALLATION_PATH/uv/bin/" UV_PRINT_QUIET=1 sh
+    fi
+    export PATH="$CW_INSTALLATION_PATH/uv/bin/:$PATH"
+    export UV_PYTHON_INSTALL_DIR="$CW_INSTALLATION_PATH/uv/python"
 fi
-print_info "Installing requirements file" 1
-python3 -m venv $_SP $CW_ENV_NAME
-source $CW_INSTALLATION_PATH/$CW_ENV_NAME/bin/activate
+
+if [[ ! -e "$env_root/bin/activate" ]]; then
+    if [[ ${CW_ENABLE_SITE_PACKAGES+defined} ]];then
+        print_info "Enabling system and user site packages" 1
+        _SP="--system-site-packages"
+    else
+        print_info "Not enabling system and user site packages" 1
+        _SP=""
+    fi
+    print_info "Creating virtual environment" 1
+    if [[ "$CW_USE_UV" == "yes" ]]; then
+        uv venv -p "$CW_PYVER" $_SP --managed-python $_NC --no-config  --link-mode=copy "$env_root"
+    else
+        python3 -m venv $_SP "$CW_ENV_NAME"
+    fi
+fi
+
+source "$env_root/bin/activate"
 
 if [[ ${CW_REQUIREMENTS_FILE+defined}  ]];then
-    pip install --disable-pip-version-check -r "$( basename $CW_REQUIREMENTS_FILE)" > $CW_BUILD_TMPDIR/_pip.log &
+    print_info "Installing requirements file" 1
+    if [[ "$CW_USE_UV" == "yes" ]]; then
+        uv pip install --link-mode=copy --compile-bytecode $_NC -r "$( basename "$CW_REQUIREMENTS_FILE")" > "$CW_BUILD_TMPDIR/_pip.log" &
+    else
+        pip install --disable-pip-version-check $_NC -r "$( basename "$CW_REQUIREMENTS_FILE")" > "$CW_BUILD_TMPDIR/_pip.log" &
+    fi
     bg_pid=$!
     wait $bg_pid
     follow_log $bg_pid $CW_BUILD_TMPDIR/_pip.log 20