From 317794bf2e9639e66c04fe7e9ff95d6d27c21a17 Mon Sep 17 00:00:00 2001 From: Xavier Abellan Ecija Date: Mon, 1 Dec 2025 12:00:49 +0000 Subject: [PATCH 1/4] Added UV support for pip-containerize. Small fix on generate_wrappers for more robust site-packages path detection. Some linting fixes and minor mamba fix also included. --- cw_common.py | 129 +++++++++++++++------------ frontends/conda-containerize.py | 119 ++++++++++++++----------- frontends/pip-containerize.py | 153 ++++++++++++++++++++------------ frontends/script_shared.py | 132 +++++++++++++++++---------- frontends/wrap-container.py | 68 +++++++------- frontends/wrap-install.py | 77 ++++++++-------- generate_wrappers.sh | 58 ++++++------ templates/conda.sh | 10 +-- templates/uvenv.sh | 44 +++++++++ templates/venv.sh | 28 +++--- 10 files changed, 492 insertions(+), 326 deletions(-) create mode 100644 templates/uvenv.sh diff --git a/cw_common.py b/cw_common.py index 856c005..e0d308a 100644 --- a/cw_common.py +++ b/cw_common.py @@ -1,98 +1,115 @@ """Utility function for printing errors and warnings while in python""" -import sys + import os +import pathlib import random +import shutil import string -import pathlib import sys -import shutil -colors={} -colors["RED"]='\033[0;31m' -colors["GREEN"]='\033[0;32m' -colors["YELLOW"]='\033[1;33m' -colors["BLUE"]="\033[1;34m" -colors["PURPLE"]='\033[0;35m' -colors["NC"]='\033[0m' # No Color +colors = {} +colors["RED"] = "\033[0;31m" +colors["GREEN"] = "\033[0;32m" +colors["YELLOW"] = "\033[1;33m" +colors["BLUE"] = "\033[1;34m" +colors["PURPLE"] = "\033[0;35m" +colors["NC"] = "\033[0m" # No Color + -def print_err(txt,err=False): +def print_err(txt, err=False): """Pretty error message, color is disabled if not in a TTY""" - if(err): + if err: if not sys.stderr.isatty(): - print("[ ERROR ] "+txt, file=sys.stderr) + print("[ ERROR ] " + txt, file=sys.stderr) else: - print("["+colors["RED"]+" ERROR "+colors["NC"]+"] "+txt,file=sys.stderr) + print( + "[" + colors["RED"] + " ERROR " + colors["NC"] + "] " + txt, + file=sys.stderr, + ) else: if not sys.stdout.isatty(): - print("[ ERROR ] "+txt) + print("[ ERROR ] " + txt) else: - print("["+colors["RED"]+" ERROR "+colors["NC"]+"] "+txt) + print("[" + colors["RED"] + " ERROR " + colors["NC"] + "] " + txt) -def print_info(txt,log_level,msg_level,err=False): + +def print_info(txt, log_level, msg_level, err=False): """Pretty info message, color is disabled if not in a TTY""" if int(log_level) <= msg_level: return if msg_level >= 2: - msg="DEBUG" - color=colors["PURPLE"] + msg = "DEBUG" + color = colors["PURPLE"] else: - msg="INFO" - color=colors["BLUE"] - if(err): + msg = "INFO" + color = colors["BLUE"] + if err: if not sys.stderr.isatty(): - print(f"[ {msg} ] "+txt,file=sys.stderr) + print(f"[ {msg} ] " + txt, file=sys.stderr) else: - print("["+color+f" {msg} "+colors["NC"]+"] "+txt,file=sys.stderr) + print("[" + color + f" {msg} " + colors["NC"] + "] " + txt, file=sys.stderr) else: if not sys.stdout.isatty(): - print(f"[ {msg} ] "+txt,file=sys.stderr) + print(f"[ {msg} ] " + txt, file=sys.stderr) else: - print("["+color+f" {msg} "+colors["NC"]+"] "+txt,file=sys.stdout) + print("[" + color + f" {msg} " + colors["NC"] + "] " + txt, file=sys.stdout) + -def print_warn(txt,err=False): - if(err): +def print_warn(txt, err=False): + if err: if not sys.stderr.isatty(): - print("[ WARNING ] "+txt, file=sys.stderr) + print("[ WARNING ] " + txt, file=sys.stderr) else: - print("["+colors["YELLOW"]+" WARNING "+colors["NC"]+"] "+txt,file=sys.stderr) + print( + "[" + colors["YELLOW"] + " WARNING " + colors["NC"] + "] " + txt, + file=sys.stderr, + ) else: if not sys.stdout.isatty(): - print("[ WARNING ] "+txt) + print("[ WARNING ] " + txt) else: - print("["+colors["YELLOW"]+" WARNING "+colors["NC"]+"] "+txt) - + print("[" + colors["YELLOW"] + " WARNING " + colors["NC"] + "] " + txt) -def expand_vars(path,rec=0): - if(rec > 10): - print_err("Max 10 shell variables allowed per value, check configuration ",True) + +def expand_vars(path, rec=0): + if rec > 10: + print_err( + "Max 10 shell variables allowed per value, check configuration ", True + ) sys.exit(1) - g=path - try: - g=string.Template(g).substitute(os.environ) + g = path + try: + g = string.Template(g).substitute(os.environ) except KeyError as E: - var=E.args[0] - return expand_vars(g.replace(f"${var}",''),rec+1) + var = E.args[0] + return expand_vars(g.replace(f"${var}", ""), rec + 1) return g + def has_apptainer(): - return shutil.which("apptainer") != None + return shutil.which("apptainer") is not None + def name_generator(size=6, chars=string.ascii_uppercase + string.digits): - return ''.join(random.choice(chars) for _ in range(size)) + return "".join(random.choice(chars) for _ in range(size)) + def installation_in_PATH(): - return [P for P in os.environ["PATH"].split(':') if is_installation(P) ] + return [P for P in os.environ["PATH"].split(":") if is_installation(P)] + def is_installation(base_path): - markers=["bin","_bin","common.sh"] - return all( pathlib.Path(base_path+'/../'+m).exists() for m in markers ) + markers = ["bin", "_bin", "common.sh"] + return all(pathlib.Path(base_path + "/../" + m).exists() for m in markers) + # UBI images are namespaced with the major version as part of the name # and not just the tag. -special={} -special["rhel"]= lambda namespace,version: namespace+version.split('.')[0] +special = {} +special["rhel"] = lambda namespace, version: namespace + version.split(".")[0] + # Get the docker image matching the host OS def get_docker_image(release_file): @@ -102,17 +119,17 @@ def get_docker_image(release_file): "rhel": "redhat/ubi", "almalinux": "almalinux", "rocky": "rockylinux/rockylinux", - "ubuntu": "ubuntu" + "ubuntu": "ubuntu", } try: - with open(os_release_file, 'r') as file: + with open(os_release_file, "r") as file: lines = file.readlines() os_info = {} for line in lines: # Lazy way to handle empty lines try: - key, value = line.strip().split('=', 1) + key, value = line.strip().split("=", 1) except: continue os_info[key] = value.strip('"') @@ -123,14 +140,14 @@ def get_docker_image(release_file): if os_id in docker_images: docker_image = docker_images[os_id] if os_id in special: - docker_image = special[os_id](docker_image,version_id) - return (True,f"{docker_image}:{version_id}") + docker_image = special[os_id](docker_image, version_id) + return (True, f"{docker_image}:{version_id}") else: # Guess what the name could be # Will most likely fail for most small distros - return (True,f"{os_id}:{version_id}") + return (True, f"{os_id}:{version_id}") except FileNotFoundError: - return (False,"OS release file not found") + return (False, "OS release file not found") except Exception as e: - return (False,f"An error occurred: {e}") + return (False, f"An error occurred: {e}") diff --git a/frontends/conda-containerize.py b/frontends/conda-containerize.py index b74e29b..a81047b 100644 --- a/frontends/conda-containerize.py +++ b/frontends/conda-containerize.py @@ -1,50 +1,70 @@ import argparse import os -import sys import pathlib -curr_dir=pathlib.Path(__file__).parent.resolve() -root_dir=pathlib.Path(curr_dir).parent.resolve() -info=sys.version_info -sys.path.insert(0,str(root_dir)) -sys.path.insert(0,str(root_dir)+"/PyDeps/lib/python{}.{}/site-packages".format(info[0],info[1])) -import yaml -from cw_common import * -from script_shared import * +import sys -sys.argv[0]=sys.argv[0].split('/')[-1].split('.')[0] -parser = argparse.ArgumentParser(description="Create or modify a Conda installation inside a container") -subparsers = parser.add_subparsers(help='subcommands',dest='command') -parser_new=add_new_pars(subparsers) -parser_new.add_argument("env_file",help="conda env file") -parser_new.add_argument("--mamba",help="use mamba for installation",action="store_true") -parser_upd=add_upd_pars(subparsers) +curr_dir = pathlib.Path(__file__).parent.resolve() +root_dir = pathlib.Path(curr_dir).parent.resolve() +info = sys.version_info +sys.path.insert(0, str(root_dir)) +sys.path.insert( + 0, str(root_dir) + "/PyDeps/lib/python{}.{}/site-packages".format(info[0], info[1]) +) +import yaml # noqa: E402 +from cw_common import print_err # noqa: E402 +from script_shared import ( # noqa E402 + add_adv_pars, + add_base_pars, + add_new_pars, + add_upd_pars, + get_old_conf, + is_valid_file, + parse_wrapper, +) + +sys.argv[0] = sys.argv[0].split("/")[-1].split(".")[0] +parser = argparse.ArgumentParser( + description="Create or modify a Conda installation inside a container" +) +subparsers = parser.add_subparsers(help="subcommands", dest="command") +parser_new = add_new_pars(subparsers) +parser_new.add_argument("env_file", help="conda env file") +parser_new.add_argument( + "--mamba", help="use mamba for installation", action="store_true" +) +parser_upd = add_upd_pars(subparsers) add_adv_pars(subparsers) -ps=[parser_new,parser_upd] +ps = [parser_new, parser_upd] for p in ps: add_base_pars(p) - p.add_argument("-r", "--requirement", type=lambda x: is_valid_file(parser, x),help="requirements file for pip") + p.add_argument( + "-r", + "--requirement", + type=lambda x: is_valid_file(x), + help="requirements file for pip", + ) if len(sys.argv) < 2: parser.print_help() sys.exit(0) args = parser.parse_args() -conf={} -conf["add_ld"]="no" +conf = {} +conf["add_ld"] = "no" if args.command == "new": - conf["env_file"]=args.env_file + conf["env_file"] = args.env_file if args.prefix: - conf["installation_prefix"]=args.prefix - conf["mode"]="conda" - if args.mamba: - conf["mamba"]="yes" + conf["installation_prefix"] = args.prefix + conf["mode"] = "conda" + if args.mamba: + conf["mamba"] = "yes" else: - conf["mamba"]="no" + conf["mamba"] = "no" elif args.command == "update": - conf["mode"]="conda_modify" - get_old_conf(args.dir,conf) + conf["mode"] = "conda_modify" + get_old_conf(args.dir, conf) else: - with open(args.yaml,'r') as y: + with open(args.yaml, "r") as y: conf.update(yaml.safe_load(y)) if args.command == "new" and not os.path.isfile(args.env_file): @@ -52,30 +72,29 @@ sys.exit(1) - -if args.command in ["update","new"]: +if args.command in ["update", "new"]: if args.environ: - conf["extra_envs"]=[{"file":args.environ}] + conf["extra_envs"] = [{"file": args.environ}] if args.post_install: - conf["post_install"]=[{"file":args.post_install}] + conf["post_install"] = [{"file": args.post_install}] if args.requirement: - conf["requirements_file"]=args.requirement + conf["requirements_file"] = args.requirement if args.pre_install: - conf["pre_install"]=[{"file":args.pre_install}] + conf["pre_install"] = [{"file": args.pre_install}] -global_conf={} -with open(os.getenv("CW_GLOBAL_YAML"),'r') as g: - global_conf=yaml.safe_load(g) - -parse_wrapper(conf,global_conf,args,False) +global_conf = {} +with open(os.getenv("CW_GLOBAL_YAML", ""), "r") as g: + global_conf = yaml.safe_load(g) + +parse_wrapper(conf, args, False) if conf["mode"] == "conda": - conf["update_installation"]="no" - conf["template_script"]="conda.sh" - conf["installation_file_paths"]=[conf["env_file"]] -elif conf["mode"]=="conda_modify": - conf["update_installation"]="yes" - conf["template_script"]="conda_modify.sh" + conf["update_installation"] = "no" + conf["template_script"] = "conda.sh" + conf["installation_file_paths"] = [conf["env_file"]] +elif conf["mode"] == "conda_modify": + conf["update_installation"] = "yes" + conf["template_script"] = "conda_modify.sh" else: print_err("No or incorrent mode set, [conda,conda_modify]") sys.exit(1) @@ -83,9 +102,7 @@ if "installation_file_paths" in conf: conf["installation_file_paths"].append(conf["requirements_file"]) else: - conf["installation_file_paths"]=conf["requirements_file"] - - -with open(os.getenv("_usr_yaml"),'a+') as f: - yaml.dump(conf,f) + conf["installation_file_paths"] = conf["requirements_file"] +with open(os.getenv("_usr_yaml", ""), "a+") as f: + yaml.dump(conf, f) diff --git a/frontends/pip-containerize.py b/frontends/pip-containerize.py index 7aaa9a9..f810724 100644 --- a/frontends/pip-containerize.py +++ b/frontends/pip-containerize.py @@ -1,93 +1,132 @@ import argparse import os -import sys import pathlib -curr_dir=pathlib.Path(__file__).parent.resolve() -root_dir=pathlib.Path(curr_dir).parent.resolve() -info=sys.version_info -sys.path.insert(0,str(root_dir)) -sys.path.insert(0,str(root_dir)+"/PyDeps/lib/python{}.{}/site-packages".format(info[0],info[1])) -import yaml -from cw_common import * -from script_shared import * - -sys.argv[0]=sys.argv[0].split('/')[-1].split('.')[0] -parser = argparse.ArgumentParser(description="Create or modify a python installation inside a container") -subparsers = parser.add_subparsers(help='subcommands',dest='command') -parser_new=add_new_pars(subparsers) -parser_new.add_argument("requirements_file", type=lambda x: is_valid_file(parser, x),help="requirements file for pip") -parser_upd=add_upd_pars(subparsers) -parser_upd.add_argument("-r","--requirements-file", type=lambda x: is_valid_file(parser, x),help="requirements file for pip") -add_adv_pars(subparsers) -parser_new.add_argument("--slim",action='store_true',help="Use minimal base python container") -parser_new.add_argument("--pyver",help="Docker tag to use for the slim python verison, e.g 3.12.9-bookworm") -parser_new.add_argument("--system-site-packages",action='store_true',help="Enable system and user site packages for the created installation") +import sys -ps=[parser_new,parser_upd] +curr_dir = pathlib.Path(__file__).parent.resolve() +root_dir = pathlib.Path(curr_dir).parent.resolve() +info = sys.version_info +sys.path.insert(0, str(root_dir)) +sys.path.insert( + 0, str(root_dir) + "/PyDeps/lib/python{}.{}/site-packages".format(info[0], info[1]) +) +import yaml # noqa: E402 +from cw_common import print_warn # noqa: E402 +from script_shared import ( # noqa: E402 + add_adv_pars, + add_base_pars, + add_new_pars, + add_upd_pars, + get_old_conf, + is_valid_file, + parse_wrapper, +) + +sys.argv[0] = sys.argv[0].split("/")[-1].split(".")[0] +parser = argparse.ArgumentParser( + description="Create or modify a python installation inside a container" +) +subparsers = parser.add_subparsers(help="subcommands", dest="command") +parser_new = add_new_pars(subparsers) +parser_new.add_argument( + "requirements_file", + type=lambda x: is_valid_file(x), + help="requirements file for pip", +) +parser_upd = add_upd_pars(subparsers) +parser_upd.add_argument( + "-r", + "--requirements-file", + type=lambda x: is_valid_file(x), + help="requirements file for pip", +) +parser_upd.add_argument( + "--nocache", help="Do not use pip/uv cache", action="store_true" +) +add_adv_pars(subparsers) +python_mode = parser_new.add_mutually_exclusive_group() +python_mode.add_argument("--uv", action="store_true", help="Use uv") +python_mode.add_argument( + "--slim", action="store_true", help="Use minimal base python container" +) +parser_new.add_argument("--pyver", help="Python version to use for slim or uv modes") +parser_new.add_argument( + "--system-site-packages", + action="store_true", + help="Enable system and user site packages for the created installation", +) +parser_new.add_argument( + "--nocache", help="Do not use pip/uv cache", action="store_true" +) + +ps = [parser_new, parser_upd] for p in ps: add_base_pars(p) - - - if len(sys.argv) < 2: parser.print_help() sys.exit(0) args = parser.parse_args() -conf={} -pyver="3.12.9-slim-bookworm" - +conf = {} +pyver = "3.12.9-slim-bookworm" if args.requirements_file: - conf["requirements_file"]=args.requirements_file - conf["installation_file_paths"]=[conf["requirements_file"]] + conf["requirements_file"] = args.requirements_file + conf["installation_file_paths"] = [conf["requirements_file"]] if args.command == "new": + conf["update_installation"] = "no" if args.system_site_packages: - conf["enable_site_packages"]="yes" + conf["enable_site_packages"] = "yes" if args.prefix: - conf["installation_prefix"]=args.prefix - conf["mode"]="venv" + conf["installation_prefix"] = args.prefix + conf["mode"] = "venv" if args.slim: if args.pyver: - pyver=args.pyver - conf["container_src"]="docker://python:{}".format(pyver) - conf["isolate"]="yes" + pyver = args.pyver + conf["container_src"] = "docker://python:{}".format(pyver) + conf["isolate"] = "yes" + elif args.uv: + conf["mode"] = "uvenv" + conf["pyver"] = args.pyver if args.pyver else "3" else: if args.pyver: - print_warn("Using --pyver without --slim does not have an effect") + print_warn("Using --pyver without --slim or --uv does not have an effect") elif args.command == "update": - conf["mode"]="venv_modify" - get_old_conf(args.dir,conf) + conf["update_installation"] = "yes" + get_old_conf(args.dir, conf) else: - with open(args.yaml,'r') as y: + with open(args.yaml, "r") as y: conf.update(yaml.safe_load(y)) - -if args.command in ["update","new"]: +if args.command in ["update", "new"]: if args.environ: - conf["extra_envs"]=[{"file":args.environ}] + conf["extra_envs"] = [{"file": args.environ}] if args.post_install: - conf["post_install"]=[{"file":args.post_install}] + conf["post_install"] = [{"file": args.post_install}] if args.pre_install: - conf["pre_install"]=[{"file":args.pre_install}] + conf["pre_install"] = [{"file": args.pre_install}] -global_conf={} -with open(os.getenv("CW_GLOBAL_YAML"),'r') as g: - global_conf=yaml.safe_load(g) - -parse_wrapper(conf,global_conf,args,False) -if conf["mode"] == "venv": - conf["update_installation"]="no" - conf["template_script"]="venv.sh" +global_conf = {} +with open(os.getenv("CW_GLOBAL_YAML", ""), "r") as g: + global_conf = yaml.safe_load(g) + +parse_wrapper(conf, args, False) + +if conf["mode"] == "uvenv": + conf["template_script"] = "uvenv.sh" else: - conf["update_installation"]="yes" - conf["template_script"]="venv_modify.sh" + conf["template_script"] = "venv.sh" + +if "pipcache" not in conf: + conf["pipcache"] = True -with open(os.getenv("_usr_yaml"),'a+') as f: - yaml.dump(conf,f) +if args.nocache is not None: + conf["pipcache"] = not args.nocache +with open(os.getenv("_usr_yaml", ""), "a+") as f: + yaml.dump(conf, f) diff --git a/frontends/script_shared.py b/frontends/script_shared.py index f906db8..3e0da21 100644 --- a/frontends/script_shared.py +++ b/frontends/script_shared.py @@ -1,68 +1,93 @@ import os -import sys import pathlib -curr_dir=pathlib.Path(__file__).parent.resolve() -root_dir=pathlib.Path(curr_dir).parent.resolve() -info=sys.version_info -sys.path.insert(0,str(root_dir)) -sys.path.insert(0,str(root_dir)+"/PyDeps/lib/python{}.{}/site-packages".format(info[0],info[1])) -import yaml -from cw_common import * -def is_valid_file(par,arg): +import sys + +curr_dir = pathlib.Path(__file__).parent.resolve() +root_dir = pathlib.Path(curr_dir).parent.resolve() +info = sys.version_info +sys.path.insert(0, str(root_dir)) +sys.path.insert( + 0, str(root_dir) + "/PyDeps/lib/python{}.{}/site-packages".format(info[0], info[1]) +) +import yaml # noqa: E402 +from cw_common import print_err # noqa: E402 + + +def is_valid_file(arg): if not os.path.exists(arg): print_err("The file %s does not exist!" % arg) sys.exit(1) else: - return arg + return arg + def add_prefix_flag(p): - p.add_argument("--prefix",type=str,help="Installation location") + p.add_argument("--prefix", type=str, help="Installation location") + def add_post_flag(par): - par.add_argument("--post-install",help="Script to run after initial setup",type=lambda x: is_valid_file(par,x)) + par.add_argument( + "--post-install", + help="Script to run after initial setup", + type=lambda x: is_valid_file(x), + ) + + def add_pre_flag(par): - par.add_argument("--pre-install",help="Script to run before initial setup",type=lambda x: is_valid_file(par,x)) + par.add_argument( + "--pre-install", + help="Script to run before initial setup", + type=lambda x: is_valid_file(x), + ) + + def add_env_flag(par): - par.add_argument("--environ",help="Script to run before each program launch ",type=lambda x: is_valid_file(par,x)) + par.add_argument( + "--environ", + help="Script to run before each program launch ", + type=lambda x: is_valid_file(x), + ) + def add_wrapper_flag(par): - par.add_argument("-w","--wrapper-paths",help='Comma separated list of paths') + par.add_argument("-w", "--wrapper-paths", help="Comma separated list of paths") + -def add_adv_pars(subpar): - parser_advanced = subpar.add_parser('advanced', help='') - parser_advanced.add_argument('yaml',type=str,help='yaml file with tool config') +def add_adv_pars(subpar): + parser_advanced = subpar.add_parser("advanced", help="") + parser_advanced.add_argument("yaml", type=str, help="yaml file with tool config") return parser_advanced + + def add_upd_pars(subpar): - parser_update = subpar.add_parser('update', help='update an existing installation') - parser_update.add_argument('dir', type=str, help='Installation to update') + parser_update = subpar.add_parser("update", help="update an existing installation") + parser_update.add_argument("dir", type=str, help="Installation to update") return parser_update + + def add_new_pars(subpar): - parser_new = subpar.add_parser('new', help='Create new installation') + parser_new = subpar.add_parser("new", help="Create new installation") add_prefix_flag(parser_new) return parser_new -def add_base_pars(par,pre_post=True): + + +def add_base_pars(par, pre_post=True): if pre_post: add_post_flag(par) add_pre_flag(par) add_env_flag(par) add_wrapper_flag(par) + # non absolute paths are relative to the installation dir -def parse_wrapper(conf,g_conf,a,req_abs): +def parse_wrapper(conf, a, req_abs): if a.wrapper_paths: - ip="" - if "installation_path" in conf: - ip = conf["installation_path"] - elif "installation_path" in g_conf["force"]: - ip = g_conf["force"]["installation_path"] - elif "installation_path" in g_conf["defaults"]: - ip = g_conf["defaults"]["installation_path"] - elif not req_abs: + if not req_abs: print_err("Failed to parse wrapper paths, missing installation path") sys.exit(1) - if not "wrapper_paths" in conf: - conf["wrapper_paths"]=[] - for p in a.wrapper_paths.split(','): + if "wrapper_paths" not in conf: + conf["wrapper_paths"] = [] + for p in a.wrapper_paths.split(","): if p[0] == "/": conf["wrapper_paths"].append(p) elif req_abs: @@ -71,27 +96,38 @@ def parse_wrapper(conf,g_conf,a,req_abs): else: conf["wrapper_paths"].append(p) -def get_old_conf(d,conf): - old_conf={} + +def get_old_conf(d, conf): + old_conf = {} try: - with open(d+"/share/conf.yaml",'r') as c: - old_conf=yaml.safe_load(c) + with open(d + "/share/conf.yaml", "r") as c: + old_conf = yaml.safe_load(c) except FileNotFoundError: - print_err("Directory {} does not exist or is not a valid installation ( missing share/conf.yaml )".format(d)) + print_err( + "Directory {} does not exist or is not a valid installation ( missing share/conf.yaml )".format( + d + ) + ) sys.exit(1) - + # If the installation uses a shared container it should # continue doing so if "share_container" in old_conf and old_conf["share_container"]: - conf["container_src"]=old_conf["container_src"] + conf["container_src"] = old_conf["container_src"] else: - conf["container_src"]=d+"/"+old_conf["container_image"] - conf["sqfs_src"]=d+"/"+old_conf["sqfs_image"] - conf["installation_path"]=old_conf["installation_path"] - conf["installation_prefix"]=d - conf["sqfs_image"]=old_conf["sqfs_image"] - conf["container_image"]=old_conf["container_image"] - conf["isolate"]=old_conf["isolate"] + conf["container_src"] = d + "/" + old_conf["container_image"] + conf["sqfs_src"] = d + "/" + old_conf["sqfs_image"] + conf["installation_path"] = old_conf["installation_path"] + conf["installation_prefix"] = d + conf["sqfs_image"] = old_conf["sqfs_image"] + conf["container_image"] = old_conf["container_image"] + conf["isolate"] = old_conf["isolate"] + conf["mode"] = old_conf["mode"] + conf["container_image"] = old_conf["container_image"] + conf["isolate"] = old_conf["isolate"] + conf["mode"] = old_conf["mode"] + if "pipcache" in old_conf: + conf["pipcache"] = old_conf["pipcache"] if "wrapper_paths" in old_conf: conf["wrapper_paths"] = old_conf["wrapper_paths"] diff --git a/frontends/wrap-container.py b/frontends/wrap-container.py index 47c8809..0aaba5b 100644 --- a/frontends/wrap-container.py +++ b/frontends/wrap-container.py @@ -1,24 +1,29 @@ import argparse import os -import sys import pathlib -curr_dir=pathlib.Path(__file__).parent.resolve() -root_dir=pathlib.Path(curr_dir).parent.resolve() -info=sys.version_info -sys.path.insert(0,str(root_dir)) -sys.path.insert(0,str(root_dir)+"/PyDeps/lib/python{}.{}/site-packages".format(info[0],info[1])) -import yaml -from cw_common import * -from script_shared import * - -sys.argv[0]=sys.argv[0].split('/')[-1].split('.')[0] -parser = argparse.ArgumentParser(description="Create wrappers for executables inside a container") -parser.add_argument("container",type=str,help="Container to wrap, can be docker/singularity url") -parser.add_argument("-y","--yaml",help="Tool yaml conf file") -add_base_pars(parser,False) -add_prefix_flag(parser) - +import sys +curr_dir = pathlib.Path(__file__).parent.resolve() +root_dir = pathlib.Path(curr_dir).parent.resolve() +info = sys.version_info +sys.path.insert(0, str(root_dir)) +sys.path.insert( + 0, str(root_dir) + "/PyDeps/lib/python{}.{}/site-packages".format(info[0], info[1]) +) +import yaml # noqa: E402 +from cw_common import print_err # noqa: E402 +from script_shared import add_base_pars, add_prefix_flag, parse_wrapper # noqa: E402 + +sys.argv[0] = sys.argv[0].split("/")[-1].split(".")[0] +parser = argparse.ArgumentParser( + description="Create wrappers for executables inside a container" +) +parser.add_argument( + "container", type=str, help="Container to wrap, can be docker/singularity url" +) +parser.add_argument("-y", "--yaml", help="Tool yaml conf file") +add_base_pars(parser, False) +add_prefix_flag(parser) if len(sys.argv) < 2: parser.print_help() @@ -27,28 +32,25 @@ if not args.wrapper_paths: print_err("Tool {} requires -w/--wrapper-paths to be used".format(sys.argv[0])) sys.exit(1) -conf={} -conf["container_src"]=args.container -conf["isolate"]="yes" -conf["mode"]="wrapcont" +conf = {} +conf["container_src"] = args.container +conf["isolate"] = "yes" +conf["mode"] = "wrapcont" if args.prefix: - conf["installation_prefix"]=args.prefix + conf["installation_prefix"] = args.prefix if args.yaml: - with open(args.yaml,'r') as y: + with open(args.yaml, "r") as y: conf.update(yaml.safe_load(y)) if args.environ: - conf["extra_envs"]=[{"file":args.environ}] - - -global_conf={} -with open(os.getenv("CW_GLOBAL_YAML"),'r') as g: - global_conf=yaml.safe_load(g) - -parse_wrapper(conf,global_conf,args,True) + conf["extra_envs"] = [{"file": args.environ}] +global_conf = {} +with open(os.getenv("CW_GLOBAL_YAML", ""), "r") as g: + global_conf = yaml.safe_load(g) -with open(os.getenv("_usr_yaml"),'a+') as f: - yaml.dump(conf,f) +parse_wrapper(conf, args, True) +with open(os.getenv("_usr_yaml", ""), "a+") as f: + yaml.dump(conf, f) diff --git a/frontends/wrap-install.py b/frontends/wrap-install.py index 95290de..0bff89a 100644 --- a/frontends/wrap-install.py +++ b/frontends/wrap-install.py @@ -1,24 +1,29 @@ import argparse import os -import sys import pathlib -curr_dir=pathlib.Path(__file__).parent.resolve() -root_dir=pathlib.Path(curr_dir).parent.resolve() -info=sys.version_info -sys.path.insert(0,str(root_dir)) -sys.path.insert(0,str(root_dir)+"/PyDeps/lib/python{}.{}/site-packages".format(info[0],info[1])) -import yaml -from cw_common import * -from script_shared import * +import sys -sys.argv[0]=sys.argv[0].split('/')[-1].split('.')[0] -parser = argparse.ArgumentParser(description="Wrap an existing installation into a container") -parser.add_argument("dir",type=str,help="Installation to wrap") +curr_dir = pathlib.Path(__file__).parent.resolve() +root_dir = pathlib.Path(curr_dir).parent.resolve() +info = sys.version_info +sys.path.insert(0, str(root_dir)) +sys.path.insert( + 0, str(root_dir) + "/PyDeps/lib/python{}.{}/site-packages".format(info[0], info[1]) +) +import yaml # noqa: E402 +from cw_common import has_apptainer, print_err # noqa: E402 +from script_shared import add_base_pars, add_prefix_flag, parse_wrapper # noqa: E402 + +sys.argv[0] = sys.argv[0].split("/")[-1].split(".")[0] +parser = argparse.ArgumentParser( + description="Wrap an existing installation into a container" +) +parser.add_argument("dir", type=str, help="Installation to wrap") add_base_pars(parser) add_prefix_flag(parser) -parser.add_argument("-y","--yaml",help="Tool yaml conf file") -parser.add_argument("--mask",action='store_true',help="Mask installation on disk") +parser.add_argument("-y", "--yaml", help="Tool yaml conf file") +parser.add_argument("--mask", action="store_true", help="Mask installation on disk") if len(sys.argv) < 2: @@ -28,42 +33,40 @@ if not args.wrapper_paths: print_err("Tool {} requires -w/--wrapper-paths to be used".format(sys.argv[0])) sys.exit(1) -conf={} -#wrapp=[ str(pathlib.Path(w).resolve()) for w in args.wrapper_paths.split(',')] -#args.wrapper_paths=",".join(wrapp) -conf["isolate"]="no" -conf["mode"]="wrapdisk" -conf["wrap_src"]=str(pathlib.Path(args.dir).resolve()) -conf["update_installation"]="no" -conf["template_script"]="wrap.sh" +conf = {} +# wrapp=[ str(pathlib.Path(w).resolve()) for w in args.wrapper_paths.split(',')] +# args.wrapper_paths=",".join(wrapp) +conf["isolate"] = "no" +conf["mode"] = "wrapdisk" +conf["wrap_src"] = str(pathlib.Path(args.dir).resolve()) +conf["update_installation"] = "no" +conf["template_script"] = "wrap.sh" if args.mask: - conf["installation_path"]=str(pathlib.Path(args.dir).resolve()) + conf["installation_path"] = str(pathlib.Path(args.dir).resolve()) if not has_apptainer(): - conf["excluded_mount_points"]="/"+conf["installation_path"].split('/')[1] + conf["excluded_mount_points"] = "/" + conf["installation_path"].split("/")[1] conf["mask_wrap_install"] = True if args.prefix: - conf["installation_prefix"]=args.prefix + conf["installation_prefix"] = args.prefix if args.yaml: - with open(args.yaml,'r') as y: + with open(args.yaml, "r") as y: conf.update(yaml.safe_load(y)) if args.environ: - conf["extra_envs"]=[{"file":args.environ}] + conf["extra_envs"] = [{"file": args.environ}] if args.post_install: - conf["post_install"]=[{"file":args.post_install}] + conf["post_install"] = [{"file": args.post_install}] if args.pre_install: - conf["pre_install"]=[{"file":args.pre_install}] - + conf["pre_install"] = [{"file": args.pre_install}] -global_conf={} -with open(os.getenv("CW_GLOBAL_YAML"),'r') as g: - global_conf=yaml.safe_load(g) - -parse_wrapper(conf,global_conf,args,False) +global_conf = {} +with open(os.getenv("CW_GLOBAL_YAML", ""), "r") as g: + global_conf = yaml.safe_load(g) -with open(os.getenv("_usr_yaml"),'a+') as f: - yaml.dump(conf,f) +parse_wrapper(conf, args, False) +with open(os.getenv("_usr_yaml", ""), "a+") as f: + yaml.dump(conf, f) diff --git a/generate_wrappers.sh b/generate_wrappers.sh index 6fb0a7d..a16cdbd 100755 --- a/generate_wrappers.sh +++ b/generate_wrappers.sh @@ -1,7 +1,7 @@ #!/bin/bash SINGULARITY_BIND="" set -e -set -u +set -u SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" source $SCRIPT_DIR/common_functions.sh @@ -26,18 +26,18 @@ else fi # Need to unset the path, otherwise we might be stuck in a nasty loop -# and exhaust the system +# and exhaust the system _REAL_PATH_CMD=' -export OLD_PATH=$PATH +export OLD_PATH=$PATH export PATH="/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/bin" -SOURCE="${BASH_SOURCE[0]}" -_O_SOURCE=$SOURCE -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" +SOURCE="${BASH_SOURCE[0]}" +_O_SOURCE=$SOURCE +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" +done +DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" ' _PRE_COMMAND="source \$DIR/../common.sh" @@ -79,13 +79,13 @@ if grep -q 'singularity/mnt/session\|apptainer/mnt/session' /proc/self/mountinf export _CW_IN_CONTAINER=Yes if [[ \"$CW_ISOLATE\" == \"yes\" && ! \"\$( stat -c '%i' \$SINGULARITY_CONTAINER)\" == \"\$( stat -c '%i' \$_C_DIR/\$CONTAINER_IMAGE)\" ]]; then echo \"[ ERROR ] wrapper called from another container. Is \$SINGULARITY_CONTAINER, should be \$_C_DIR/\$CONTAINER_IMAGE \" - exit 1 + exit 1 fi if [[ ! -e $CW_INSTALLATION_PATH ]]; then - echo \"[ ERROR ] Installation for \$_C_DIR/ is not mounted. Wrapper called from another container?\" + echo \"[ ERROR ] Installation for \$_C_DIR/ is not mounted. Wrapper called from another container?\" exit 1 fi - + else unset _CW_IN_CONTAINER export _CW_IS_ISOLATED=$CW_ISOLATE @@ -185,21 +185,21 @@ for wrapper_path in "${CW_WRAPPER_PATHS[@]}";do # so that quote are maintainted # e.g python -c "print('Hello')" works # The test is there as printf returns '' if $@ is empty - # passing '' is not wanted behavior + # passing '' is not wanted behavior print_info "Checking if conda installation" 3 unset CONDA_CMD - if $_CONTAINER_EXEC test -f $wrapper_path/../../../bin/conda ; then + if $_CONTAINER_EXEC test -f $wrapper_path/../../../bin/conda ; then export CONDA_CMD=1 print_info "Inserting conda activation into wrappers" 3 env_name=$(basename $(realpath -m $wrapper_path/../ )) conda_path=$(realpath -m $wrapper_path/../../../bin/conda) _cws="bash -c \"eval \\\"\\\$($conda_path shell.bash hook )\\\" && conda activate $env_name &>/dev/null && " else - print_info "Does not look like a conda installation" 3 + print_info "Does not look like a conda installation" 3 fi fi - + for target in "${targets[@]}"; do print_info "Creating wrapper for $target" 3 @@ -221,19 +221,19 @@ else" >> _deploy/bin/$target _v_in_use=\$? if [[ ( \$_v_in_use -eq 0 && ! \${CW_FORCE_CONDA_ACTIVATE+defined} ) || \${CW_NO_CONDA_ACTIVATE+defined} ]];then export PATH=\"\$OLD_PATH\" - $_RUN_CMD $_default_cws exec -a \$_O_SOURCE \$DIR/$target $_cwe + $_RUN_CMD $_default_cws exec -a \$_O_SOURCE \$DIR/$target $_cwe else export PATH=\"\$OLD_PATH\" _venv_act=\":\" - test 0 -eq \$_v_in_use && _venv_act=\"source \$_venvd/activate\" - $_RUN_CMD $_cws \$_venv_act && exec -a \$_O_SOURCE \$DIR/$target $_cwe + test 0 -eq \$_v_in_use && _venv_act=\"source \$_venvd/activate\" + $_RUN_CMD $_cws \$_venv_act && exec -a \$_O_SOURCE \$DIR/$target $_cwe fi fi " >> _deploy/bin/$target - else + else echo " export PATH=\"\$OLD_PATH\" - $_RUN_CMD $_cws exec -a \$_O_SOURCE \$DIR/$target $_cwe + $_RUN_CMD $_cws exec -a \$_O_SOURCE \$DIR/$target $_cwe fi" >> _deploy/bin/$target fi chmod +x _deploy/bin/$target @@ -242,7 +242,7 @@ fi" >> _deploy/bin/$target if [[ ! -z "$($_CONTAINER_EXEC ls $wrapper_path/../pyvenv.cfg 2>/dev/null )" ]]; then print_info "Target is a venv" 2 $_CONTAINER_EXEC cat $wrapper_path/../pyvenv.cfg > _deploy/pyvenv.cfg - _pyd=$($_CONTAINER_EXEC ls $wrapper_path/../lib) + _pyd=$(basename "$($_CONTAINER_EXEC find $wrapper_path/../lib -maxdepth 1 -type d -regex '.*/python[0-9]+\.[0-9]+$')") mkdir -p _deploy/lib/$_pyd/ (cd _deploy/lib/$_pyd && ln -s $wrapper_path/../lib/$_pyd/site-packages site-packages ) (cd _deploy && ln -s lib lib64 ) @@ -258,7 +258,7 @@ echo "$_PRE_COMMAND" >> _deploy/bin/$target echo " if [[ -z \"\$SINGULARITY_NAME\" ]];then - $_SHELL_CMD \"\$@\" + $_SHELL_CMD \"\$@\" fi" >> _deploy/bin/$target chmod +x _deploy/bin/$target @@ -268,7 +268,7 @@ echo "$_REAL_PATH_CMD" >> _deploy/bin/$target echo "$_PRE_COMMAND" >> _deploy/bin/$target echo " if [[ -z \"\$SINGULARITY_NAME\" ]];then - $_RUN_CMD \"\$@\" + $_RUN_CMD \"\$@\" fi" >> _deploy/bin/$target chmod +x _deploy/bin/$target @@ -282,9 +282,9 @@ printf -- '%s\n' "SINGULARITYENV_PATH=\$(echo \"\${_tmp_arr[@]}\" | /usr/bin/tr printf -- '%s\n' "_tmp_arr=(\$(echo \$SINGULARITYENV_LD_LIBRARY_PATH | /usr/bin/tr ':' '\n' ))" >> _deploy/common.sh printf -- '%s\n' "SINGULARITYENV_LD_LIBRARY_PATH=\$(echo \"\${_tmp_arr[@]}\" | /usr/bin/tr ' ' ':')" >> _deploy/common.sh -# Keep Venv path if we wrap a container and create +# Keep Venv path if we wrap a container and create # So the expectation if a user creates a venv on top of a wrapped container -# would be that invoking the venv python which launches shell commands again +# would be that invoking the venv python which launches shell commands again # calling the python interpreter would call the venv python and not the base python if [[ "$CW_MODE" == "wrapcont" ]];then echo ' @@ -296,10 +296,10 @@ test -e $_venvd/../pyvenv.cfg fi if [[ -f _extra_envs.sh ]];then - cat _extra_envs.sh >> _deploy/common.sh + cat _extra_envs.sh >> _deploy/common.sh fi if [[ -f _extra_user_envs.sh ]];then - cat _extra_user_envs.sh >> _deploy/common.sh + cat _extra_user_envs.sh >> _deploy/common.sh fi chmod o+r _deploy chmod o+x _deploy diff --git a/templates/conda.sh b/templates/conda.sh index e6eb2a6..f053b87 100644 --- a/templates/conda.sh +++ b/templates/conda.sh @@ -24,7 +24,7 @@ eval "$($CW_INSTALLATION_PATH/miniforge/bin/conda shell.bash hook)" cd $CW_WORKDIR source $CW_INSTALLATION_PATH/_pre_install.sh -if [[ ! -z "$(echo "$CW_ENV_FILE" | grep ".*\.yaml\|.*\.yml")" ]];then +if [[ ! -z "$(echo "$CW_ENV_FILE" | grep ".*\.yaml\|.*\.yml")" ]]; then _EC="env" _FF="-f" else @@ -34,14 +34,14 @@ cd $CW_INSTALLATION_PATH print_info "Creating env, full log in $CW_BUILD_TMPDIR/build.log" 1 if [[ ${CW_MAMBA} == "yes" ]] ;then - print_info "Using mamba to install packages" 1 - mamba $_EC create --name $CW_ENV_NAME $_FF $( basename $CW_ENV_FILE ) &>> $CW_BUILD_TMPDIR/build.log & + print_info "Using mamba to install packages" 1 + mamba $_EC create -y --name $CW_ENV_NAME $_FF $( basename $CW_ENV_FILE ) &>> $CW_BUILD_TMPDIR/build.log & else - conda $_EC create --name $CW_ENV_NAME $_FF $( basename $CW_ENV_FILE ) &>> $CW_BUILD_TMPDIR/build.log & + conda $_EC create -y --name $CW_ENV_NAME $_FF $( basename $CW_ENV_FILE ) &>> $CW_BUILD_TMPDIR/build.log & fi inst_pid=$! -follow_log $inst_pid $CW_BUILD_TMPDIR/build.log 20 +follow_log $inst_pid $CW_BUILD_TMPDIR/build.log 20 wait $inst_pid conda activate $CW_ENV_NAME if [[ ${CW_REQUIREMENTS_FILE+defined} ]];then diff --git a/templates/uvenv.sh b/templates/uvenv.sh new file mode 100644 index 0000000..2b90d69 --- /dev/null +++ b/templates/uvenv.sh @@ -0,0 +1,44 @@ +#!/bin/bash +set -e + +cd "$CW_BUILD_TMPDIR" +echo "export env_root=$CW_INSTALLATION_PATH/$CW_ENV_NAME/" >> _extra_envs.sh +echo "export env_root=$CW_INSTALLATION_PATH/$CW_ENV_NAME/" >> _vars.sh +export env_root="$CW_INSTALLATION_PATH/$CW_ENV_NAME/" + +cd "$CW_INSTALLATION_PATH" + +if [[ ! -f "$CW_INSTALLATION_PATH/uv/bin/uv" ]]; then + curl -LsSf https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL="$CW_INSTALLATION_PATH/uv/bin/" UV_PRINT_QUIET=1 sh +fi +export PATH="$CW_INSTALLATION_PATH/uv/bin/:$PATH" + +cd "$CW_WORKDIR" +source "$CW_INSTALLATION_PATH/_pre_install.sh" + +cd "$CW_INSTALLATION_PATH" + +print_info "Installing requirements file" 1 +export UV_PYTHON_INSTALL_DIR="$CW_INSTALLATION_PATH/uv/python" + +NOCACHE_FLAG="" +if [[ "$CW_PIPCACHE" != "yes" ]]; then + NOCACHE_FLAG="-n" +fi + +if [[ ! -e "$env_root/bin/activate" ]]; then + uv venv -p "$CW_PYVER" --managed-python $NOCACHE_FLAG --no-config --link-mode=copy "$env_root" +fi +source "$env_root/bin/activate" + +if [[ ${CW_REQUIREMENTS_FILE+defined} ]];then + uv pip install --link-mode=copy --compile-bytecode $NOCACHE_FLAG -r "$( basename "$CW_REQUIREMENTS_FILE")" > "$CW_BUILD_TMPDIR/_uv.log" & + bg_pid=$! + wait $bg_pid + follow_log $bg_pid "$CW_BUILD_TMPDIR/_uv.log" 20 +fi +cd "$CW_WORKDIR" +print_info "Running user supplied commands" 1 +source "$CW_INSTALLATION_PATH/_post_install.sh" + +echo "CW_WRAPPER_PATHS+=( \"$CW_INSTALLATION_PATH/$CW_ENV_NAME/bin/\" )" >> "$CW_BUILD_TMPDIR"/_vars.sh diff --git a/templates/venv.sh b/templates/venv.sh index 03816f9..d7d13ef 100644 --- a/templates/venv.sh +++ b/templates/venv.sh @@ -14,19 +14,27 @@ cd $CW_WORKDIR source $CW_INSTALLATION_PATH/_pre_install.sh cd $CW_INSTALLATION_PATH -if [[ ${CW_ENABLE_SITE_PACKAGES+defined} ]];then - print_info "Enabling system and user site packages" 1 - _SP="--system-site-packages" -else - print_info "Not enabling system and user site packages" 1 - _SP="" +if [[ ! -e $env_root/bin/activate ]]; then + if [[ ${CW_ENABLE_SITE_PACKAGES+defined} ]];then + print_info "Enabling system and user site packages" 1 + _SP="--system-site-packages" + else + print_info "Not enabling system and user site packages" 1 + _SP="" + fi + print_info "Installing requirements file" 1 + python3 -m venv $_SP $CW_ENV_NAME +fi + +source $env_root/bin/activate + +NOCACHE_FLAG="" +if [[ "$CW_PIPCACHE" != "yes" ]]; then + NOCACHE_FLAG="--no-cache-dir" fi -print_info "Installing requirements file" 1 -python3 -m venv $_SP $CW_ENV_NAME -source $CW_INSTALLATION_PATH/$CW_ENV_NAME/bin/activate if [[ ${CW_REQUIREMENTS_FILE+defined} ]];then - pip install --disable-pip-version-check -r "$( basename $CW_REQUIREMENTS_FILE)" > $CW_BUILD_TMPDIR/_pip.log & + pip install --disable-pip-version-check $NOCACHE_FLAG -r "$( basename $CW_REQUIREMENTS_FILE)" > $CW_BUILD_TMPDIR/_pip.log & bg_pid=$! wait $bg_pid follow_log $bg_pid $CW_BUILD_TMPDIR/_pip.log 20 From da92e9c15391ac25f901f9d2bfb4f0f9d5b4f8b0 Mon Sep 17 00:00:00 2001 From: Xavier Abellan Ecija Date: Tue, 2 Dec 2025 17:09:20 +0000 Subject: [PATCH 2/4] Further support for uv and rationalise templates - conda-containerize can now also do uv for pip packages - consolidated templates for venv/uvenv and conda, removing the specific ones for updates - included some tests for the new functionality --- README.md | 3 +- frontends/conda-containerize.py | 40 ++++++++---- frontends/pip-containerize.py | 21 ++++--- frontends/script_shared.py | 4 ++ templates/conda.sh | 106 ++++++++++++++++++++------------ templates/conda_modify.sh | 34 ---------- templates/uvenv.sh | 44 ------------- templates/venv.sh | 55 ++++++++++------- templates/venv_modify.sh | 28 --------- tests/tests.sh | 53 +++++++++++----- tests/validate_slim.sh | 6 +- 11 files changed, 188 insertions(+), 206 deletions(-) delete mode 100644 templates/conda_modify.sh delete mode 100644 templates/uvenv.sh delete mode 100644 templates/venv_modify.sh mode change 100644 => 100755 tests/tests.sh diff --git a/README.md b/README.md index 99c3802..e637a14 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,7 @@ on the sqfs image itself. This behavior can be disabled by setting `CW_NO_FIX_PE - `pip-containerize` - Wrap new venv installation or edit existing - Will by default use currently available python + - Option to use uv to manage python and venv - Option to also use slim container image (will then not mount full host) - `wrap-container` - Generate wrappers for existing container. Mainly @@ -291,4 +292,4 @@ the conf to allow for very "creative" usages of the tool Technically updating masked disk installations is not an issue, but let's not do that until there is a specific request. The tool now drops the full path leading to the target -from the bind list, if more binds are needed a yaml input needs to be constructed. +from the bind list, if more binds are needed a yaml input needs to be constructed. diff --git a/frontends/conda-containerize.py b/frontends/conda-containerize.py index a81047b..da0a7fa 100644 --- a/frontends/conda-containerize.py +++ b/frontends/conda-containerize.py @@ -11,7 +11,7 @@ 0, str(root_dir) + "/PyDeps/lib/python{}.{}/site-packages".format(info[0], info[1]) ) import yaml # noqa: E402 -from cw_common import print_err # noqa: E402 +from cw_common import print_err, print_warn # noqa: E402 from script_shared import ( # noqa E402 add_adv_pars, add_base_pars, @@ -32,8 +32,17 @@ parser_new.add_argument( "--mamba", help="use mamba for installation", action="store_true" ) +parser_new.add_argument( + "--uv", help="use uv for pip dependencies (only with mamba)", action="store_true" +) +parser_new.add_argument( + "--nocache", help="Do not use pip/uv cache", action="store_true" +) parser_upd = add_upd_pars(subparsers) add_adv_pars(subparsers) +parser_upd.add_argument( + "--nocache", help="Do not use pip/uv cache", action="store_true" +) ps = [parser_new, parser_upd] for p in ps: @@ -51,8 +60,13 @@ args = parser.parse_args() conf = {} conf["add_ld"] = "no" +conf["use_uv"] = "no" +conf["mode"] = "conda" +conf["template_script"] = "conda.sh" if args.command == "new": conf["env_file"] = args.env_file + conf["update_installation"] = "no" + conf["installation_file_paths"] = [conf["env_file"]] if args.prefix: conf["installation_prefix"] = args.prefix conf["mode"] = "conda" @@ -60,8 +74,12 @@ conf["mamba"] = "yes" else: conf["mamba"] = "no" + if args.uv and args.mamba: + conf["use_uv"] = "yes" + elif args.uv and not args.mamba: + print_warn("Using --uv without --mamba does not have an effect") elif args.command == "update": - conf["mode"] = "conda_modify" + conf["update_installation"] = "yes" get_old_conf(args.dir, conf) else: with open(args.yaml, "r") as y: @@ -71,7 +89,6 @@ print_err("Env file {} does not exist".format(args.env_file)) sys.exit(1) - if args.command in ["update", "new"]: if args.environ: conf["extra_envs"] = [{"file": args.environ}] @@ -88,21 +105,18 @@ global_conf = yaml.safe_load(g) parse_wrapper(conf, args, False) -if conf["mode"] == "conda": - conf["update_installation"] = "no" - conf["template_script"] = "conda.sh" - conf["installation_file_paths"] = [conf["env_file"]] -elif conf["mode"] == "conda_modify": - conf["update_installation"] = "yes" - conf["template_script"] = "conda_modify.sh" -else: - print_err("No or incorrent mode set, [conda,conda_modify]") - sys.exit(1) + if "requirements_file" in conf: if "installation_file_paths" in conf: conf["installation_file_paths"].append(conf["requirements_file"]) else: conf["installation_file_paths"] = conf["requirements_file"] +if "pipcache" not in conf: + conf["pipcache"] = True + +if args.nocache is not None: + conf["pipcache"] = not args.nocache + with open(os.getenv("_usr_yaml", ""), "a+") as f: yaml.dump(conf, f) diff --git a/frontends/pip-containerize.py b/frontends/pip-containerize.py index f810724..fc61f1c 100644 --- a/frontends/pip-containerize.py +++ b/frontends/pip-containerize.py @@ -68,27 +68,31 @@ sys.exit(0) args = parser.parse_args() conf = {} -pyver = "3.12.9-slim-bookworm" - if args.requirements_file: conf["requirements_file"] = args.requirements_file conf["installation_file_paths"] = [conf["requirements_file"]] if args.command == "new": + conf["mode"] = "venv" + conf["use_uv"] = "no" conf["update_installation"] = "no" if args.system_site_packages: conf["enable_site_packages"] = "yes" if args.prefix: conf["installation_prefix"] = args.prefix - conf["mode"] = "venv" if args.slim: + conf["pyver"] = "slim" if args.pyver: - pyver = args.pyver - conf["container_src"] = "docker://python:{}".format(pyver) + if "-slim" in args.pyver or args.pyver == "slim": + conf["pyver"] = args.pyver + else: + conf["pyver"] = args.pyver + "-slim" + + conf["container_src"] = "docker://python:{}".format(conf["pyver"]) conf["isolate"] = "yes" elif args.uv: - conf["mode"] = "uvenv" + conf["use_uv"] = "yes" conf["pyver"] = args.pyver if args.pyver else "3" else: if args.pyver: @@ -117,10 +121,7 @@ parse_wrapper(conf, args, False) -if conf["mode"] == "uvenv": - conf["template_script"] = "uvenv.sh" -else: - conf["template_script"] = "venv.sh" +conf["template_script"] = "venv.sh" if "pipcache" not in conf: conf["pipcache"] = True diff --git a/frontends/script_shared.py b/frontends/script_shared.py index 3e0da21..6b986e6 100644 --- a/frontends/script_shared.py +++ b/frontends/script_shared.py @@ -127,6 +127,10 @@ def get_old_conf(d, conf): conf["container_image"] = old_conf["container_image"] conf["isolate"] = old_conf["isolate"] conf["mode"] = old_conf["mode"] + if "mamba" in old_conf: + conf["mamba"] = old_conf["mamba"] + if "use_uv" in old_conf: + conf["use_uv"] = old_conf["use_uv"] if "pipcache" in old_conf: conf["pipcache"] = old_conf["pipcache"] if "wrapper_paths" in old_conf: diff --git a/templates/conda.sh b/templates/conda.sh index f053b87..ff84c84 100644 --- a/templates/conda.sh +++ b/templates/conda.sh @@ -1,65 +1,95 @@ #!/bin/bash -set -e +#set -e - -cd $CW_BUILD_TMPDIR +cd "$CW_BUILD_TMPDIR" echo "export env_root=$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/" >> _extra_envs.sh echo "export env_root=$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/" >> _vars.sh -export env_root=$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/ +export env_root="$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/" + +cd "$CW_INSTALLATION_PATH" -cd $CW_INSTALLATION_PATH +if [[ ! -e $CW_INSTALLATION_PATH/miniforge/ ]]; then + if [[ -e $CW_INSTALLATION_PATH/miniconda/ ]]; then + print_info "Updating older installation which is using miniconda and not miniforge\nCreating symlink miniforge -> miniconda" 1 + ln -s "$CW_INSTALLATION_PATH/miniconda" "$CW_INSTALLATION_PATH/miniforge" + else + [ "$CW_CONDA_VERSION" = "latest" ] && CW_CONDA_VERSION=$(curl -s https://api.github.com/repos/conda-forge/miniforge/releases/latest | grep "tag_name" | cut -d: -f2 | tr -d \" | tr -d , | tr -d " ") -[ "$CW_CONDA_VERSION" = "latest" ] && CW_CONDA_VERSION=$(curl -s https://api.github.com/repos/conda-forge/miniforge/releases/latest | grep "tag_name" | cut -d: -f2 | tr -d \" | tr -d , | tr -d " ") + print_info "Using miniforge version Miniforge3-$CW_CONDA_VERSION-$CW_CONDA_ARCH" 1 + print_info "Downloading miniforge " 2 + curl -sL "https://github.com/conda-forge/miniforge/releases/download/$CW_CONDA_VERSION/Miniforge3-$CW_CONDA_VERSION-$CW_CONDA_ARCH.sh" --output Miniforge_inst.sh &>/dev/null + print_info "Installing miniforge " 1 + bash Miniforge_inst.sh -b -p "$CW_INSTALLATION_PATH/miniforge" > "$CW_BUILD_TMPDIR/_inst_miniforge.log" & + inst_pid=$! -print_info "Using miniforge version Miniforge3-$CW_CONDA_VERSION-$CW_CONDA_ARCH" 1 -print_info "Downloading miniforge " 2 -curl -sL https://github.com/conda-forge/miniforge/releases/download/$CW_CONDA_VERSION/Miniforge3-$CW_CONDA_VERSION-$CW_CONDA_ARCH.sh --output Miniforge_inst.sh &>/dev/null -print_info "Installing miniforge " 1 -bash Miniforge_inst.sh -b -p $CW_INSTALLATION_PATH/miniforge > $CW_BUILD_TMPDIR/_inst_miniforge.log & -inst_pid=$! + follow_log $inst_pid "$CW_BUILD_TMPDIR/_inst_miniforge.log" 20 + rm Miniforge_inst.sh + fi +fi -follow_log $inst_pid $CW_BUILD_TMPDIR/_inst_miniforge.log 20 -rm Miniforge_inst.sh -eval "$($CW_INSTALLATION_PATH/miniforge/bin/conda shell.bash hook)" +eval "$("$CW_INSTALLATION_PATH/miniforge/bin/conda" shell.bash hook)" -cd $CW_WORKDIR -source $CW_INSTALLATION_PATH/_pre_install.sh +cd "$CW_WORKDIR" +source "$CW_INSTALLATION_PATH/_pre_install.sh" if [[ ! -z "$(echo "$CW_ENV_FILE" | grep ".*\.yaml\|.*\.yml")" ]]; then _EC="env" _FF="-f" else _FF="--file" fi -cd $CW_INSTALLATION_PATH -print_info "Creating env, full log in $CW_BUILD_TMPDIR/build.log" 1 +cd "$CW_INSTALLATION_PATH" -if [[ ${CW_MAMBA} == "yes" ]] ;then - print_info "Using mamba to install packages" 1 - mamba $_EC create -y --name $CW_ENV_NAME $_FF $( basename $CW_ENV_FILE ) &>> $CW_BUILD_TMPDIR/build.log & -else - conda $_EC create -y --name $CW_ENV_NAME $_FF $( basename $CW_ENV_FILE ) &>> $CW_BUILD_TMPDIR/build.log & +if [[ "$CW_PIPCACHE" != "yes" ]]; then + export PIP_NO_CACHE_DIR=1 + export UV_NO_CACHE=1 +fi + +_UV="" +if [[ ${CW_USE_UV} == "yes" ]] ; then + export UV_LINK_MODE=copy + _UV="--use-uv" fi -inst_pid=$! -follow_log $inst_pid $CW_BUILD_TMPDIR/build.log 20 -wait $inst_pid -conda activate $CW_ENV_NAME +if [[ ! -e "$env_root" ]]; then + print_info "Creating env, full log in $CW_BUILD_TMPDIR/build.log" 1 + + if [[ ${CW_MAMBA} == "yes" ]] ; then + print_info "Using mamba to install packages" 1 + mamba $_EC create -y $_UV --name "$CW_ENV_NAME" $_FF "$( basename "$CW_ENV_FILE" )" &>> "$CW_BUILD_TMPDIR/build.log" & + else + conda $_EC create -y --name "$CW_ENV_NAME" $_FF "$( basename "$CW_ENV_FILE" )" &>> "$CW_BUILD_TMPDIR/build.log" & + fi + inst_pid=$! + follow_log $inst_pid "$CW_BUILD_TMPDIR/build.log" 20 + wait $inst_pid +fi + +conda activate "$CW_ENV_NAME" + if [[ ${CW_REQUIREMENTS_FILE+defined} ]];then - pip install -r $( basename "$CW_REQUIREMENTS_FILE" ) + print_info "Installing requirements file" 1 + if [[ ${CW_USE_UV} == "yes" ]] ; then + uv pip install -r "$( basename "$CW_REQUIREMENTS_FILE" )" > "$CW_BUILD_TMPDIR/_pip.log" & + else + pip install -r "$( basename "$CW_REQUIREMENTS_FILE" )"> "$CW_BUILD_TMPDIR/_pip.log" & + fi + bg_pid=$! + wait $bg_pid + follow_log $bg_pid "$CW_BUILD_TMPDIR/_pip.log" 20 fi -cd $CW_WORKDIR + +cd "$CW_WORKDIR" print_info "Running user supplied commands" 1 -source $CW_INSTALLATION_PATH/_post_install.sh -if [[ -d $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/ ]];then - echo 'echo "' > $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages - conda list >> $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages - echo '"' >> $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages - chmod +x $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages +source "$CW_INSTALLATION_PATH/_post_install.sh" +if [[ -d "$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/" ]];then + echo 'echo "' > "$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages" + conda list >> "$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages" + echo '"' >> "$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages" + chmod +x "$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages" else print_warn "Created env is empty" fi - # Set here as they are dynamic # Could also set them in construct.py... -echo "CW_WRAPPER_PATHS+=( \"$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/\" )" >> $CW_BUILD_TMPDIR/_vars.sh +echo "CW_WRAPPER_PATHS+=( \"$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/\" )" >> "$CW_BUILD_TMPDIR/_vars.sh" diff --git a/templates/conda_modify.sh b/templates/conda_modify.sh deleted file mode 100644 index 0283a6f..0000000 --- a/templates/conda_modify.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - - -## Fix for updating older installations - -if [[ ! -e $CW_INSTALLATION_PATH/miniforge/ ]];then - print_info "Updating older installation which is using miniconda and not miniforge\nCreating symlink miniforge -> miniconda" 1 - ln -s "$CW_INSTALLATION_PATH/miniconda" "$CW_INSTALLATION_PATH/miniforge" -fi - -cd $CW_BUILD_TMPDIR -echo "export env_root=$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/" >> _extra_envs.sh -echo "export env_root=$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/" >> _vars.sh -cd $CW_INSTALLATION_PATH -export env_root=$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/ -eval "$($CW_INSTALLATION_PATH/miniforge/bin/conda shell.bash hook)" -cd $CW_WORKDIR -source $CW_INSTALLATION_PATH/_pre_install.sh -conda activate $CW_ENV_NAME -if [[ ${CW_REQUIREMENTS_FILE+defined} ]];then - print_info "Installing requirements file" 1 - pip install -r $( basename "$CW_REQUIREMENTS_FILE" ) > $CW_BUILD_TMPDIR/_pip.log & - bg_pid=$! - wait $bg_pid - follow_log $bg_pid $CW_BUILD_TMPDIR/_pip.log 20 -fi -cd $CW_WORKDIR -source $CW_INSTALLATION_PATH/_post_install.sh -echo 'echo "' > $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages -conda list >> $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages -echo '"' >> $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages -chmod +x $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages - -echo "CW_WRAPPER_PATHS+=( \"$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/\" )" >> $CW_BUILD_TMPDIR/_vars.sh diff --git a/templates/uvenv.sh b/templates/uvenv.sh deleted file mode 100644 index 2b90d69..0000000 --- a/templates/uvenv.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -set -e - -cd "$CW_BUILD_TMPDIR" -echo "export env_root=$CW_INSTALLATION_PATH/$CW_ENV_NAME/" >> _extra_envs.sh -echo "export env_root=$CW_INSTALLATION_PATH/$CW_ENV_NAME/" >> _vars.sh -export env_root="$CW_INSTALLATION_PATH/$CW_ENV_NAME/" - -cd "$CW_INSTALLATION_PATH" - -if [[ ! -f "$CW_INSTALLATION_PATH/uv/bin/uv" ]]; then - curl -LsSf https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL="$CW_INSTALLATION_PATH/uv/bin/" UV_PRINT_QUIET=1 sh -fi -export PATH="$CW_INSTALLATION_PATH/uv/bin/:$PATH" - -cd "$CW_WORKDIR" -source "$CW_INSTALLATION_PATH/_pre_install.sh" - -cd "$CW_INSTALLATION_PATH" - -print_info "Installing requirements file" 1 -export UV_PYTHON_INSTALL_DIR="$CW_INSTALLATION_PATH/uv/python" - -NOCACHE_FLAG="" -if [[ "$CW_PIPCACHE" != "yes" ]]; then - NOCACHE_FLAG="-n" -fi - -if [[ ! -e "$env_root/bin/activate" ]]; then - uv venv -p "$CW_PYVER" --managed-python $NOCACHE_FLAG --no-config --link-mode=copy "$env_root" -fi -source "$env_root/bin/activate" - -if [[ ${CW_REQUIREMENTS_FILE+defined} ]];then - uv pip install --link-mode=copy --compile-bytecode $NOCACHE_FLAG -r "$( basename "$CW_REQUIREMENTS_FILE")" > "$CW_BUILD_TMPDIR/_uv.log" & - bg_pid=$! - wait $bg_pid - follow_log $bg_pid "$CW_BUILD_TMPDIR/_uv.log" 20 -fi -cd "$CW_WORKDIR" -print_info "Running user supplied commands" 1 -source "$CW_INSTALLATION_PATH/_post_install.sh" - -echo "CW_WRAPPER_PATHS+=( \"$CW_INSTALLATION_PATH/$CW_ENV_NAME/bin/\" )" >> "$CW_BUILD_TMPDIR"/_vars.sh diff --git a/templates/venv.sh b/templates/venv.sh index d7d13ef..e5108eb 100644 --- a/templates/venv.sh +++ b/templates/venv.sh @@ -1,20 +1,29 @@ #!/bin/bash set -e - -cd $CW_BUILD_TMPDIR +cd "$CW_BUILD_TMPDIR" echo "export env_root=$CW_INSTALLATION_PATH/$CW_ENV_NAME/" >> _extra_envs.sh echo "export env_root=$CW_INSTALLATION_PATH/$CW_ENV_NAME/" >> _vars.sh -export env_root=$CW_INSTALLATION_PATH/$CW_ENV_NAME/ +export env_root="$CW_INSTALLATION_PATH/$CW_ENV_NAME/" -cd $CW_INSTALLATION_PATH +cd "$CW_INSTALLATION_PATH" +source "$CW_INSTALLATION_PATH/_pre_install.sh" +_NC="" +if [[ "$CW_PIPCACHE" != "yes" ]]; then + _NC="-n" +fi -cd $CW_WORKDIR -source $CW_INSTALLATION_PATH/_pre_install.sh -cd $CW_INSTALLATION_PATH +if [[ "$CW_USE_UV" == "yes" ]]; then + if [[ ! -f "$CW_INSTALLATION_PATH/uv/bin/uv" ]]; then + print_info "Installing uv package manager" 1 + curl -LsSf https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL="$CW_INSTALLATION_PATH/uv/bin/" UV_PRINT_QUIET=1 sh + fi + export PATH="$CW_INSTALLATION_PATH/uv/bin/:$PATH" + export UV_PYTHON_INSTALL_DIR="$CW_INSTALLATION_PATH/uv/python" +fi -if [[ ! -e $env_root/bin/activate ]]; then +if [[ ! -e "$env_root/bin/activate" ]]; then if [[ ${CW_ENABLE_SITE_PACKAGES+defined} ]];then print_info "Enabling system and user site packages" 1 _SP="--system-site-packages" @@ -22,25 +31,29 @@ if [[ ! -e $env_root/bin/activate ]]; then print_info "Not enabling system and user site packages" 1 _SP="" fi - print_info "Installing requirements file" 1 - python3 -m venv $_SP $CW_ENV_NAME + print_info "Creating virtual environment" 1 + if [[ "$CW_USE_UV" == "yes" ]]; then + uv venv -p "$CW_PYVER" $_SP --managed-python $_NC --no-config --link-mode=copy "$env_root" + else + python3 -m venv $_SP "$CW_ENV_NAME" + fi fi -source $env_root/bin/activate - -NOCACHE_FLAG="" -if [[ "$CW_PIPCACHE" != "yes" ]]; then - NOCACHE_FLAG="--no-cache-dir" -fi +source "$env_root/bin/activate" if [[ ${CW_REQUIREMENTS_FILE+defined} ]];then - pip install --disable-pip-version-check $NOCACHE_FLAG -r "$( basename $CW_REQUIREMENTS_FILE)" > $CW_BUILD_TMPDIR/_pip.log & + print_info "Installing requirements file" 1 + if [[ "$CW_USE_UV" == "yes" ]]; then + uv pip install --link-mode=copy --compile-bytecode $_NC -r "$( basename "$CW_REQUIREMENTS_FILE")" > "$CW_BUILD_TMPDIR/_pip.log" & + else + pip install --disable-pip-version-check $_NC -r "$( basename "$CW_REQUIREMENTS_FILE")" > "$CW_BUILD_TMPDIR/_pip.log" & + fi bg_pid=$! wait $bg_pid - follow_log $bg_pid $CW_BUILD_TMPDIR/_pip.log 20 + follow_log $bg_pid "$CW_BUILD_TMPDIR/_pip.log" 20 fi -cd $CW_WORKDIR +cd "$CW_WORKDIR" print_info "Running user supplied commands" 1 -source $CW_INSTALLATION_PATH/_post_install.sh +source "$CW_INSTALLATION_PATH/_post_install.sh" -echo "CW_WRAPPER_PATHS+=( \"$CW_INSTALLATION_PATH/$CW_ENV_NAME/bin/\" )" >> $CW_BUILD_TMPDIR/_vars.sh +echo "CW_WRAPPER_PATHS+=( \"$CW_INSTALLATION_PATH/$CW_ENV_NAME/bin/\" )" >> "$CW_BUILD_TMPDIR"/_vars.sh diff --git a/templates/venv_modify.sh b/templates/venv_modify.sh deleted file mode 100644 index d8cd259..0000000 --- a/templates/venv_modify.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -set -e - - -cd $CW_BUILD_TMPDIR -echo "export env_root=$CW_INSTALLATION_PATH/$CW_ENV_NAME/" >> _extra_envs.sh -echo "export env_root=$CW_INSTALLATION_PATH/$CW_ENV_NAME/" >> _vars.sh -export env_root=$CW_INSTALLATION_PATH/$CW_ENV_NAME/ - -cd $CW_INSTALLATION_PATH - - -cd $CW_WORKDIR -source $CW_INSTALLATION_PATH/_pre_install.sh -cd $CW_INSTALLATION_PATH -source $CW_INSTALLATION_PATH/$CW_ENV_NAME/bin/activate -if [[ ${CW_REQUIREMENTS_FILE+defined} ]];then - print_info "Installing requirements file" 1 - pip install --disable-pip-version-check -r "$( basename $CW_REQUIREMENTS_FILE)" > $CW_BUILD_TMPDIR/_pip.log & - bg_pid=$! - wait $bg_pid - follow_log $bg_pid $CW_BUILD_TMPDIR/_pip.log 20 -fi -cd $CW_WORKDIR -#print_info "Running user supplied commands" 1 -source $CW_INSTALLATION_PATH/_post_install.sh - -echo "CW_WRAPPER_PATHS+=( \"$CW_INSTALLATION_PATH/$CW_ENV_NAME/bin/\" )" >> $CW_BUILD_TMPDIR/_vars.sh diff --git a/tests/tests.sh b/tests/tests.sh old mode 100644 new mode 100755 index f13a4d6..91d2912 --- a/tests/tests.sh +++ b/tests/tests.sh @@ -1,8 +1,9 @@ -#!/bin/bash -eu +#!/bin/bash -u SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -source $SCRIPT_DIR/setup.sh +source "$SCRIPT_DIR/setup.sh" # Test are run in current directory +#cd "${TMPDIR:-/tmp}" rm -fr TEST_DIR mkdir TEST_DIR cd TEST_DIR @@ -32,6 +33,7 @@ done echo "pyyaml" > req.txt echo "pyyyaml" > req_typo.txt echo "pip install requests" > post.sh +echo "uv pip install requests" > post_uv.sh echo " channels: @@ -55,6 +57,16 @@ dependencies: " > conda_broken.yaml echo "GARBAGE" > conda_env.txt +echo " +channels: + - conda-forge +dependencies: + - numpy + - uv + - pip: + - requests +" > hybrid.yaml + t_run "conda-containerize new conda_base.yml --prefix NOT_A_DIR" "Missing install dir is created" mkdir A_DIR_NO_WRITE chmod ugo-w A_DIR_NO_WRITE @@ -66,21 +78,21 @@ t_run "conda-containerize new conda_base.yml --prefix A_DIR_NO_EXE | grep ERROR" mkdir CONDA_INSTALL_DIR t_run "conda-containerize new conda_broken.yaml --prefix CONDA_INSTALL_DIR | tee conda_inst.out | grep 'ResolvePackageNotFound\|PackagesNotFoundError'" "Conda errors are propagated to the user" -t_run "grep ERROR conda_inst.out" "Failed run contains error" +t_run "grep ERROR conda_inst.out" "Failed run contains error" t_run "grep INFO conda_inst.out" "Info is present" t_run "test -z \"\$(grep ' DEBUG ' conda_inst.out )\" " "Default no debug message" tmp_dir=$(cat conda_inst.out | grep -o "[^ ]*/cw-[A-Z,0-9]\{6\} ") t_run "\[ ! -e $tmp_dir \]" "Build dir is deleted on error" export CW_DEBUG_KEEP_FILES=1 -conda-containerize new conda_broken.yaml --prefix CONDA_INSTALL_DIR > conda_inst.out +conda-containerize new conda_broken.yaml --prefix CONDA_INSTALL_DIR &> conda_inst.out tmp_dir=$(cat conda_inst.out | grep -o "[^ ]*/cw-[A-Z,0-9]\{6\} ") t_run "\[ -e $tmp_dir \]" "Build dir is saved if CW_DEBUG_KEEP_FILES set" -test -d $tmp_dir && rm -rf $tmp_dir +test -d "$tmp_dir" && rm -rf "$tmp_dir" unset CW_DEBUG_KEEP_FILES unset CW_ENABLE_CONDARC echo "conda config --show-sources;conda config --show pkgs_dirs;exit 1" > pre.sh -rc_res=$(conda-containerize new --pre-install=pre.sh conda_base.yml --prefix CONDA_INSTALL_DIR | grep -o $HOME/.conda/pkgs) +rc_res=$(conda-containerize new --pre-install=pre.sh conda_base.yml --prefix CONDA_INSTALL_DIR 2>&1 | grep -o "$HOME/.conda/pkgs") t_run "test -z $rc_res" "User .condarc is ignored" export CW_ENABLE_CONDARC=1 t_run "conda-containerize new --pre-install=pre.sh conda_base.yml --prefix CONDA_INSTALL_DIR | grep -q $HOME/.conda/pkgs" "User .condarc can be enabled" @@ -90,19 +102,23 @@ t_run "CONDA_INSTALL_DIR/bin/python -m venv VE " "Virtual environment creation w t_run "VE/bin/python -c 'import sys;sys.exit( sys.prefix == sys.base_prefix )'" "Virtual environment is correct" t_run "VE/bin/pip install requests" "pip works for a venv" t_run "VE/bin/python -c 'import requests;print(requests.__file__)' | grep -q VE " "Package is installed correctly to venv" -CONDA_INSTALL_DIR/bin/_debug_exec bash -c "\$(readlink -f \$env_root)/../../bin/conda list --explicit" > explicit_env.txt +CONDA_INSTALL_DIR/bin/_debug_exec bash -c "\$(readlink -f \$env_root)/../../bin/conda list --explicit" > explicit_env.txt t_run "CONDA_INSTALL_DIR/bin/python -c 'import yaml'" "Package added by -r is there" t_run "conda-containerize update CONDA_INSTALL_DIR --post-install post.sh" "Update works" t_run "CONDA_INSTALL_DIR/bin/python -c 'import requests'" "Package added by update is there" rm -fr CONDA_INSTALL_DIR && mkdir CONDA_INSTALL_DIR t_run "conda-containerize new --mamba explicit_env.txt --prefix CONDA_INSTALL_DIR &>/dev/null" "Explicit env file works" +rm -fr CONDA_INSTALL_DIR && mkdir CONDA_INSTALL_DIR +t_run "conda-containerize new --mamba --uv --prefix CONDA_INSTALL_DIR hybrid.yaml 2>&1 | grep -o 'Installing uv packages:'" "Hybrid conda/pip environment creation with uv" +t_run "conda-containerize update -r req.txt CONDA_INSTALL_DIR 2>&1 | grep -o 'Resolved 1 package in'" "Hybrid conda/pip environment update with uv" + rm -fr CONDA_INSTALL_DIR && mkdir CONDA_INSTALL_DIR t_run "conda-containerize new --mamba dask_env.yaml --prefix CONDA_INSTALL_DIR &>/dev/null" "yaml ending is also supported" OLD_PATH=$PATH PATH="CONDA_INSTALL_DIR/bin:$PATH" t_run " \[ $(which python)==$(_debug_exec which python) \] " "Which returns same in and out" -str1="$(python -c "print('Hello world --a g -b \ ')" )" +str1="$(python -c "print('Hello world --a g -b \\\ ')" )" str2="Hello world --a g -b \ " t_run "\[ \"$str1\" = \"$str2\" \]" "Wrapper passed quotes correctly" t_run "python -c \"import os; os.environ['CONDA_DEFAULT_ENV']\"" "Conda is activated" @@ -134,14 +150,14 @@ cluster = dask_jobqueue.SLURMCluster( print(cluster.job_script()) ' > dask_test.py -t_run "python dask_test.py | grep \"$(realpath -s $PWD/CONDA_INSTALL_DIR/bin/python )\"" "Dask uses correct python path" +t_run "python dask_test.py | grep \"$(realpath -s "$PWD/CONDA_INSTALL_DIR/bin/python")\"" "Dask uses correct python path" PATH=$OLD_PATH -OLD_PATH=$PATH -mkdir PIP_INSTALL_DIR +# OLD_PATH=$PATH +rm -fr PIP_INSTALL_DIR && mkdir PIP_INSTALL_DIR -cat ../../default_config/config.yaml | sed 's/container_image.*$/container_image: My_very_cool_name.sif/g' > my_config.yaml +cat "$SCRIPT_DIR/../default_config/config.yaml" | sed 's/container_image.*$/container_image: My_very_cool_name.sif/g' > my_config.yaml t_run "pip-containerize new --prefix PIP_INSTALL_DIR req_typo.txt 2>&1 | grep 'No matching distribution'" "pip error shown to user" export CW_GLOBAL_YAML=my_config.yaml t_run "pip-containerize new --prefix PIP_INSTALL_DIR req.txt " "pip install works" @@ -150,6 +166,13 @@ PATH="PIP_INSTALL_DIR/bin:$PATH" t_run "python -c 'import requests'" "Package added in update available" t_run "\[ -e PIP_INSTALL_DIR/My_very_cool_name.sif \]" "Using custom conf" t_run "python -c 'import sys;sys.exit(sys.prefix == sys.base_prefix)'" "Installation is venv" -in_p=$(python3 -c 'import sys;print(sys.executable)') -out_p="$PWD/PIP_INSTALL_DIR/bin/python3" -t_run "[[ $in_p == $out_p ]]" "Executable name is same on in and out" +in_p=$(python3 -c 'import sys;print(sys.executable)') +out_p="$PWD/PIP_INSTALL_DIR/bin/python3" +t_run "[[ $in_p == $out_p ]]" "Executable name is same on in and out" + +rm -fr PIP_INSTALL_DIR && mkdir PIP_INSTALL_DIR +t_run "pip-containerize new --uv --prefix PIP_UV_INSTALL_DIR req_typo.txt 2>&1 | grep -o 'not found in the package registry'" "uv error shown to user" +t_run "pip-containerize new --uv --prefix PIP_UV_INSTALL_DIR req.txt " "pip install with uv works" +t_run "pip-containerize update PIP_UV_INSTALL_DIR --post-install post_uv.sh" "Update with uv works" +t_run "PIP_UV_INSTALL_DIR/bin/python -c 'import requests'" "Package added in uv update available" +t_run "PIP_UV_INSTALL_DIR/bin/python -c 'import sys;sys.exit(sys.prefix == sys.base_prefix)'" "UV Installation is venv" diff --git a/tests/validate_slim.sh b/tests/validate_slim.sh index d6dc14b..8a0016b 100644 --- a/tests/validate_slim.sh +++ b/tests/validate_slim.sh @@ -3,15 +3,17 @@ SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" source $SCRIPT_DIR/setup.sh +#cd "${TMPDIR:-/tmp}" rm -rf TYKKY_V_TEST mkdir -p TYKKY_V_TEST cd TYKKY_V_TEST -echo "csvkit" > requirements.txt +echo "distro" > requirements.txt echo "pip install lxml" > extra.txt t_run "pip-containerize new --slim --prefix tykky_test requirements.txt" "Creating slim container works" -t_run "singularity exec tykky_test/container.sif cat /etc/os-release | grep 'Debian'" "Slim container is actually using debian" +#t_run "singularity exec tykky_test/container.sif cat /etc/os-release | grep 'Debian'" "Slim container is actually using debian" +t_run "tykky_test/bin/python -c 'import distro; print(distro.id())' | grep 'debian'" "Slim container is actually using debian" t_run "pip-containerize update --post-install extra.txt tykky_test" "Updating a slim container works" t_run "pip-containerize new --slim --pyver 3.13.2-slim-bullseye --prefix tykky_test2 requirements.txt" "--pyver flag does not break" t_run "tykky_test2/bin/python --version | grep 'Python 3.13.2'" "Correct python version used" From a19b679778a85131efcfcd734c66440f9d6ff484 Mon Sep 17 00:00:00 2001 From: Xavier Abellan Ecija Date: Thu, 4 Dec 2025 15:22:16 +0000 Subject: [PATCH 3/4] Reverted proper formatting changes --- README.md | 2 +- cw_common.py | 129 ++++++++++++++------------------ frontends/conda-containerize.py | 116 +++++++++++++--------------- frontends/pip-containerize.py | 104 ++++++++++--------------- frontends/script_shared.py | 129 ++++++++++++-------------------- frontends/wrap-container.py | 68 ++++++++--------- frontends/wrap-install.py | 77 +++++++++---------- generate_wrappers.sh | 56 +++++++------- templates/conda.sh | 37 ++++----- templates/venv.sh | 21 ++++-- tests/tests.sh | 23 +++--- tests/validate_slim.sh | 2 +- 12 files changed, 340 insertions(+), 424 deletions(-) mode change 100755 => 100644 tests/tests.sh diff --git a/README.md b/README.md index e637a14..f46b8f0 100644 --- a/README.md +++ b/README.md @@ -292,4 +292,4 @@ the conf to allow for very "creative" usages of the tool Technically updating masked disk installations is not an issue, but let's not do that until there is a specific request. The tool now drops the full path leading to the target -from the bind list, if more binds are needed a yaml input needs to be constructed. +from the bind list, if more binds are needed a yaml input needs to be constructed. diff --git a/cw_common.py b/cw_common.py index e0d308a..856c005 100644 --- a/cw_common.py +++ b/cw_common.py @@ -1,115 +1,98 @@ """Utility function for printing errors and warnings while in python""" - +import sys import os -import pathlib import random -import shutil import string +import pathlib import sys +import shutil -colors = {} -colors["RED"] = "\033[0;31m" -colors["GREEN"] = "\033[0;32m" -colors["YELLOW"] = "\033[1;33m" -colors["BLUE"] = "\033[1;34m" -colors["PURPLE"] = "\033[0;35m" -colors["NC"] = "\033[0m" # No Color - +colors={} +colors["RED"]='\033[0;31m' +colors["GREEN"]='\033[0;32m' +colors["YELLOW"]='\033[1;33m' +colors["BLUE"]="\033[1;34m" +colors["PURPLE"]='\033[0;35m' +colors["NC"]='\033[0m' # No Color -def print_err(txt, err=False): +def print_err(txt,err=False): """Pretty error message, color is disabled if not in a TTY""" - if err: + if(err): if not sys.stderr.isatty(): - print("[ ERROR ] " + txt, file=sys.stderr) + print("[ ERROR ] "+txt, file=sys.stderr) else: - print( - "[" + colors["RED"] + " ERROR " + colors["NC"] + "] " + txt, - file=sys.stderr, - ) + print("["+colors["RED"]+" ERROR "+colors["NC"]+"] "+txt,file=sys.stderr) else: if not sys.stdout.isatty(): - print("[ ERROR ] " + txt) + print("[ ERROR ] "+txt) else: - print("[" + colors["RED"] + " ERROR " + colors["NC"] + "] " + txt) + print("["+colors["RED"]+" ERROR "+colors["NC"]+"] "+txt) - -def print_info(txt, log_level, msg_level, err=False): +def print_info(txt,log_level,msg_level,err=False): """Pretty info message, color is disabled if not in a TTY""" if int(log_level) <= msg_level: return if msg_level >= 2: - msg = "DEBUG" - color = colors["PURPLE"] + msg="DEBUG" + color=colors["PURPLE"] else: - msg = "INFO" - color = colors["BLUE"] - if err: + msg="INFO" + color=colors["BLUE"] + if(err): if not sys.stderr.isatty(): - print(f"[ {msg} ] " + txt, file=sys.stderr) + print(f"[ {msg} ] "+txt,file=sys.stderr) else: - print("[" + color + f" {msg} " + colors["NC"] + "] " + txt, file=sys.stderr) + print("["+color+f" {msg} "+colors["NC"]+"] "+txt,file=sys.stderr) else: if not sys.stdout.isatty(): - print(f"[ {msg} ] " + txt, file=sys.stderr) + print(f"[ {msg} ] "+txt,file=sys.stderr) else: - print("[" + color + f" {msg} " + colors["NC"] + "] " + txt, file=sys.stdout) - + print("["+color+f" {msg} "+colors["NC"]+"] "+txt,file=sys.stdout) -def print_warn(txt, err=False): - if err: +def print_warn(txt,err=False): + if(err): if not sys.stderr.isatty(): - print("[ WARNING ] " + txt, file=sys.stderr) + print("[ WARNING ] "+txt, file=sys.stderr) else: - print( - "[" + colors["YELLOW"] + " WARNING " + colors["NC"] + "] " + txt, - file=sys.stderr, - ) + print("["+colors["YELLOW"]+" WARNING "+colors["NC"]+"] "+txt,file=sys.stderr) else: if not sys.stdout.isatty(): - print("[ WARNING ] " + txt) + print("[ WARNING ] "+txt) else: - print("[" + colors["YELLOW"] + " WARNING " + colors["NC"] + "] " + txt) + print("["+colors["YELLOW"]+" WARNING "+colors["NC"]+"] "+txt) + - -def expand_vars(path, rec=0): - if rec > 10: - print_err( - "Max 10 shell variables allowed per value, check configuration ", True - ) +def expand_vars(path,rec=0): + if(rec > 10): + print_err("Max 10 shell variables allowed per value, check configuration ",True) sys.exit(1) - g = path - try: - g = string.Template(g).substitute(os.environ) + g=path + try: + g=string.Template(g).substitute(os.environ) except KeyError as E: - var = E.args[0] - return expand_vars(g.replace(f"${var}", ""), rec + 1) + var=E.args[0] + return expand_vars(g.replace(f"${var}",''),rec+1) return g - def has_apptainer(): - return shutil.which("apptainer") is not None - + return shutil.which("apptainer") != None def name_generator(size=6, chars=string.ascii_uppercase + string.digits): - return "".join(random.choice(chars) for _ in range(size)) - + return ''.join(random.choice(chars) for _ in range(size)) def installation_in_PATH(): - return [P for P in os.environ["PATH"].split(":") if is_installation(P)] - + return [P for P in os.environ["PATH"].split(':') if is_installation(P) ] def is_installation(base_path): - markers = ["bin", "_bin", "common.sh"] - return all(pathlib.Path(base_path + "/../" + m).exists() for m in markers) - + markers=["bin","_bin","common.sh"] + return all( pathlib.Path(base_path+'/../'+m).exists() for m in markers ) # UBI images are namespaced with the major version as part of the name # and not just the tag. -special = {} -special["rhel"] = lambda namespace, version: namespace + version.split(".")[0] - +special={} +special["rhel"]= lambda namespace,version: namespace+version.split('.')[0] # Get the docker image matching the host OS def get_docker_image(release_file): @@ -119,17 +102,17 @@ def get_docker_image(release_file): "rhel": "redhat/ubi", "almalinux": "almalinux", "rocky": "rockylinux/rockylinux", - "ubuntu": "ubuntu", + "ubuntu": "ubuntu" } try: - with open(os_release_file, "r") as file: + with open(os_release_file, 'r') as file: lines = file.readlines() os_info = {} for line in lines: # Lazy way to handle empty lines try: - key, value = line.strip().split("=", 1) + key, value = line.strip().split('=', 1) except: continue os_info[key] = value.strip('"') @@ -140,14 +123,14 @@ def get_docker_image(release_file): if os_id in docker_images: docker_image = docker_images[os_id] if os_id in special: - docker_image = special[os_id](docker_image, version_id) - return (True, f"{docker_image}:{version_id}") + docker_image = special[os_id](docker_image,version_id) + return (True,f"{docker_image}:{version_id}") else: # Guess what the name could be # Will most likely fail for most small distros - return (True, f"{os_id}:{version_id}") + return (True,f"{os_id}:{version_id}") except FileNotFoundError: - return (False, "OS release file not found") + return (False,"OS release file not found") except Exception as e: - return (False, f"An error occurred: {e}") + return (False,f"An error occurred: {e}") diff --git a/frontends/conda-containerize.py b/frontends/conda-containerize.py index da0a7fa..7a7652a 100644 --- a/frontends/conda-containerize.py +++ b/frontends/conda-containerize.py @@ -1,116 +1,107 @@ import argparse import os -import pathlib import sys +import pathlib +curr_dir=pathlib.Path(__file__).parent.resolve() +root_dir=pathlib.Path(curr_dir).parent.resolve() +info=sys.version_info +sys.path.insert(0,str(root_dir)) +sys.path.insert(0,str(root_dir)+"/PyDeps/lib/python{}.{}/site-packages".format(info[0],info[1])) +import yaml +from cw_common import * +from script_shared import * -curr_dir = pathlib.Path(__file__).parent.resolve() -root_dir = pathlib.Path(curr_dir).parent.resolve() -info = sys.version_info -sys.path.insert(0, str(root_dir)) -sys.path.insert( - 0, str(root_dir) + "/PyDeps/lib/python{}.{}/site-packages".format(info[0], info[1]) -) -import yaml # noqa: E402 -from cw_common import print_err, print_warn # noqa: E402 -from script_shared import ( # noqa E402 - add_adv_pars, - add_base_pars, - add_new_pars, - add_upd_pars, - get_old_conf, - is_valid_file, - parse_wrapper, -) - -sys.argv[0] = sys.argv[0].split("/")[-1].split(".")[0] -parser = argparse.ArgumentParser( - description="Create or modify a Conda installation inside a container" -) -subparsers = parser.add_subparsers(help="subcommands", dest="command") -parser_new = add_new_pars(subparsers) -parser_new.add_argument("env_file", help="conda env file") -parser_new.add_argument( - "--mamba", help="use mamba for installation", action="store_true" -) +sys.argv[0]=sys.argv[0].split('/')[-1].split('.')[0] +parser = argparse.ArgumentParser(description="Create or modify a Conda installation inside a container") +subparsers = parser.add_subparsers(help='subcommands',dest='command') +parser_new=add_new_pars(subparsers) +parser_new.add_argument("env_file",help="conda env file") +parser_new.add_argument("--mamba",help="use mamba for installation",action="store_true") parser_new.add_argument( "--uv", help="use uv for pip dependencies (only with mamba)", action="store_true" ) parser_new.add_argument( "--nocache", help="Do not use pip/uv cache", action="store_true" ) -parser_upd = add_upd_pars(subparsers) +parser_upd=add_upd_pars(subparsers) add_adv_pars(subparsers) parser_upd.add_argument( "--nocache", help="Do not use pip/uv cache", action="store_true" ) -ps = [parser_new, parser_upd] +ps=[parser_new,parser_upd] for p in ps: add_base_pars(p) - p.add_argument( - "-r", - "--requirement", - type=lambda x: is_valid_file(x), - help="requirements file for pip", - ) + p.add_argument("-r", "--requirement", type=lambda x: is_valid_file(parser, x),help="requirements file for pip") if len(sys.argv) < 2: parser.print_help() sys.exit(0) args = parser.parse_args() -conf = {} -conf["add_ld"] = "no" +conf={} +conf["add_ld"]="no" conf["use_uv"] = "no" conf["mode"] = "conda" conf["template_script"] = "conda.sh" if args.command == "new": - conf["env_file"] = args.env_file + conf["env_file"]=args.env_file conf["update_installation"] = "no" conf["installation_file_paths"] = [conf["env_file"]] if args.prefix: - conf["installation_prefix"] = args.prefix - conf["mode"] = "conda" - if args.mamba: - conf["mamba"] = "yes" + conf["installation_prefix"]=args.prefix + conf["mode"]="conda" + if args.mamba: + conf["mamba"]="yes" else: - conf["mamba"] = "no" + conf["mamba"]="no" if args.uv and args.mamba: conf["use_uv"] = "yes" elif args.uv and not args.mamba: print_warn("Using --uv without --mamba does not have an effect") elif args.command == "update": conf["update_installation"] = "yes" - get_old_conf(args.dir, conf) + get_old_conf(args.dir,conf) else: - with open(args.yaml, "r") as y: + with open(args.yaml,'r') as y: conf.update(yaml.safe_load(y)) if args.command == "new" and not os.path.isfile(args.env_file): print_err("Env file {} does not exist".format(args.env_file)) sys.exit(1) -if args.command in ["update", "new"]: + + +if args.command in ["update","new"]: if args.environ: - conf["extra_envs"] = [{"file": args.environ}] + conf["extra_envs"]=[{"file":args.environ}] if args.post_install: - conf["post_install"] = [{"file": args.post_install}] + conf["post_install"]=[{"file":args.post_install}] if args.requirement: - conf["requirements_file"] = args.requirement + conf["requirements_file"]=args.requirement if args.pre_install: - conf["pre_install"] = [{"file": args.pre_install}] - - -global_conf = {} -with open(os.getenv("CW_GLOBAL_YAML", ""), "r") as g: - global_conf = yaml.safe_load(g) + conf["pre_install"]=[{"file":args.pre_install}] -parse_wrapper(conf, args, False) +global_conf={} +with open(os.getenv("CW_GLOBAL_YAML"),'r') as g: + global_conf=yaml.safe_load(g) + +parse_wrapper(conf,global_conf,args,False) +if conf["mode"] == "conda": + conf["update_installation"]="no" + conf["template_script"]="conda.sh" + conf["installation_file_paths"]=[conf["env_file"]] +elif conf["mode"]=="conda_modify": + conf["update_installation"]="yes" + conf["template_script"]="conda_modify.sh" +else: + print_err("No or incorrent mode set, [conda,conda_modify]") + sys.exit(1) if "requirements_file" in conf: if "installation_file_paths" in conf: conf["installation_file_paths"].append(conf["requirements_file"]) else: - conf["installation_file_paths"] = conf["requirements_file"] + conf["installation_file_paths"]=conf["requirements_file"] if "pipcache" not in conf: conf["pipcache"] = True @@ -118,5 +109,6 @@ if args.nocache is not None: conf["pipcache"] = not args.nocache -with open(os.getenv("_usr_yaml", ""), "a+") as f: - yaml.dump(conf, f) +with open(os.getenv("_usr_yaml"),'a+') as f: + yaml.dump(conf,f) + diff --git a/frontends/pip-containerize.py b/frontends/pip-containerize.py index fc61f1c..fb060ce 100644 --- a/frontends/pip-containerize.py +++ b/frontends/pip-containerize.py @@ -1,45 +1,23 @@ import argparse import os -import pathlib import sys - -curr_dir = pathlib.Path(__file__).parent.resolve() -root_dir = pathlib.Path(curr_dir).parent.resolve() -info = sys.version_info -sys.path.insert(0, str(root_dir)) -sys.path.insert( - 0, str(root_dir) + "/PyDeps/lib/python{}.{}/site-packages".format(info[0], info[1]) -) -import yaml # noqa: E402 -from cw_common import print_warn # noqa: E402 -from script_shared import ( # noqa: E402 - add_adv_pars, - add_base_pars, - add_new_pars, - add_upd_pars, - get_old_conf, - is_valid_file, - parse_wrapper, -) - -sys.argv[0] = sys.argv[0].split("/")[-1].split(".")[0] -parser = argparse.ArgumentParser( - description="Create or modify a python installation inside a container" -) -subparsers = parser.add_subparsers(help="subcommands", dest="command") -parser_new = add_new_pars(subparsers) -parser_new.add_argument( - "requirements_file", - type=lambda x: is_valid_file(x), - help="requirements file for pip", -) -parser_upd = add_upd_pars(subparsers) -parser_upd.add_argument( - "-r", - "--requirements-file", - type=lambda x: is_valid_file(x), - help="requirements file for pip", -) +import pathlib +curr_dir=pathlib.Path(__file__).parent.resolve() +root_dir=pathlib.Path(curr_dir).parent.resolve() +info=sys.version_info +sys.path.insert(0,str(root_dir)) +sys.path.insert(0,str(root_dir)+"/PyDeps/lib/python{}.{}/site-packages".format(info[0],info[1])) +import yaml +from cw_common import * +from script_shared import * + +sys.argv[0]=sys.argv[0].split('/')[-1].split('.')[0] +parser = argparse.ArgumentParser(description="Create or modify a python installation inside a container") +subparsers = parser.add_subparsers(help='subcommands',dest='command') +parser_new=add_new_pars(subparsers) +parser_new.add_argument("requirements_file", type=lambda x: is_valid_file(parser, x),help="requirements file for pip") +parser_upd=add_upd_pars(subparsers) +parser_upd.add_argument("-r","--requirements-file", type=lambda x: is_valid_file(parser, x),help="requirements file for pip") parser_upd.add_argument( "--nocache", help="Do not use pip/uv cache", action="store_true" ) @@ -50,16 +28,10 @@ "--slim", action="store_true", help="Use minimal base python container" ) parser_new.add_argument("--pyver", help="Python version to use for slim or uv modes") -parser_new.add_argument( - "--system-site-packages", - action="store_true", - help="Enable system and user site packages for the created installation", -) -parser_new.add_argument( - "--nocache", help="Do not use pip/uv cache", action="store_true" -) +parser_new.add_argument("--system-site-packages",action='store_true',help="Enable system and user site packages for the created installation") +parser_new.add_argument("--nocache", help="Do not use pip/uv cache", action="store_true") -ps = [parser_new, parser_upd] +ps=[parser_new,parser_upd] for p in ps: add_base_pars(p) @@ -70,17 +42,17 @@ conf = {} if args.requirements_file: - conf["requirements_file"] = args.requirements_file - conf["installation_file_paths"] = [conf["requirements_file"]] + conf["requirements_file"]=args.requirements_file + conf["installation_file_paths"]=[conf["requirements_file"]] if args.command == "new": conf["mode"] = "venv" conf["use_uv"] = "no" conf["update_installation"] = "no" if args.system_site_packages: - conf["enable_site_packages"] = "yes" + conf["enable_site_packages"]="yes" if args.prefix: - conf["installation_prefix"] = args.prefix + conf["installation_prefix"]=args.prefix if args.slim: conf["pyver"] = "slim" if args.pyver: @@ -100,26 +72,27 @@ elif args.command == "update": conf["update_installation"] = "yes" - get_old_conf(args.dir, conf) + get_old_conf(args.dir,conf) else: - with open(args.yaml, "r") as y: + with open(args.yaml,'r') as y: conf.update(yaml.safe_load(y)) -if args.command in ["update", "new"]: + +if args.command in ["update","new"]: if args.environ: - conf["extra_envs"] = [{"file": args.environ}] + conf["extra_envs"]=[{"file":args.environ}] if args.post_install: - conf["post_install"] = [{"file": args.post_install}] + conf["post_install"]=[{"file":args.post_install}] if args.pre_install: - conf["pre_install"] = [{"file": args.pre_install}] + conf["pre_install"]=[{"file":args.pre_install}] -global_conf = {} -with open(os.getenv("CW_GLOBAL_YAML", ""), "r") as g: - global_conf = yaml.safe_load(g) - -parse_wrapper(conf, args, False) +global_conf={} +with open(os.getenv("CW_GLOBAL_YAML"),'r') as g: + global_conf=yaml.safe_load(g) + +parse_wrapper(conf,global_conf,args,False) conf["template_script"] = "venv.sh" @@ -129,5 +102,6 @@ if args.nocache is not None: conf["pipcache"] = not args.nocache -with open(os.getenv("_usr_yaml", ""), "a+") as f: - yaml.dump(conf, f) +with open(os.getenv("_usr_yaml"),'a+') as f: + yaml.dump(conf,f) + diff --git a/frontends/script_shared.py b/frontends/script_shared.py index 6b986e6..f993396 100644 --- a/frontends/script_shared.py +++ b/frontends/script_shared.py @@ -1,93 +1,68 @@ import os -import pathlib import sys - -curr_dir = pathlib.Path(__file__).parent.resolve() -root_dir = pathlib.Path(curr_dir).parent.resolve() -info = sys.version_info -sys.path.insert(0, str(root_dir)) -sys.path.insert( - 0, str(root_dir) + "/PyDeps/lib/python{}.{}/site-packages".format(info[0], info[1]) -) -import yaml # noqa: E402 -from cw_common import print_err # noqa: E402 - - -def is_valid_file(arg): +import pathlib +curr_dir=pathlib.Path(__file__).parent.resolve() +root_dir=pathlib.Path(curr_dir).parent.resolve() +info=sys.version_info +sys.path.insert(0,str(root_dir)) +sys.path.insert(0,str(root_dir)+"/PyDeps/lib/python{}.{}/site-packages".format(info[0],info[1])) +import yaml +from cw_common import * +def is_valid_file(par,arg): if not os.path.exists(arg): print_err("The file %s does not exist!" % arg) sys.exit(1) else: - return arg - + return arg def add_prefix_flag(p): - p.add_argument("--prefix", type=str, help="Installation location") - + p.add_argument("--prefix",type=str,help="Installation location") def add_post_flag(par): - par.add_argument( - "--post-install", - help="Script to run after initial setup", - type=lambda x: is_valid_file(x), - ) - - + par.add_argument("--post-install",help="Script to run after initial setup",type=lambda x: is_valid_file(par,x)) def add_pre_flag(par): - par.add_argument( - "--pre-install", - help="Script to run before initial setup", - type=lambda x: is_valid_file(x), - ) - - + par.add_argument("--pre-install",help="Script to run before initial setup",type=lambda x: is_valid_file(par,x)) def add_env_flag(par): - par.add_argument( - "--environ", - help="Script to run before each program launch ", - type=lambda x: is_valid_file(x), - ) - + par.add_argument("--environ",help="Script to run before each program launch ",type=lambda x: is_valid_file(par,x)) def add_wrapper_flag(par): - par.add_argument("-w", "--wrapper-paths", help="Comma separated list of paths") + par.add_argument("-w","--wrapper-paths",help='Comma separated list of paths') - -def add_adv_pars(subpar): - parser_advanced = subpar.add_parser("advanced", help="") - parser_advanced.add_argument("yaml", type=str, help="yaml file with tool config") +def add_adv_pars(subpar): + parser_advanced = subpar.add_parser('advanced', help='') + parser_advanced.add_argument('yaml',type=str,help='yaml file with tool config') return parser_advanced - - def add_upd_pars(subpar): - parser_update = subpar.add_parser("update", help="update an existing installation") - parser_update.add_argument("dir", type=str, help="Installation to update") + parser_update = subpar.add_parser('update', help='update an existing installation') + parser_update.add_argument('dir', type=str, help='Installation to update') return parser_update - - def add_new_pars(subpar): - parser_new = subpar.add_parser("new", help="Create new installation") + parser_new = subpar.add_parser('new', help='Create new installation') add_prefix_flag(parser_new) return parser_new - - -def add_base_pars(par, pre_post=True): +def add_base_pars(par,pre_post=True): if pre_post: add_post_flag(par) add_pre_flag(par) add_env_flag(par) add_wrapper_flag(par) - # non absolute paths are relative to the installation dir -def parse_wrapper(conf, a, req_abs): +def parse_wrapper(conf,g_conf,a,req_abs): if a.wrapper_paths: - if not req_abs: + ip="" + if "installation_path" in conf: + ip = conf["installation_path"] + elif "installation_path" in g_conf["force"]: + ip = g_conf["force"]["installation_path"] + elif "installation_path" in g_conf["defaults"]: + ip = g_conf["defaults"]["installation_path"] + elif not req_abs: print_err("Failed to parse wrapper paths, missing installation path") sys.exit(1) - if "wrapper_paths" not in conf: - conf["wrapper_paths"] = [] - for p in a.wrapper_paths.split(","): + if not "wrapper_paths" in conf: + conf["wrapper_paths"]=[] + for p in a.wrapper_paths.split(','): if p[0] == "/": conf["wrapper_paths"].append(p) elif req_abs: @@ -96,36 +71,28 @@ def parse_wrapper(conf, a, req_abs): else: conf["wrapper_paths"].append(p) - -def get_old_conf(d, conf): - old_conf = {} +def get_old_conf(d,conf): + old_conf={} try: - with open(d + "/share/conf.yaml", "r") as c: - old_conf = yaml.safe_load(c) + with open(d+"/share/conf.yaml",'r') as c: + old_conf=yaml.safe_load(c) except FileNotFoundError: - print_err( - "Directory {} does not exist or is not a valid installation ( missing share/conf.yaml )".format( - d - ) - ) + print_err("Directory {} does not exist or is not a valid installation ( missing share/conf.yaml )".format(d)) sys.exit(1) - + # If the installation uses a shared container it should # continue doing so if "share_container" in old_conf and old_conf["share_container"]: - conf["container_src"] = old_conf["container_src"] + conf["container_src"]=old_conf["container_src"] else: - conf["container_src"] = d + "/" + old_conf["container_image"] - conf["sqfs_src"] = d + "/" + old_conf["sqfs_image"] - conf["installation_path"] = old_conf["installation_path"] - conf["installation_prefix"] = d - conf["sqfs_image"] = old_conf["sqfs_image"] - conf["container_image"] = old_conf["container_image"] - conf["isolate"] = old_conf["isolate"] - conf["mode"] = old_conf["mode"] - conf["container_image"] = old_conf["container_image"] - conf["isolate"] = old_conf["isolate"] + conf["container_src"]=d+"/"+old_conf["container_image"] + conf["sqfs_src"]=d+"/"+old_conf["sqfs_image"] + conf["installation_path"]=old_conf["installation_path"] + conf["installation_prefix"]=d + conf["sqfs_image"]=old_conf["sqfs_image"] + conf["container_image"]=old_conf["container_image"] + conf["isolate"]=old_conf["isolate"] conf["mode"] = old_conf["mode"] if "mamba" in old_conf: conf["mamba"] = old_conf["mamba"] diff --git a/frontends/wrap-container.py b/frontends/wrap-container.py index 0aaba5b..47c8809 100644 --- a/frontends/wrap-container.py +++ b/frontends/wrap-container.py @@ -1,30 +1,25 @@ import argparse import os -import pathlib import sys - -curr_dir = pathlib.Path(__file__).parent.resolve() -root_dir = pathlib.Path(curr_dir).parent.resolve() -info = sys.version_info -sys.path.insert(0, str(root_dir)) -sys.path.insert( - 0, str(root_dir) + "/PyDeps/lib/python{}.{}/site-packages".format(info[0], info[1]) -) -import yaml # noqa: E402 -from cw_common import print_err # noqa: E402 -from script_shared import add_base_pars, add_prefix_flag, parse_wrapper # noqa: E402 - -sys.argv[0] = sys.argv[0].split("/")[-1].split(".")[0] -parser = argparse.ArgumentParser( - description="Create wrappers for executables inside a container" -) -parser.add_argument( - "container", type=str, help="Container to wrap, can be docker/singularity url" -) -parser.add_argument("-y", "--yaml", help="Tool yaml conf file") -add_base_pars(parser, False) +import pathlib +curr_dir=pathlib.Path(__file__).parent.resolve() +root_dir=pathlib.Path(curr_dir).parent.resolve() +info=sys.version_info +sys.path.insert(0,str(root_dir)) +sys.path.insert(0,str(root_dir)+"/PyDeps/lib/python{}.{}/site-packages".format(info[0],info[1])) +import yaml +from cw_common import * +from script_shared import * + +sys.argv[0]=sys.argv[0].split('/')[-1].split('.')[0] +parser = argparse.ArgumentParser(description="Create wrappers for executables inside a container") +parser.add_argument("container",type=str,help="Container to wrap, can be docker/singularity url") +parser.add_argument("-y","--yaml",help="Tool yaml conf file") +add_base_pars(parser,False) add_prefix_flag(parser) + + if len(sys.argv) < 2: parser.print_help() sys.exit(0) @@ -32,25 +27,28 @@ if not args.wrapper_paths: print_err("Tool {} requires -w/--wrapper-paths to be used".format(sys.argv[0])) sys.exit(1) -conf = {} -conf["container_src"] = args.container -conf["isolate"] = "yes" -conf["mode"] = "wrapcont" +conf={} +conf["container_src"]=args.container +conf["isolate"]="yes" +conf["mode"]="wrapcont" if args.prefix: - conf["installation_prefix"] = args.prefix + conf["installation_prefix"]=args.prefix if args.yaml: - with open(args.yaml, "r") as y: + with open(args.yaml,'r') as y: conf.update(yaml.safe_load(y)) if args.environ: - conf["extra_envs"] = [{"file": args.environ}] + conf["extra_envs"]=[{"file":args.environ}] + + +global_conf={} +with open(os.getenv("CW_GLOBAL_YAML"),'r') as g: + global_conf=yaml.safe_load(g) + +parse_wrapper(conf,global_conf,args,True) -global_conf = {} -with open(os.getenv("CW_GLOBAL_YAML", ""), "r") as g: - global_conf = yaml.safe_load(g) -parse_wrapper(conf, args, True) +with open(os.getenv("_usr_yaml"),'a+') as f: + yaml.dump(conf,f) -with open(os.getenv("_usr_yaml", ""), "a+") as f: - yaml.dump(conf, f) diff --git a/frontends/wrap-install.py b/frontends/wrap-install.py index 0bff89a..95290de 100644 --- a/frontends/wrap-install.py +++ b/frontends/wrap-install.py @@ -1,29 +1,24 @@ import argparse import os -import pathlib import sys +import pathlib +curr_dir=pathlib.Path(__file__).parent.resolve() +root_dir=pathlib.Path(curr_dir).parent.resolve() +info=sys.version_info +sys.path.insert(0,str(root_dir)) +sys.path.insert(0,str(root_dir)+"/PyDeps/lib/python{}.{}/site-packages".format(info[0],info[1])) +import yaml +from cw_common import * +from script_shared import * -curr_dir = pathlib.Path(__file__).parent.resolve() -root_dir = pathlib.Path(curr_dir).parent.resolve() -info = sys.version_info -sys.path.insert(0, str(root_dir)) -sys.path.insert( - 0, str(root_dir) + "/PyDeps/lib/python{}.{}/site-packages".format(info[0], info[1]) -) -import yaml # noqa: E402 -from cw_common import has_apptainer, print_err # noqa: E402 -from script_shared import add_base_pars, add_prefix_flag, parse_wrapper # noqa: E402 - -sys.argv[0] = sys.argv[0].split("/")[-1].split(".")[0] -parser = argparse.ArgumentParser( - description="Wrap an existing installation into a container" -) -parser.add_argument("dir", type=str, help="Installation to wrap") +sys.argv[0]=sys.argv[0].split('/')[-1].split('.')[0] +parser = argparse.ArgumentParser(description="Wrap an existing installation into a container") +parser.add_argument("dir",type=str,help="Installation to wrap") add_base_pars(parser) add_prefix_flag(parser) -parser.add_argument("-y", "--yaml", help="Tool yaml conf file") -parser.add_argument("--mask", action="store_true", help="Mask installation on disk") +parser.add_argument("-y","--yaml",help="Tool yaml conf file") +parser.add_argument("--mask",action='store_true',help="Mask installation on disk") if len(sys.argv) < 2: @@ -33,40 +28,42 @@ if not args.wrapper_paths: print_err("Tool {} requires -w/--wrapper-paths to be used".format(sys.argv[0])) sys.exit(1) -conf = {} -# wrapp=[ str(pathlib.Path(w).resolve()) for w in args.wrapper_paths.split(',')] -# args.wrapper_paths=",".join(wrapp) -conf["isolate"] = "no" -conf["mode"] = "wrapdisk" -conf["wrap_src"] = str(pathlib.Path(args.dir).resolve()) -conf["update_installation"] = "no" -conf["template_script"] = "wrap.sh" +conf={} +#wrapp=[ str(pathlib.Path(w).resolve()) for w in args.wrapper_paths.split(',')] +#args.wrapper_paths=",".join(wrapp) +conf["isolate"]="no" +conf["mode"]="wrapdisk" +conf["wrap_src"]=str(pathlib.Path(args.dir).resolve()) +conf["update_installation"]="no" +conf["template_script"]="wrap.sh" if args.mask: - conf["installation_path"] = str(pathlib.Path(args.dir).resolve()) + conf["installation_path"]=str(pathlib.Path(args.dir).resolve()) if not has_apptainer(): - conf["excluded_mount_points"] = "/" + conf["installation_path"].split("/")[1] + conf["excluded_mount_points"]="/"+conf["installation_path"].split('/')[1] conf["mask_wrap_install"] = True if args.prefix: - conf["installation_prefix"] = args.prefix + conf["installation_prefix"]=args.prefix if args.yaml: - with open(args.yaml, "r") as y: + with open(args.yaml,'r') as y: conf.update(yaml.safe_load(y)) if args.environ: - conf["extra_envs"] = [{"file": args.environ}] + conf["extra_envs"]=[{"file":args.environ}] if args.post_install: - conf["post_install"] = [{"file": args.post_install}] + conf["post_install"]=[{"file":args.post_install}] if args.pre_install: - conf["pre_install"] = [{"file": args.pre_install}] + conf["pre_install"]=[{"file":args.pre_install}] + +global_conf={} +with open(os.getenv("CW_GLOBAL_YAML"),'r') as g: + global_conf=yaml.safe_load(g) + +parse_wrapper(conf,global_conf,args,False) -global_conf = {} -with open(os.getenv("CW_GLOBAL_YAML", ""), "r") as g: - global_conf = yaml.safe_load(g) -parse_wrapper(conf, args, False) +with open(os.getenv("_usr_yaml"),'a+') as f: + yaml.dump(conf,f) -with open(os.getenv("_usr_yaml", ""), "a+") as f: - yaml.dump(conf, f) diff --git a/generate_wrappers.sh b/generate_wrappers.sh index a16cdbd..63e1f27 100755 --- a/generate_wrappers.sh +++ b/generate_wrappers.sh @@ -1,7 +1,7 @@ #!/bin/bash SINGULARITY_BIND="" set -e -set -u +set -u SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" source $SCRIPT_DIR/common_functions.sh @@ -26,18 +26,18 @@ else fi # Need to unset the path, otherwise we might be stuck in a nasty loop -# and exhaust the system +# and exhaust the system _REAL_PATH_CMD=' -export OLD_PATH=$PATH +export OLD_PATH=$PATH export PATH="/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/bin" -SOURCE="${BASH_SOURCE[0]}" -_O_SOURCE=$SOURCE -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" +SOURCE="${BASH_SOURCE[0]}" +_O_SOURCE=$SOURCE +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" +done +DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" ' _PRE_COMMAND="source \$DIR/../common.sh" @@ -79,13 +79,13 @@ if grep -q 'singularity/mnt/session\|apptainer/mnt/session' /proc/self/mountinf export _CW_IN_CONTAINER=Yes if [[ \"$CW_ISOLATE\" == \"yes\" && ! \"\$( stat -c '%i' \$SINGULARITY_CONTAINER)\" == \"\$( stat -c '%i' \$_C_DIR/\$CONTAINER_IMAGE)\" ]]; then echo \"[ ERROR ] wrapper called from another container. Is \$SINGULARITY_CONTAINER, should be \$_C_DIR/\$CONTAINER_IMAGE \" - exit 1 + exit 1 fi if [[ ! -e $CW_INSTALLATION_PATH ]]; then - echo \"[ ERROR ] Installation for \$_C_DIR/ is not mounted. Wrapper called from another container?\" + echo \"[ ERROR ] Installation for \$_C_DIR/ is not mounted. Wrapper called from another container?\" exit 1 fi - + else unset _CW_IN_CONTAINER export _CW_IS_ISOLATED=$CW_ISOLATE @@ -185,21 +185,21 @@ for wrapper_path in "${CW_WRAPPER_PATHS[@]}";do # so that quote are maintainted # e.g python -c "print('Hello')" works # The test is there as printf returns '' if $@ is empty - # passing '' is not wanted behavior + # passing '' is not wanted behavior print_info "Checking if conda installation" 3 unset CONDA_CMD - if $_CONTAINER_EXEC test -f $wrapper_path/../../../bin/conda ; then + if $_CONTAINER_EXEC test -f $wrapper_path/../../../bin/conda ; then export CONDA_CMD=1 print_info "Inserting conda activation into wrappers" 3 env_name=$(basename $(realpath -m $wrapper_path/../ )) conda_path=$(realpath -m $wrapper_path/../../../bin/conda) _cws="bash -c \"eval \\\"\\\$($conda_path shell.bash hook )\\\" && conda activate $env_name &>/dev/null && " else - print_info "Does not look like a conda installation" 3 + print_info "Does not look like a conda installation" 3 fi fi - + for target in "${targets[@]}"; do print_info "Creating wrapper for $target" 3 @@ -221,19 +221,19 @@ else" >> _deploy/bin/$target _v_in_use=\$? if [[ ( \$_v_in_use -eq 0 && ! \${CW_FORCE_CONDA_ACTIVATE+defined} ) || \${CW_NO_CONDA_ACTIVATE+defined} ]];then export PATH=\"\$OLD_PATH\" - $_RUN_CMD $_default_cws exec -a \$_O_SOURCE \$DIR/$target $_cwe + $_RUN_CMD $_default_cws exec -a \$_O_SOURCE \$DIR/$target $_cwe else export PATH=\"\$OLD_PATH\" _venv_act=\":\" - test 0 -eq \$_v_in_use && _venv_act=\"source \$_venvd/activate\" - $_RUN_CMD $_cws \$_venv_act && exec -a \$_O_SOURCE \$DIR/$target $_cwe + test 0 -eq \$_v_in_use && _venv_act=\"source \$_venvd/activate\" + $_RUN_CMD $_cws \$_venv_act && exec -a \$_O_SOURCE \$DIR/$target $_cwe fi fi " >> _deploy/bin/$target - else + else echo " export PATH=\"\$OLD_PATH\" - $_RUN_CMD $_cws exec -a \$_O_SOURCE \$DIR/$target $_cwe + $_RUN_CMD $_cws exec -a \$_O_SOURCE \$DIR/$target $_cwe fi" >> _deploy/bin/$target fi chmod +x _deploy/bin/$target @@ -258,7 +258,7 @@ echo "$_PRE_COMMAND" >> _deploy/bin/$target echo " if [[ -z \"\$SINGULARITY_NAME\" ]];then - $_SHELL_CMD \"\$@\" + $_SHELL_CMD \"\$@\" fi" >> _deploy/bin/$target chmod +x _deploy/bin/$target @@ -268,7 +268,7 @@ echo "$_REAL_PATH_CMD" >> _deploy/bin/$target echo "$_PRE_COMMAND" >> _deploy/bin/$target echo " if [[ -z \"\$SINGULARITY_NAME\" ]];then - $_RUN_CMD \"\$@\" + $_RUN_CMD \"\$@\" fi" >> _deploy/bin/$target chmod +x _deploy/bin/$target @@ -282,9 +282,9 @@ printf -- '%s\n' "SINGULARITYENV_PATH=\$(echo \"\${_tmp_arr[@]}\" | /usr/bin/tr printf -- '%s\n' "_tmp_arr=(\$(echo \$SINGULARITYENV_LD_LIBRARY_PATH | /usr/bin/tr ':' '\n' ))" >> _deploy/common.sh printf -- '%s\n' "SINGULARITYENV_LD_LIBRARY_PATH=\$(echo \"\${_tmp_arr[@]}\" | /usr/bin/tr ' ' ':')" >> _deploy/common.sh -# Keep Venv path if we wrap a container and create +# Keep Venv path if we wrap a container and create # So the expectation if a user creates a venv on top of a wrapped container -# would be that invoking the venv python which launches shell commands again +# would be that invoking the venv python which launches shell commands again # calling the python interpreter would call the venv python and not the base python if [[ "$CW_MODE" == "wrapcont" ]];then echo ' @@ -296,10 +296,10 @@ test -e $_venvd/../pyvenv.cfg fi if [[ -f _extra_envs.sh ]];then - cat _extra_envs.sh >> _deploy/common.sh + cat _extra_envs.sh >> _deploy/common.sh fi if [[ -f _extra_user_envs.sh ]];then - cat _extra_user_envs.sh >> _deploy/common.sh + cat _extra_user_envs.sh >> _deploy/common.sh fi chmod o+r _deploy chmod o+x _deploy diff --git a/templates/conda.sh b/templates/conda.sh index ff84c84..8774f0c 100644 --- a/templates/conda.sh +++ b/templates/conda.sh @@ -1,12 +1,13 @@ #!/bin/bash -#set -e +set -e -cd "$CW_BUILD_TMPDIR" + +cd $CW_BUILD_TMPDIR echo "export env_root=$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/" >> _extra_envs.sh echo "export env_root=$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/" >> _vars.sh -export env_root="$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/" +export env_root=$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/ -cd "$CW_INSTALLATION_PATH" +cd $CW_INSTALLATION_PATH if [[ ! -e $CW_INSTALLATION_PATH/miniforge/ ]]; then if [[ -e $CW_INSTALLATION_PATH/miniconda/ ]]; then @@ -27,17 +28,17 @@ if [[ ! -e $CW_INSTALLATION_PATH/miniforge/ ]]; then fi fi -eval "$("$CW_INSTALLATION_PATH/miniforge/bin/conda" shell.bash hook)" +eval "$($CW_INSTALLATION_PATH/miniforge/bin/conda shell.bash hook)" -cd "$CW_WORKDIR" -source "$CW_INSTALLATION_PATH/_pre_install.sh" -if [[ ! -z "$(echo "$CW_ENV_FILE" | grep ".*\.yaml\|.*\.yml")" ]]; then +cd $CW_WORKDIR +source $CW_INSTALLATION_PATH/_pre_install.sh +if [[ ! -z "$(echo "$CW_ENV_FILE" | grep ".*\.yaml\|.*\.yml")" ]];then _EC="env" _FF="-f" else _FF="--file" fi -cd "$CW_INSTALLATION_PATH" +cd $CW_INSTALLATION_PATH if [[ "$CW_PIPCACHE" != "yes" ]]; then export PIP_NO_CACHE_DIR=1 @@ -77,19 +78,19 @@ if [[ ${CW_REQUIREMENTS_FILE+defined} ]];then wait $bg_pid follow_log $bg_pid "$CW_BUILD_TMPDIR/_pip.log" 20 fi - -cd "$CW_WORKDIR" +cd $CW_WORKDIR print_info "Running user supplied commands" 1 -source "$CW_INSTALLATION_PATH/_post_install.sh" -if [[ -d "$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/" ]];then - echo 'echo "' > "$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages" - conda list >> "$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages" - echo '"' >> "$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages" - chmod +x "$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages" +source $CW_INSTALLATION_PATH/_post_install.sh +if [[ -d $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/ ]];then + echo 'echo "' > $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages + conda list >> $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages + echo '"' >> $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages + chmod +x $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages else print_warn "Created env is empty" fi + # Set here as they are dynamic # Could also set them in construct.py... -echo "CW_WRAPPER_PATHS+=( \"$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/\" )" >> "$CW_BUILD_TMPDIR/_vars.sh" +echo "CW_WRAPPER_PATHS+=( \"$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/\" )" >> $CW_BUILD_TMPDIR/_vars.sh diff --git a/templates/venv.sh b/templates/venv.sh index e5108eb..19ed478 100644 --- a/templates/venv.sh +++ b/templates/venv.sh @@ -1,13 +1,18 @@ #!/bin/bash set -e -cd "$CW_BUILD_TMPDIR" + +cd $CW_BUILD_TMPDIR echo "export env_root=$CW_INSTALLATION_PATH/$CW_ENV_NAME/" >> _extra_envs.sh echo "export env_root=$CW_INSTALLATION_PATH/$CW_ENV_NAME/" >> _vars.sh -export env_root="$CW_INSTALLATION_PATH/$CW_ENV_NAME/" +export env_root=$CW_INSTALLATION_PATH/$CW_ENV_NAME/ + +cd $CW_INSTALLATION_PATH + +cd $CW_WORKDIR +source $CW_INSTALLATION_PATH/_pre_install.sh +cd $CW_INSTALLATION_PATH -cd "$CW_INSTALLATION_PATH" -source "$CW_INSTALLATION_PATH/_pre_install.sh" _NC="" if [[ "$CW_PIPCACHE" != "yes" ]]; then @@ -50,10 +55,10 @@ if [[ ${CW_REQUIREMENTS_FILE+defined} ]];then fi bg_pid=$! wait $bg_pid - follow_log $bg_pid "$CW_BUILD_TMPDIR/_pip.log" 20 + follow_log $bg_pid $CW_BUILD_TMPDIR/_pip.log 20 fi -cd "$CW_WORKDIR" +cd $CW_WORKDIR print_info "Running user supplied commands" 1 -source "$CW_INSTALLATION_PATH/_post_install.sh" +source $CW_INSTALLATION_PATH/_post_install.sh -echo "CW_WRAPPER_PATHS+=( \"$CW_INSTALLATION_PATH/$CW_ENV_NAME/bin/\" )" >> "$CW_BUILD_TMPDIR"/_vars.sh +echo "CW_WRAPPER_PATHS+=( \"$CW_INSTALLATION_PATH/$CW_ENV_NAME/bin/\" )" >> $CW_BUILD_TMPDIR/_vars.sh diff --git a/tests/tests.sh b/tests/tests.sh old mode 100755 new mode 100644 index 91d2912..179aa22 --- a/tests/tests.sh +++ b/tests/tests.sh @@ -1,6 +1,6 @@ #!/bin/bash -u SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -source "$SCRIPT_DIR/setup.sh" +source $SCRIPT_DIR/setup.sh # Test are run in current directory #cd "${TMPDIR:-/tmp}" @@ -78,7 +78,7 @@ t_run "conda-containerize new conda_base.yml --prefix A_DIR_NO_EXE | grep ERROR" mkdir CONDA_INSTALL_DIR t_run "conda-containerize new conda_broken.yaml --prefix CONDA_INSTALL_DIR | tee conda_inst.out | grep 'ResolvePackageNotFound\|PackagesNotFoundError'" "Conda errors are propagated to the user" -t_run "grep ERROR conda_inst.out" "Failed run contains error" +t_run "grep ERROR conda_inst.out" "Failed run contains error" t_run "grep INFO conda_inst.out" "Info is present" t_run "test -z \"\$(grep ' DEBUG ' conda_inst.out )\" " "Default no debug message" tmp_dir=$(cat conda_inst.out | grep -o "[^ ]*/cw-[A-Z,0-9]\{6\} ") @@ -87,7 +87,7 @@ export CW_DEBUG_KEEP_FILES=1 conda-containerize new conda_broken.yaml --prefix CONDA_INSTALL_DIR &> conda_inst.out tmp_dir=$(cat conda_inst.out | grep -o "[^ ]*/cw-[A-Z,0-9]\{6\} ") t_run "\[ -e $tmp_dir \]" "Build dir is saved if CW_DEBUG_KEEP_FILES set" -test -d "$tmp_dir" && rm -rf "$tmp_dir" +test -d $tmp_dir && rm -rf $tmp_dir unset CW_DEBUG_KEEP_FILES unset CW_ENABLE_CONDARC @@ -102,7 +102,7 @@ t_run "CONDA_INSTALL_DIR/bin/python -m venv VE " "Virtual environment creation w t_run "VE/bin/python -c 'import sys;sys.exit( sys.prefix == sys.base_prefix )'" "Virtual environment is correct" t_run "VE/bin/pip install requests" "pip works for a venv" t_run "VE/bin/python -c 'import requests;print(requests.__file__)' | grep -q VE " "Package is installed correctly to venv" -CONDA_INSTALL_DIR/bin/_debug_exec bash -c "\$(readlink -f \$env_root)/../../bin/conda list --explicit" > explicit_env.txt +CONDA_INSTALL_DIR/bin/_debug_exec bash -c "\$(readlink -f \$env_root)/../../bin/conda list --explicit" > explicit_env.txt t_run "CONDA_INSTALL_DIR/bin/python -c 'import yaml'" "Package added by -r is there" t_run "conda-containerize update CONDA_INSTALL_DIR --post-install post.sh" "Update works" t_run "CONDA_INSTALL_DIR/bin/python -c 'import requests'" "Package added by update is there" @@ -150,14 +150,14 @@ cluster = dask_jobqueue.SLURMCluster( print(cluster.job_script()) ' > dask_test.py -t_run "python dask_test.py | grep \"$(realpath -s "$PWD/CONDA_INSTALL_DIR/bin/python")\"" "Dask uses correct python path" +t_run "python dask_test.py | grep \"$(realpath -s $PWD/CONDA_INSTALL_DIR/bin/python )\"" "Dask uses correct python path" PATH=$OLD_PATH -# OLD_PATH=$PATH -rm -fr PIP_INSTALL_DIR && mkdir PIP_INSTALL_DIR +OLD_PATH=$PATH +mkdir PIP_INSTALL_DIR -cat "$SCRIPT_DIR/../default_config/config.yaml" | sed 's/container_image.*$/container_image: My_very_cool_name.sif/g' > my_config.yaml +cat ../../default_config/config.yaml | sed 's/container_image.*$/container_image: My_very_cool_name.sif/g' > my_config.yaml t_run "pip-containerize new --prefix PIP_INSTALL_DIR req_typo.txt 2>&1 | grep 'No matching distribution'" "pip error shown to user" export CW_GLOBAL_YAML=my_config.yaml t_run "pip-containerize new --prefix PIP_INSTALL_DIR req.txt " "pip install works" @@ -166,10 +166,9 @@ PATH="PIP_INSTALL_DIR/bin:$PATH" t_run "python -c 'import requests'" "Package added in update available" t_run "\[ -e PIP_INSTALL_DIR/My_very_cool_name.sif \]" "Using custom conf" t_run "python -c 'import sys;sys.exit(sys.prefix == sys.base_prefix)'" "Installation is venv" -in_p=$(python3 -c 'import sys;print(sys.executable)') -out_p="$PWD/PIP_INSTALL_DIR/bin/python3" -t_run "[[ $in_p == $out_p ]]" "Executable name is same on in and out" - +in_p=$(python3 -c 'import sys;print(sys.executable)') +out_p="$PWD/PIP_INSTALL_DIR/bin/python3" +t_run "[[ $in_p == $out_p ]]" "Executable name is same on in and out" rm -fr PIP_INSTALL_DIR && mkdir PIP_INSTALL_DIR t_run "pip-containerize new --uv --prefix PIP_UV_INSTALL_DIR req_typo.txt 2>&1 | grep -o 'not found in the package registry'" "uv error shown to user" t_run "pip-containerize new --uv --prefix PIP_UV_INSTALL_DIR req.txt " "pip install with uv works" diff --git a/tests/validate_slim.sh b/tests/validate_slim.sh index 8a0016b..d26d50a 100644 --- a/tests/validate_slim.sh +++ b/tests/validate_slim.sh @@ -12,8 +12,8 @@ echo "distro" > requirements.txt echo "pip install lxml" > extra.txt t_run "pip-containerize new --slim --prefix tykky_test requirements.txt" "Creating slim container works" -#t_run "singularity exec tykky_test/container.sif cat /etc/os-release | grep 'Debian'" "Slim container is actually using debian" t_run "tykky_test/bin/python -c 'import distro; print(distro.id())' | grep 'debian'" "Slim container is actually using debian" + t_run "pip-containerize update --post-install extra.txt tykky_test" "Updating a slim container works" t_run "pip-containerize new --slim --pyver 3.13.2-slim-bullseye --prefix tykky_test2 requirements.txt" "--pyver flag does not break" t_run "tykky_test2/bin/python --version | grep 'Python 3.13.2'" "Correct python version used" From 30318eddebe84eb968922f13129249c9384fde5f Mon Sep 17 00:00:00 2001 From: Xavier Abellan Ecija Date: Thu, 4 Dec 2025 15:50:19 +0000 Subject: [PATCH 4/4] Minor problems after formatting reversal --- frontends/conda-containerize.py | 11 +---------- tests/tests.sh | 2 +- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/frontends/conda-containerize.py b/frontends/conda-containerize.py index 7a7652a..032d627 100644 --- a/frontends/conda-containerize.py +++ b/frontends/conda-containerize.py @@ -87,16 +87,7 @@ global_conf=yaml.safe_load(g) parse_wrapper(conf,global_conf,args,False) -if conf["mode"] == "conda": - conf["update_installation"]="no" - conf["template_script"]="conda.sh" - conf["installation_file_paths"]=[conf["env_file"]] -elif conf["mode"]=="conda_modify": - conf["update_installation"]="yes" - conf["template_script"]="conda_modify.sh" -else: - print_err("No or incorrent mode set, [conda,conda_modify]") - sys.exit(1) + if "requirements_file" in conf: if "installation_file_paths" in conf: conf["installation_file_paths"].append(conf["requirements_file"]) diff --git a/tests/tests.sh b/tests/tests.sh index 179aa22..ae4acb7 100644 --- a/tests/tests.sh +++ b/tests/tests.sh @@ -157,7 +157,7 @@ PATH=$OLD_PATH OLD_PATH=$PATH mkdir PIP_INSTALL_DIR -cat ../../default_config/config.yaml | sed 's/container_image.*$/container_image: My_very_cool_name.sif/g' > my_config.yaml +cat $SCRIPT_DIR/../default_config/config.yaml | sed 's/container_image.*$/container_image: My_very_cool_name.sif/g' > my_config.yaml t_run "pip-containerize new --prefix PIP_INSTALL_DIR req_typo.txt 2>&1 | grep 'No matching distribution'" "pip error shown to user" export CW_GLOBAL_YAML=my_config.yaml t_run "pip-containerize new --prefix PIP_INSTALL_DIR req.txt " "pip install works"