diff --git a/.github/workflows/deploy-book.yml b/.github/workflows/deploy-book.yml index afff8a1..8ed8018 100644 --- a/.github/workflows/deploy-book.yml +++ b/.github/workflows/deploy-book.yml @@ -19,7 +19,7 @@ jobs: environment-file: environment.yml auto-activate-base: false activate-environment: arcdocs-jb - python-version: 3.9 + python-version: 3.14 # Build the book - name: Build the book diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 658a0aa..0e03a1e 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -19,7 +19,7 @@ jobs: environment-file: environment.yml auto-activate-base: false activate-environment: arcdocs-jb - python-version: 3.9 + python-version: 3.14 # Build the book - name: Build the book diff --git a/README.md b/README.md index c3f41fc..0f03690 100644 --- a/README.md +++ b/README.md @@ -31,13 +31,12 @@ $ git clone https://github.com/ARCTraining/template-jb-docs.git $ conda env create -f environment.yml ``` -To build the html content locally you can use the `jupyter-book` command line tool: +To build the html content locally and serve on http://localhost:3000 you can use the `jupyter-book` command line tool: ```{bash} # navigate to the repository root -$ cd template-jb-docs -# sometimes worth running jupyter-book clean book/ to remove old files -$ jupyter-book build book/ +$ cd book +$ jupyter-book start ``` ### Windows diff --git a/book/_config.yml b/book/_config.yml deleted file mode 100644 index 10e2893..0000000 --- a/book/_config.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Book settings -title : "HPC2: Installing and Managing Applications on the HPC" -author : University of Leeds Research Computing Team -logo : ./assets/img/logo/logo.png -copyright : "2025" - -# execute settings -execute: - execute_notebooks : auto - -# HTML-specific settings -html: - favicon : "./assets/img/logo/favicon-32x32.png" # A path to a favicon image - navbar_number_sections : False # Add a number to each section in your left navbar - google_analytics_id : "" # A GA id that can be used to track book views. - home_page_in_navbar : true # Whether to include your home page in the left Navigation Bar - use_repository_button : true - use_issues_button : true - use_edit_page_button : true - baseurl : "https://arctraining.github.io/hpc2-software/" # The base URL where your book will be hosted. Used for creating image previews and social links. e.g.: https://mypage.com/mybook/ - -# Launch button settings -launch_buttons: - notebook_interface : "classic" # The interface interactive links will activate ["classic", "jupyterlab"] - binderhub_url : "https://mybinder.org" # The URL of the BinderHub (e.g., https://mybinder.org) - jupyterhub_url : "" # The URL of the JupyterHub (e.g., https://datahub.berkeley.edu) - thebelab : false # Add a thebelab button to pages (requires the repository to run on Binder) - -repository: - url : https://github.com/ARCTraining/hpc2-software # The URL to your book's repository - path_to_book : "book/" # A path to your book's folder, relative to the repository root. - branch : main # Which branch of the repository should be used when creating links diff --git a/book/_toc.yml b/book/_toc.yml deleted file mode 100644 index 3ab222d..0000000 --- a/book/_toc.yml +++ /dev/null @@ -1,30 +0,0 @@ -# these comments are borrowed from https://github.com/alan-turing-institute/the-turing-way/blob/master/book/website/_toc.yml -# Each entry has the following schema: -# -# - title: mytitle # Title of chapter or subchapter -# file: myfile # filename relative to the folder. -# sections: # Contains a list of more entries that make up the chapter's subchapters -# -# Below are some special values that trigger specific behavior: -# - header: My Header # Will insert a header with no link in the sidebar - -format: jb-article -root: welcome -sections: -- file: course/theory -- file: course/conda -- file: course/autotools -- file: course/cmake -- file: course/containers -- file: course/spack - sections: - - file: course/spack/installing - - file: course/spack/testinstall - - file: course/spack/existing - - file: course/spack/advanced - - file: course/spack/containers - - file: course/spack/environments - - file: course/spack/recipes - - file: course/spack/modules -- url: https://arc.leeds.ac.uk - title: Research Computing Website diff --git a/book/course/conda.md b/book/course/conda.md index 1781336..05fa9af 100644 --- a/book/course/conda.md +++ b/book/course/conda.md @@ -4,7 +4,7 @@ Content from this lesson has been inspired and adapted from a number of sources - [Conda documentation](https://docs.conda.io/en/latest/) - [Carpentries Incubator: Introduction to Conda for data scientists](https://carpentries-incubator.github.io/introduction-to-conda-for-data-scientists/) -(introduction)= +(intro)= ## Introduction Conda is an open source package management and environment management system that runs on multiple operating systems (Windows, Linux, macOS). Its features include: @@ -450,7 +450,7 @@ We've discovered we also need the `statsmodels` package for some extra work we w ````{admonition} Searching for packages -Conda has a command-line search functionality that we describe below in the section [Use Conda to search for a package](#searching-for-packages); you can also use the [`conda-forge` repository](https://anaconda.org/conda-forge) or [`bioconda` repository](https://anaconda.org/bioconda) to search for packages. +Conda has a command-line search functionality that we describe below in the section [Using Conda to search for a package](searching-for-packages); you can also use the [`conda-forge` repository](https://anaconda.org/conda-forge) or [`bioconda` repository](https://anaconda.org/bioconda) to search for packages. ```` Once you have the name (and possibly version) of the package you want to install, again there are two different ways to add these packages, much like there were two ways to create the environment to begin with. @@ -459,7 +459,7 @@ Once you have the name (and possibly version) of the package you want to install You can add new packages directly from the command line using the `install` subcommand with the format `conda install PACKAGE`, where `PACKAGE` is the name of the package you wish to install. -To install packages into an existing environment we need to activate it with the [subcommand shown above](activating-environments). +To install packages into an existing environment we need to activate it with the [subcommand shown above](#activating-environments). ```bash $ conda activate data-sci-env @@ -614,7 +614,7 @@ The `--prune` argument here clears out old unused libraries and is key to keepin ### Removing a Conda environment It is also possible to delete a Conda environment through the `remove` subcommand. -This [command is outlined below](removing-packages) in relation to removing specific packages but can also be used to delete an entire Conda environment. +This [command is outlined below](#removing-packages) in relation to removing specific packages but can also be used to delete an entire Conda environment. To remove the `py39-env` we created earlier we use the command: @@ -1494,19 +1494,19 @@ Again, this can also be easily done by updating our `environment.yaml` file to c ```{important} -- [Introduces Conda](introduction) as a cross-platform package and environment manager -- Highlights options for [how to install Conda](installing-conda) +- [Introduces Conda](#intro) as a cross-platform package and environment manager +- Highlights options for [how to install Conda](#installing-conda) - Introduces Conda environments for separating specific package dependencies on a project-by-project basis - - How to [create an environment using Conda](creating-environments) - - How to use a created Conda environment through [environment activation](activating-environments) - - Leaving an environment through [deactivation](deactivating-environments) - - [Listing available Conda environments](listing-current-environments) - - [Deleting Conda environments](removing-a-conda-environment) - - [Exporting and sharing Conda Environments](sharing-conda-environments) + - How to [create an environment using Conda](#creating-environments) + - How to use a created Conda environment through [environment activation](#activating-environments) + - Leaving an environment through [deactivation](#deactivating-environments) + - [Listing available Conda environments](#listing-current-environments) + - [Deleting Conda environments](#removing-a-conda-environment) + - [Exporting and sharing Conda Environments](#sharing-conda-environments) - Shows how Conda can be used for managing packages - - Using Conda to [search Conda repositories for a package](searching-for-packages) - - Using Conda to [install a package](installing-packages) - - Using Conda to [remove a package](removing-packages) - - Using Conda to [update a package](updating-a-package) + - Using Conda to [search Conda repositories for a package](#searching-for-packages) + - Using Conda to [install a package](#installing-packages) + - Using Conda to [remove a package](#removing-packages) + - Using Conda to [update a package](#updating-a-package) ``` diff --git a/book/course/containers.md b/book/course/containers.md index 45a85a7..09215a7 100644 --- a/book/course/containers.md +++ b/book/course/containers.md @@ -319,25 +319,25 @@ You can now experiment inside this sandbox to work out what you need for your re ## Summary ```{important} -- [What are containers](containers:what)? -- [Why do I want a container](containers:why)? -- [Can I use Docker on HPC](containers:docker-hpc)? -- [Singularity vs Apptainer](containers:singularity-vs-apptainer) -- [Example of running a container](containers:example) and a look into the steps involved: - - [Pulling a container image down from Docker Hub](containers:pulling-from-docker) - - [Converting it into a SIF image](containers:converting-to-sif) - - [Running software within a container](containers:running-within) +- [What are containers](#containers:what)? +- [Why do I want a container](#containers:why)? +- [Can I use Docker on HPC](#containers:docker-hpc)? +- [Singularity vs Apptainer](#containers:singularity-vs-apptainer) +- [Example of running a container](#containers:example) and a look into the steps involved: + - [Pulling a container image down from Docker Hub](#containers:pulling-from-docker) + - [Converting it into a SIF image](#containers:converting-to-sif) + - [Running software within a container](#containers:running-within) - What else can I do with containers? - - [Access host storage within a container](containers:host-storage) - - [Run an alternative command within a container](containers:run-alternative-command) - - [Have an interactive shell inside the container](containers:shell) - - [Use GPUs within a container](containers:gpu) + - [Access host storage within a container](#containers:host-storage) + - [Run an alternative command within a container](#containers:run-alternative-command) + - [Have an interactive shell inside the container](#containers:shell) + - [Use GPUs within a container](#containers:gpu) - Building a container - - [Creating a recipe](containers:recipe) - - [Converting a Dockerfile to a Singularity recipe](containers:convert-from-dockerfile) - - [Generate a SIF image from a recipe](containers:generate-sif) - - [Test the image we've created](containers:test) -- [Submitting jobs with containers](containers:submit-job) + - [Creating a recipe](#containers:recipe) + - [Converting a Dockerfile to a Singularity recipe](#containers:convert-from-dockerfile) + - [Generate a SIF image from a recipe](#containers:generate-sif) + - [Test the image we've created](#containers:test) +- [Submitting jobs with containers](#containers:submit-job) - Bonus section - - [Sandboxes](containers:sandbox) + - [Sandboxes](#containers:sandbox) ``` diff --git a/book/course/spack.md b/book/course/spack.md index a408823..87bdadc 100644 --- a/book/course/spack.md +++ b/book/course/spack.md @@ -9,13 +9,13 @@ I'll introduce Spack by quoting the first paragraph of their documentation: > specific microarchitectures. So what does this mean in practice? You've already learnt about build systems -like [Autotools](autotools) and [CMake](cmake), and may be familiar with +like [Autotools](./autotools) and [CMake](./cmake), and may be familiar with [module systems](https://arcdocs.leeds.ac.uk/aire/software/start.html#modules) as used on Aire. You may also have worked through the section on -[containers](containers) and wondered if there was an easier way of building a +[containers](./containers) and wondered if there was an easier way of building a potentially deep stack of software for a particular project. -You may have already tried [Conda](conda) and found it wonderful, but for the +You may have already tried [Conda](./conda) and found it wonderful, but for the fact that the piece of software you want isn't built with the options you wanted, so you can't use it. Or maybe you just can't make the conda installed version work with the MPI setup on the HPC cluster you're using. diff --git a/book/course/spack/advanced.md b/book/course/spack/advanced.md index 7dca5eb..b0bfaa0 100644 --- a/book/course/spack/advanced.md +++ b/book/course/spack/advanced.md @@ -9,129 +9,160 @@ different packages if you know better than the defaults. ## Variants Variants are where the package has options how to build it. Let's pick on -`libtiff` as a nice example. You can get information on a package in Spack, -which will tell you about how it can be built: +`clingo` as an example. You can get information on a package in Spack, which +will tell you about how it can be built: ```bash -$ spack info libtiff -CMakePackage: libtiff +$ spack info clingo +CMakePackage: clingo Description: - LibTIFF - Tag Image File Format (TIFF) Library and Utilities. - -Homepage: http://www.simplesystems.org/libtiff/ - -Preferred version: - 4.7.0 https://download.osgeo.org/libtiff/tiff-4.7.0.tar.gz - -Safe versions: - 4.7.0 https://download.osgeo.org/libtiff/tiff-4.7.0.tar.gz - -Deprecated versions: - 4.6.0 https://download.osgeo.org/libtiff/tiff-4.6.0.tar.gz - 4.5.1 https://download.osgeo.org/libtiff/tiff-4.5.1.tar.gz - 4.5.0 https://download.osgeo.org/libtiff/tiff-4.5.0.tar.gz - 4.4.0 https://download.osgeo.org/libtiff/tiff-4.4.0.tar.gz - 4.3.0 https://download.osgeo.org/libtiff/tiff-4.3.0.tar.gz - 4.2.0 https://download.osgeo.org/libtiff/tiff-4.2.0.tar.gz - 4.1.0 https://download.osgeo.org/libtiff/tiff-4.1.0.tar.gz - 4.0.10 https://download.osgeo.org/libtiff/tiff-4.0.10.tar.gz - 4.0.9 https://download.osgeo.org/libtiff/tiff-4.0.9.tar.gz - 4.0.8 https://download.osgeo.org/libtiff/tiff-4.0.8.tar.gz - 4.0.7 https://download.osgeo.org/libtiff/tiff-4.0.7.tar.gz - 4.0.6 https://download.osgeo.org/libtiff/tiff-4.0.6.tar.gz - 4.0.5 https://download.osgeo.org/libtiff/tiff-4.0.5.tar.gz - 4.0.4 https://download.osgeo.org/libtiff/tiff-4.0.4.tar.gz - 3.9.7 https://download.osgeo.org/libtiff/tiff-3.9.7.tar.gz + Clingo: A grounder and solver for logic programs Clingo is part of the + Potassco project for Answer Set Programming (ASP). ASP offers a simple + and powerful modeling language to describe combinatorial problems as + logic programs. The clingo system then takes such a logic program and + computes answer sets representing solutions to the given problem. + +Homepage: https://potassco.org/clingo/ + +Preferred version: + 5.7.1 https://github.com/potassco/clingo/archive/v5.7.1.tar.gz + +Safe versions: + develop [git] https://github.com/potassco/clingo.git on branch wip-20 + master [git] https://github.com/potassco/clingo.git on branch master + 5.8.0 https://github.com/potassco/clingo/archive/v5.8.0.tar.gz + 5.7.1 https://github.com/potassco/clingo/archive/v5.7.1.tar.gz + 5.6.2 https://github.com/potassco/clingo/archive/v5.6.2.tar.gz + 5.5.2 https://github.com/potassco/clingo/archive/v5.5.2.tar.gz + 5.4.1 https://github.com/potassco/clingo/archive/v5.4.1.tar.gz + spack [git] https://github.com/potassco/clingo.git at commit 2a025667090d71b2c9dce60fe924feb6bde8f667 + +Deprecated versions: + 5.7.0 https://github.com/potassco/clingo/archive/v5.7.0.tar.gz + 5.5.1 https://github.com/potassco/clingo/archive/v5.5.1.tar.gz + 5.5.0 https://github.com/potassco/clingo/archive/v5.5.0.tar.gz + 5.4.0 https://github.com/potassco/clingo/archive/v5.4.0.tar.gz + 5.3.0 https://github.com/potassco/clingo/archive/v5.3.0.tar.gz + 5.2.2 https://github.com/potassco/clingo/archive/v5.2.2.tar.gz Variants: - build_system [cmake] autotools, cmake + apps [true] false, true + build command line applications + + build_system [cmake] cmake Build systems supported by the package - ccitt [true] false, true - support for CCITT Group 3 & 4 algorithms - jbig [false] false, true - use ISO JBIG compression - jpeg [true] false, true - use libjpeg - logluv [true] false, true - support for LogLuv high dynamic range algorithm - lzw [true] false, true - support for LZW algorithm - next [true] false, true - support for NeXT 2-bit RLE algorithm - old-jpeg [false] false, true - support for Old JPEG compression - packbits [true] false, true - support for Macintosh PackBits algorithm - pic [false] false, true - Enable position-independent code (PIC) - pixarlog [false] false, true - support for Pixar log-format algorithm - shared [true] false, true - Build shared - thunder [true] false, true - support for ThunderScan 4-bit RLE algorithm - zlib [true] false, true - use zlib - - when build_system=cmake - build_type [Release] Debug, MinSizeRel, RelWithDebInfo, Release - CMake build type - generator [make] none - the build system generator to use - - when build_system=cmake ^cmake@3.9: - ipo [false] false, true - CMake interprocedural optimization - - when @4.5,4.7: - opengl [false] false, true - use OpenGL (required for tiffgt viewer) - - when @4.2: - libdeflate [false] false, true - use libdeflate - - when @4: - jpeg12 [false] false, true - enable libjpeg 8/12-bit dual mode - lzma [false] false, true - use liblzma - - when @4.3: - lerc [false] false, true - use libLerc - - when @4.0.10: - webp [false] false, true - use libwebp - zstd [false] false, true - use libzstd - -Build Dependencies: - cmake gmake gnuconfig jbigkit jpeg lerc libwebp ninja xz zlib-api zstd - -Link Dependencies: - jbigkit jpeg lerc libwebp xz zlib-api zstd - -Run Dependencies: - None -Licenses: - libtiff + build_type [Release] Debug, MinSizeRel, RelWithDebInfo, Release + when build_system=cmake + CMake build type + + docs [false] false, true + build documentation with Doxygen + + generator [make] none + when build_system=cmake + the build system generator to use + + ipo [false] false, true + when build_system=cmake + CMake interprocedural optimization + + python [true] false, true + build with python bindings + + +Dependencies: + bison@2.5: build + when @5.6:5.8,master platform=linux + + bison@2.5: build + when @5.6:5.8,master platform=darwin + + bison@2.5: build + when @5.6:5.8,master platform=freebsd + + bison@2.5: build + when @spack platform=linux + + bison@2.5: build + when @spack platform=darwin + + bison@2.5: build + when @spack platform=freebsd + + c build + + cmake@:3 build + when @:5.7 + + cmake@3.1: build + + cmake@3.18: build + when @5.5: + + cmake@3.22.1: build + when @develop + + cxx build + + doxygen build + when +docs + + gmake build + when build_system=cmake generator=make + + ninja build + when build_system=cmake generator=ninja + + py-cffi@1.14: build, run + when @5.5:+python platform=linux + + py-cffi@1.14: build, run + when @5.5:+python platform=darwin + + py-cffi@1.14: build, run + when @5.5:+python platform=freebsd + + python@3.6: build, link, run + when +python + + python-venv build, run + when +python + + re2c@0.13: build + + re2c@0.13: build + when @spack + + re2c@0.13: build + when platform=windows + + re2c@1.1.1: build + when @5.6:5.8,master + + re2c@2: build + when @develop + + winbison@2.4.12: build, link + when platform=windows + + +Licenses: + MIT ``` From this you can see that there's lots of versions available, and there's a number of variants listed, along with dependencies. Let's say I wanted to -build version 4.4.0, and we want webp support to be on, jpeg support to be -off, and a build\_type of Debug. Basically, be as awkward as possible and -change everything: +build version 5.7.0, and we want to build the documentation, not build +python bindings, and with a build\_type of Debug. Basically, be as awkward as +possible and change everything: ```bash -spack install --deprecated libtiff@4.4.0 +webp -jpeg build_type=Debug +spack install --deprecated clingo@5.7.0+docs~python build_type=Debug ``` -There we're just turned on a feature (webp), turned off a feature (jpeg), and +There we're just turned on a feature (docs), turned off a feature (python), and told it we want the Debug build type. Have a look through the references to get further information, but this is actually the basics of choosing variants of packages. @@ -148,7 +179,7 @@ You may want to build a package with a different compiler, that you already have configured. ```bash -spack install libtiff@4.7.0 +webp -jpeg %gcc@11.4.1 +spack install clingo@5.7.1+docs~python %gcc@14.2.0 ``` Now Spack will happily rebuild that specific variant you've asked for, with the @@ -160,13 +191,18 @@ You can also specify particular versions and variants of dependencies. We can take our previous build even further: ```bash -spack install libtiff@4.7.0 +webp -jpeg build_type=Release ^perl@5.36.0 %gcc@11.4.1 +spack install clingo@5.7.1+docs~python build_type=Release ^bison@3.8.1 %gcc@14.2.0 ``` -This would build using Perl 5.36.0, without JPEG support, and with WebP -support, using the GCC 11.4.1 compiler. Note that we have used a ^ to say which -version of perl we want to use because it's a package that we depend on, -whereas we have to use % to say which compiler to use. +This would build clingo with documentation, without Python bindings, choosing the +release build of it, using the GCC 14.2.0 compiler to build the bison +dependency. When selecting dependencies, there's a subtle difference between +`%` and `^`. `%` means a direct dependency (build perl with the gcc@14.2.0 +compiler), whereas `^` means a transitive depedency, so when building libtiff, +and you're satisfying dependencies, use this package. + +Take a look at the references for further reading on how this all works, along +with even more ways of defining these. ## Previewing a software install @@ -174,16 +210,18 @@ Before installing a piece of software, you can review what Spack is planning on doing, and which dependencies it's going to rely on. ```bash -$ spack spec atop - - atop@2.5.0%gcc@14.2.0 build_system=generic arch=linux-rocky9-zen4 -[+] ^gcc-runtime@14.2.0%gcc@14.2.0 build_system=generic arch=linux-rocky9-zen4 -[e] ^glibc@2.34%gcc@14.2.0 build_system=autotools arch=linux-rocky9-zen4 -[+] ^ncurses@6.5%gcc@14.2.0~symlinks+termlib abi=none build_system=autotools patches=7a351bc arch=linux-rocky9-zen4 -[+] ^gmake@4.4.1%gcc@11.4.1~guile build_system=generic arch=linux-rocky9-zen4 -[+] ^pkgconf@2.2.0%gcc@14.2.0 build_system=autotools arch=linux-rocky9-zen4 -[+] ^zlib-ng@2.2.1%gcc@11.4.1+compat+new_strategies+opt+pic+shared build_system=autotools arch=linux-rocky9-zen4 -[+] ^gcc-runtime@11.4.1%gcc@11.4.1 build_system=generic arch=linux-rocky9-zen4 -[e] ^glibc@2.34%gcc@11.4.1 build_system=autotools arch=linux-rocky9-zen4 +$ spack spec atop %gcc@14.2.0 + - atop@2.5.0 build_system=generic platform=linux os=rocky9 target=zen4 %c=gcc@14.2.0 +[+] ^compiler-wrapper@1.0 build_system=generic platform=linux os=rocky9 target=zen4 +[e] ^gcc@14.2.0~binutils+bootstrap~graphite~mold~nvptx~piclibs~profiled~strip build_system=autotools build_type=RelWithDebInfo languages:='c,c++,fortran' platform=linux os=rocky9 target=x86_64 +[+] ^gcc-runtime@14.2.0 build_system=generic platform=linux os=rocky9 target=zen4 +[e] ^glibc@2.34 build_system=autotools platform=linux os=rocky9 target=x86_64 +[+] ^gmake@4.4.1~guile build_system=generic platform=linux os=rocky9 target=zen4 %c=gcc@11.4.1 +[e] ^gcc@11.4.1~binutils+bootstrap~graphite~nvptx~piclibs~profiled~strip build_system=autotools build_type=RelWithDebInfo languages:='c,c++' platform=linux os=rocky9 target=x86_64 +[+] ^gcc-runtime@11.4.1 build_system=generic platform=linux os=rocky9 target=zen4 +[+] ^ncurses@6.5-20250705~symlinks+termlib abi=none build_system=autotools patches:=7a351bc platform=linux os=rocky9 target=zen4 %c,cxx=gcc@11.4.1 +[+] ^pkgconf@2.5.1 build_system=autotools platform=linux os=rocky9 target=zen4 %c=gcc@11.4.1 +[+] ^zlib-ng@2.2.4+compat+new_strategies+opt+pic+shared build_system=autotools platform=linux os=rocky9 target=zen4 %c,cxx=gcc@11.4.1 ``` This way you can verify which versions it's expecting to use, and can adjust @@ -197,14 +235,6 @@ asked. In that instance, you can simply add the argument `--fresh` which tells Spack to work out dependencies ignoring what's already been built, and select what it thinks is optimal. -You'll note in our section on simplifying the modules system, we simplified it -to the point that the naming scheme used was name/version. This loses all -details about variants, making it impossible to distinguish between variants. -If you only ever build one particular instance of a module, this isn't a -problem, but if you're wanting to switch between multiple variants, or with -different compilers, you'll either want to stick with the original scheme, or -look at alternative naming schemes that can capture the necessary detail. - ## Exercise Install tmux version 3.2a with UTF-8 support. @@ -229,13 +259,16 @@ Description: Homepage: https://tmux.github.io -Preferred version: - 3.4 https://github.com/tmux/tmux/releases/download/3.4/tmux-3.4.tar.gz +Preferred version: + 3.5a https://github.com/tmux/tmux/releases/download/3.5a/tmux-3.5a.tar.gz -Safe versions: +Safe versions: master [git] https://github.com/tmux/tmux.git on branch master + 3.5a https://github.com/tmux/tmux/releases/download/3.5a/tmux-3.5a.tar.gz + 3.5 https://github.com/tmux/tmux/releases/download/3.5/tmux-3.5.tar.gz 3.4 https://github.com/tmux/tmux/releases/download/3.4/tmux-3.4.tar.gz 3.3a https://github.com/tmux/tmux/releases/download/3.3a/tmux-3.3a.tar.gz + 3.3 https://github.com/tmux/tmux/releases/download/3.3/tmux-3.3.tar.gz 3.2a https://github.com/tmux/tmux/releases/download/3.2a/tmux-3.2a.tar.gz 3.2 https://github.com/tmux/tmux/releases/download/3.2/tmux-3.2.tar.gz 3.1c https://github.com/tmux/tmux/releases/download/3.1c/tmux-3.1c.tar.gz @@ -256,27 +289,62 @@ Safe versions: 2.1 https://github.com/tmux/tmux/releases/download/2.1/tmux-2.1.tar.gz 1.9a https://github.com/tmux/tmux/releases/download/1.9a/tmux-1.9a.tar.gz -Deprecated versions: +Deprecated versions: None Variants: build_system [autotools] autotools Build systems supported by the package + + jemalloc [false] false, true + when @3.5: + Use jemalloc for memory allocation + static [false] false, true Create a static build + utf8proc [false] false, true Build with UTF-8 support from utf8proc library -Build Dependencies: - autoconf automake gmake gnuconfig libevent ncurses pkgconfig utf8proc yacc -Link Dependencies: - autoconf automake libevent ncurses utf8proc +Dependencies: + autoconf build, link + when @master + + automake build, link + when @master + + c build + + gmake build + when build_system=autotools + + gnuconfig build + when build_system=autotools target=ppc64le: + + gnuconfig build + when build_system=autotools target=aarch64: + + gnuconfig build + when build_system=autotools target=riscv64: + + jemalloc build, link + when +jemalloc + + libevent build, link + + ncurses build, link + + pkgconfig build + + utf8proc build, link + when +utf8proc + + yacc build + when @3: -Run Dependencies: - None -Licenses: +Licenses: ISC ``` @@ -295,8 +363,8 @@ $ spack load tmux $ tmux -V tmux 3.2a $ spack find -v tmux --- linux-rocky9-zen4 / gcc@14.2.0 ------------------------------- -tmux@3.2a~static+utf8proc build_system=autotools patches=c1b61a1 +-- linux-rocky9-zen4 / %c=gcc@11.4.1 ---------------------------- +tmux@3.2a~static+utf8proc build_system=autotools patches:=c1b61a1 ==> 1 installed package ``` @@ -307,6 +375,7 @@ confirm that it was built with UTF-8 support included. ## References -- [Basic Usage](https://spack.readthedocs.io/en/latest/basic_usage.html) +- [Basic Usage](https://spack.readthedocs.io/en/latest/package_fundamentals.html) - [Installing packages](https://spack-tutorial.readthedocs.io/en/latest/tutorial_basics.html#installing-packages) - [Module tutorial](https://spack-tutorial.readthedocs.io/en/latest/tutorial_modules.html) +- [Spec Syntax](https://spack.readthedocs.io/en/latest/spec_syntax.html) diff --git a/book/course/spack/containers.md b/book/course/spack/containers.md index 9be9bb5..e0972a8 100644 --- a/book/course/spack/containers.md +++ b/book/course/spack/containers.md @@ -8,7 +8,7 @@ Spack install software within. To define your container, you create a YAML file that describes what you want. Here's I'm saying I want spack to install gromacs with MPI enabled, and OpenMPI. I want it to create the container for Singularity/Apptainer, and I -want it to make sure the libgomp1 package is installed onto the OS. +want it to make sure the libgomp package is installed onto the OS. spack.yaml: @@ -21,10 +21,13 @@ spack: # What I want it to create container: format: singularity + images: + os: almalinux:9 + spack: 1.1.0 # Bonus OS packages I'd like installed os_packages: final: - - libgomp1 + - libgomp # Additional config for the build process config: mount proc: yes @@ -57,18 +60,10 @@ As a proof of life, let's run gromacs from within that container: ```bash $ apptainer run gromacs-openmpi.sif mpirun -np 4 gmx_mpi - :-) GROMACS - gmx_mpi, 2024.4-spack (-: + :-) GROMACS - gmx_mpi, 2025.3-spack (-: -Executable: /opt/software/linux-zen4/gromacs-2024.4-af5srsrjr7w2alftldvyek2chqu4rtls/bin/gmx_mpi -Data prefix: /opt/software/linux-zen4/gromacs-2024.4-af5srsrjr7w2alftldvyek2chqu4rtls -Working dir: /users/example/hpc2 -Command line: - gmx_mpi - -SYNOPSIS - -gmx [-[no]h] [-[no]quiet] [-[no]version] [-[no]copyright] [-nice ] - [-[no]backup] +Executable: /opt/software/linux-zen4/gromacs-2025.3-ipqq3w22geizv4y6pdroneco3hq33xkj/bin/gmx_mpi +Data prefix: /opt/software/linux-zen4/gromacs-2025.3-ipqq3w22geizv4y6pdroneco3hq33xkj ... ``` diff --git a/book/course/spack/environments.md b/book/course/spack/environments.md index 34194ae..001e5c3 100644 --- a/book/course/spack/environments.md +++ b/book/course/spack/environments.md @@ -4,6 +4,10 @@ Environments in Spack feel a whole lot like environments in Conda; they allow you to bundle up a number of different pieces of software into a single entity, making a single coherent collection. +This is a very simple introduction, and really doesn't cover all the things +you can do with environments, so if you're interested in these, please read +the main documentation links in the [References](environments:references) section. + ## Creating an environment Here's a minimal workflow for creating an environment: @@ -13,17 +17,21 @@ Here's a minimal workflow for creating an environment: ```bash spack env create myenv ``` +- Activate it + ```bash + spack env activate myenv + ``` - Add a list of software to it ```bash - spack -e myenv add bash@5 python py-numpy py-scipy py-matplotlib openssl@3.0.15 + spack add bash@5 python py-numpy py-scipy py-matplotlib openssl@3.0.7 ``` - Tweak the config if required ```bash - spack -e myenv config edit + spack config edit ``` Please review the references for more information on options here, but nothing @@ -32,7 +40,7 @@ Here's a minimal workflow for creating an environment: - Validate the installation ```bash - spack -e myenv concretize + spack concretize ``` [Concretizing](https://spack.readthedocs.io/en/latest/environments.html#spec-concretization) @@ -42,7 +50,7 @@ Here's a minimal workflow for creating an environment: - Build and install the software ```bash - spack -e myenv install + spack install ``` Note this follows the normal behaviour of Spack, reusing previously built @@ -51,7 +59,8 @@ software, and external software you've told it to use. ## Using an environment Again, this feels very familiar to people used to Conda; you have to activate -an environment before using it: +an environment before using it (although we did actually do this earlier when +we created it): ```bash spack env activate myenv @@ -63,8 +72,8 @@ Once activated, you can confirm that it's really live: $ which bash ~/spack/var/spack/environments/myenv/.spack-env/view/bin/bash $ bash --version -GNU bash, version 5.2.26(1)-release (x86_64-pc-linux-gnu) -Copyright (C) 2022 Free Software Foundation, Inc. +GNU bash, version 5.3.3(1)-release (x86_64-pc-linux-gnu) +Copyright (C) 2025 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software; you are free to change and redistribute it. @@ -79,7 +88,8 @@ You can leave an environment using: spack env deactivate ``` +(environments:references)= ## References -- [Using Spack to Replace Homebrew/Conda](https://spack.readthedocs.io/en/latest/replace_conda_homebrew.html) +- [Environment Basics](https://spack.readthedocs.io/en/latest/environments_basics.html) - [Environments](https://spack.readthedocs.io/en/latest/environments.html) diff --git a/book/course/spack/existing.md b/book/course/spack/existing.md index 7a81658..36068a8 100644 --- a/book/course/spack/existing.md +++ b/book/course/spack/existing.md @@ -1,7 +1,7 @@ # Using existing software Spack by default will quite happily just build everything itself, as long as a -limited number of [system requirements](https://spack.readthedocs.io/en/latest/getting_started.html#system-prerequisites) +limited number of [system requirements](https://spack.readthedocs.io/en/latest/installing_prerequisites.html) are satisfied. But you might want to use a compiler and MPI library from a host system. This is often the case if packages take a long time to build, or if you've got access to optimised versions of software that you don't want to @@ -24,22 +24,13 @@ newer GCC compiler than is available by default: ```bash $ module add gcc/14.2.0 $ spack compiler add -==> Added 1 new compiler to /users/example/.spack/linux/compilers.yaml +==> Added 1 new compiler to /users/example/spack/etc/spack/packages.yaml gcc@14.2.0 ==> Compilers are defined in the following files: - /users/example/.spack/linux/compilers.yaml + /users/example/spack/etc/spack/packages.yaml ``` -Spack now knows about this compiler. If this was a compiler in a module, you -can just load the module beforehand, and Spack will add that compiler to its -known list. - -If you want to set that as your preferred compiler, you can do this with a -single command: - -```bash -spack config add packages:all:compiler:[gcc@14.2.0] -``` +Spack now knows about this compiler. You can confirm your config, or make other changes with: @@ -49,8 +40,21 @@ spack config edit packages ```yaml packages: - all: - compiler: [gcc@14.2.0] + gcc: + externals: + - spec: gcc@14.2.0 languages:='c,c++,fortran' + prefix: /opt/apps/pkg/compilers/gcc/14.2.0 + extra_attributes: + compilers: + c: /opt/apps/pkg/compilers/gcc/14.2.0/bin/gcc + cxx: /opt/apps/pkg/compilers/gcc/14.2.0/bin/g++ + fortran: /opt/apps/pkg/compilers/gcc/14.2.0/bin/gfortran + - spec: gcc@11.4.1 languages:='c,c++' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ ``` ## Other software @@ -66,66 +70,70 @@ spack spec phylobayesmpi
Full spec -``` - - phylobayesmpi@1.9%gcc@14.2.0 build_system=makefile arch=linux-rocky9-zen4 - - ^gcc-runtime@14.2.0%gcc@14.2.0 build_system=generic arch=linux-rocky9-zen4 -[e] ^glibc@2.34%gcc@14.2.0 build_system=autotools arch=linux-rocky9-zen4 -[+] ^gmake@4.4.1%gcc@11.4.1~guile build_system=generic arch=linux-rocky9-zen4 -[+] ^gcc-runtime@11.4.1%gcc@11.4.1 build_system=generic arch=linux-rocky9-zen4 -[e] ^glibc@2.34%gcc@11.4.1 build_system=autotools arch=linux-rocky9-zen4 - - ^openmpi@5.0.5%gcc@14.2.0+atomics~cuda~debug~gpfs~internal-hwloc~internal-libevent~internal-pmix~java~lustre~memchecker~openshmem~romio+rsh~static~two_level_namespace+vt+wrapper-rpath build_system=autotools fabrics=none romio-filesystem=none schedulers=none arch=linux-rocky9-zen4 - - ^autoconf@2.72%gcc@14.2.0 build_system=autotools arch=linux-rocky9-zen4 - - ^m4@1.4.19%gcc@14.2.0+sigsegv build_system=autotools patches=9dc5fbd,bfdffa7 arch=linux-rocky9-zen4 - - ^diffutils@3.10%gcc@14.2.0 build_system=autotools arch=linux-rocky9-zen4 - - ^libsigsegv@2.14%gcc@14.2.0 build_system=autotools arch=linux-rocky9-zen4 - - ^automake@1.16.5%gcc@14.2.0 build_system=autotools arch=linux-rocky9-zen4 - - ^hwloc@2.11.1%gcc@14.2.0~cairo~cuda~gl~libudev+libxml2~nvml~oneapi-level-zero~opencl+pci~rocm build_system=autotools libs=shared,static arch=linux-rocky9-zen4 - - ^libpciaccess@0.17%gcc@14.2.0 build_system=autotools arch=linux-rocky9-zen4 - - ^util-macros@1.20.1%gcc@14.2.0 build_system=autotools arch=linux-rocky9-zen4 - - ^libxml2@2.13.4%gcc@14.2.0+pic~python+shared build_system=autotools arch=linux-rocky9-zen4 - - ^libiconv@1.17%gcc@14.2.0 build_system=autotools libs=shared,static arch=linux-rocky9-zen4 - - ^xz@5.4.6%gcc@14.2.0~pic build_system=autotools libs=shared,static arch=linux-rocky9-zen4 - - ^ncurses@6.5%gcc@14.2.0~symlinks+termlib abi=none build_system=autotools patches=7a351bc arch=linux-rocky9-zen4 - - ^libevent@2.1.12%gcc@14.2.0+openssl build_system=autotools arch=linux-rocky9-zen4 - - ^openssl@3.4.0%gcc@14.2.0~docs+shared build_system=generic certs=mozilla arch=linux-rocky9-zen4 - - ^ca-certificates-mozilla@2023-05-30%gcc@14.2.0 build_system=generic arch=linux-rocky9-zen4 - - ^libtool@2.4.7%gcc@14.2.0 build_system=autotools arch=linux-rocky9-zen4 - - ^findutils@4.9.0%gcc@14.2.0 build_system=autotools patches=440b954 arch=linux-rocky9-zen4 - - ^numactl@2.0.18%gcc@14.2.0 build_system=autotools arch=linux-rocky9-zen4 - - ^openssh@9.9p1%gcc@14.2.0+gssapi build_system=autotools arch=linux-rocky9-zen4 - - ^krb5@1.21.3%gcc@14.2.0+shared build_system=autotools arch=linux-rocky9-zen4 - - ^bison@3.8.2%gcc@14.2.0~color build_system=autotools arch=linux-rocky9-zen4 - - ^gettext@0.22.5%gcc@14.2.0+bzip2+curses+git~libunistring+libxml2+pic+shared+tar+xz build_system=autotools arch=linux-rocky9-zen4 - - ^tar@1.34%gcc@14.2.0 build_system=autotools zip=pigz arch=linux-rocky9-zen4 -[+] ^pigz@2.8%gcc@11.4.1 build_system=makefile arch=linux-rocky9-zen4 - - ^zstd@1.5.6%gcc@14.2.0+programs build_system=makefile compression=none libs=shared,static arch=linux-rocky9-zen4 - - ^libedit@3.1-20240808%gcc@14.2.0 build_system=autotools arch=linux-rocky9-zen4 - - ^libxcrypt@4.4.35%gcc@14.2.0~obsolete_api build_system=autotools patches=4885da3 arch=linux-rocky9-zen4 - - ^perl@5.40.0%gcc@14.2.0+cpanm+opcode+open+shared+threads build_system=generic arch=linux-rocky9-zen4 - - ^berkeley-db@18.1.40%gcc@14.2.0+cxx~docs+stl build_system=autotools patches=26090f4,b231fcc arch=linux-rocky9-zen4 - - ^bzip2@1.0.8%gcc@14.2.0~debug~pic+shared build_system=generic arch=linux-rocky9-zen4 - - ^gdbm@1.23%gcc@14.2.0 build_system=autotools arch=linux-rocky9-zen4 - - ^readline@8.2%gcc@14.2.0 build_system=autotools patches=bbf97f1 arch=linux-rocky9-zen4 - - ^pkgconf@2.2.0%gcc@14.2.0 build_system=autotools arch=linux-rocky9-zen4 - - ^pmix@5.0.3%gcc@14.2.0~munge~python~restful build_system=autotools arch=linux-rocky9-zen4 -[+] ^zlib-ng@2.2.1%gcc@11.4.1+compat+new_strategies+opt+pic+shared build_system=autotools arch=linux-rocky9-zen4 +```bash + - phylobayesmpi@1.9 build_system=makefile platform=linux os=rocky9 target=zen4 %cxx=gcc@14.2.0 +[+] ^compiler-wrapper@1.0 build_system=generic platform=linux os=rocky9 target=zen4 +[e] ^gcc@14.2.0~binutils+bootstrap~graphite~mold~nvptx~piclibs~profiled~strip build_system=autotools build_type=RelWithDebInfo languages:='c,c++,fortran' platform=linux os=rocky9 target=x86_64 + - ^gcc-runtime@14.2.0 build_system=generic platform=linux os=rocky9 target=zen4 +[e] ^glibc@2.34 build_system=autotools platform=linux os=rocky9 target=x86_64 +[+] ^gmake@4.4.1~guile build_system=generic platform=linux os=rocky9 target=zen4 %c=gcc@11.4.1 +[e] ^gcc@11.4.1~binutils+bootstrap~graphite~nvptx~piclibs~profiled~strip build_system=autotools build_type=RelWithDebInfo languages:='c,c++' platform=linux os=rocky9 target=x86_64 +[+] ^gcc-runtime@11.4.1 build_system=generic platform=linux os=rocky9 target=zen4 + - ^openmpi@5.0.8+atomics~cuda~debug+fortran~gpfs~internal-hwloc~internal-libevent~internal-pmix~ipv6~java~lustre~memchecker~openshmem~rocm~romio+rsh~static~two_level_namespace+vt+wrapper-rpath build_system=autotools fabrics:=none romio-filesystem:=none schedulers:=none platform=linux os=rocky9 target=zen4 %c,cxx,fortran=gcc@14.2.0 + - ^autoconf@2.72 build_system=autotools platform=linux os=rocky9 target=zen4 + - ^m4@1.4.20+sigsegv build_system=autotools platform=linux os=rocky9 target=zen4 %c,cxx=gcc@14.2.0 +[+] ^diffutils@3.12 build_system=autotools platform=linux os=rocky9 target=zen4 %c=gcc@11.4.1 + - ^libsigsegv@2.14 build_system=autotools platform=linux os=rocky9 target=zen4 %c=gcc@14.2.0 + - ^automake@1.16.5 build_system=autotools platform=linux os=rocky9 target=zen4 %c=gcc@14.2.0 + - ^hwloc@2.12.2~cairo~cuda~gl~level_zero~libudev+libxml2~nvml~opencl+pci~rocm build_system=autotools libs:=shared,static platform=linux os=rocky9 target=zen4 %c,cxx=gcc@14.2.0 + - ^libpciaccess@0.17 build_system=autotools platform=linux os=rocky9 target=zen4 %c=gcc@14.2.0 + - ^util-macros@1.20.1 build_system=autotools platform=linux os=rocky9 target=zen4 + - ^libxml2@2.13.5~http+pic~python+shared build_system=autotools platform=linux os=rocky9 target=zen4 %c=gcc@14.2.0 +[+] ^libiconv@1.18 build_system=autotools libs:=shared,static platform=linux os=rocky9 target=zen4 %c=gcc@11.4.1 + - ^xz@5.6.3~pic build_system=autotools libs:=shared,static platform=linux os=rocky9 target=zen4 %c=gcc@14.2.0 +[+] ^ncurses@6.5-20250705~symlinks+termlib abi=none build_system=autotools patches:=7a351bc platform=linux os=rocky9 target=zen4 %c,cxx=gcc@11.4.1 + - ^libevent@2.1.12+openssl build_system=autotools platform=linux os=rocky9 target=zen4 %c=gcc@14.2.0 + - ^openssl@3.6.0~docs+shared build_system=generic certs=mozilla platform=linux os=rocky9 target=zen4 %c,cxx=gcc@14.2.0 + - ^ca-certificates-mozilla@2025-08-12 build_system=generic platform=linux os=rocky9 target=zen4 + - ^libtool@2.4.7 build_system=autotools platform=linux os=rocky9 target=zen4 %c=gcc@14.2.0 + - ^findutils@4.10.0 build_system=autotools patches:=440b954 platform=linux os=rocky9 target=zen4 %c=gcc@14.2.0 + - ^gettext@0.23.1+bzip2+curses+git~libunistring+libxml2+pic+shared+tar+xz build_system=autotools platform=linux os=rocky9 target=zen4 %c,cxx=gcc@14.2.0 + - ^tar@1.35 build_system=autotools zip=pigz platform=linux os=rocky9 target=zen4 %c=gcc@14.2.0 +[+] ^pigz@2.8 build_system=makefile platform=linux os=rocky9 target=zen4 %c=gcc@11.4.1 + - ^zstd@1.5.7+programs build_system=makefile compression:=none libs:=shared,static platform=linux os=rocky9 target=zen4 %c,cxx=gcc@14.2.0 + - ^numactl@2.0.18 build_system=autotools platform=linux os=rocky9 target=zen4 %c=gcc@14.2.0 + - ^openssh@9.9p1+gssapi build_system=autotools platform=linux os=rocky9 target=zen4 %c,cxx=gcc@14.2.0 + - ^krb5@1.21.3+shared build_system=autotools platform=linux os=rocky9 target=zen4 %c,cxx=gcc@14.2.0 + - ^bison@3.8.2~color build_system=autotools platform=linux os=rocky9 target=zen4 %c,cxx=gcc@14.2.0 + - ^libedit@3.1-20240808 build_system=autotools platform=linux os=rocky9 target=zen4 %c=gcc@14.2.0 + - ^libxcrypt@4.4.38~obsolete_api build_system=autotools platform=linux os=rocky9 target=zen4 %c=gcc@14.2.0 +[+] ^perl@5.42.0+cpanm+opcode+open+shared+threads build_system=generic platform=linux os=rocky9 target=zen4 %c=gcc@11.4.1 +[+] ^berkeley-db@18.1.40+cxx~docs+stl build_system=autotools patches:=26090f4,b231fcc platform=linux os=rocky9 target=zen4 %c,cxx=gcc@11.4.1 +[+] ^bzip2@1.0.8~debug~pic+shared build_system=generic platform=linux os=rocky9 target=zen4 %c=gcc@11.4.1 +[+] ^gdbm@1.25 build_system=autotools platform=linux os=rocky9 target=zen4 %c=gcc@11.4.1 +[+] ^readline@8.3 build_system=autotools patches:=21f0a03 platform=linux os=rocky9 target=zen4 %c=gcc@11.4.1 +[+] ^pkgconf@2.5.1 build_system=autotools platform=linux os=rocky9 target=zen4 %c=gcc@11.4.1 + - ^pmix@6.0.0~munge~python build_system=autotools platform=linux os=rocky9 target=zen4 %c=gcc@14.2.0 + - ^prrte@4.0.0 build_system=autotools schedulers:=none platform=linux os=rocky9 target=zen4 %c=gcc@14.2.0 + - ^flex@2.6.3+lex~nls build_system=autotools platform=linux os=rocky9 target=zen4 %c,cxx=gcc@14.2.0 +[+] ^zlib-ng@2.2.4+compat+new_strategies+opt+pic+shared build_system=autotools platform=linux os=rocky9 target=zen4 %c,cxx=gcc@11.4.1 ```
-You don't need to read the full spec above, but on Aire currently, this adds up to -43 packages, which I'm sure would take a fair old amount of time to build. So -let's see if we can cut that down by using software already installed on my -system. +You don't need to read the full spec above, but on Aire currently, this adds +up to 37 missing packages, which I'm sure would take a fair old amount of time +to build. So let's see if we can cut that down by using software already +installed on my system. It can find certain software in your environment without you having to configure anything by hand. So if I load an openmpi module, and run: ```bash -$ module add openmpi/5.0.6/gcc-13.2.0_cuda-12.6.2 +$ module add openmpi/5.0.6/gcc-14.2.0 $ spack external find openmpi -==> The following specs have been detected on this system and added to /home/home02/me/.spack/packages.yaml --- no arch / gcc@13.2.0 ----------------------------------------- +==> The following specs have been detected on this system and added to /users/example/spack/etc/spack/packages.yaml +-- no arch / no compilers --------------------------------------- openmpi@5.0.6 ``` @@ -151,11 +159,23 @@ My full packages.yaml now looks like this, if looked at with `spack config edit packages: openmpi: externals: - - spec: openmpi@5.0.6%gcc@=13.2.0+cuda~java~memchecker~static~wrapper-rpath fabrics=ofi,psm2,ucx - schedulers=none - prefix: /opt/apps/pkg/libraries/openmpi/5.0.6/gcc-13.2.0+cuda-12.6.2 - all: - compiler: [gcc@14.2.0] + - spec: openmpi@5.0.6~cuda~java~memchecker~rocm~static~wrapper-rpath fabrics=ofi,psm2,ucx schedulers=none + prefix: /opt/apps/pkg/libraries/openmpi/5.0.6/gcc-14.2.0 + gcc: + externals: + - spec: gcc@14.2.0 languages:='c,c++,fortran' + prefix: /opt/apps/pkg/compilers/gcc/14.2.0 + extra_attributes: + compilers: + c: /opt/apps/pkg/compilers/gcc/14.2.0/bin/gcc + cxx: /opt/apps/pkg/compilers/gcc/14.2.0/bin/g++ + fortran: /opt/apps/pkg/compilers/gcc/14.2.0/bin/gfortran + - spec: gcc@11.4.1 languages:='c,c++' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ ``` ```{admonition} OpenMPI on Red Hat oddity @@ -176,48 +196,41 @@ look again now at building mpiwrapper, and see what that looks like: ```bash $ spack spec phylobayesmpi -$ spack spec phylobayesmpi - - phylobayesmpi@1.9%gcc@13.2.0 build_system=makefile arch=linux-rocky9-zen4 - - ^gcc-runtime@13.2.0%gcc@13.2.0 build_system=generic arch=linux-rocky9-zen4 -[e] ^glibc@2.34%gcc@13.2.0 build_system=autotools arch=linux-rocky9-zen4 -[+] ^gmake@4.4.1%gcc@11.4.1~guile build_system=generic arch=linux-rocky9-zen4 -[+] ^gcc-runtime@11.4.1%gcc@11.4.1 build_system=generic arch=linux-rocky9-zen4 -[e] ^glibc@2.34%gcc@11.4.1 build_system=autotools arch=linux-rocky9-zen4 -[e] ^openmpi@5.0.6%gcc@13.2.0+atomics+cuda~debug~gpfs~internal-hwloc~internal-libevent~internal-pmix~java~lustre~memchecker~openshmem~romio+rsh~static~two_level_namespace+vt~wrapper-rpath build_system=autotools cuda_arch=none fabrics=ofi,psm2,ucx romio-filesystem=none schedulers=none arch=linux-rocky9-zen4 + - phylobayesmpi@1.9 build_system=makefile platform=linux os=rocky9 target=zen4 %cxx=gcc@11.4.1 +[+] ^compiler-wrapper@1.0 build_system=generic platform=linux os=rocky9 target=zen4 +[e] ^gcc@11.4.1~binutils+bootstrap~graphite~nvptx~piclibs~profiled~strip build_system=autotools build_type=RelWithDebInfo languages:='c,c++' platform=linux os=rocky9 target=x86_64 +[+] ^gcc-runtime@11.4.1 build_system=generic platform=linux os=rocky9 target=zen4 +[e] ^glibc@2.34 build_system=autotools platform=linux os=rocky9 target=x86_64 +[+] ^gmake@4.4.1~guile build_system=generic platform=linux os=rocky9 target=zen4 %c=gcc@11.4.1 +[e] ^openmpi@5.0.6+atomics~cuda~debug+fortran~gpfs~internal-hwloc~internal-libevent~internal-pmix~ipv6~java~lustre~memchecker~openshmem~rocm+romio+rsh~static~two_level_namespace+vt~wrapper-rpath build_system=autotools fabrics:=ofi,psm2,ucx romio-filesystem:=none schedulers:=none platform=linux os=rocky9 target=x86_64 ``` -So now Spack believes that it can reuse my openmpi from the system and other -than a little shim package (gcc-runtime), it's only actually going to build -the piece of software I've asked for, rather than the huge list of -dependencies we had before. +So now Spack believes that it can reuse my openmpi from the system and it's +only actually going to build the piece of software I've asked for, rather than +the huge list of dependencies we had before. ```bash $ spack install phylobayesmpi -[+] /usr (external glibc-2.34-yzotqqevluwtq3jprsigxmtetjc3s3c7) -[+] /usr (external glibc-2.34-nuyxhw7kdup423xfoh3erg5yl7c3xrlh) -[+] /opt/apps/pkg/libraries/openmpi/5.0.6/gcc-13.2.0+cuda-12.6.2 (external openmpi-5.0.6-73sqmyovadjm2mdbuqrmpd5vm3ihygyo) -==> Installing gcc-runtime-13.2.0-r7anquf3s2qyszutz3dprsgfjg2baypt [4/7] -==> No binary for gcc-runtime-13.2.0-r7anquf3s2qyszutz3dprsgfjg2baypt found: installing from source -==> No patches needed for gcc-runtime -==> gcc-runtime: Executing phase: 'install' -==> gcc-runtime: Successfully installed gcc-runtime-13.2.0-r7anquf3s2qyszutz3dprsgfjg2baypt - Stage: 0.00s. Install: 0.22s. Post-install: 0.12s. Total: 0.37s -[+] /users/example/spack/opt/spack/linux-rocky9-zen4/gcc-13.2.0/gcc-runtime-13.2.0-r7anquf3s2qyszutz3dprsgfjg2baypt -[+] /users/example/spack/opt/spack/linux-rocky9-zen4/gcc-11.4.1/gcc-runtime-11.4.1-7hex6dyh2ttbdeywfkq5vbsinmnhjoub -[+] /users/example/spack/opt/spack/linux-rocky9-zen4/gcc-11.4.1/gmake-4.4.1-36fbslt63hhoisn7shlrkgd5fsb2awmz -==> Installing phylobayesmpi-1.9-6267nytqg6se4wqmg6p37ftescnz2jpz [7/7] -==> No binary for phylobayesmpi-1.9-6267nytqg6se4wqmg6p37ftescnz2jpz found: installing from source +[+] /usr (external glibc-2.34-z4aqp7yhzcmlwfcydu32wbqxounooohb) +[+] /opt/apps/pkg/libraries/openmpi/5.0.6/gcc-14.2.0 (external openmpi-5.0.6-xjyqqbweqxrrioc72tktzn3zipubmjy5) +[+] /usr (external gcc-11.4.1-fhj3ubjk6iqnexhdlnbcmlzjuo3iofwa) +[+] /users/example/spack/opt/spack/linux-zen4/compiler-wrapper-1.0-et3ejxhc3t5yoqbw74pqjv4izptvuj3q +[+] /users/example/spack/opt/spack/linux-zen4/gcc-runtime-11.4.1-dqguk7uvu3bh5ehxvi7sv4qaartrartc +[+] /users/example/spack/opt/spack/linux-zen4/gmake-4.4.1-2fg6qftd7dmzej4hzzydnwqwnbl65zac +==> No binary for phylobayesmpi-1.9-ajhjgnvilq36v2idl6ukpn7dxsckh5qn found: installing from source +==> Installing phylobayesmpi-1.9-ajhjgnvilq36v2idl6ukpn7dxsckh5qn [7/7] ==> Fetching https://mirror.spack.io/_source-cache/archive/56/567d8db995f23b2b0109c1e6088a7e5621e38fec91d6b2f27abd886b90ea31ce.tar.gz + [100%] 701.19 KB @ 1.3 MB/s ==> No patches needed for phylobayesmpi ==> phylobayesmpi: Executing phase: 'edit' ==> phylobayesmpi: Executing phase: 'build' ==> phylobayesmpi: Executing phase: 'install' -==> phylobayesmpi: Successfully installed phylobayesmpi-1.9-6267nytqg6se4wqmg6p37ftescnz2jpz - Stage: 1.18s. Edit: 0.00s. Build: 6.59s. Install: 0.05s. Post-install: 0.06s. Total: 7.95s -[+] /users/example/spack/opt/spack/linux-rocky9-zen4/gcc-13.2.0/phylobayesmpi-1.9-6267nytqg6se4wqmg6p37ftescnz2jpz +==> phylobayesmpi: Successfully installed phylobayesmpi-1.9-ajhjgnvilq36v2idl6ukpn7dxsckh5qn + Stage: 1.16s. Edit: 0.00s. Build: 5.30s. Install: 0.01s. Post-install: 0.02s. Total: 6.55s +[+] /users/example/spack/opt/spack/linux-zen4/phylobayesmpi-1.9-ajhjgnvilq36v2idl6ukpn7dxsckh5qn ``` -So we've just install phylobayesmpi with Spack, using dependencies already +So we've just installed phylobayesmpi with Spack, using dependencies already installed on the system, in seconds. Lovely. ## Exercise @@ -243,16 +256,18 @@ want it to use. ```bash $ spack external find openssl -==> The following specs have been detected on this system and added to /users/example/.spack/packages.yaml +==> The following specs have been detected on this system and added to /users/example/spack/etc/spack/packages.yaml +-- no arch / no compilers --------------------------------------- openssl@3.0.7 openssl@3.3.2 ``` Here we find it's detected both the system OpenSSL, along with a version -lurking within miniforge. If you found it picked up a package you didn't want -it to use, you can use `spack config edit packages` to adjust this after it's -detected packages. +lurking within miniforge, as I had that module loaded when I ran this. If you +found it picked up a package you didn't want it to use, you can use `spack +config edit packages` to adjust this after it's detected packages. -In this case, you may choose to remove these two lines, to stop it from using the miniforge version: +In this case, you may choose to remove these two lines, to stop it from using +the miniforge version: ```yaml - spec: openssl@3.3.2 prefix: /opt/apps/pkg/interpreters/miniforge/24.7.1 diff --git a/book/course/spack/installing.md b/book/course/spack/installing.md index 7801730..851cfa3 100644 --- a/book/course/spack/installing.md +++ b/book/course/spack/installing.md @@ -12,7 +12,7 @@ deliberately pick a specific stable version: ```bash $ cd spack -$ git checkout v0.23.1 +$ git checkout v1.1.0 $ cd .. ``` @@ -32,11 +32,13 @@ $ . spack/share/spack/setup-env.sh ``` You need to make sure you have Python 3 available in your PATH, but there are -few other [required dependencies](https://spack.readthedocs.io/en/latest/getting_started.html#system-prerequisites). +few other [required dependencies](https://spack.readthedocs.io/en/latest/installing_prerequisites.html). Spack is now alive in your terminal, and you can query available packages, or -look at one to see what options you have for building it. We can now have a -look at what packages are available to install: +look at one to see what options you have for building it. Note the first +command you run might take a little while, as it also has to download the +separate spack packages repository, which by default ends up in `~/.spack`. +We can now have a look at what packages are available to install: ```bash $ spack list @@ -53,7 +55,7 @@ abyss ... ``` -Now that's quite a list (8307 different pieces of software at the time of +Now that's quite a list (8611 different pieces of software at the time of writing). So let's move on to look at the next section, where we install one to show how diff --git a/book/course/spack/modules.md b/book/course/spack/modules.md index 77634ab..a003d3d 100644 --- a/book/course/spack/modules.md +++ b/book/course/spack/modules.md @@ -42,7 +42,7 @@ We can confirm now that we can see the module with the module command: ```bash $ module avail pigz ------ /users/example/spack/share/spack/modules/linux-rocky9-zen4 ------ -pigz/2.8-gcc-11.4.1-gqwhh2i +pigz/2.8-gcc-11.4.1-rpmloas # Make sure Spack has no packages enabled $ spack unload -a @@ -50,9 +50,9 @@ $ spack unload -a # On Aire, pigz is installed, but on your system you may find it's not. $ pigz --version pigz 2.5 -$ module add pigz/2.8-gcc-11.4.1-gqwhh2i -Loading pigz/2.8-gcc-11.4.1-gqwhh2i - Loading requirement: glibc/2.34-gcc-11.4.1-nuyxhw7 gcc-runtime/11.4.1-gcc-11.4.1-7hex6dy zlib-ng/2.2.1-gcc-11.4.1-5rrpd7 +$ module add pigz/2.8-gcc-11.4.1-rpmloas +Loading pigz/2.8-gcc-11.4.1-rpmloas + Loading requirement: glibc/2.34-none-none-z4aqp7y gcc-runtime/11.4.1-none-none-dqguk7u zlib-ng/2.2.4-gcc-11.4.1-ro6otkl $ pigz --version pigz 2.8 ``` @@ -70,7 +70,7 @@ One option is to simplify this a great deal, hiding a lot of that information, which might be appealing if you know that information is not important to you. So if you're using one compiler, you don't really care about the compiler used, you don't need the hash at the end if you're only dealing with a single variant -(covered [later](advanced:spack:variants)), and you'd probably like the modules +(covered [later](#advanced:spack:variants)), and you'd probably like the modules names more in line with how they have traditionally been named, with the version separate from the package name. @@ -109,7 +109,7 @@ Testing this out now we can see all is how we wanted it: ```bash $ module avail pigz ------ /users/example/spack/share/spack/modules/linux-rocky9-zen4 ------ -pigz/2.8-gqwh +pigz/2.8-rpml ``` We now have tidier module files, named in a simpler fashion, which is probably @@ -118,7 +118,7 @@ you would on other systems: ```bash $ module add pigz -$ module add pigz/2.8-gqwh +$ module add pigz/2.8-rpml ``` Now you may be wondering why we bothered including this hash, as it'd clearly @@ -129,20 +129,18 @@ in one way, but elsewhere we may have modules built with different compilers, dependencies and variants. So for example: ```bash -$ module avail zlib-ng +$ module avail ncurses ------ /users/example/spack/share/spack/modules/linux-rocky9-zen4 ------ -zlib-ng/2.0.7-pjab zlib-ng/2.2.1-5rrp zlib-ng/2.2.1-xjlb -$ spack find zlib-ng --- linux-rocky9-zen4 / gcc@11.4.1 ------------------------------- -zlib-ng@2.0.7 zlib-ng@2.2.1 - --- linux-rocky9-zen4 / gcc@14.2.0 ------------------------------- -zlib-ng@2.2.1 -==> 3 installed packages +ncurses/6.5-20250705-lcco ncurses/6.5-20250705-tx2b +$ spack find -v ncurses +-- linux-rocky9-zen4 / %c,cxx=gcc@11.4.1 ------------------------ +ncurses@6.5-20250705~symlinks~termlib abi=none build_system=autotools patches:=7a351bc ncurses@6.5-20250705~symlinks+termlib abi=none build_system=autotools patches:=7a351bc +==> 2 installed packages ``` -You can see here we have two versions of zlib-ng built with two different -compilers, which would clearly class with a simpler scheme. +You can see here we have two versions of ncurses, in this case built with the +same compiler, but with two different variants, which would clearly class with +a simpler scheme. If you want more complicated structures, supporting multiple compilers, MPI implementations, numerical libraries, this is all possible, and covered in the diff --git a/book/course/spack/recipes.md b/book/course/spack/recipes.md index a1ba343..6bc2803 100644 --- a/book/course/spack/recipes.md +++ b/book/course/spack/recipes.md @@ -273,11 +273,11 @@ This way it knows to use the master branch to build a "develop" release. This the leaves us with this minimal config: ```python -# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. +# Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.cmake import CMakePackage from spack.package import * class CmakeTutorial(CMakePackage): @@ -322,11 +322,11 @@ We make the same basic edits done in the Cmake section, to tell it we're using git, and which branch to use for the `develop` version. This leaves us with: ```python -# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. +# Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.autotools import AutotoolsPackage from spack.package import * class AutotoolsExample(AutotoolsPackage): @@ -341,6 +341,7 @@ class AutotoolsExample(AutotoolsPackage): depends_on("m4", type="build") def autoreconf(self, spec, prefix): + # FIXME: Modify the autoreconf method as necessary autoreconf("--install", "--verbose", "--force") ``` diff --git a/book/course/spack/testinstall.md b/book/course/spack/testinstall.md index 9af57d9..ea94a61 100644 --- a/book/course/spack/testinstall.md +++ b/book/course/spack/testinstall.md @@ -8,53 +8,71 @@ dependencies, so won't take long to try out: ```bash $ spack install pigz -[+] /usr (external glibc-2.34-nuyxhw7kdup423xfoh3erg5yl7c3xrlh) -==> Installing gcc-runtime-11.4.1-7hex6dyh2ttbdeywfkq5vbsinmnhjoub [2/5] -==> No binary for gcc-runtime-11.4.1-7hex6dyh2ttbdeywfkq5vbsinmnhjoub found: installing from source +==> Compilers have been configured automatically from PATH inspection +==> Fetching https://ghcr.io/v2/spack/bootstrap-buildcache-v2.2/blobs/sha256:2010a2a50b9620c2bda7c5fa4e9ce137a115dbba35094857fecc819d9a00a789 +==> Fetching https://ghcr.io/v2/spack/bootstrap-buildcache-v2.2/blobs/sha256:31f1649728e2d58902eb62d1c2e37b1cfc73e007089322a17463b3cb5777cb98 +==> Installing "clingo-bootstrap@=spack~apps~docs+ipo+optimized+python+static_libstdcpp build_system=cmake build_type=Release commit=2a025667090d71b2c9dce60fe924feb6bde8f667 generator=make patches:=bebb819,ec99431 platform=linux os=centos7 target=x86_64" from a buildcache +[+] /usr (external gcc-11.4.1-fhj3ubjk6iqnexhdlnbcmlzjuo3iofwa) +[+] /usr (external glibc-2.34-z4aqp7yhzcmlwfcydu32wbqxounooohb) +==> No binary for compiler-wrapper-1.0-et3ejxhc3t5yoqbw74pqjv4izptvuj3q found: installing from source +==> Installing compiler-wrapper-1.0-et3ejxhc3t5yoqbw74pqjv4izptvuj3q [3/7] +==> Fetching https://mirror.spack.io/_source-cache/archive/a5/a5ff4fcdbeda284a7993b87f294b6338434cffc84ced31e4d04008ed5ea389bf + [100%] 30.08 KB @ 14.2 MB/s +==> No patches needed for compiler-wrapper +==> compiler-wrapper: Executing phase: 'install' +==> compiler-wrapper: Successfully installed compiler-wrapper-1.0-et3ejxhc3t5yoqbw74pqjv4izptvuj3q + Stage: 0.10s. Install: 0.00s. Post-install: 0.01s. Total: 0.13s +[+] /users/example/spack/opt/spack/linux-zen4/compiler-wrapper-1.0-et3ejxhc3t5yoqbw74pqjv4izptvuj3q +==> No binary for gcc-runtime-11.4.1-dqguk7uvu3bh5ehxvi7sv4qaartrartc found: installing from source +==> Installing gcc-runtime-11.4.1-dqguk7uvu3bh5ehxvi7sv4qaartrartc [4/7] ==> No patches needed for gcc-runtime ==> gcc-runtime: Executing phase: 'install' -==> gcc-runtime: Successfully installed gcc-runtime-11.4.1-7hex6dyh2ttbdeywfkq5vbsinmnhjoub - Stage: 0.00s. Install: 0.05s. Post-install: 0.04s. Total: 0.11s -[+] /users/example/spack/opt/spack/linux-rocky9-zen4/gcc-11.4.1/gcc-runtime-11.4.1-7hex6dyh2ttbdeywfkq5vbsinmnhjoub -==> Installing gmake-4.4.1-36fbslt63hhoisn7shlrkgd5fsb2awmz [3/5] -==> No binary for gmake-4.4.1-36fbslt63hhoisn7shlrkgd5fsb2awmz found: installing from source +==> gcc-runtime: Successfully installed gcc-runtime-11.4.1-dqguk7uvu3bh5ehxvi7sv4qaartrartc + Stage: 0.00s. Install: 0.03s. Post-install: 0.01s. Total: 0.06s +[+] /users/example/spack/opt/spack/linux-zen4/gcc-runtime-11.4.1-dqguk7uvu3bh5ehxvi7sv4qaartrartc +==> No binary for gmake-4.4.1-2fg6qftd7dmzej4hzzydnwqwnbl65zac found: installing from source +==> Installing gmake-4.4.1-2fg6qftd7dmzej4hzzydnwqwnbl65zac [5/7] ==> Fetching https://mirror.spack.io/_source-cache/archive/dd/dd16fb1d67bfab79a72f5e8390735c49e3e8e70b4945a15ab1f81ddb78658fb3.tar.gz + [100%] 2.35 MB @ 21.5 MB/s ==> No patches needed for gmake ==> gmake: Executing phase: 'install' -==> gmake: Successfully installed gmake-4.4.1-36fbslt63hhoisn7shlrkgd5fsb2awmz - Stage: 0.21s. Install: 10.08s. Post-install: 0.02s. Total: 10.36s -[+] /users/example/spack/opt/spack/linux-rocky9-zen4/gcc-11.4.1/gmake-4.4.1-36fbslt63hhoisn7shlrkgd5fsb2awmz -==> Installing zlib-ng-2.2.1-5rrpd7bzlpd7tgfp6nfw2z23dpzpcme7 [4/5] -==> No binary for zlib-ng-2.2.1-5rrpd7bzlpd7tgfp6nfw2z23dpzpcme7 found: installing from source -==> Fetching https://mirror.spack.io/_source-cache/archive/ec/ec6a76169d4214e2e8b737e0850ba4acb806c69eeace6240ed4481b9f5c57cdf.tar.gz +==> gmake: Successfully installed gmake-4.4.1-2fg6qftd7dmzej4hzzydnwqwnbl65zac + Stage: 0.23s. Install: 9.98s. Post-install: 0.00s. Total: 10.24s +[+] /users/example/spack/opt/spack/linux-zen4/gmake-4.4.1-2fg6qftd7dmzej4hzzydnwqwnbl65zac +==> No binary for zlib-ng-2.2.4-ro6otklzkonqr26ittllhxc72qpr37le found: installing from source +==> Installing zlib-ng-2.2.4-ro6otklzkonqr26ittllhxc72qpr37le [6/7] +==> Fetching https://mirror.spack.io/_source-cache/archive/a7/a73343c3093e5cdc50d9377997c3815b878fd110bf6511c2c7759f2afb90f5a3.tar.gz + [100%] 2.42 MB @ 23.8 MB/s ==> No patches needed for zlib-ng ==> zlib-ng: Executing phase: 'autoreconf' ==> zlib-ng: Executing phase: 'configure' ==> zlib-ng: Executing phase: 'build' ==> zlib-ng: Executing phase: 'install' -==> zlib-ng: Successfully installed zlib-ng-2.2.1-5rrpd7bzlpd7tgfp6nfw2z23dpzpcme7 - Stage: 0.18s. Autoreconf: 0.00s. Configure: 2.81s. Build: 0.97s. Install: 0.08s. Post-install: 0.04s. Total: 4.18s -[+] /users/example/spack/opt/spack/linux-rocky9-zen4/gcc-11.4.1/zlib-ng-2.2.1-5rrpd7bzlpd7tgfp6nfw2z23dpzpcme7 -==> Installing pigz-2.8-gqwhh2itznlqmnlv7qfddsckiogws3jh [5/5] -==> No binary for pigz-2.8-gqwhh2itznlqmnlv7qfddsckiogws3jh found: installing from source +==> zlib-ng: Successfully installed zlib-ng-2.2.4-ro6otklzkonqr26ittllhxc72qpr37le + Stage: 0.22s. Autoreconf: 0.00s. Configure: 2.89s. Build: 0.83s. Install: 0.37s. Post-install: 0.01s. Total: 4.39s +[+] /users/example/spack/opt/spack/linux-zen4/zlib-ng-2.2.4-ro6otklzkonqr26ittllhxc72qpr37le +==> No binary for pigz-2.8-rpmloasg4nsdwdo4lvyskke7xex5nvdd found: installing from source +==> Installing pigz-2.8-rpmloasg4nsdwdo4lvyskke7xex5nvdd [7/7] ==> Fetching https://mirror.spack.io/_source-cache/archive/2f/2f7f6a6986996d21cb8658535fff95f1c7107ddce22b5324f4b41890e2904706.tar.gz + [100%] 128.77 KB @ 3.1 MB/s ==> No patches needed for pigz ==> pigz: Executing phase: 'edit' ==> pigz: Executing phase: 'build' ==> pigz: Executing phase: 'install' -==> pigz: Successfully installed pigz-2.8-gqwhh2itznlqmnlv7qfddsckiogws3jh - Stage: 0.09s. Edit: 0.00s. Build: 1.02s. Install: 0.01s. Post-install: 0.03s. Total: 1.23s -[+] /users/example/spack/opt/spack/linux-rocky9-zen4/gcc-11.4.1/pigz-2.8-gqwhh2itznlqmnlv7qfddsckiogws3jh +==> pigz: Successfully installed pigz-2.8-rpmloasg4nsdwdo4lvyskke7xex5nvdd + Stage: 0.12s. Edit: 0.00s. Build: 0.98s. Install: 0.00s. Post-install: 0.00s. Total: 1.15s +[+] /users/example/spack/opt/spack/linux-zen4/pigz-2.8-rpmloasg4nsdwdo4lvyskke7xex5nvdd ``` -That is quite wordy, but to note, it's installed pigz, and also installed -dependencies of it, gcc-runtime, gmake, zlib-ng and pigz. Just asking for it -to install that one package has led to it downloading and installing several, -and it's quite happy doing that without needing any guidance. +That is quite wordy, but you can see that it has installed pigz, after first +installing its dependencies, compiler-wrapper, gcc-runtime, gmake, zlib-ng. +Just asking for it to install that one package has led to it downloading and +installing several, and it's quite happy doing that without needing any +guidance. ```bash $ spack find pigz --- linux-rocky9-zen4 / gcc@11.4.1 ------------------------------- +-- linux-rocky9-zen4 / %c=gcc@11.4.1 ---------------------------- pigz@2.8 ==> 1 installed package ``` @@ -63,9 +81,12 @@ At this point we can see there's a version available for pigz. I can now load this, and test it to confirm it's worked: ```bash +$ pigz --version +pigz 2.5 $ spack load pigz $ pigz --version pigz 2.8 +$ spack unload pigz ``` Excellent. To recap, by this stage we've: @@ -80,11 +101,11 @@ much work yet. ## Exercise -Install and use the `k8` software, so we can find out the square root of 25 -using javascript. The command you can run to test your install is: +Install and use the `cowsay` software, to produce an ascii picture of a cow +saying "Moooo!". The command you can run to test your install is: ``` -k8 -e 'print( "Square root of 25 is: " + Math.sqrt(25) );' +cowsay Moooo! ```
@@ -92,32 +113,40 @@ k8 -e 'print( "Square root of 25 is: " + Math.sqrt(25) );' ### Solution -- Install k8 +- Install cowsay - ``` - spack install k8 + ```bash + spack install cowsay ``` - Find the name of the module - ``` - $ spack find k8 - -- linux-rocky9-zen4 / gcc@11.4.1 ------------------------------- - k8@0.2.4 + ```bash + $ spack find cowsay + -- linux-rocky9-zen4 / no compilers ----------------------------- + cowsay@3.04 ==> 1 installed package ``` - Load the software - ``` - $ spack load k8 + ```bash + spack load cowsay ``` - Test the software - ``` - $ k8 -e 'print( "Square root of 25 is: " + Math.sqrt(25) );' - Square root of 25 is: 5 + ```bash + $ cowsay Moooo! + ________ + < Moooo! > + -------- + \ ^__^ + \ (oo)\_______ + (__)\ )\/\ + ||----w | + || || + $ spack unload cowsay ```
diff --git a/book/course/theory.md b/book/course/theory.md index 11f1f7d..1fc91e8 100644 --- a/book/course/theory.md +++ b/book/course/theory.md @@ -27,7 +27,7 @@ Below are the key types of files: * **executables** - these types of files contain compiled binary code. When opened, the operating system will execute the instructions of the code to run the program. * **libraries** - In programming, a library contains common functions that can be reused in programs. -```{Note} +```{note} The term library is also used in computing for the name of a collection of related files, such as in /usr/share or as a package, eg tensorflow. ``` @@ -69,7 +69,7 @@ Software usually requires installation before it can be run. This is usually by * archive with instructions of how to extract and install and run. * source archives with build instructions and scripts. -```{Note} +```{note} It's possible to have an executable program or script that can be run directly without installation. ``` @@ -132,7 +132,7 @@ They can contain options for conditional builds such as for debug and release bu ### Packaging -```{Note} +```{note} Users do not have sufficient permissions to install system packages on our HPC systems. Containers provide a mechanism for users to install system packages in a user managed environment. ``` @@ -165,7 +165,7 @@ They utilise an **Operating System** (eg Windows, OSX, Android Linux) to control ### Virtual Machines -```{Note} +```{note} Aire cannot run virtual machines. ``` diff --git a/book/myst.yml b/book/myst.yml new file mode 100644 index 0000000..0aaf2c2 --- /dev/null +++ b/book/myst.yml @@ -0,0 +1,41 @@ +version: 1 +project: + title: 'HPC2: Installing and Managing Applications on the HPC' + authors: + - name: Research Computing Team + affiliation: University of Leeds + url: https://arc.leeds.ac.uk + copyright: '2025' + github: ARCTraining/hpc2-software + thebe: + binder: + repo: ARCTraining/hpc2-software + provider: github + url: https://mybinder.org + ref: main + toc: + - file: welcome.md + - file: course/theory.md + - file: course/conda.md + - file: course/autotools.md + - file: course/cmake.md + - file: course/containers.md + - file: course/spack.md + children: + - file: course/spack/installing.md + - file: course/spack/testinstall.md + - file: course/spack/existing.md + - file: course/spack/advanced.md + - file: course/spack/containers.md + - file: course/spack/environments.md + - file: course/spack/recipes.md + - file: course/spack/modules.md + - url: https://arc.leeds.ac.uk + title: Research Computing Website +site: + options: + logo: ./assets/img/logo/logo.png + favicon: ./assets/img/logo/favicon-32x32.png + analytics_google: '' + folders: true + template: book-theme diff --git a/book/welcome.md b/book/welcome.md index 291792a..1bbd40a 100644 --- a/book/welcome.md +++ b/book/welcome.md @@ -12,14 +12,14 @@ The purpose of the workshop is to introduce the application development environm At the end of this workshop, learners will be able to: -1. [ ] Understand what an executable is -2. [ ] Understand the how filesystem configuration and environment variables relate to software -3. [ ] Develop hands on experience with: - 1. [ ] Using Conda package manager - 2. [ ] Building software using GNU Autotools - 3. [ ] Building software using CMake - 4. [ ] Using Singularity/Apptainer containers - 5. [ ] Using Spack package manager +- Understand what an executable is +- Understand the how filesystem configuration and environment variables relate to software +- Develop hands on experience with: + - Using Conda package manager + - Building software using GNU Autotools + - Building software using CMake + - Using Singularity/Apptainer containers + - Using Spack package manager ## Prerequisites diff --git a/environment.yml b/environment.yml index 41e6f52..0b7257c 100644 --- a/environment.yml +++ b/environment.yml @@ -3,6 +3,6 @@ channels: - defaults - conda-forge dependencies: - - python=3.9 - - jinja2=3.0.3 - - jupyter-book==0.15.1 + - jinja2=3.1.6 + - jupyter-book=2.0.2 + - python=3.14