From c88cc0256be407e07e9e74be7ff7272ddbde5316 Mon Sep 17 00:00:00 2001 From: cssherman Date: Tue, 21 Oct 2025 10:00:12 -0700 Subject: [PATCH 01/10] Testing clean version of pygeosx install --- .github/workflows/docker_build_tpls.yml | 118 +++++++------- docker/rocky-spack-pygeosx.yaml | 153 ++++++++++++++++++ docker/rocky-spack.yaml | 2 +- ...ockylinux-clang-cuda-12-pygeosx.Dockerfile | 116 +++++++++++++ .../spack_packages/packages/geosx/package.py | 5 +- 5 files changed, 335 insertions(+), 59 deletions(-) create mode 100644 docker/rocky-spack-pygeosx.yaml create mode 100644 docker/tpl-rockylinux-clang-cuda-12-pygeosx.Dockerfile diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index b4181e62..abc1eacf 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -19,68 +19,72 @@ jobs: matrix: include: - - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3) - DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 - TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - DOCKER_ROOT_IMAGE: ubuntu:20.04 - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=9" - RUNS_ON: ubuntu-latest - - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces - DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 - TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - DOCKER_ROOT_IMAGE: mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=10" - RUNS_ON: ubuntu-latest - - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2) - DOCKER_ROOT_IMAGE: ubuntu:22.04 - DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 - TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=11" - RUNS_ON: ubuntu-latest - - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2) - DOCKER_ROOT_IMAGE: ubuntu:22.04 - DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12 - TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=12" - RUNS_ON: ubuntu-latest - - name: Ubuntu (22.04, clang 15.0.7 + gcc 11.4.0, open-mpi 4.1.2) - DOCKER_ROOT_IMAGE: ubuntu:22.04 - DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15 - TPL_DOCKERFILE: docker/tpl-ubuntu-clang.Dockerfile - DOCKER_COMPILER_BUILD_ARG: "--build-arg CLANG_MAJOR_VERSION=15 --build-arg GCC_MAJOR_VERSION=11" - RUNS_ON: ubuntu-latest - - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) - DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 - TPL_DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile - RUNS_ON: ubuntu-latest + # - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3) + # DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 + # TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + # DOCKER_ROOT_IMAGE: ubuntu:20.04 + # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=9" + # RUNS_ON: ubuntu-latest + # - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces + # DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 + # TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + # DOCKER_ROOT_IMAGE: mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 + # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=10" + # RUNS_ON: ubuntu-latest + # - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2) + # DOCKER_ROOT_IMAGE: ubuntu:22.04 + # DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 + # TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=11" + # RUNS_ON: ubuntu-latest + # - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2) + # DOCKER_ROOT_IMAGE: ubuntu:22.04 + # DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12 + # TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=12" + # RUNS_ON: ubuntu-latest + # - name: Ubuntu (22.04, clang 15.0.7 + gcc 11.4.0, open-mpi 4.1.2) + # DOCKER_ROOT_IMAGE: ubuntu:22.04 + # DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15 + # TPL_DOCKERFILE: docker/tpl-ubuntu-clang.Dockerfile + # DOCKER_COMPILER_BUILD_ARG: "--build-arg CLANG_MAJOR_VERSION=15 --build-arg GCC_MAJOR_VERSION=11" + # RUNS_ON: ubuntu-latest + # - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) + # DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 + # TPL_DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile + # RUNS_ON: ubuntu-latest - name: Rockylinux (8, gcc 13.3, cuda 12.9.1) DOCKER_REPOSITORY: geosx/rockylinux8-gcc13-cuda12.9.1 TPL_DOCKERFILE: docker/tpl-rockylinux-gcc-cuda-12.Dockerfile RUNS_ON: Runner_4core_16GB - - name: Rockylinux (8, clang 17.0.6, cuda 12.9.1) - DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.9.1 - TPL_DOCKERFILE: docker/tpl-rockylinux-clang-cuda-12.Dockerfile + - name: Rockylinux (8, gcc 13.3, cuda 12.9.1, pygeosx) + DOCKER_REPOSITORY: geosx/rockylinux8-gcc13-cuda12.9.1 + TPL_DOCKERFILE: docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile RUNS_ON: Runner_4core_16GB - - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10) - DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18 - TPL_DOCKERFILE: docker/TotalEnergies/Pangea3.Dockerfile - DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4 - HOST_ARCH: ppc64le - RUNS_ON: streak2 - - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) - DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 - TPL_DOCKERFILE: docker/Stanford/Dockerfile - DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 - SPEC: "%gcc@10.1.0~pygeosx~openmp" - INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU - RUNS_ON: ubuntu-latest - - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 12.4.0, openblas 0.3.10, zlib 1.2.11) - DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11 - TPL_DOCKERFILE: docker/Stanford/Dockerfile - DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 - SPEC: "%gcc@10.1.0~pygeosx~openmp cuda_arch=70 ^cuda@12.4.0+allow-unsupported-compilers" - INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU - RUNS_ON: ubuntu-latest + # - name: Rockylinux (8, clang 17.0.6, cuda 12.9.1) + # DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.9.1 + # TPL_DOCKERFILE: docker/tpl-rockylinux-clang-cuda-12.Dockerfile + # RUNS_ON: Runner_4core_16GB + # - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10) + # DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18 + # TPL_DOCKERFILE: docker/TotalEnergies/Pangea3.Dockerfile + # DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4 + # HOST_ARCH: ppc64le + # RUNS_ON: streak2 + # - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) + # DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 + # TPL_DOCKERFILE: docker/Stanford/Dockerfile + # DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 + # SPEC: "%gcc@10.1.0~pygeosx~openmp" + # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU + # RUNS_ON: ubuntu-latest + # - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 12.4.0, openblas 0.3.10, zlib 1.2.11) + # DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11 + # TPL_DOCKERFILE: docker/Stanford/Dockerfile + # DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 + # SPEC: "%gcc@10.1.0~pygeosx~openmp cuda_arch=70 ^cuda@12.4.0+allow-unsupported-compilers" + # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU + # RUNS_ON: ubuntu-latest steps: - name: Checkout diff --git a/docker/rocky-spack-pygeosx.yaml b/docker/rocky-spack-pygeosx.yaml new file mode 100644 index 00000000..9c0c7af3 --- /dev/null +++ b/docker/rocky-spack-pygeosx.yaml @@ -0,0 +1,153 @@ +spack: + config: + install_tree: + root: $spack/.. + projections: + all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + misc_cache: $spack/../misc_cache + test_stage: $spack/../test_stage + build_stage:: + - $spack/../build_stage + + # Regular TPLs do not need views + view: false + + # Include shared variants and versions + include: + - ../defaults.yaml + - ../versions.yaml + + compilers:: + - compiler: + extra_rpaths: [] + flags: + cflags: -pthread + cxxflags: -pthread + modules: [] + operating_system: rocky8 + paths: + cc: /usr/bin/clang + cxx: /usr/bin/clang++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + spec: clang@17.0.6 + target: x86_64 + - compiler: + extra_rpaths: [] + flags: + cflags: -pthread + cxxflags: -pthread + modules: [] + operating_system: rocky8 + paths: + cc: /opt/rh/gcc-toolset-13/root/usr/bin/gcc + cxx: /opt/rh/gcc-toolset-13/root/usr/bin/g++ + f77: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran + fc: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran + spec: gcc@13.3.1 + target: x86_64 + + packages: + all: + target: [x86_64] + compiler: [gcc, clang] + providers: + blas: [netlib-lapack] + lapack: [netlib-lapack] + mpi: [openmpi] + + autoconf: + version: [2.71] + buildable: false + externals: + - spec: autoconf@2.71 + prefix: /usr + automake: + version: [1.16.5] + buildable: false + externals: + - spec: automake@1.16.5 + prefix: /usr + cmake: + version: [3.28.6] + buildable: false + externals: + - spec: cmake@3.28.6 + prefix: /usr/local + cuda: + buildable: False + externals: + - spec: cuda@12.9.1%gcc@13.3.1 +allow-unsupported-compilers + prefix: /usr/local/cuda + - spec: cuda@12.9.1%clang@17.0.6 +allow-unsupported-compilers + prefix: /usr/local/cuda + m4: + buildable: false + externals: + - spec: m4@1.4.18 + prefix: /usr + + # Spack may grab for mpi & we don't want to use them + mpi: + buildable: false + + netlib-lapack: + buildable: false + externals: + - spec: netlib-lapack@3.8.0 + prefix: /usr + netlib-blas: + buildable: false + externals: + - spec: netlib-blas@3.8.0 + prefix: /usr + openmpi: + externals: + - spec: openmpi@4.1.1 + prefix: /usr/lib64/openmpi + perl: + buildable: false + externals: + - spec: perl@5.26.3 + prefix: /usr + pkg-config: + buildable: false + externals: + - spec: pkg-config@1.4.2 + prefix: /usr + # python: + # buildable: false + # externals: + # - spec: python@3.6.8 + # prefix: /usr + tar: + buildable: false + externals: + - spec: tar@1.34 + prefix: /usr + unzip: + buildable: false + externals: + - spec: unzip@6.0 + prefix: /usr + xz: + buildable: false + externals: + - spec: xz@5.2.5 + prefix: /usr + # Pygeosx dependencies + openssl: + buildable: false + externals: + - spec: openssl@1.1.1 + prefix: /usr + tk: + buildable: false + externals: + - spec: tk@8.6.8 + prefix: /usr + tcl: + buildable: false + externals: + - spec: tcl@8.6.8 + prefix: /usr diff --git a/docker/rocky-spack.yaml b/docker/rocky-spack.yaml index 94025319..a11b9dcd 100644 --- a/docker/rocky-spack.yaml +++ b/docker/rocky-spack.yaml @@ -118,7 +118,7 @@ spack: python: buildable: false externals: - - spec: python@3.6.8 + - spec: python@3.9.20 prefix: /usr tar: buildable: false diff --git a/docker/tpl-rockylinux-clang-cuda-12-pygeosx.Dockerfile b/docker/tpl-rockylinux-clang-cuda-12-pygeosx.Dockerfile new file mode 100644 index 00000000..1fc04eee --- /dev/null +++ b/docker/tpl-rockylinux-clang-cuda-12-pygeosx.Dockerfile @@ -0,0 +1,116 @@ +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/thirdPartyLibs +ARG BLD_DIR=$TMP_DIR/build + +FROM nvidia/cuda:12.9.1-devel-rockylinux8 AS tpl_toolchain_intersect_geosx_toolchain +ARG SRC_DIR + +ARG INSTALL_DIR +ENV GEOSX_TPL_DIR=$INSTALL_DIR + +# Installing dependencies +RUN dnf clean all && \ + dnf -y update && \ + dnf -y install \ + which \ + gcc-toolset-13 \ + python3.9 \ + zlib-devel \ + tbb \ + blas \ + lapack \ + openmpi \ + openmpi-devel \ + # Additional spack dependencies + python3-pip \ + python3-tkinter \ + unzip \ + mpfr-devel \ + bzip2 \ + gnupg \ + xz \ + python3-virtualenv + +# Install clingo for Spack +RUN python3 -m pip install --upgrade pip && \ + python3 -m pip install clingo + +# Custom install script for CMake or other tools +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh + +# Installing TPL's +FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain +ARG SRC_DIR +ARG BLD_DIR + +# Install additional required packages +RUN dnf clean all && \ + dnf -y update && \ + dnf -y install \ + tbb-devel \ + bc \ + file \ + patch \ + ca-certificates \ + autoconf \ + automake \ + m4 \ + git + +# Run uberenv +# Have to create install directory first for uberenv +# -k flag is to ignore SSL errors +RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ + mkdir -p ${GEOSX_TPL_DIR} && \ +# Create symlink to openmpi include directory + ln -s /usr/include/openmpi-x86_64 /usr/lib64/openmpi/include && \ +# Create symlinks to blas/lapack libraries + ln -s /usr/lib64/libblas.so.3 /usr/lib64/libblas.so && \ + ln -s /usr/lib64/liblapack.so.3 /usr/lib64/liblapack.so && \ + ./scripts/uberenv/uberenv.py \ + --spec "%gcc@13.3.1+cuda~uncrustify~openmp+pygeosxdeps cuda_arch=70 ^cuda@12.9.1+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ + --spack-env-file=${SRC_DIR}/docker/rocky-spack-pygeosx.yaml \ + --project-json=.uberenv_config.json \ + --prefix ${GEOSX_TPL_DIR} \ + -k && \ +# Remove host-config generated for LvArray + rm lvarray* && \ +# Rename and copy spack-generated host-config to root directory + cp *.cmake /spack-generated.cmake && \ +# Remove extraneous spack files + cd ${GEOSX_TPL_DIR} && \ + rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + +# Extract only TPL's from the previous stage +FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain +ARG SRC_DIR + +COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR + +# Extract the generated host-config +COPY --from=tpl_toolchain /spack-generated.cmake / + +# Final installation of packages and tools +RUN dnf clean all && \ + rm -rf /var/cache/dnf && \ + dnf -y install dnf-plugins-core && \ + dnf config-manager --set-enabled devel && \ + dnf -y update && \ + dnf -y install \ + openssh-clients \ + ca-certificates \ + curl \ + python3 \ + texlive \ + graphviz \ + ninja-build \ + git && \ +# Regenerate symlink to openmpi include directory + ln -s /usr/include/openmpi-x86_64 /usr/lib64/openmpi/include && \ +# Regenerate symlinks to blas/lapack libraries + ln -s /usr/lib64/libblas.so.3 /usr/lib64/libblas.so && \ + ln -s /usr/lib64/liblapack.so.3 /usr/lib64/liblapack.so + +# Install sccache +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh +ENV SCCACHE=/opt/sccache/bin/sccache diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index 6405d322..7f2cb3c3 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -190,7 +190,10 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): # Python # depends_on('python') - + with when("+pygeosxdeps"): + depends_on('python@3.10:+shared+pic+tkinter+optimizations') + depends_on("py-numpy@1.21.0:1.23.4+blas+lapack") + depends_on('py-mpi4py') # # Dev tools From e62ce7b6560086ee2bef6fb28e39cb237348b6f6 Mon Sep 17 00:00:00 2001 From: cssherman Date: Tue, 21 Oct 2025 10:07:50 -0700 Subject: [PATCH 02/10] Adding additional dockerfile --- ...-rockylinux-gcc-cuda-12-pygeosx.Dockerfile | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile diff --git a/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile b/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile new file mode 100644 index 00000000..1fc04eee --- /dev/null +++ b/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile @@ -0,0 +1,116 @@ +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/thirdPartyLibs +ARG BLD_DIR=$TMP_DIR/build + +FROM nvidia/cuda:12.9.1-devel-rockylinux8 AS tpl_toolchain_intersect_geosx_toolchain +ARG SRC_DIR + +ARG INSTALL_DIR +ENV GEOSX_TPL_DIR=$INSTALL_DIR + +# Installing dependencies +RUN dnf clean all && \ + dnf -y update && \ + dnf -y install \ + which \ + gcc-toolset-13 \ + python3.9 \ + zlib-devel \ + tbb \ + blas \ + lapack \ + openmpi \ + openmpi-devel \ + # Additional spack dependencies + python3-pip \ + python3-tkinter \ + unzip \ + mpfr-devel \ + bzip2 \ + gnupg \ + xz \ + python3-virtualenv + +# Install clingo for Spack +RUN python3 -m pip install --upgrade pip && \ + python3 -m pip install clingo + +# Custom install script for CMake or other tools +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh + +# Installing TPL's +FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain +ARG SRC_DIR +ARG BLD_DIR + +# Install additional required packages +RUN dnf clean all && \ + dnf -y update && \ + dnf -y install \ + tbb-devel \ + bc \ + file \ + patch \ + ca-certificates \ + autoconf \ + automake \ + m4 \ + git + +# Run uberenv +# Have to create install directory first for uberenv +# -k flag is to ignore SSL errors +RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ + mkdir -p ${GEOSX_TPL_DIR} && \ +# Create symlink to openmpi include directory + ln -s /usr/include/openmpi-x86_64 /usr/lib64/openmpi/include && \ +# Create symlinks to blas/lapack libraries + ln -s /usr/lib64/libblas.so.3 /usr/lib64/libblas.so && \ + ln -s /usr/lib64/liblapack.so.3 /usr/lib64/liblapack.so && \ + ./scripts/uberenv/uberenv.py \ + --spec "%gcc@13.3.1+cuda~uncrustify~openmp+pygeosxdeps cuda_arch=70 ^cuda@12.9.1+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ + --spack-env-file=${SRC_DIR}/docker/rocky-spack-pygeosx.yaml \ + --project-json=.uberenv_config.json \ + --prefix ${GEOSX_TPL_DIR} \ + -k && \ +# Remove host-config generated for LvArray + rm lvarray* && \ +# Rename and copy spack-generated host-config to root directory + cp *.cmake /spack-generated.cmake && \ +# Remove extraneous spack files + cd ${GEOSX_TPL_DIR} && \ + rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + +# Extract only TPL's from the previous stage +FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain +ARG SRC_DIR + +COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR + +# Extract the generated host-config +COPY --from=tpl_toolchain /spack-generated.cmake / + +# Final installation of packages and tools +RUN dnf clean all && \ + rm -rf /var/cache/dnf && \ + dnf -y install dnf-plugins-core && \ + dnf config-manager --set-enabled devel && \ + dnf -y update && \ + dnf -y install \ + openssh-clients \ + ca-certificates \ + curl \ + python3 \ + texlive \ + graphviz \ + ninja-build \ + git && \ +# Regenerate symlink to openmpi include directory + ln -s /usr/include/openmpi-x86_64 /usr/lib64/openmpi/include && \ +# Regenerate symlinks to blas/lapack libraries + ln -s /usr/lib64/libblas.so.3 /usr/lib64/libblas.so && \ + ln -s /usr/lib64/liblapack.so.3 /usr/lib64/liblapack.so + +# Install sccache +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh +ENV SCCACHE=/opt/sccache/bin/sccache From 8ddb24475a2c5a34e511252628a48003c9ef4b5a Mon Sep 17 00:00:00 2001 From: cssherman Date: Tue, 21 Oct 2025 10:10:08 -0700 Subject: [PATCH 03/10] Fixing package bug --- scripts/spack_packages/packages/geosx/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index 7f2cb3c3..a32b9fef 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -73,6 +73,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): multi=False) variant('grpc', default=False, description='Enable gRPC.') variant('pygeosx', default=True, description='Enable pygeosx.') + variant('pygeosxdeps', default=False, description='Build pygeosx dependencies.') # SPHINX_END_VARIANTS From e8d563f2981ea3790a6aad02b16457b8f23ed1c5 Mon Sep 17 00:00:00 2001 From: cssherman Date: Tue, 21 Oct 2025 14:40:55 -0700 Subject: [PATCH 04/10] Fixing package definition --- scripts/spack_packages/packages/geosx/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index a32b9fef..ce5a047b 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -193,7 +193,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): depends_on('python') with when("+pygeosxdeps"): depends_on('python@3.10:+shared+pic+tkinter+optimizations') - depends_on("py-numpy@1.21.0:1.23.4+blas+lapack") + depends_on("py-numpy@1.23.0:") depends_on('py-mpi4py') # From 71d5e58c24f8b17e6c0accf3c259c35d8e4037cd Mon Sep 17 00:00:00 2001 From: cssherman Date: Thu, 23 Oct 2025 09:49:02 -0700 Subject: [PATCH 05/10] Adding additional external package to pygeosx build --- docker/rocky-spack-pygeosx.yaml | 5 +++++ docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile | 3 +++ 2 files changed, 8 insertions(+) diff --git a/docker/rocky-spack-pygeosx.yaml b/docker/rocky-spack-pygeosx.yaml index 9c0c7af3..2cb825e1 100644 --- a/docker/rocky-spack-pygeosx.yaml +++ b/docker/rocky-spack-pygeosx.yaml @@ -151,3 +151,8 @@ spack: externals: - spec: tcl@8.6.8 prefix: /usr + libxcrypt: + buildable: false + externals: + - spec: libxcrypt@4.4.35 + prefix: /usr diff --git a/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile b/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile index 1fc04eee..ec3d6779 100644 --- a/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile +++ b/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile @@ -29,8 +29,11 @@ RUN dnf clean all && \ bzip2 \ gnupg \ xz \ + libxcrypt \ python3-virtualenv +RUN ls -lah /usr/lib | grep xcrypt + # Install clingo for Spack RUN python3 -m pip install --upgrade pip && \ python3 -m pip install clingo From 0eba8c68a65aff222de262a4c9dad694ce13ae8d Mon Sep 17 00:00:00 2001 From: cssherman Date: Thu, 23 Oct 2025 11:42:53 -0700 Subject: [PATCH 06/10] Adding additional external package to pygeosx build --- docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile b/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile index ec3d6779..e1777492 100644 --- a/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile +++ b/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile @@ -32,7 +32,8 @@ RUN dnf clean all && \ libxcrypt \ python3-virtualenv -RUN ls -lah /usr/lib | grep xcrypt +RUN ls -lah /usr | grep xcrypt && \ + ls -lah /usr/lib64 | grep xcrypt # Install clingo for Spack RUN python3 -m pip install --upgrade pip && \ From 72942a37a1f6d19d3d94006cc6fe646a22cb5950 Mon Sep 17 00:00:00 2001 From: cssherman Date: Thu, 23 Oct 2025 17:01:12 -0700 Subject: [PATCH 07/10] Testing bug fix for pygeosx build --- docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile b/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile index e1777492..723f9479 100644 --- a/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile +++ b/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile @@ -32,9 +32,6 @@ RUN dnf clean all && \ libxcrypt \ python3-virtualenv -RUN ls -lah /usr | grep xcrypt && \ - ls -lah /usr/lib64 | grep xcrypt - # Install clingo for Spack RUN python3 -m pip install --upgrade pip && \ python3 -m pip install clingo From 7380f8e346806643104043ad969cf6bb6f30473d Mon Sep 17 00:00:00 2001 From: cssherman Date: Fri, 24 Oct 2025 11:33:52 -0700 Subject: [PATCH 08/10] Updating pygeosx environment --- docker/rocky-spack-pygeosx.yaml | 4 ++-- docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile | 12 ++++++++++-- scripts/spack_packages/packages/geosx/package.py | 3 ++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/docker/rocky-spack-pygeosx.yaml b/docker/rocky-spack-pygeosx.yaml index 2cb825e1..38b75c77 100644 --- a/docker/rocky-spack-pygeosx.yaml +++ b/docker/rocky-spack-pygeosx.yaml @@ -139,7 +139,7 @@ spack: openssl: buildable: false externals: - - spec: openssl@1.1.1 + - spec: openssl@1.1.1k prefix: /usr tk: buildable: false @@ -154,5 +154,5 @@ spack: libxcrypt: buildable: false externals: - - spec: libxcrypt@4.4.35 + - spec: libxcrypt@4.1.1 prefix: /usr diff --git a/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile b/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile index 723f9479..81e34cb2 100644 --- a/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile +++ b/docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile @@ -56,7 +56,9 @@ RUN dnf clean all && \ autoconf \ automake \ m4 \ - git + git \ + rsync \ + openssl-devel # Run uberenv # Have to create install directory first for uberenv @@ -80,7 +82,13 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ cp *.cmake /spack-generated.cmake && \ # Remove extraneous spack files cd ${GEOSX_TPL_DIR} && \ - rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ && \ +# Move python packages into the correct view + cd gcc-13.3.1 && \ + rsync -avh py-*/ $(ls | grep -- python-3)/ && \ + cd $(ls | grep -- python-3)/bin && \ + ./python -m pip install virtualenv + # Extract only TPL's from the previous stage FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index ce5a047b..af6a5b45 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -192,9 +192,10 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): # depends_on('python') with when("+pygeosxdeps"): - depends_on('python@3.10:+shared+pic+tkinter+optimizations') + depends_on('python@3.10:+shared+pic+tkinter+optimizations+ssl') depends_on("py-numpy@1.23.0:") depends_on('py-mpi4py') + depends_on('py-virtualenv') # # Dev tools From 23a1e62544c392fb0714d91fde43c068af3e8b2f Mon Sep 17 00:00:00 2001 From: cssherman Date: Fri, 24 Oct 2025 14:21:39 -0700 Subject: [PATCH 09/10] Testing dane config for pygeosx --- .../spack_configs/toss_4_x86_64_ib/spack.yaml | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index ebb477b9..f205a15c 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -198,10 +198,10 @@ spack: # This needs to be the prefix to the pygeosx LC installation # or system install with pygeosx dependencies python: - buildable: False + buildable: True externals: - - spec: python@3.10.8 - prefix: /usr/gapps/GEOSX/thirdPartyLibs/python/quartz-gcc-python/python/ + - spec: python@3.13.2~shared + prefix: /usr/apps/python-3.13.2 # This needs to be the prefix to the pygeosx LC installation # or system install with pygeosx dependencies @@ -230,3 +230,20 @@ spack: externals: - spec: gettext@0.19.8.1 prefix: /usr/bin/ + + # pygeosx dependencies + openssl: + buildable: false + externals: + - spec: openssl@1.1.1k + prefix: /usr + tk: + buildable: false + externals: + - spec: tk@8.6.8 + prefix: /usr + tcl: + buildable: false + externals: + - spec: tcl@8.6.8 + prefix: /usr From 9690c669ee4b9a3d39964dff8b5d168b912b93d2 Mon Sep 17 00:00:00 2001 From: cssherman Date: Fri, 24 Oct 2025 14:56:23 -0700 Subject: [PATCH 10/10] Reenabling full test build --- .github/workflows/docker_build_tpls.yml | 116 ++++----- .../toss_4_x86_64_ib/spack-pygeosx.yaml | 234 ++++++++++++++++++ .../spack_configs/toss_4_x86_64_ib/spack.yaml | 19 +- 3 files changed, 293 insertions(+), 76 deletions(-) create mode 100644 scripts/spack_configs/toss_4_x86_64_ib/spack-pygeosx.yaml diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index abc1eacf..be9c744b 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -19,40 +19,40 @@ jobs: matrix: include: - # - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3) - # DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 - # TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - # DOCKER_ROOT_IMAGE: ubuntu:20.04 - # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=9" - # RUNS_ON: ubuntu-latest - # - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces - # DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 - # TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - # DOCKER_ROOT_IMAGE: mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 - # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=10" - # RUNS_ON: ubuntu-latest - # - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2) - # DOCKER_ROOT_IMAGE: ubuntu:22.04 - # DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 - # TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=11" - # RUNS_ON: ubuntu-latest - # - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2) - # DOCKER_ROOT_IMAGE: ubuntu:22.04 - # DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12 - # TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=12" - # RUNS_ON: ubuntu-latest - # - name: Ubuntu (22.04, clang 15.0.7 + gcc 11.4.0, open-mpi 4.1.2) - # DOCKER_ROOT_IMAGE: ubuntu:22.04 - # DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15 - # TPL_DOCKERFILE: docker/tpl-ubuntu-clang.Dockerfile - # DOCKER_COMPILER_BUILD_ARG: "--build-arg CLANG_MAJOR_VERSION=15 --build-arg GCC_MAJOR_VERSION=11" - # RUNS_ON: ubuntu-latest - # - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) - # DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 - # TPL_DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile - # RUNS_ON: ubuntu-latest + - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3) + DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 + TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + DOCKER_ROOT_IMAGE: ubuntu:20.04 + DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=9" + RUNS_ON: ubuntu-latest + - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces + DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 + TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + DOCKER_ROOT_IMAGE: mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 + DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=10" + RUNS_ON: ubuntu-latest + - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2) + DOCKER_ROOT_IMAGE: ubuntu:22.04 + DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 + TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=11" + RUNS_ON: ubuntu-latest + - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2) + DOCKER_ROOT_IMAGE: ubuntu:22.04 + DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12 + TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=12" + RUNS_ON: ubuntu-latest + - name: Ubuntu (22.04, clang 15.0.7 + gcc 11.4.0, open-mpi 4.1.2) + DOCKER_ROOT_IMAGE: ubuntu:22.04 + DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15 + TPL_DOCKERFILE: docker/tpl-ubuntu-clang.Dockerfile + DOCKER_COMPILER_BUILD_ARG: "--build-arg CLANG_MAJOR_VERSION=15 --build-arg GCC_MAJOR_VERSION=11" + RUNS_ON: ubuntu-latest + - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) + DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 + TPL_DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile + RUNS_ON: ubuntu-latest - name: Rockylinux (8, gcc 13.3, cuda 12.9.1) DOCKER_REPOSITORY: geosx/rockylinux8-gcc13-cuda12.9.1 TPL_DOCKERFILE: docker/tpl-rockylinux-gcc-cuda-12.Dockerfile @@ -61,30 +61,30 @@ jobs: DOCKER_REPOSITORY: geosx/rockylinux8-gcc13-cuda12.9.1 TPL_DOCKERFILE: docker/tpl-rockylinux-gcc-cuda-12-pygeosx.Dockerfile RUNS_ON: Runner_4core_16GB - # - name: Rockylinux (8, clang 17.0.6, cuda 12.9.1) - # DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.9.1 - # TPL_DOCKERFILE: docker/tpl-rockylinux-clang-cuda-12.Dockerfile - # RUNS_ON: Runner_4core_16GB - # - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10) - # DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18 - # TPL_DOCKERFILE: docker/TotalEnergies/Pangea3.Dockerfile - # DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4 - # HOST_ARCH: ppc64le - # RUNS_ON: streak2 - # - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) - # DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 - # TPL_DOCKERFILE: docker/Stanford/Dockerfile - # DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 - # SPEC: "%gcc@10.1.0~pygeosx~openmp" - # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU - # RUNS_ON: ubuntu-latest - # - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 12.4.0, openblas 0.3.10, zlib 1.2.11) - # DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11 - # TPL_DOCKERFILE: docker/Stanford/Dockerfile - # DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 - # SPEC: "%gcc@10.1.0~pygeosx~openmp cuda_arch=70 ^cuda@12.4.0+allow-unsupported-compilers" - # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU - # RUNS_ON: ubuntu-latest + - name: Rockylinux (8, clang 17.0.6, cuda 12.9.1) + DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.9.1 + TPL_DOCKERFILE: docker/tpl-rockylinux-clang-cuda-12.Dockerfile + RUNS_ON: Runner_4core_16GB + - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10) + DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18 + TPL_DOCKERFILE: docker/TotalEnergies/Pangea3.Dockerfile + DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4 + HOST_ARCH: ppc64le + RUNS_ON: streak2 + - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) + DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 + TPL_DOCKERFILE: docker/Stanford/Dockerfile + DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 + SPEC: "%gcc@10.1.0~pygeosx~openmp" + INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU + RUNS_ON: ubuntu-latest + - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 12.4.0, openblas 0.3.10, zlib 1.2.11) + DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11 + TPL_DOCKERFILE: docker/Stanford/Dockerfile + DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 + SPEC: "%gcc@10.1.0~pygeosx~openmp cuda_arch=70 ^cuda@12.4.0+allow-unsupported-compilers" + INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU + RUNS_ON: ubuntu-latest steps: - name: Checkout diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack-pygeosx.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack-pygeosx.yaml new file mode 100644 index 00000000..79ff2f8f --- /dev/null +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack-pygeosx.yaml @@ -0,0 +1,234 @@ +#------------------------------------------------------------------------------------------------------------ +# SPDX-License-Identifier: LGPL-2.1-only +# +# Copyright (c) 2018-2020 Lawrence Livermore National Security LLC +# Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University +# Copyright (c) 2018-2020 TotalEnergies +# Copyright (c) 2019- GEOSX Contributors +# All rights reserved +# +# See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. +#------------------------------------------------------------------------------------------------------------ + +# geosx@develop%clang@14.0.6 +# geosx@develop%gcc@12.1.1 +# geosx@develop%gcc@noAVX +# +# Uberenv command to build geos dependencies: +# python3 ./scripts/uberenv/uberenv.py --spec="%clang@14.0.6 +docs" +# +# python3 ./scripts/uberenv/uberenv.py --spec="%gcc@12.1.1 +docs" +# +# No AVX instructions +# python3 ./scripts/uberenv/uberenv.py --spec="%gcc@12noAVX +docs" + + +spack: + config: + install_tree: + root: $spack/.. + projections: + all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + misc_cache: $spack/../misc_cache + test_stage: $spack/../test_stage + build_stage:: + - $spack/../build_stage + + # Regular TPLs do not need views + view: false + + # Include shared variants and versions + include: + - ../defaults.yaml + - ../versions.yaml + + compilers:: + - compiler: + spec: clang@14.0.6 + paths: + cc: /usr/tce/packages/clang/clang-14.0.6-magic/bin/clang + cxx: /usr/tce/packages/clang/clang-14.0.6-magic/bin/clang++ + f77: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran + fc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran + operating_system: rhel8 + target: x86_64 + modules: [] + extra_rpaths: [] + - compiler: + spec: clang@19.1.3 + paths: + cc: /usr/tce/packages/clang/clang-19.1.3-magic/bin/clang + cxx: /usr/tce/packages/clang/clang-19.1.3-magic/bin/clang++ + f77: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran + fc: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran + operating_system: rhel8 + target: x86_64 + modules: [] + extra_rpaths: [] + - compiler: + spec: gcc@12.1.1 + paths: + cc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/g++ + f77: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran + fc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran + operating_system: rhel8 + target: x86_64 + modules: [] + extra_rpaths: [] + - compiler: + spec: gcc@12noAVX + paths: + cc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/g++ + f77: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran + fc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran + flags: + cxxflags: -march=x86-64-v2 -mno-avx512f + operating_system: rhel8 + target: x86_64 + modules: [] + extra_rpaths: [] + - compiler: + spec: gcc@13.3.1 + paths: + cc: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/g++ + f77: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran + fc: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: # Needed for scotch + BISON: bison + FLEX: flex + extra_rpaths: [] + - compiler: + spec: gcc@13noAVX + paths: + cc: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/g++ + f77: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran + fc: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran + flags: + cxxflags: -march=x86-64-v2 -mno-avx512f + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: # Needed for scotch + BISON: bison + FLEX: flex + extra_rpaths: [] + + packages: + all: + target: [ivybridge] + compiler: [gcc, clang] + providers: + mpi: [mvapich2] + blas: [intel-oneapi-mkl] + lapack: [intel-oneapi-mkl] + + # Lock down which MPI we are using + mvapich2: + buildable: False + externals: + - spec: mvapich2@2.3%gcc@12.1.1 process_managers=slurm + prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1-magic + - spec: mvapich2@2.3%gcc@13.3.1 process_managers=slurm + prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-13.3.1-magic + - spec: mvapich2@2.3%gcc@12noAVX process_managers=slurm + prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1-magic + - spec: mvapich2@2.3%clang@14.0.6 process_managers=slurm + prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-14.0.6-magic + - spec: mvapich2@2.3%clang@19.1.3 process_managers=slurm + prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-19.1.3-magic + + intel-oneapi-mkl: + externals: + - spec: intel-oneapi-mkl@2022.1.0 threads=openmp + prefix: /usr/tce/packages/mkl/mkl-2022.1.0/ + buildable: False + cuda: + buildable: False + externals: + - spec: cuda@12.6.0 +allow-unsupported-compilers + prefix: /usr/tce/packages/cuda/cuda-12.6.0 + - spec: cuda@12.9.1 +allow-unsupported-compilers + prefix: /usr/tce/packages/cuda/cuda-12.9.1 + # System level packages to not build + papi: + buildable: False + externals: + - spec: papi@6.0.0.1 + prefix: /usr/tce/packages/papi/papi-6.0.0.1/ + cmake: + version: [3.26.3] + buildable: false + externals: + - spec: cmake@3.26.3 + prefix: /usr/tce/packages/cmake/cmake-3.26.3 + readline: + externals: + - spec: readline@7.0 + prefix: /collab/usr/gapps/python/build/spack-toss3.3/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/readline-7.0-e5jqqjmcjknidgwvi353pd6umpixzxr2 + buildable: false + m4: + buildable: False + externals: + - spec: m4@1.4.18 + prefix: /usr + perl: + buildable: false + externals: + - spec: perl@5.26.3 + prefix: /usr + pkg-config: + buildable: false + externals: + - spec: pkg-config@1.8.0 + prefix: /usr + diffutils: + buildable: False + externals: + - spec: diffutils@3.3 + prefix: /usr/bin + autoconf: + buildable: False + externals: + - spec: autoconf@2.69 + prefix: /usr + automake: + buildable: False + externals: + - spec: automake@1.16.1 + prefix: /usr + libtool: + buildable: False + externals: + - spec: libtool@2.4.6 + prefix: /usr + gettext: + buildable: False + externals: + - spec: gettext@0.19.8.1 + prefix: /usr/bin/ + + # pygeosx dependencies + openssl: + buildable: false + externals: + - spec: openssl@1.1.1k + prefix: /usr + tk: + buildable: false + externals: + - spec: tk@8.6.8 + prefix: /usr + tcl: + buildable: false + externals: + - spec: tcl@8.6.8 + prefix: /usr diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index f205a15c..da28d684 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -209,7 +209,7 @@ spack: buildable: False externals: - spec: py-sphinx@1.6.3 - prefix: /usr/gapps/GEOSX/thirdPartyLibs/python/quartz-gcc-python/python/ + prefix: /usr/apps/python-3.13.2 autoconf: buildable: False externals: @@ -230,20 +230,3 @@ spack: externals: - spec: gettext@0.19.8.1 prefix: /usr/bin/ - - # pygeosx dependencies - openssl: - buildable: false - externals: - - spec: openssl@1.1.1k - prefix: /usr - tk: - buildable: false - externals: - - spec: tk@8.6.8 - prefix: /usr - tcl: - buildable: false - externals: - - spec: tcl@8.6.8 - prefix: /usr