From ffcce7736146ce89ab3467350d77080774f943d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CHenrique?= Date: Thu, 22 Jan 2026 10:26:29 +0000 Subject: [PATCH 1/4] Add ampere build script --- scripts/build_csd3_ampere.sh | 70 ++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 scripts/build_csd3_ampere.sh diff --git a/scripts/build_csd3_ampere.sh b/scripts/build_csd3_ampere.sh new file mode 100644 index 0000000..a4334d4 --- /dev/null +++ b/scripts/build_csd3_ampere.sh @@ -0,0 +1,70 @@ +#!/bin/bash +#SBATCH --job-name=moose_cuda_build_ampere +#SBATCH --output=moose_cuda_build_log.txt +#SBATCH --error=moose_cuda_build_err.txt +#SBATCH --account= +#SBATCH --partition=ampere +#SBATCH --gres=gpu:1 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --time=02:00:00 + +## This script builds MOOSE with MFEM configured for CUDA support. It is designed for the Ampere partition on the CSD3 machine +## To run, substitute with your project account before submitting the job + +export METHOD=dbg +export MOOSE_JOBS=32 +export STDCXX_PATH=/usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-x86_64_v3/gcc-8.5.0/gcc-11.3.0-r7adil6umxm6geju4jjmtwo4ecbkzpeo/lib64 + +module purge +module load rhel8/slurm +module use /usr/local/software/spack/spack-modules/rocky8-a100-20230831/linux-rocky8-zen3 +module load openmpi/4.1.5/gcc/33z33ovn + +git clone https://github.com/spack/spack.git +. spack/share/spack/setup-env.sh + +spack install cmake +spack install python +spack install py-pandas^/$(spack find --format '{hash}' python) +spack install py-pyaml +spack install py-packaging +spack install py-jinja2 +spack install py-setuptools +spack install py-deepdiff +spack install py-xmltodict + +spack load cmake +spack load python +spack load py-pandas +spack load py-pyaml +spack load py-packaging +spack load py-jinja2 +spack load py-setuptools +spack load py-deepdiff +spack load py-xmltodict + +git clone https://github.com/Heinrich-BR/moose.git moose_cuda +cd moose_cuda + +./scripts/update_and_rebuild_petsc.sh --with-cuda --with-cuda-arch=80 +./scripts/update_and_rebuild_libmesh.sh +./scripts/update_and_rebuild_conduit.sh +./scripts/update_and_rebuild_wasp.sh +./scripts/update_and_rebuild_mfem.sh -DMFEM_USE_CUDA=YES -DCUDA_ARCH=sm_80 -DCMAKE_BUILD_TYPE=Debug + +./configure --with-mfem + +cd framework +cp contrib/mfem/build-dbg/config/config.mk contrib/mfem/build-dbg/config/config.mk.bak +cp contrib/mfem/build-dbg/config/config-install.mk contrib/mfem/build-dbg/config/config-install.mk.bak +cp contrib/mfem/installed/share/mfem/config.mk contrib/mfem/installed/share/mfem/config.mk.bak +sed -i.bak 's/\$<\$:SHELL:-Xcompiler *>//g' contrib/mfem/build-dbg/config/config.mk +sed -i.bak 's/\$<\$:SHELL:-Xcompiler *>//g' contrib/mfem/build-dbg/config/config-install.mk +sed -i.bak 's/\$<\$:SHELL:-Xcompiler *>//g' contrib/mfem/installed/share/mfem/config.mk +make -j $MOOSE_JOBS +cd ../test +make -j $MOOSE_JOBS + +export LD_LIBRARY_PATH=${STDCXX_PATH}:$LD_LIBRARY_PATH + From 6eb1b29c0db977ad6aa3dd1cd52ba7b3dab180e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CHenrique?= Date: Thu, 22 Jan 2026 10:58:34 +0000 Subject: [PATCH 2/4] Change moose git link Linting Linting Linting --- scripts/build_csd3_ampere.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) mode change 100644 => 100755 scripts/build_csd3_ampere.sh diff --git a/scripts/build_csd3_ampere.sh b/scripts/build_csd3_ampere.sh old mode 100644 new mode 100755 index a4334d4..ddf823c --- a/scripts/build_csd3_ampere.sh +++ b/scripts/build_csd3_ampere.sh @@ -22,11 +22,12 @@ module use /usr/local/software/spack/spack-modules/rocky8-a100-20230831/linux-ro module load openmpi/4.1.5/gcc/33z33ovn git clone https://github.com/spack/spack.git +# shellcheck source=/dev/null . spack/share/spack/setup-env.sh spack install cmake spack install python -spack install py-pandas^/$(spack find --format '{hash}' python) +spack install py-pandas^/"$(spack find --format '{hash}' python)" spack install py-pyaml spack install py-packaging spack install py-jinja2 @@ -44,8 +45,8 @@ spack load py-setuptools spack load py-deepdiff spack load py-xmltodict -git clone https://github.com/Heinrich-BR/moose.git moose_cuda -cd moose_cuda +git clone https://github.com/idaholab/moose.git moose_cuda +cd moose_cuda || exit ./scripts/update_and_rebuild_petsc.sh --with-cuda --with-cuda-arch=80 ./scripts/update_and_rebuild_libmesh.sh @@ -55,7 +56,7 @@ cd moose_cuda ./configure --with-mfem -cd framework +cd framework || exit cp contrib/mfem/build-dbg/config/config.mk contrib/mfem/build-dbg/config/config.mk.bak cp contrib/mfem/build-dbg/config/config-install.mk contrib/mfem/build-dbg/config/config-install.mk.bak cp contrib/mfem/installed/share/mfem/config.mk contrib/mfem/installed/share/mfem/config.mk.bak @@ -63,8 +64,7 @@ sed -i.bak 's/\$<\$:SHELL:-Xcompiler *>//g' con sed -i.bak 's/\$<\$:SHELL:-Xcompiler *>//g' contrib/mfem/build-dbg/config/config-install.mk sed -i.bak 's/\$<\$:SHELL:-Xcompiler *>//g' contrib/mfem/installed/share/mfem/config.mk make -j $MOOSE_JOBS -cd ../test +cd ../test || exit make -j $MOOSE_JOBS export LD_LIBRARY_PATH=${STDCXX_PATH}:$LD_LIBRARY_PATH - From d35fc6b0f0bbc17cc7d2067a5b60c17558253d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CHenrique?= Date: Fri, 23 Jan 2026 09:57:48 +0000 Subject: [PATCH 3/4] Changed build to opt --- scripts/build_csd3_ampere.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_csd3_ampere.sh b/scripts/build_csd3_ampere.sh index ddf823c..1ce9738 100755 --- a/scripts/build_csd3_ampere.sh +++ b/scripts/build_csd3_ampere.sh @@ -12,7 +12,7 @@ ## This script builds MOOSE with MFEM configured for CUDA support. It is designed for the Ampere partition on the CSD3 machine ## To run, substitute with your project account before submitting the job -export METHOD=dbg +export METHOD=opt export MOOSE_JOBS=32 export STDCXX_PATH=/usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-x86_64_v3/gcc-8.5.0/gcc-11.3.0-r7adil6umxm6geju4jjmtwo4ecbkzpeo/lib64 @@ -52,7 +52,7 @@ cd moose_cuda || exit ./scripts/update_and_rebuild_libmesh.sh ./scripts/update_and_rebuild_conduit.sh ./scripts/update_and_rebuild_wasp.sh -./scripts/update_and_rebuild_mfem.sh -DMFEM_USE_CUDA=YES -DCUDA_ARCH=sm_80 -DCMAKE_BUILD_TYPE=Debug +./scripts/update_and_rebuild_mfem.sh -DMFEM_USE_CUDA=YES -DCUDA_ARCH=sm_80 ./configure --with-mfem From d9cc67b34c2192344b26ff684ad07a5114e0fb77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CHenrique?= Date: Mon, 26 Jan 2026 15:06:41 +0000 Subject: [PATCH 4/4] Fix opt/dbg directory name --- scripts/build_csd3_ampere.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/build_csd3_ampere.sh b/scripts/build_csd3_ampere.sh index 1ce9738..3077b60 100755 --- a/scripts/build_csd3_ampere.sh +++ b/scripts/build_csd3_ampere.sh @@ -57,11 +57,11 @@ cd moose_cuda || exit ./configure --with-mfem cd framework || exit -cp contrib/mfem/build-dbg/config/config.mk contrib/mfem/build-dbg/config/config.mk.bak -cp contrib/mfem/build-dbg/config/config-install.mk contrib/mfem/build-dbg/config/config-install.mk.bak +cp contrib/mfem/build-opt/config/config.mk contrib/mfem/build-opt/config/config.mk.bak +cp contrib/mfem/build-opt/config/config-install.mk contrib/mfem/build-opt/config/config-install.mk.bak cp contrib/mfem/installed/share/mfem/config.mk contrib/mfem/installed/share/mfem/config.mk.bak -sed -i.bak 's/\$<\$:SHELL:-Xcompiler *>//g' contrib/mfem/build-dbg/config/config.mk -sed -i.bak 's/\$<\$:SHELL:-Xcompiler *>//g' contrib/mfem/build-dbg/config/config-install.mk +sed -i.bak 's/\$<\$:SHELL:-Xcompiler *>//g' contrib/mfem/build-opt/config/config.mk +sed -i.bak 's/\$<\$:SHELL:-Xcompiler *>//g' contrib/mfem/build-opt/config/config-install.mk sed -i.bak 's/\$<\$:SHELL:-Xcompiler *>//g' contrib/mfem/installed/share/mfem/config.mk make -j $MOOSE_JOBS cd ../test || exit