From 734720e381f1af5cd6bc19c705d39ce6c22d86d2 Mon Sep 17 00:00:00 2001 From: settgast1 Date: Thu, 6 Jun 2024 01:06:15 -0700 Subject: [PATCH 01/16] add ubi dockerfile using base images --- .github/workflows/docker_build_tpls.yml | 7 +++ docker/tpl-ubi-gcc-cuda.Dockerfile | 84 +++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 docker/tpl-ubi-gcc-cuda.Dockerfile diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 350c42c4..7aaeaf2f 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -7,6 +7,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true + +env: + BASE_IMAGE_TAG_HASH: b5b06ad736cc8f2df227dabf8ff1da1a2f538939 + jobs: build_images: name: ${{ matrix.name }} @@ -80,6 +84,9 @@ jobs: - name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.89) DOCKER_REPOSITORY: geosx/centos7.7-gcc8.3.1-cuda11.8.89 DOCKERFILE: docker/tpl-centos-gcc-cuda.Dockerfile + - name: ubi (8.9, gcc 8.3.1, open-mpi 1.10.7, cuda 12.4.1) + DOCKER_REPOSITORY: geosx/ubi:8.9-cuda12.4 + DOCKERFILE: docker/tpl-ubi-gcc-cuda.Dockerfile steps: - name: Checkout diff --git a/docker/tpl-ubi-gcc-cuda.Dockerfile b/docker/tpl-ubi-gcc-cuda.Dockerfile new file mode 100644 index 00000000..4c3d38ae --- /dev/null +++ b/docker/tpl-ubi-gcc-cuda.Dockerfile @@ -0,0 +1,84 @@ +# NOTE: see docker/tpl-ubuntu-gcc.Dockerfile for detailed comments +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/thirdPartyLibs +ARG BLD_DIR=$TMP_DIR/build + +FROM geosx/ubi:8.9-cuda12.4 AS tpl_toolchain_intersect_geosx_toolchain +ARG SRC_DIR + +ARG INSTALL_DIR +ENV GEOSX_TPL_DIR=$INSTALL_DIR + +# Installing dependencies +RUN yum -y install \ + ca-certificates \ + tbb \ + blas-devel \ + lapack-devel \ + zlib-devel \ + openmpi-devel + +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh + +ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc \ + CXX=/opt/rh/devtoolset-8/root/usr/bin/g++ \ + MPICC=/usr/lib64/openmpi/bin/mpicc \ + MPICXX=/usr/lib64/openmpi/bin/mpicxx \ + MPIEXEC=/usr/lib64/openmpi/bin/mpirun +ENV OMPI_CC=$CC \ + OMPI_CXX=$CXX +ENV ENABLE_CUDA=ON \ + CMAKE_CUDA_FLAGS="-restrict -arch sm_70 --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations" + +# Installing TPL's +FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain +ARG SRC_DIR +ARG BLD_DIR + +ENV FC=/opt/rh/devtoolset-8/root/usr/bin/gfortran \ + MPIFC=/usr/lib64/openmpi/bin/mpifort +ENV OMPI_FC=$FC + +RUN yum install -y \ + tbb-devel \ + bc \ + file \ + bison \ + flex \ + patch + +ARG HOST_CONFIG + +ARG CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda +ARG CUDA_ARCH=sm_70 +ARG CMAKE_CUDA_COMPILER=$CUDA_TOOLKIT_ROOT_DIR/bin/nvcc +ARG CMAKE_CUDA_ARCHITECTURES=70 + +ENV HYPRE_CUDA_SM=70 +ENV CUDA_HOME=$CUDA_TOOLKIT_ROOT_DIR + +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh \ + -DENABLE_CUDA=$ENABLE_CUDA \ + -DENABLE_HYPRE_DEVICE="CUDA" \ + -DCUDA_TOOLKIT_ROOT_DIR=$CUDA_TOOLKIT_ROOT_DIR \ + -DCUDA_ARCH=$CUDA_ARCH \ + -DCMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES \ + -DCMAKE_CUDA_COMPILER=$CMAKE_CUDA_COMPILER +WORKDIR $BLD_DIR +RUN --mount=src=.,dst=$SRC_DIR make + +# Extract only TPL's from previous stage +FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain +ARG SRC_DIR + +COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR +RUN yum install -y \ + openssh-client \ + ca-certificates \ + texlive \ + graphviz + +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-ninja.sh + +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh +ENV SCCACHE=/opt/sccache/bin/sccache From 56a27f2721c428fd36ff1e0eaf7b587ec1ea4d44 Mon Sep 17 00:00:00 2001 From: settgast1 Date: Thu, 6 Jun 2024 01:09:06 -0700 Subject: [PATCH 02/16] turn off old builds --- .github/workflows/docker_build_tpls.yml | 120 ++++++++++++------------ 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 7aaeaf2f..5bf9d4a9 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -24,66 +24,66 @@ jobs: matrix: include: - - name: Pangea 2 (centos 7.6, gcc 8.3.0, open-mpi 2.1.5, mkl 2019.3) - DOCKER_REPOSITORY: geosx/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3 - DOCKERFILE: docker/TotalEnergies/Dockerfile - DOCKER_ROOT_IMAGE: totogaz/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3-no-geosx:0.0.1 - INSTALL_DIR_ROOT: /workrd/SCR/GEOSX/install/gcc8 - - name: Pecan GPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5, cuda 11.5.119) - DOCKER_REPOSITORY: geosx/pecan-gpu-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119 - DOCKERFILE: docker/TotalEnergies/Dockerfile - DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 - HOST_CONFIG: docker/TotalEnergies/pecan-GPU.cmake - INSTALL_DIR_ROOT: /shared/data1/Users/j0436735/travis-deployments/GPU - - name: Pecan CPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5) - DOCKER_REPOSITORY: geosx/pecan-cpu-gcc8.2.0-openmpi4.0.1-mkl2019.5 - DOCKERFILE: docker/TotalEnergies/Dockerfile - DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 - HOST_CONFIG: docker/TotalEnergies/pecan-CPU.cmake - INSTALL_DIR_ROOT: /shared/data1/Users/j0436735/travis-deployments/CPU - - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) - DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 - DOCKERFILE: docker/Stanford/Dockerfile - DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11-no-geosx:0.0.2 - HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake - INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU - - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 11.7.1, openblas 0.3.10, zlib 1.2.11) - DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11 - DOCKERFILE: docker/Stanford/Dockerfile - DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11-no-geosx:0.0.2 - HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake - INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU - - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3) - DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 - DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - DOCKER_ROOT_IMAGE: ubuntu:20.04 - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=9" - - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces - DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 - DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - DOCKER_ROOT_IMAGE: mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=10" - - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2) - DOCKER_ROOT_IMAGE: ubuntu:22.04 - DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 - DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=11" - - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2) - DOCKER_ROOT_IMAGE: ubuntu:22.04 - DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12 - DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=12" - - name: Ubuntu (22.04, clang 15.0.7 + gcc 11.4.0, open-mpi 4.1.2) - DOCKER_ROOT_IMAGE: ubuntu:22.04 - DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15 - DOCKERFILE: docker/tpl-ubuntu-clang.Dockerfile - DOCKER_COMPILER_BUILD_ARG: "--build-arg CLANG_MAJOR_VERSION=15 --build-arg GCC_MAJOR_VERSION=11" - - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) - DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 - DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile - - name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.89) - DOCKER_REPOSITORY: geosx/centos7.7-gcc8.3.1-cuda11.8.89 - DOCKERFILE: docker/tpl-centos-gcc-cuda.Dockerfile + # - name: Pangea 2 (centos 7.6, gcc 8.3.0, open-mpi 2.1.5, mkl 2019.3) + # DOCKER_REPOSITORY: geosx/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3 + # DOCKERFILE: docker/TotalEnergies/Dockerfile + # DOCKER_ROOT_IMAGE: totogaz/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3-no-geosx:0.0.1 + # INSTALL_DIR_ROOT: /workrd/SCR/GEOSX/install/gcc8 + # - name: Pecan GPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5, cuda 11.5.119) + # DOCKER_REPOSITORY: geosx/pecan-gpu-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119 + # DOCKERFILE: docker/TotalEnergies/Dockerfile + # DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 + # HOST_CONFIG: docker/TotalEnergies/pecan-GPU.cmake + # INSTALL_DIR_ROOT: /shared/data1/Users/j0436735/travis-deployments/GPU + # - name: Pecan CPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5) + # DOCKER_REPOSITORY: geosx/pecan-cpu-gcc8.2.0-openmpi4.0.1-mkl2019.5 + # DOCKERFILE: docker/TotalEnergies/Dockerfile + # DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 + # HOST_CONFIG: docker/TotalEnergies/pecan-CPU.cmake + # INSTALL_DIR_ROOT: /shared/data1/Users/j0436735/travis-deployments/CPU + # - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) + # DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 + # DOCKERFILE: docker/Stanford/Dockerfile + # DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11-no-geosx:0.0.2 + # HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake + # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU + # - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 11.7.1, openblas 0.3.10, zlib 1.2.11) + # DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11 + # DOCKERFILE: docker/Stanford/Dockerfile + # DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11-no-geosx:0.0.2 + # HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake + # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU + # - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3) + # DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 + # DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + # DOCKER_ROOT_IMAGE: ubuntu:20.04 + # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=9" + # - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces + # DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 + # DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + # DOCKER_ROOT_IMAGE: mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 + # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=10" + # - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2) + # DOCKER_ROOT_IMAGE: ubuntu:22.04 + # DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 + # DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=11" + # - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2) + # DOCKER_ROOT_IMAGE: ubuntu:22.04 + # DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12 + # DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=12" + # - name: Ubuntu (22.04, clang 15.0.7 + gcc 11.4.0, open-mpi 4.1.2) + # DOCKER_ROOT_IMAGE: ubuntu:22.04 + # DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15 + # DOCKERFILE: docker/tpl-ubuntu-clang.Dockerfile + # DOCKER_COMPILER_BUILD_ARG: "--build-arg CLANG_MAJOR_VERSION=15 --build-arg GCC_MAJOR_VERSION=11" + # - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) + # DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 + # DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile + # - name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.89) + # DOCKER_REPOSITORY: geosx/centos7.7-gcc8.3.1-cuda11.8.89 + # DOCKERFILE: docker/tpl-centos-gcc-cuda.Dockerfile - name: ubi (8.9, gcc 8.3.1, open-mpi 1.10.7, cuda 12.4.1) DOCKER_REPOSITORY: geosx/ubi:8.9-cuda12.4 DOCKERFILE: docker/tpl-ubi-gcc-cuda.Dockerfile From 977268a6e27cb67a51da5d8bcfee08b0b5ce11dd Mon Sep 17 00:00:00 2001 From: settgast1 Date: Thu, 6 Jun 2024 01:14:28 -0700 Subject: [PATCH 03/16] specity root hash --- .github/workflows/docker_build_tpls.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 5bf9d4a9..2ec3e788 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -9,7 +9,7 @@ concurrency: env: - BASE_IMAGE_TAG_HASH: b5b06ad736cc8f2df227dabf8ff1da1a2f538939 + ROOT_IMAGE_TAG_HASH: b5b06ad736cc8f2df227dabf8ff1da1a2f538939 jobs: build_images: @@ -90,16 +90,16 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4.1.6 with: submodules: true lfs: true - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3.3.0 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3.2.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} @@ -114,7 +114,7 @@ jobs: DOCKER_COMPILER_BUILD_ARG: ${{ matrix.DOCKER_COMPILER_BUILD_ARG }} INSTALL_DIR_ROOT: ${{ matrix.INSTALL_DIR_ROOT || '/opt/GEOS' }} HOST_CONFIG: ${{ matrix.HOST_CONFIG || 'host-configs/environment.cmake'}} - DOCKER_ROOT_IMAGE: ${{ matrix.DOCKER_ROOT_IMAGE || 'undefined' }} + DOCKER_ROOT_IMAGE: ${{ matrix.DOCKER_ROOT_IMAGE || 'undefined' }}-${{ROOT_IMAGE_TAG_HASHZ}} PULL_REQUEST_NUMBER: ${{ github.event.number }} BUILD_NUMBER: ${{ github.run_number }} COMMIT: ${{ github.sha }} From 35fb591f58e99979997a38fe44c733fff18e80b1 Mon Sep 17 00:00:00 2001 From: settgast1 Date: Thu, 6 Jun 2024 01:15:44 -0700 Subject: [PATCH 04/16] specity root hash --- .github/workflows/docker_build_tpls.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 2ec3e788..d60a2bb6 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -114,7 +114,7 @@ jobs: DOCKER_COMPILER_BUILD_ARG: ${{ matrix.DOCKER_COMPILER_BUILD_ARG }} INSTALL_DIR_ROOT: ${{ matrix.INSTALL_DIR_ROOT || '/opt/GEOS' }} HOST_CONFIG: ${{ matrix.HOST_CONFIG || 'host-configs/environment.cmake'}} - DOCKER_ROOT_IMAGE: ${{ matrix.DOCKER_ROOT_IMAGE || 'undefined' }}-${{ROOT_IMAGE_TAG_HASHZ}} + DOCKER_ROOT_IMAGE: ${{ matrix.DOCKER_ROOT_IMAGE || 'undefined' }}-${{ROOT_IMAGE_TAG_HASH}} PULL_REQUEST_NUMBER: ${{ github.event.number }} BUILD_NUMBER: ${{ github.run_number }} COMMIT: ${{ github.sha }} From 412a216cc8e6816d22a98d7b73680fb48763ebb4 Mon Sep 17 00:00:00 2001 From: settgast1 Date: Thu, 6 Jun 2024 01:25:59 -0700 Subject: [PATCH 05/16] specity root hash --- .github/workflows/docker_build_tpls.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index d60a2bb6..22270e02 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -114,7 +114,7 @@ jobs: DOCKER_COMPILER_BUILD_ARG: ${{ matrix.DOCKER_COMPILER_BUILD_ARG }} INSTALL_DIR_ROOT: ${{ matrix.INSTALL_DIR_ROOT || '/opt/GEOS' }} HOST_CONFIG: ${{ matrix.HOST_CONFIG || 'host-configs/environment.cmake'}} - DOCKER_ROOT_IMAGE: ${{ matrix.DOCKER_ROOT_IMAGE || 'undefined' }}-${{ROOT_IMAGE_TAG_HASH}} + DOCKER_ROOT_IMAGE: ${{ matrix.DOCKER_ROOT_IMAGE || 'undefined' }}-${{env.ROOT_IMAGE_TAG_HASH}} PULL_REQUEST_NUMBER: ${{ github.event.number }} BUILD_NUMBER: ${{ github.run_number }} COMMIT: ${{ github.sha }} From 8c69b856edbd405261983b457263de4cb7c78e97 Mon Sep 17 00:00:00 2001 From: settgast1 Date: Thu, 6 Jun 2024 01:29:20 -0700 Subject: [PATCH 06/16] specity root hash --- .github/workflows/docker_build_tpls.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 22270e02..95aca2bb 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -85,7 +85,8 @@ jobs: # DOCKER_REPOSITORY: geosx/centos7.7-gcc8.3.1-cuda11.8.89 # DOCKERFILE: docker/tpl-centos-gcc-cuda.Dockerfile - name: ubi (8.9, gcc 8.3.1, open-mpi 1.10.7, cuda 12.4.1) - DOCKER_REPOSITORY: geosx/ubi:8.9-cuda12.4 + DOCKER_ROOT_IMAGE: geosx/ubi:8.9-cuda12. + DOCKER_REPOSITORY: geosx/ubi:8.9-gcccuda12.4 DOCKERFILE: docker/tpl-ubi-gcc-cuda.Dockerfile steps: From deaf6ddb82f5c909b70de59130cd7b85da515c4f Mon Sep 17 00:00:00 2001 From: settgast1 Date: Thu, 6 Jun 2024 02:29:11 -0700 Subject: [PATCH 07/16] specity root hash --- .github/workflows/docker_build_tpls.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 95aca2bb..77e79e5a 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -84,9 +84,9 @@ jobs: # - name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.89) # DOCKER_REPOSITORY: geosx/centos7.7-gcc8.3.1-cuda11.8.89 # DOCKERFILE: docker/tpl-centos-gcc-cuda.Dockerfile - - name: ubi (8.9, gcc 8.3.1, open-mpi 1.10.7, cuda 12.4.1) - DOCKER_ROOT_IMAGE: geosx/ubi:8.9-cuda12. - DOCKER_REPOSITORY: geosx/ubi:8.9-gcccuda12.4 + - name: ubi (8.9, gcc 13.2.1, open-mpi 1.10.7, cuda 12.4.1) + DOCKER_ROOT_IMAGE: geosx/ubi:8.9-cuda12.4 + DOCKER_REPOSITORY: geosx/ubi:8.9-gcc13-cuda12.4 DOCKERFILE: docker/tpl-ubi-gcc-cuda.Dockerfile steps: From da644418077845952bf07ad955d498a51047c236 Mon Sep 17 00:00:00 2001 From: settgast1 Date: Thu, 6 Jun 2024 03:23:54 -0700 Subject: [PATCH 08/16] specity root hash --- .github/workflows/docker_build_tpls.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 77e79e5a..2e9d2b46 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -86,7 +86,7 @@ jobs: # DOCKERFILE: docker/tpl-centos-gcc-cuda.Dockerfile - name: ubi (8.9, gcc 13.2.1, open-mpi 1.10.7, cuda 12.4.1) DOCKER_ROOT_IMAGE: geosx/ubi:8.9-cuda12.4 - DOCKER_REPOSITORY: geosx/ubi:8.9-gcc13-cuda12.4 + DOCKER_REPOSITORY: geosx/ubi8.9-gcc13-cuda12.4 DOCKERFILE: docker/tpl-ubi-gcc-cuda.Dockerfile steps: From 7f24ceeb02206683db0e593760ad11abd65e085f Mon Sep 17 00:00:00 2001 From: settgast1 Date: Thu, 6 Jun 2024 03:32:39 -0700 Subject: [PATCH 09/16] specify compilers in tpl dockerfile --- .github/workflows/docker_build_tpls.yml | 7 ++++++- docker/tpl-ubi-gcc-cuda.Dockerfile | 9 ++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 2e9d2b46..f7108b99 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -88,7 +88,9 @@ jobs: DOCKER_ROOT_IMAGE: geosx/ubi:8.9-cuda12.4 DOCKER_REPOSITORY: geosx/ubi8.9-gcc13-cuda12.4 DOCKERFILE: docker/tpl-ubi-gcc-cuda.Dockerfile - + CMAKE_CXX_COMPILER: /opt/rh/gcc-toolset-13/root/bin/g++ + CMAKE_C_COMPILER: /opt/rh/gcc-toolset-13/root/bin/gcc + CMAKE_FORTRAN_COMPILER: /opt/rh/gcc-toolset-13/root/bin/gfortran steps: - name: Checkout uses: actions/checkout@v4.1.6 @@ -112,6 +114,9 @@ jobs: env: DOCKERFILE: ${{ matrix.DOCKERFILE }} DOCKER_REPOSITORY: ${{ matrix.DOCKER_REPOSITORY }} + CMAKE_CXX_COMPILER: ${{ matrix.CMAKE_CXX_COMPILER }} + CMAKE_C_COMPILER: ${{ matrix.CMAKE_C_COMPILER }} + CMAKE_FORTRAN_COMPILER: ${{ matrix.CMAKE_FORTRAN_COMPILER }} DOCKER_COMPILER_BUILD_ARG: ${{ matrix.DOCKER_COMPILER_BUILD_ARG }} INSTALL_DIR_ROOT: ${{ matrix.INSTALL_DIR_ROOT || '/opt/GEOS' }} HOST_CONFIG: ${{ matrix.HOST_CONFIG || 'host-configs/environment.cmake'}} diff --git a/docker/tpl-ubi-gcc-cuda.Dockerfile b/docker/tpl-ubi-gcc-cuda.Dockerfile index 4c3d38ae..45f5d83f 100644 --- a/docker/tpl-ubi-gcc-cuda.Dockerfile +++ b/docker/tpl-ubi-gcc-cuda.Dockerfile @@ -2,6 +2,9 @@ ARG TMP_DIR=/tmp ARG SRC_DIR=$TMP_DIR/thirdPartyLibs ARG BLD_DIR=$TMP_DIR/build +ARG CMAKE_C_COMPILER +ARG CMAKE_CXX_COMPILER +ARG CMAKE_FORTRAN_COMPILER FROM geosx/ubi:8.9-cuda12.4 AS tpl_toolchain_intersect_geosx_toolchain ARG SRC_DIR @@ -20,8 +23,8 @@ RUN yum -y install \ RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh -ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc \ - CXX=/opt/rh/devtoolset-8/root/usr/bin/g++ \ +ENV CC=$CMAKE_C_COMPILER \ + CXX=$CMAKE_CXX_COMPILER \ MPICC=/usr/lib64/openmpi/bin/mpicc \ MPICXX=/usr/lib64/openmpi/bin/mpicxx \ MPIEXEC=/usr/lib64/openmpi/bin/mpirun @@ -35,7 +38,7 @@ FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain ARG SRC_DIR ARG BLD_DIR -ENV FC=/opt/rh/devtoolset-8/root/usr/bin/gfortran \ +ENV FC=$CMAKE_FORTRAN_COMPILER \ MPIFC=/usr/lib64/openmpi/bin/mpifort ENV OMPI_FC=$FC From 7500bc78fe8f704ad352771277ace4fed7c0029f Mon Sep 17 00:00:00 2001 From: settgast1 Date: Thu, 6 Jun 2024 03:39:54 -0700 Subject: [PATCH 10/16] hardcode hash --- docker/tpl-ubi-gcc-cuda.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/tpl-ubi-gcc-cuda.Dockerfile b/docker/tpl-ubi-gcc-cuda.Dockerfile index 45f5d83f..dc6fa9a0 100644 --- a/docker/tpl-ubi-gcc-cuda.Dockerfile +++ b/docker/tpl-ubi-gcc-cuda.Dockerfile @@ -6,7 +6,7 @@ ARG CMAKE_C_COMPILER ARG CMAKE_CXX_COMPILER ARG CMAKE_FORTRAN_COMPILER -FROM geosx/ubi:8.9-cuda12.4 AS tpl_toolchain_intersect_geosx_toolchain +FROM geosx/ubi:8.9-cuda12.4-b5b06ad736cc8f2df227dabf8ff1da1a2f538939 AS tpl_toolchain_intersect_geosx_toolchain ARG SRC_DIR ARG INSTALL_DIR From 7f48cb59071db5d4169e1128d0c81724ce4814a7 Mon Sep 17 00:00:00 2001 From: settgast1 Date: Fri, 7 Jun 2024 04:03:48 -0700 Subject: [PATCH 11/16] separate out yum install to debug --- docker/tpl-ubi-gcc-cuda.Dockerfile | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docker/tpl-ubi-gcc-cuda.Dockerfile b/docker/tpl-ubi-gcc-cuda.Dockerfile index dc6fa9a0..60e4fd9c 100644 --- a/docker/tpl-ubi-gcc-cuda.Dockerfile +++ b/docker/tpl-ubi-gcc-cuda.Dockerfile @@ -13,13 +13,12 @@ ARG INSTALL_DIR ENV GEOSX_TPL_DIR=$INSTALL_DIR # Installing dependencies -RUN yum -y install \ - ca-certificates \ - tbb \ - blas-devel \ - lapack-devel \ - zlib-devel \ - openmpi-devel +RUN yum -y install ca-certificates +RUN yum -y install tbb +RUN yum -y install blas-devel +RUN yum -y install lapack-devel +RUN yum -y install zlib-devel +RUN yum -y install openmpi-devel RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh From 61cff06c9a10e040f1438350d931af95619dcccc Mon Sep 17 00:00:00 2001 From: settgast1 Date: Fri, 7 Jun 2024 08:56:44 -0700 Subject: [PATCH 12/16] search for tbb in yum --- docker/tpl-ubi-gcc-cuda.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/tpl-ubi-gcc-cuda.Dockerfile b/docker/tpl-ubi-gcc-cuda.Dockerfile index 60e4fd9c..e2039e9a 100644 --- a/docker/tpl-ubi-gcc-cuda.Dockerfile +++ b/docker/tpl-ubi-gcc-cuda.Dockerfile @@ -12,8 +12,8 @@ ARG SRC_DIR ARG INSTALL_DIR ENV GEOSX_TPL_DIR=$INSTALL_DIR +RUN yum search tbb # Installing dependencies -RUN yum -y install ca-certificates RUN yum -y install tbb RUN yum -y install blas-devel RUN yum -y install lapack-devel From 76f7004ae7ecd7ff0dace811678bca9b8ca2f621 Mon Sep 17 00:00:00 2001 From: settgast1 Date: Fri, 7 Jun 2024 11:14:17 -0700 Subject: [PATCH 13/16] remove tbb --- docker/tpl-ubi-gcc-cuda.Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/tpl-ubi-gcc-cuda.Dockerfile b/docker/tpl-ubi-gcc-cuda.Dockerfile index e2039e9a..dc948628 100644 --- a/docker/tpl-ubi-gcc-cuda.Dockerfile +++ b/docker/tpl-ubi-gcc-cuda.Dockerfile @@ -12,9 +12,8 @@ ARG SRC_DIR ARG INSTALL_DIR ENV GEOSX_TPL_DIR=$INSTALL_DIR -RUN yum search tbb # Installing dependencies -RUN yum -y install tbb +#RUN yum -y install tbb RUN yum -y install blas-devel RUN yum -y install lapack-devel RUN yum -y install zlib-devel From 97dc3a48f6edb3ea950d04d72454f11e72b913f7 Mon Sep 17 00:00:00 2001 From: settgast1 Date: Fri, 7 Jun 2024 11:48:29 -0700 Subject: [PATCH 14/16] search for packages --- docker/tpl-ubi-gcc-cuda.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/tpl-ubi-gcc-cuda.Dockerfile b/docker/tpl-ubi-gcc-cuda.Dockerfile index dc948628..f85bb1a4 100644 --- a/docker/tpl-ubi-gcc-cuda.Dockerfile +++ b/docker/tpl-ubi-gcc-cuda.Dockerfile @@ -12,6 +12,8 @@ ARG SRC_DIR ARG INSTALL_DIR ENV GEOSX_TPL_DIR=$INSTALL_DIR + +RUN yum -y search blas lapack zlib openmpi # Installing dependencies #RUN yum -y install tbb RUN yum -y install blas-devel From 45129257f98aafb897a5aaedf2d9cc00f74f90b7 Mon Sep 17 00:00:00 2001 From: settgast1 Date: Fri, 7 Jun 2024 11:56:59 -0700 Subject: [PATCH 15/16] search for packages --- docker/tpl-ubi-gcc-cuda.Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/tpl-ubi-gcc-cuda.Dockerfile b/docker/tpl-ubi-gcc-cuda.Dockerfile index f85bb1a4..079a78f5 100644 --- a/docker/tpl-ubi-gcc-cuda.Dockerfile +++ b/docker/tpl-ubi-gcc-cuda.Dockerfile @@ -13,7 +13,10 @@ ARG INSTALL_DIR ENV GEOSX_TPL_DIR=$INSTALL_DIR -RUN yum -y search blas lapack zlib openmpi +RUN yum search blas +RUN yum search lapack +RUN yum search zlib +RUN yum search openmpi # Installing dependencies #RUN yum -y install tbb RUN yum -y install blas-devel From 8e02f365f185a4a02bab03941e59ea0abfd09ee0 Mon Sep 17 00:00:00 2001 From: settgast1 Date: Fri, 7 Jun 2024 16:19:11 -0700 Subject: [PATCH 16/16] install openmpi from rpm --- docker/tpl-ubi-gcc-cuda.Dockerfile | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/docker/tpl-ubi-gcc-cuda.Dockerfile b/docker/tpl-ubi-gcc-cuda.Dockerfile index 079a78f5..6608cf38 100644 --- a/docker/tpl-ubi-gcc-cuda.Dockerfile +++ b/docker/tpl-ubi-gcc-cuda.Dockerfile @@ -13,16 +13,18 @@ ARG INSTALL_DIR ENV GEOSX_TPL_DIR=$INSTALL_DIR -RUN yum search blas -RUN yum search lapack -RUN yum search zlib -RUN yum search openmpi +# RUN dnf search blas +# RUN dnf search lapack +RUN dnf search zlib +RUN dnf search openmpi # Installing dependencies -#RUN yum -y install tbb -RUN yum -y install blas-devel -RUN yum -y install lapack-devel -RUN yum -y install zlib-devel -RUN yum -y install openmpi-devel +#RUN dnf -y install tbb +RUN dnf -y install openblas +#RUN dnf -y install lapack-devel +RUN dnf -y install zlib-devel +#RUN dnf -y install openmpi-devel +RUN dnf -y install https://download.open-mpi.org/release/open-mpi/v5.0/openmpi-5.0.3-1.src.rpm + RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh @@ -45,7 +47,7 @@ ENV FC=$CMAKE_FORTRAN_COMPILER \ MPIFC=/usr/lib64/openmpi/bin/mpifort ENV OMPI_FC=$FC -RUN yum install -y \ +RUN dnf install -y \ tbb-devel \ bc \ file \ @@ -78,7 +80,7 @@ FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain ARG SRC_DIR COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR -RUN yum install -y \ +RUN dnf install -y \ openssh-client \ ca-certificates \ texlive \