Merge pull request #443 from CrayLabs/develop

This PR brings master up to date with develop before releasing v0.6.0.

Author: al-rigazzi

.github/workflows/run_tests.yml

@@ -44,23 +44,17 @@ env:
 
 jobs:
   run_tests:
-    name: Run tests with ${{ matrix.os }}, Python ${{ matrix.py_v}}, RedisAI ${{ matrix.rai }}
+    name: Run tests ${{ matrix.subset }} with ${{ matrix.os }}, Python ${{ matrix.py_v}}, RedisAI ${{ matrix.rai }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
+        subset: [backends, slow_tests, group_a, group_b]
         os: [macos-12, ubuntu-20.04] # Operating systems
         compiler: [8] # GNU compiler version
-        rai: [1.2.5, 1.2.7] # Redis AI versions
+        rai: [1.2.7] # Redis AI versions
         py_v: [3.8, 3.9, '3.10'] # Python versions
-        exclude:
-          # Do not build with Redis AI 1.2.5 on MacOS
-          - os: macos-12
-            rai: 1.2.5
-          # Do not build Redis AI 1.2.5 with py3.10
-          # as wheels for dependecies are not availble
-          - py_v: '3.10'
-            rai: 1.2.5
+
 
     env:
       SMARTSIM_REDISAI: ${{ matrix.rai }}
@@ -86,7 +80,7 @@ jobs:
           echo "$(brew --prefix)/opt/make/libexec/gnubin" >> $GITHUB_PATH
 
       - name: Build Singularity from source
-        if: contains( matrix.os, 'ubuntu' ) && matrix.py_v == 3.9 && matrix.rai == '1.2.5'
+        if: contains( matrix.os, 'ubuntu' ) && matrix.py_v == 3.9
         run: |
           sudo apt-get install -y libseccomp-dev pkg-config squashfs-tools cryptsetup curl git # wget build-essential
           echo 'export PATH=/usr/local/go/bin:$PATH' >> ~/.bashrc
@@ -100,16 +94,18 @@ jobs:
           sudo make -C builddir install
 
       - name: singularity pull test container # This lets us time how long the pull takes
-        if: contains( matrix.os, 'ubuntu' ) && matrix.py_v == 3.9 && matrix.rai == '1.2.5'
+        if: contains( matrix.os, 'ubuntu' ) && matrix.py_v == 3.9
         run: singularity pull docker://alrigazzi/smartsim-testing
 
       # Note: The develop branch of smartredis is installed first to ensure that any tests that depend
       # on developments of the client are brought in.
       - name: Install SmartSim (with ML backends)
         run: |
+
           python -m pip install git+https://github.com/CrayLabs/SmartRedis.git@develop#egg=smartredis
           python -m pip install .[dev,ml]
 
+
       - name: Install ML Runtimes with Smart (with pt, tf, and onnx support)
         if: (matrix.py_v != '3.10')
         run: smart build --device cpu --onnx -v
@@ -119,22 +115,43 @@ jobs:
         run: smart build --device cpu -v
 
       - name: Run mypy
-        # TF 2.6.2 has a dep conflict with new mypy versions
-        if: (matrix.rai != '1.2.5')
         run: |
           python -m pip install .[mypy]
           make check-mypy
 
       - name: Run Pylint
         run: make check-lint
 
+      # Run pytest (backends subdirectory)
       - name: Run Pytest
+        if: (matrix.subset == 'backends')
         run: |
           echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV
-          py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ ./tests/
+          py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ ./tests/backends
+
+      # Run pytest (test subsets)
+      - name: Run Pytest
+        if: "!contains(matrix.subset, 'backends')" # if not running backend tests
+        run: |
+          echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV
+          py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ -m ${{ matrix.subset }} ./tests
+
+      # Upload artifacts on failure, ignoring binary files
+      - name: Upload Artifact
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test_artifact
+          path: |
+            tests/test_output
+            !**/*.so
+            !**/*.pb
+            !**/*.pt
+            !**/core
+          retention-days: 5
 
       - name: Upload Pytest coverage to Codecov
         uses: codecov/codecov-action@v2
         with:
-          fail_ci_if_error: true
+          fail_ci_if_error: false
           files: ./coverage.xml

.gitignore

@@ -7,7 +7,7 @@ __pycache__
 htmlcov
 smartsim.egg-info
 tests/test_output
-docs/*
+.DS_Store
 
 # Dependencies
 smartsim/_core/.third-party
@@ -31,3 +31,8 @@ smartsim/_core/bin/*-cli
 
 # created upon install
 smartsim/_core/lib
+
+**/manifest/
+**/*.err
+**/*.out
+**/.smartsim/*

.pylintrc

@@ -325,7 +325,7 @@ valid-metaclass-classmethod-first-arg=mcs
 max-args=9
 
 # Maximum number of locals for function / method body
-max-locals=19
+max-locals=20
 
 # Maximum number of return / yield for function / method body
 max-returns=11

.wci.yml

@@ -22,8 +22,8 @@
     language: Python
 
     release:
-      version: 0.5.1
-      date: 2023-09-13
+      version: 0.6.0
+      date: 2023-12-18
 
     documentation:
       general: https://www.craylabs.org/docs/overview.html

MANIFEST.in

@@ -1,3 +1,4 @@
 include Makefile
 graft smartsim/
-
+global-exclude __pycache__
+global-exclude *.py[co]

Makefile

@@ -146,11 +146,11 @@ tutorials-dev:
 	@docker compose build tutorials-dev
 	@docker run -p 8888:8888 smartsim-tutorials:dev-latest
 
-# help: tutorials-prod                 - Build and start a docker container to run the tutorials (v0.4.2)
+# help: tutorials-prod                 - Build and start a docker container to run the tutorials (v0.6.0)
 .PHONY: tutorials-prod
 tutorials-prod:
 	@docker compose build tutorials-prod
-	@docker run -p 8888:8888 smartsim-tutorials:v0.4.2
+	@docker run -p 8888:8888 smartsim-tutorials:v0.6.0
 
 
 # help:

README.md

@@ -6,10 +6,11 @@
     <br />
 <div display="inline-block">
     <a href="https://github.com/CrayLabs/SmartSim"><b>Home</b></a>&nbsp;&nbsp;&nbsp;
-    <a href="https://www.craylabs.org/docs/installation.html"><b>Install</b></a>&nbsp;&nbsp;&nbsp;
+    <a href="https://www.craylabs.org/docs/installation_instructions/basic.html"><b>Install</b></a>&nbsp;&nbsp;&nbsp;
     <a href="https://www.craylabs.org/docs/overview.html"><b>Documentation</b></a>&nbsp;&nbsp;&nbsp;
-    <a href="https://join.slack.com/t/craylabs/shared_invite/zt-nw3ag5z5-5PS4tIXBfufu1bIvvr71UA"><b>Slack Invite</b></a>&nbsp;&nbsp;&nbsp;
     <a href="https://github.com/CrayLabs"><b>Cray Labs</b></a>&nbsp;&nbsp;&nbsp;
+    <a href="mailto:craylabs@hpe.com"><b>Contact</b></a>&nbsp;&nbsp;&nbsp;
+    <a href="https://join.slack.com/t/craylabs/shared_invite/zt-nw3ag5z5-5PS4tIXBfufu1bIvvr71UA"><b>Join us on Slack!</b></a>&nbsp;&nbsp;&nbsp;
   </div>
     <br />
     <br />
@@ -649,18 +650,6 @@ from C, C++, Fortran and Python with the SmartRedis Clients:
       <td>ONNX</td>
       <td>1.11.x</td>
     </tr>
-      <td rowspan="3">1.2.5</td>
-      <td>PyTorch</td>
-      <td>1.9.x</td>
-    </tr>
-    <tr>
-      <td>TensorFlow\Keras</td>
-      <td>2.6.x</td>
-    </tr>
-    <tr>
-      <td>ONNX</td>
-      <td>1.9.x</td>
-    </tr>
   </tbody>
 </table>