Merge pull request #440 from CrayLabs/develop

This PR brings master up to date with develop for v0.5.0 release.

[ committed by @al-rigazzi ]
[ reviewed by @billschereriii @MattToast ]

Author: al-rigazzi

…ithub/workflows/run_post_merge_tests.yml …ub/workflows/run_static_and_examples.yml.github/workflows/run_post_merge_tests.yml renamed to .github/workflows/run_static_and_examples.yml .github/workflows/run_post_merge_tests.yml renamed to .github/workflows/run_static_and_examples.yml

@@ -26,25 +26,24 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 
-name: run_post_merge_tests
-
-# This file is for tests that are to be run rarely, once on each merge.
+name: run_static_and_examples
 
 on:
   push:
     branches:
       - master
       - develop
-
+  pull_request:
+    branches:
+      - master
+      - develop
 env:
   HOMEBREW_NO_ANALYTICS: "ON" # Make Homebrew installation a little quicker
   HOMEBREW_NO_AUTO_UPDATE: "ON"
   HOMEBREW_NO_BOTTLE_SOURCE_FALLBACK: "ON"
   HOMEBREW_NO_GITHUB_API: "ON"
   HOMEBREW_NO_INSTALL_CLEANUP: "ON"
   DEBIAN_FRONTEND: "noninteractive" # disable interactive apt installs
-  SSDB: "127.0.0.1:6379"
-  SMARTREDIS_TEST_CLUSTER: False
 
 jobs:
 
@@ -56,7 +55,7 @@ jobs:
       matrix:
         os: [ubuntu-20.04] # cannot test on macOS as docker isn't supported on Mac
         compiler: [intel, 8, 9, 10, 11] # intel compiler, and versions of GNU compiler
-        rai_v: [1.2.4, 1.2.5] # versions of RedisAI
+        rai_v: [1.2.7] # version(s) of RedisAI
         py_v: ['3.7.x', '3.8.x', '3.9.x', '3.10.x'] # versions of Python
     env:
       FC: gfortran-${{ matrix.compiler }}
@@ -97,12 +96,15 @@ jobs:
           rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB &&
           echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list &&
           sudo apt-get update -y &&
-          sudo apt-get install -y intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic
+          sudo apt-get install -y intel-oneapi-compiler-fortran &&
           source /opt/intel/oneapi/setvars.sh &&
+          sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic
+          sudo apt-get install -y intel-oneapi-common-vars &&
+          source /opt/intel/oneapi/setvars.sh --force &&
           printenv >> $GITHUB_ENV &&
           echo "CC=icx" >> $GITHUB_ENV &&
           echo "CXX=icpx" >> $GITHUB_ENV &&
-          echo "FC=ifort" >> $GITHUB_ENV
+          echo "FC=ifx" >> $GITHUB_ENV
 
       # Install additional dependencies
       - name: Install Cmake Linux

.github/workflows/run_tests.yml

@@ -59,7 +59,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-20.04] # cannot test on macOS as docker isn't supported on Mac
-        rai_v: [1.2.4, 1.2.5] # versions of RedisAI
+        rai_v: [1.2.7] # versions of RedisAI
         py_v: ['3.7.x', '3.8.x', '3.9.x', '3.10.x'] # versions of Python
         compiler: [intel, 8, 9, 10, 11] # intel compiler, and versions of GNU compiler
     env:
@@ -75,6 +75,13 @@ jobs:
         with:
           python-version: ${{ matrix.py_v }}
 
+      # Free up some disk space
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet &&
+          sudo rm -rf /opt/ghc &&
+          sudo rm -rf "/usr/local/share/boost"
+
       # Install compilers (Intel or GCC)
       - name: Install GCC
         if: "!contains( matrix.compiler, 'intel' )" # if using GNU compiler
@@ -101,12 +108,34 @@ jobs:
           rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB &&
           echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list &&
           sudo apt-get update -y &&
-          sudo apt-get install -y intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic
+          sudo apt-get install -y intel-oneapi-compiler-fortran &&
           source /opt/intel/oneapi/setvars.sh &&
+          sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic
+          sudo apt-get install -y intel-oneapi-common-vars &&
+          source /opt/intel/oneapi/setvars.sh --force &&
           printenv >> $GITHUB_ENV &&
           echo "CC=icx" >> $GITHUB_ENV &&
           echo "CXX=icpx" >> $GITHUB_ENV &&
-          echo "FC=ifort" >> $GITHUB_ENV
+          echo "FC=ifx" >> $GITHUB_ENV
+
+      # Set up perl environment for LCOV
+      - uses: actions/checkout@v3
+      - name: Setup perl
+        uses: shogo82148/actions-setup-perl@v1
+        with:
+          perl-version: '5.30'
+          install-modules: Memory::Process
+
+      # Install additional perl Modules
+      - name: Add perl modules
+        run: |
+          sudo apt install libcapture-tiny-perl && \
+          sudo apt install libdatetime-perl && \
+          sudo apt install libdevel-cover-perl && \
+          sudo apt install libdigest-md5-perl && \
+          sudo apt install libfile-spec-perl && \
+          sudo apt install libjson-xs-perl && \
+          sudo apt install libtime-hires-perl
 
       # Install additional dependencies
       - name: Install Cmake Linux
@@ -167,7 +196,7 @@ jobs:
 
       # Process and upload code coverage (Python was collected during pytest)
       - name: Collect coverage from C/C++/Fortran testers
-        run: third-party/lcov/install/usr/local/bin/lcov -c -d build/Coverage/CMakeFiles -o coverage.info
+        run: third-party/lcov/install/bin/lcov --ignore-errors gcov,mismatch --keep-going -c -d build/Coverage/CMakeFiles -o coverage.info
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3

CMakeLists.txt

@@ -31,7 +31,7 @@ endif (POLICY CMP0048)
 
 # Project definition for the SmartRedis project
 cmake_minimum_required(VERSION 3.13)
-project(SmartRedis VERSION "0.4.2")
+project(SmartRedis VERSION "0.5.0")
 
 # Configure options for the SmartRedis project
 option(SR_PYTHON  "Build the python module" OFF)

CONTRIBUTING.md

@@ -0,0 +1,3 @@
+SmartRedis and SmartSim share the same contributor guidelines. Please refer to
+[CONTRIBUTING.rst](https://github.com/CrayLabs/SmartSim/blob/develop/CONTRIBUTING.rst)
+in the SmartSim repo or at CrayLabs[https://www.craylabs.org/docs/contributing.html]

Makefile

@@ -27,6 +27,7 @@
 # General settings
 MAKEFLAGS += --no-print-directory
 SHELL:=/bin/bash
+CWD := $(shell pwd)
 
 # Params for third-party software
 HIREDIS_URL := https://github.com/redis/hiredis.git
@@ -42,7 +43,7 @@ REDISAI_URL := https://github.com/RedisAI/RedisAI.git
 CATCH2_URL := https://github.com/catchorg/Catch2.git
 CATCH2_VER := v2.13.6
 LCOV_URL := https://github.com/linux-test-project/lcov.git
-LCOV_VER := v1.15
+LCOV_VER := v2.0
 
 # Build variables
 NPROC := $(shell nproc 2>/dev/null || python -c "import multiprocessing as mp; print (mp.cpu_count())" 2>/dev/null || echo 4)
@@ -58,7 +59,7 @@ SR_TEST_REDIS_MODE := Clustered
 SR_TEST_UDS_FILE := /tmp/redis.sock
 SR_TEST_PORT := 6379
 SR_TEST_NODES := 3
-SR_TEST_REDISAI_VER := v1.2.3
+SR_TEST_REDISAI_VER := v1.2.7
 SR_TEST_DEVICE := cpu
 SR_TEST_PYTEST_FLAGS := -vv -s
 
@@ -318,7 +319,7 @@ SSDB_STRING := $(shell echo $(SSDB_STRING) | tr -d " ")
 # 	1: the test directory in which to run tests
 define run_smartredis_tests_with_standalone_server
 	echo "Launching standalone Redis server" && \
-	export SR_TEST_DEVICE=$(SR_TEST_DEVICE) SR_SERVER_MODE=Standalone && \
+	export SR_TEST_DEVICE=$(SR_TEST_DEVICE) SR_DB_TYPE=Standalone && \
 	export SMARTREDIS_TEST_CLUSTER=False SMARTREDIS_TEST_DEVICE=$(SR_TEST_DEVICE) && \
 	export SSDB=127.0.0.1:$(SR_TEST_PORT) && \
 	python utils/launch_redis.py --port $(SR_TEST_PORT) --nodes 1 \
@@ -340,7 +341,7 @@ endef
 # 	1: the test directory in which to run tests
 define run_smartredis_tests_with_clustered_server
 	echo "Launching clustered Redis server" && \
-	export SR_TEST_DEVICE=$(SR_TEST_DEVICE) SR_SERVER_MODE=Clustered && \
+	export SR_TEST_DEVICE=$(SR_TEST_DEVICE) SR_DB_TYPE=Clustered && \
 	export SMARTREDIS_TEST_CLUSTER=True SMARTREDIS_TEST_DEVICE=$(SR_TEST_DEVICE) && \
 	export SSDB=$(SSDB_STRING) && \
 	python utils/launch_redis.py --port $(SR_TEST_PORT) --nodes $(SR_TEST_NODES) \
@@ -364,7 +365,7 @@ endef
 # 	1: the test directory in which to run tests
 define run_smartredis_tests_with_uds_server
 	echo "Launching standalone Redis server with Unix Domain Socket support"
-	export SR_TEST_DEVICE=$(SR_TEST_DEVICE) SR_SERVER_MODE=Standalone && \
+	export SR_TEST_DEVICE=$(SR_TEST_DEVICE) SR_DB_TYPE=Standalone && \
 	export SMARTREDIS_TEST_CLUSTER=False SMARTREDIS_TEST_DEVICE=$(SR_TEST_DEVICE) && \
 	export SSDB=unix://$(SR_TEST_UDS_FILE) && \
 	python utils/launch_redis.py --port $(SR_TEST_PORT) --nodes 1 \
@@ -596,12 +597,13 @@ third-party/catch/single_include/catch2/catch.hpp:
 
 # LCOV (hidden test target)
 .PHONY: lcov
-lcov: third-party/lcov/install/usr/local/bin/lcov
-third-party/lcov/install/usr/local/bin/lcov:
+lcov: third-party/lcov/install/bin/lcov
+third-party/lcov/install/bin/lcov:
+	@echo Installing LCOV
 	@mkdir -p third-party
 	@cd third-party && \
 	git clone $(LCOV_URL) lcov --branch $(LCOV_VER) --depth=1
 	@cd third-party/lcov && \
 	mkdir -p install && \
-	CC=gcc CXX=g++ DESTDIR="install/" make install && \
+	CC=gcc CXX=g++ make PREFIX=$(CWD)/third-party/lcov/install/ install && \
 	echo "Finished installing LCOV"

conftest.py

@@ -53,10 +53,6 @@
     np.uint64,
 ]
 
-@pytest.fixture
-def use_cluster():
-    return os.getenv('SMARTREDIS_TEST_CLUSTER', "").lower() == 'true'
-
 @pytest.fixture
 def mock_data():
     return MockTestData

doc/advanced_topics.rst

@@ -5,7 +5,7 @@ Advanced Topics
 This page of documentation is reserved for advanced topics
 that may not be needed for all users.
 
-.. _advanced_topics_dataset_aggregation:
+.. _advanced-topics-dataset-aggregation:
 
 Dataset Aggregation
 ===================
@@ -92,15 +92,15 @@ predicate is matched on the length of the list:
 
 .. code-block:: cpp
 
-    # Block until the list reaches a specific length
+    // Block until the list reaches a specific length
     bool poll_list_length(const std::string& name, int list_length,
                             int poll_frequency_ms, int num_tries);
 
-    # Block until the list reaches or exceeds a specific length
+    // Block until the list reaches or exceeds a specific length
     bool poll_list_length_gte(const std::string& name, int list_length,
                                 int poll_frequency_ms, int num_tries);
 
-    # Block until the list is no longer than a specific length
+    // Block until the list is no longer than a specific length
     bool poll_list_length_lte(const std::string& name, int list_length,
                                 int poll_frequency_ms, int num_tries);
 
@@ -114,13 +114,72 @@ lead to race conditions:
 
 .. code-block:: cpp
 
-    # Copy an aggregation list
+    // Copy an aggregation list
     void copy_list(const std::string& src_name,
                     const std::string& dest_name);
 
-    # Rename an aggregation list
+    // Rename an aggregation list
     void rename_list(const std::string& src_name,
                         const std::string& dest_name);
 
-    # Delete an aggregation list
+    // Delete an aggregation list
     void delete_list(const std::string& list_name);
+
+.. _advanced-topics-multi-db:
+
+Multiple Database Support
+=========================
+
+SmartRedis offers clients the ability to interact with multiple databases
+concurrently. Each Client represents a connection to a specific database,
+but an application with multiple clients can have each one connected to a
+different database.
+
+Differentiating databases via environment variables
+---------------------------------------------------
+
+In order to differentiate the databases that clients connect to, SmartRedis
+relies on differentiation in the environment variables that the client uses
+to initialize itself. Of primary importance here are the SSDB and SR_DB_TYPE
+variables, but all environment variables (other than SR_LOG_LEVEL and
+SR_LOG_FILE, which are shared for all databases) are duplicated in order to
+represent additional databases.
+
+This duplication is done via suffixing: an underscore and the identifier for
+the database are suffixed to the base variable names to derive a set of
+environment variables specific to each database. For example, SSDB_INPUT
+and SR_DB_TYPE_INPUT reflect a database named ``INPUT``.
+
+For backward compatibility, the default database is anonymous and thus its
+environment variables use neither an underscore nor a database name. This
+behavior exactly matches earlier releases of SmartRedis.
+
+Instantiating Clients for named databases
+-----------------------------------------
+
+Beginning with version 0.5.0 of SmartRedis, users can initialize Clients
+using a new construction method that accepts a ConfigOptions object as
+an input parameter. In turn, the ConfigOptions object can be constructed
+via the ConfigOptions create_from_environment() factory method, which
+accepts the suffix to be applied to environment variables when looking
+them up (or an empty string, to indicate that the default names should be
+used, as for an anonymous database). Depending on the programming language
+for the SmartRedis client, variously None, NULL, or skipping the
+ConfigOptions parameter altogether also implicitly requests an anonymous
+database.
+
+For example, to create a Client for a database named ``INPUT`` in C++,
+one would write the following code:
+
+.. code-block:: cpp
+
+    // Create a ConfigOptions object
+    auto co = ConfigOptions::create_from_environment("INPUT");
+
+    // Pass it to the Client constructor along with an identifier for logging
+    Client* input_client = new Client(co, "input_client");
+
+Note that with the Client constructor that accepts a ConfigOptions object,
+there is no parameter for whether the database is clustered or not. This is
+because the type of database is now read in from the SR_DB_TYPE environment
+variable (with optional {_suffix}).