Merge with upstream and made replication as an optional support

.github/workflows/build_dependencies.yml

@@ -211,6 +211,8 @@ jobs:
 
     - name: Code Coverage Run
       run: |
+        du -sh ~/.conan2/p/*
+        df -h
         conan build \
             -o "sisl/*:prerelease=${{ inputs.prerelease }}" \
             -o "sisl/*:malloc_impl=${{ inputs.malloc-impl }}" \

.jenkins/Dockerfile

@@ -1,5 +1,5 @@
 # ##########   #######   ############
-FROM hub.tess.io/sds/sds_develop:4.x-latest
+FROM hub.tess.io/sds/sds_develop:7.x-latest
 LABEL description="Automated HomeStore compilation"
 
 WORKDIR /output

.jenkins/jenkinsfile_nightly

@@ -1,5 +1,5 @@
 pipeline {
-    agent { label 'sds-builder-2204' }
+    agent { label 'sds-builder-v5' }
     triggers {
           cron('TZ=US/Pacific\nH H(0-2) * * *')
     }
@@ -8,7 +8,7 @@ pipeline {
         ORG = 'sds'
         ECR_URL = 'hub.tess.io'
         ARTIFACTORY_PASS = credentials('ARTIFACTORY_PASS')
-        CONAN_USER = 'sds'
+        CONAN_USER = 'oss'
         failed_stage = ""
     }
     stages {
@@ -26,6 +26,7 @@ pipeline {
                     VER = sh(script: "grep -m 1 ' version =' conanfile.py | awk '{print \$3}' | tr -d '\n' | tr -d '\"'", returnStdout: true)
                     NIGHTLY_TAG = "master-nightly-debug-4.0"
                     ECR_PATH = "${ECR_URL}/${ORG}/${PROJECT}"
+                    CONAN_FLAGS="--name ${PROJECT} --user ${CONAN_USER} --channel ${NIGHTLY_TAG}"
                     failed_stage = ""
                 }
             }
@@ -40,20 +41,25 @@ pipeline {
                 }
                 stage("Build") {
                     steps {
-                        sh "conan create --build missing -o homestore:sanitize=True -pr debug . ${PROJECT}/${VER}@"
-                        sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_index_btree' -exec cp {} .jenkins/test_index_btree \\;"
-                        sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_index_crash_recovery' -exec cp {} .jenkins/test_index_crash_recovery \\;"
-                        sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_meta_blk_mgr' -exec cp {} .jenkins/test_meta_blk_mgr \\;"
-                        sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_log_store' -exec cp {} .jenkins/test_log_store \\;"
-                        sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_home_raft_logstore' -exec cp {} .jenkins/test_home_raft_logstore \\;"
-                        sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_log_store_long_run' -exec cp {} .jenkins/test_log_store_long_run \\;"
-                        sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_data_service' -exec cp {} .jenkins/test_data_service \\;"
-                        sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_raft_repl_dev' -exec cp {} .jenkins/test_raft_repl_dev \\;"
-                        sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_solo_repl_dev' -exec cp {} .jenkins/test_solo_repl_dev \\;"
-                        sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/scripts/index_test.py' -exec install -Dm755 {} .jenkins/index_test.py \\; "
-                        sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/scripts/log_meta_test.py' -exec install -Dm755 {} .jenkins/log_meta_test.py \\; "
-                        sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/scripts/data_test.py' -exec install -Dm755 {} .jenkins/data_test.py \\; "
-                        sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/scripts/long_running.py' -exec install -Dm755 {} .jenkins/long_running.py \\; "
+                        sh '''
+                            hostname
+                            echo $NODE_NAME
+                            conan create --build missing -s:h build_type=Debug -o ${PROJECT}/*:sanitize=True ${CONAN_FLAGS} .
+
+                            find /home/jenkins -type f -wholename '*/test_index_btree' -exec cp {} .jenkins/test_index_btree \\;
+                            find /home/jenkins -type f -wholename '*/test_index_crash_recovery' -exec cp {} .jenkins/test_index_crash_recovery \\;
+                            find /home/jenkins -type f -wholename '*/test_meta_blk_mgr' -exec cp {} .jenkins/test_meta_blk_mgr \\;
+                            find /home/jenkins -type f -wholename '*/test_log_store' -exec cp {} .jenkins/test_log_store \\;
+                            find /home/jenkins -type f -wholename '*/test_home_raft_logstore' -exec cp {} .jenkins/test_home_raft_logstore \\;
+                            find /home/jenkins -type f -wholename '*/test_log_store_long_run' -exec cp {} .jenkins/test_log_store_long_run \\;
+                            find /home/jenkins -type f -wholename '*/test_data_service' -exec cp {} .jenkins/test_data_service \\;
+                            find /home/jenkins -type f -wholename '*/test_raft_repl_dev' -exec cp {} .jenkins/test_raft_repl_dev \\;
+                            find /home/jenkins -type f -wholename '*/test_solo_repl_dev' -exec cp {} .jenkins/test_solo_repl_dev \\;
+                            find /home/jenkins -type f -wholename '*/test_scripts/index_test.py' -exec install -Dm755 {} .jenkins/index_test.py \\;
+                            find /home/jenkins -type f -wholename '*/test_scripts/log_meta_test.py' -exec install -Dm755 {} .jenkins/log_meta_test.py \\;
+                            find /home/jenkins -type f -wholename '*/test_scripts/data_test.py' -exec install -Dm755 {} .jenkins/data_test.py \\;
+                            find /home/jenkins -type f -wholename '*/test_scripts/long_running.py' -exec install -Dm755 {} .jenkins/long_running.py \\;
+                        '''
                     }
                     post {
                         failure {

CMakeLists.txt

@@ -87,6 +87,18 @@ endif ()
 add_flags("-DPACKAGE_NAME=\\\"${PROJECT_NAME}\\\"")
 add_flags("-DPACKAGE_VERSION=\\\"${PACKAGE_REVISION}\\\"")
 
+# add replication flag
+if (DEFINED REPLICATION)
+  if (${REPLICATION} STREQUAL "ON")
+    add_flags("-DREPLICATION_SUPPORT")
+    message(STATUS "Building with REPLICATION enabled")
+  else()
+    message(STATUS "Building with REPLICATION disabled")
+  endif()
+else()
+  message(STATUS "Building with REPLICATION disabled")
+endif()
+
 if(UNIX)
     # enable proper pread/pwrite and large file
     add_flags("-D_POSIX_C_SOURCE=200809L -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE")

cmake/test_mode.cmake

@@ -39,6 +39,9 @@ if (DEFINED TEST_TARGET)
           set(${ret} true)
       endif()
   endmacro()
+  macro(can_build_repl_tests ret)
+      set(${ret} false)
+  endmacro()
 else()
   macro(can_build_io_tests ret)
       set(${ret} false)
@@ -55,4 +58,7 @@ else()
   macro(can_build_epoll_io_tests ret)
       set(${ret} false)
   endmacro()
+  macro(can_build_repl_tests ret)
+      set(${ret} false)
+  endmacro()
 endif()

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "5.2.2"
+    version = "5.3.1"
 
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"
@@ -25,13 +25,15 @@ class HomestoreConan(ConanFile):
                 "coverage": ['True', 'False'],
                 "sanitize": ['True', 'False'],
                 "testing" : ['full', 'min', 'off', 'epoll_mode', 'spdk_mode'],
+                "replication" : ['off', 'on'],
             }
     default_options = {
                 'shared':       False,
                 'fPIC':         True,
                 'coverage':     False,
                 'sanitize':     False,
                 'testing':      'epoll_mode',
+                'replication':  'off',
             }
 
     exports_sources = "cmake/*", "src/*", "CMakeLists.txt", "test_wrap.sh", "LICENSE"
@@ -54,18 +56,27 @@ def build_requirements(self):
     def requirements(self):
         self.requires("iomgr/[^12.1]@oss/master", transitive_headers=True)
         self.requires("sisl/[^13.3]@oss/master", transitive_headers=True)
-        self.requires("nuraft_mesg/[^4.1]@oss/main", transitive_headers=True)
+        if str(self.options.replication) == "on":
+            self.requires("nuraft_mesg/[^4.1]@oss/main", transitive_headers=True)
 
         self.requires("farmhash/cci.20190513@", transitive_headers=True)
         if self.settings.arch in ['x86', 'x86_64']:
             self.requires("isa-l/2.30.0", transitive_headers=True)
 
+        # Tests require OpenSSL 3.x
+        self.requires("openssl/[^3.1]", override=True)
+
     def imports(self):
         self.copy(root_package="sisl", pattern="*", dst="bin/scripts/python/flip/", src="bindings/flip/python/", keep_path=False)
 
     def layout(self):
         self.folders.source = "."
-        self.folders.build = join("build", str(self.settings.build_type))
+        if self.options.get_safe("sanitize"):
+            self.folders.build = join("build", "Sanitized")
+        elif self.options.get_safe("coverage"):
+            self.folders.build = join("build", "Coverage")
+        else:
+            self.folders.build = join("build", str(self.settings.build_type))
         self.folders.generators = join(self.folders.build, "generators")
 
         self.cpp.source.includedirs = ["src/include"]
@@ -94,6 +105,12 @@ def generate(self):
                 tc.variables['BUILD_COVERAGE'] = 'ON'
             elif self.options.get_safe("sanitize"):
                 tc.variables['MEMORY_SANITIZER_ON'] = 'ON'
+        tc.variables["CONAN_PACKAGE_NAME"] = self.name
+        tc.variables["CONAN_PACKAGE_VERSION"] = self.version
+        if str(self.options.replication) == "on":
+            tc.variables["REPLICATION"] = "ON"
+        else:
+            tc.variables["REPLICATION"] = "OFF"
         tc.generate()
 
         # This generates "boost-config.cmake" and "grpc-config.cmake" etc in self.generators_folder

src/CMakeLists.txt

@@ -8,15 +8,27 @@ find_package(isa-l QUIET)
 find_package(iomgr QUIET REQUIRED)
 find_package(farmhash QUIET REQUIRED)
 find_package(GTest QUIET REQUIRED)
-find_package(NuraftMesg QUIET REQUIRED)
+if (DEFINED REPLICATION)
+  if (${REPLICATION} STREQUAL "ON")
+    find_package(NuraftMesg QUIET REQUIRED)
+  endif()
+endif()
 
 list(APPEND COMMON_DEPS
     iomgr::iomgr
     farmhash::farmhash
-    nuraft_mesg::proto
-    nuraft::nuraft
     sisl::sisl
 )
+
+if (DEFINED REPLICATION)
+  if (${REPLICATION} STREQUAL "ON")
+    list(APPEND COMMON_DEPS 
+        nuraft_mesg::proto
+        nuraft::nuraft
+    )
+  endif()
+endif()
+
 if (${isa-l_FOUND})
     list(APPEND COMMON_DEPS isa-l::isa-l)
 else ()
@@ -42,7 +54,11 @@ add_subdirectory(lib/logstore)
 add_subdirectory(lib/meta)
 add_subdirectory(lib/index)
 add_subdirectory(lib/blkdata_svc/)
-add_subdirectory(lib/replication/)
+if (DEFINED REPLICATION)
+  if (${REPLICATION} STREQUAL "ON")
+    add_subdirectory(lib/replication/)
+  endif()
+endif()
 
 if(NOT DEFINED BUILD_TESTING OR BUILD_TESTING)
     add_subdirectory(tests)
@@ -59,20 +75,19 @@ set(HOMESTORE_OBJECTS
     $<TARGET_OBJECTS:hs_mem_btree>
     $<TARGET_OBJECTS:hs_cow_btree>
     $<TARGET_OBJECTS:hs_datasvc>
-    $<TARGET_OBJECTS:hs_replication>
     lib/homestore.cpp
     lib/crc.cpp
 )
-#target_link_libraries(homestore_objs ${COMMON_DEPS})
-if (${CMAKE_BUILD_TYPE} STREQUAL "Debug")
-    add_library(homestore STATIC
-        ${HOMESTORE_OBJECTS} 
-    )
-else()
-    add_library(homestore STATIC 
-        ${HOMESTORE_OBJECTS} 
-    )
+
+if (DEFINED REPLICATION)
+  if (${REPLICATION} STREQUAL "ON")
+    list(APPEND HOMESTORE_OBJECTS $<TARGET_OBJECTS:hs_replication>)
+  endif()
 endif()
+#target_link_libraries(homestore_objs ${COMMON_DEPS})
 
+add_library(homestore STATIC 
+    ${HOMESTORE_OBJECTS} 
+)
 target_compile_definitions (homestore PRIVATE LOG_MODS_V2_SUPPORT)
 target_link_libraries(homestore PRIVATE ${COMMON_DEPS})

src/include/homestore/blk.h

@@ -251,9 +251,14 @@ VENUM(BlkAllocStatus, uint32_t,
 
 struct blk_alloc_hints {
     blk_temp_t desired_temp{0};                  // Temperature hint for the device
-    std::optional< uint32_t > pdev_id_hint{std::nullopt};      // which physical device to pick (hint if any)
-    std::optional< chunk_num_t > chunk_id_hint{std::nullopt};  // any specific chunk id to pick for this allocation
+    std::optional< uint32_t > reserved_blks{std::nullopt};    // Reserved blks in a chunk
+    std::optional< uint32_t > pdev_id_hint{std::nullopt};     // which physical device to pick (hint if any)
+    std::optional< chunk_num_t > chunk_id_hint{std::nullopt}; // any specific chunk id to pick for this allocation
+    std::optional< MultiBlkId > committed_blk_id{
+        std::nullopt}; //  blk id indicates the blk was already allocated and committed, don't allocate and commit again
     std::optional< stream_id_t > stream_id_hint{std::nullopt}; // any specific stream to pick
+    std::optional< uint64_t > application_hint{
+        std::nullopt};                           // hints in uint64 what will be passed opaque to select_chunk
     bool can_look_for_other_chunk{true};         // If alloc on device not available can I pick other device
     bool is_contiguous{true};                    // Should the entire allocation be one contiguous block
     bool partial_alloc_ok{false};   // ok to allocate only portion of nblks? Mutually exclusive with is_contiguous

src/include/homestore/blkdata_service.hpp

@@ -56,17 +56,19 @@ class BlkDataService {
 
     /**
      * @brief Creates a new virtual device with the specified size and block size, using the specified
-     * block allocator and chunk selector types. The virtual device will be composed of the specified
-     * number of chunks.
+     * block allocator and chunk selector types. The virtual device will be composed of a number of chunks.
+     * Either `num_chunks` or `chunk_size` must be specified.
+     * Prioritize `num_chunks` over `chunk_size` if both are provided.
      *
      * @param size The size of the virtual device, in bytes.
      * @param blk_size The size of each block in the virtual device, in bytes.
      * @param alloc_type The type of block allocator to use for the virtual device.
      * @param chunk_sel_type The type of chunk selector to use for the virtual device.
      * @param num_chunks The number of chunks to use for the virtual device.
+     * @param chunk_size The size of chunks to use for the virtual device, in bytes.
      */
     void create_vdev(uint64_t size, HSDevType devType, uint32_t blk_size, blk_allocator_type_t alloc_type,
-                     chunk_selector_type_t chunk_sel_type, uint32_t num_chunks);
+                     chunk_selector_type_t chunk_sel_type, uint32_t num_chunks, uint32_t chunk_size);
 
     /**
      * @brief Opens a virtual device with the specified virtual device information.
@@ -112,6 +114,18 @@ class BlkDataService {
     folly::Future< std::error_code > async_write(sisl::sg_list const& sgs, MultiBlkId const& in_blkids,
                                                  bool part_of_batch = false);
 
+    /**
+     * @brief : asynchronous write with input block ids;
+     *
+     * @param sgs : the data buffer that needs to be written
+     * @param hints : blk alloc hints
+     * @param in_blkids : input block ids that this write should be written to;
+     * @param cb : callback that will be triggered after write completes
+     * @param part_of_batch : is this write part of a batch;
+     */
+    folly::Future< std::error_code > async_write(sisl::sg_list const& sgs, std::vector< MultiBlkId > const& in_blkids,
+                                                 bool part_of_batch = false);
+
     /**
      * @brief Asynchronously reads data from the specified block ID into the provided buffer.
      *
@@ -137,6 +151,13 @@ class BlkDataService {
     folly::Future< std::error_code > async_read(MultiBlkId const& bid, sisl::sg_list& sgs, uint32_t size,
                                                 bool part_of_batch = false);
 
+    /**
+     * @brief Submit the io batch, which is a mandatory method to be called if read/write are issued with part_of_batch
+     * is set to true. In those cases, without this method, IOs might not be even issued. No-op if previous io requests
+     * are not part of batch.
+     * */
+    void submit_io_batch();
+
     /**
      * @brief Commits the block with the given MultiBlkId.
      *
@@ -145,7 +166,8 @@ class BlkDataService {
     BlkAllocStatus commit_blk(MultiBlkId const& bid);
 
     /**
-     * @brief Allocates a contiguous block of disk space of the given size.
+     * @brief Allocates a contiguous block of disk space of the given size. This API should be called that when consumer
+     * is expecting blks only allocated on same chunk.
      *
      * @param size The size of the block to allocate, in bytes.
      * @param hints Hints for how to allocate the block.
@@ -154,6 +176,17 @@ class BlkDataService {
      */
     BlkAllocStatus alloc_blks(uint32_t size, blk_alloc_hints const& hints, MultiBlkId& out_blkids);
 
+    /**
+     * @brief Allocates blocks of disk space of the given size.This API should be called when consumer is expecting blk
+     * allocation happen on different chunks is possible and acceptable.
+     *
+     * @param size The size of the block to allocate, in bytes.
+     * @param hints Hints for how to allocate the block.
+     * @param out_blkids Output parameter that will be filled with the IDs of the allocated blocks.
+     * @return The status of the block allocation attempt.
+     */
+    BlkAllocStatus alloc_blks(uint32_t size, blk_alloc_hints const& hints, std::vector< BlkId >& out_blkids);
+
     /**
      * @brief Asynchronously frees the specified block IDs.
      * It is asynchronous because it might need to wait for pending read to complete if same block is being read and not
@@ -194,10 +227,35 @@ class BlkDataService {
      */
     void start();
 
+    /**
+     * @brief Gets the total capacity of the block data service.
+     *
+     * This function returns the total capacity of the block data service, in bytes.
+     *
+     * @return The total capacity of the block data service, in bytes.
+     */
     uint64_t get_total_capacity() const;
 
+    /**
+     * @brief Gets the used capacity of the block data service.
+     *
+     * This function returns the used capacity of the block data service, in bytes.
+     *
+     * @return The used capacity of the block data service, in bytes.
+     */
     uint64_t get_used_capacity() const;
 
+    /**
+     * @brief Gets the drive type of the data service.
+     *
+     * Data Service doesn't support mixed drive types.
+     *
+     * @return The drive type of the data service, HDD or NVME.
+     */
+    HSDevType get_dev_type() const;
+
+    void stop();
+
 private:
     /**
      * @brief Initializes the block data service.