Workload fixes for total_iodepth

Signed-off-by: CHris Harris(harriscr@uk.ibm.com)

Author: Chris Harris

benchmark/librbdfio.py

@@ -9,7 +9,6 @@
 import settings
 import monitoring
 
-from typing import Optional
 from .benchmark import Benchmark
 
 logger = logging.getLogger("cbt")
@@ -33,7 +32,6 @@ def __init__(self, archive_dir, cluster, config):
         # would be nice to have them as a separate class -- future PR
         self.time_based = bool(config.get('time_based', False))
         self.ramp = config.get('ramp', None)
-        self.iodepth = config.get('iodepth', 16)
         self.numjobs = config.get('numjobs', 1)
         self.end_fsync = config.get('end_fsync', 0)
         self.mode = config.get('mode', 'write')
@@ -52,12 +50,11 @@ def __init__(self, archive_dir, cluster, config):
         self.fio_out_format = config.get('fio_out_format', 'json,normal')
         self.data_pool = None
 
-        self._ioddepth_per_volume: dict[int, int] = {}
-        total_iodepth: Optional[str] = config.get("total_iodepth", None)
-        if total_iodepth is not None:
-            self._ioddepth_per_volume = self._calculate_iodepth_per_volume(
-                int(self.volumes_per_client), int(total_iodepth)
-            )
+        iodepth_key: str = self._get_iodepth_key(config.keys())  # type: ignore[arg-type]
+        self.iodepth: str = config.get(iodepth_key, "16")
+        self._iodepth_per_volume: dict[int, int] = self._calculate_iodepth_per_volume(
+            self.volumes_per_client, int(self.iodepth), iodepth_key
+        )
 
         # use_existing_volumes needs to be true to set the pool and rbd names
         self.use_existing_volumes = bool(config.get('use_existing_volumes', False))
@@ -172,18 +169,23 @@ def run_workloads(self):
                 enable_monitor = bool(test['monitor'])
             # TODO: simplify this loop to have a single iterator for general queu depth
             for job in test['numjobs']:
-                for iod in test['iodepth']:
+                iodepth_key: str = self._get_iodepth_key(test.keys())  # type: ignore[arg-type]
+                for iodepth_value in test[iodepth_key]:
+                    self._iodepth_per_volume = self._calculate_iodepth_per_volume(
+                        int(self.volumes_per_client), int(iodepth_value), iodepth_key
+                    )
                     self.mode = test['mode']
                     if 'op_size' in test:
                         self.op_size = test['op_size']
                     self.mode = test['mode']
                     self.numjobs = job
-                    self.iodepth = iod
+                    self.iodepth = iodepth_value
                     self.run_dir =  ( f'{self.base_run_dir}/{self.mode}_{int(self.op_size)}/'
                                      f'iodepth-{int(self.iodepth):03d}/numjobs-{int(self.numjobs):03d}' )
                     common.make_remote_dir(self.run_dir)
 
-                    for i in range(self.volumes_per_client):
+                    number_of_volumes: int = len(self._iodepth_per_volume.keys())
+                    for i in range(number_of_volumes):
                         fio_cmd = self.mkfiocmd(i)
                         p = common.pdsh(settings.getnodes('clients'), fio_cmd)
                         ps.append(p)
@@ -235,7 +237,8 @@ def run(self):
             monitoring.start(self.run_dir)
             logger.info('Running rbd fio %s test.', self.mode)
             ps = []
-            for i in range(self.volumes_per_client):
+            number_of_volumes: int = len(self._iodepth_per_volume.keys())
+            for i in range(number_of_volumes):
                 fio_cmd = self.mkfiocmd(i)
                 p = common.pdsh(settings.getnodes('clients'), fio_cmd)
                 ps.append(p)
@@ -284,10 +287,8 @@ def mkfiocmd(self, volnum: int) -> str:
         fio_cmd += ' --direct=1'
         fio_cmd += ' --bs=%dB' % self.op_size
 
-        iodepth: str = f"{self.iodepth}"
-        if self._ioddepth_per_volume != {}:
-            iodepth = f"{self._ioddepth_per_volume[volnum]}"
-
+        iodepth: str = f"{self._iodepth_per_volume[volnum]}"
+        
         fio_cmd += ' --iodepth=%s' % iodepth
         fio_cmd += ' --end_fsync=%d' % self.end_fsync
 #        if self.vol_size:
@@ -415,7 +416,33 @@ def analyze(self, out_dir):
         logger.info('Convert results to json format.')
         self.parse(out_dir)
 
-    def _calculate_iodepth_per_volume(self, number_of_volumes: int, total_desired_iodepth: int) -> dict[int, int]:
+    def _get_iodepth_key(self, configuration_keys: list[str]) -> str:
+        """
+        Get the string that represents the key to use when reading the iodepth
+        values from the configuration. This will be 'total_iodepth' if it is
+        present, otherwise iodepth
+        """
+        iodepth_key: str = "iodepth"
+        if "total_iodepth" in configuration_keys:
+            iodepth_key = "total_iodepth"
+
+        return iodepth_key
+
+    def _calculate_iodepth_per_volume(self, number_of_volumes: int, iodepth: int, iodepth_key: str) -> dict[int, int]:
+        """
+        Calculate the desired iodepth per volume for a single benchmark run.
+        If total_iodepth is to be used calculate what the iodepth per volume
+        should be and return that, otherwise return the iodepth value for each
+        volume
+        """
+        if iodepth_key == "total_iodepth":
+            return self._calculate_iodepth_per_volume_from_total_iodepth(number_of_volumes, iodepth)
+        else:
+            return self._set_iodepth_for_every_volume(number_of_volumes, iodepth)
+
+    def _calculate_iodepth_per_volume_from_total_iodepth(
+        self, number_of_volumes: int, total_desired_iodepth: int
+    ) -> dict[int, int]:
         """
         Given the total desired iodepth and the number of volumes from the
         configuration yaml file, calculate the iodepth for each volume
@@ -436,7 +463,6 @@ def _calculate_iodepth_per_volume(self, number_of_volumes: int, total_desired_io
                 "Number of volumes per client will be reduced from %s to %s", number_of_volumes, total_desired_iodepth
             )
             number_of_volumes = total_desired_iodepth
-            self.volumes_per_client = number_of_volumes
 
         iodepth_per_volume: int = total_desired_iodepth // number_of_volumes
         remainder: int = total_desired_iodepth % number_of_volumes
@@ -451,5 +477,16 @@ def _calculate_iodepth_per_volume(self, number_of_volumes: int, total_desired_io
 
         return queue_depths
 
+    def _set_iodepth_for_every_volume(self, number_of_volumes: int, iodepth: int) -> dict[int, int]:
+        """
+        Given an iodepth value and the number of volumes return a dictionary
+        that contains the desired iodepth value for each volume
+        """
+        queue_depths: dict[int, int] = {}
+        for volume_id in range(number_of_volumes):
+            queue_depths[volume_id] = iodepth
+
+        return queue_depths
+    
     def __str__(self):
         return "%s\n%s\n%s" % (self.run_dir, self.out_dir, super(LibrbdFio, self).__str__())