ENH Add support for RoiBinSz compression

pyproject.toml

@@ -32,7 +32,7 @@ classifiers = [
   "Natural Language :: English",
   "Intended Audience :: Science/Research",
 ]
-dependencies = [ "h5py", "numpy<2", "pillow", "pyzmq", "scipy", "cython>=3.1.2,<4", "fabio>=2024.9.0,<2025", "hdf5plugin>=5.1.0,<6", "msgpack>=1.1.1,<2", "msgpack-numpy>=0.4.8,<0.5", "pydantic>=2.11.7,<3", "pyyaml>=6.0.2,<7", "rich>=14.0.0,<15", "ruamel-yaml>=0.18.14,<0.19", "typer>=0.16.0,<0.17"]
+dependencies = [ "h5py", "numpy", "pillow", "pyzmq", "scipy", "cython>=3.1.2,<4", "fabio>=2024.9.0,<2025", "hdf5plugin>=5.1.0,<6", "msgpack>=1.1.1,<2", "msgpack-numpy>=0.4.8,<0.5", "pydantic>=2.10.0,<3", "pyyaml>=6.0.2,<7", "rich>=14.0.0,<15", "ruamel-yaml>=0.18.14,<0.19", "typer>=0.16.0,<0.17"]
 
 [project.urls]
 Homepage = "https://www.ondamonitor.com"

src/cython/peakfinder8.cpp

@@ -972,7 +972,6 @@ int peakfinder8(tPeakList *peaklist, float *data, char *mask, float *pix_r,
 	struct peakfinder_peak_data *pkdata;
 	int iterations;
 	int num_pix_fs, num_pix_ss;
-	int num_pix_tot;
 	int max_num_peaks;
 	int num_found_peaks;
 	int ret;
@@ -984,7 +983,6 @@ int peakfinder8(tPeakList *peaklist, float *data, char *mask, float *pix_r,
 	// Derived values
 	num_pix_fs = asic_nx * nasics_x;
 	num_pix_ss = asic_ny * nasics_y;
-	num_pix_tot = num_pix_fs * num_pix_ss;
 
 	// Compute radial statistics as 1 function (O.Y.)
 	iterations = 5;

src/om/algorithms/calibration.py

@@ -61,10 +61,10 @@ def __init__(
         """
 
         num_panels: int = len(dark_filenames)
-        self._dark: NDArray[numpy.float_] = numpy.ndarray(
+        self._dark: NDArray[numpy.float64] = numpy.ndarray(
             (3, 512 * num_panels, 1024), dtype=numpy.float32
         )
-        self._gain: NDArray[numpy.float_] = numpy.ndarray(
+        self._gain: NDArray[numpy.float64] = numpy.ndarray(
             (3, 512 * num_panels, 1024), dtype=numpy.float32
         )
 
@@ -87,7 +87,7 @@ def __init__(
 
         self._photon_energy_kev: float = photon_energy_kev
 
-    def apply_calibration(self, *, data: NDArray[numpy.int_]) -> NDArray[numpy.float_]:
+    def apply_calibration(self, *, data: NDArray[numpy.int_]) -> NDArray[numpy.float64]:
         """
         Applies the calibration to a detector data frame.
 
@@ -103,7 +103,7 @@ def apply_calibration(self, *, data: NDArray[numpy.int_]) -> NDArray[numpy.float
 
             The calibrated data frame.
         """
-        calibrated_data: NDArray[numpy.float_] = data.astype(numpy.float_)
+        calibrated_data: NDArray[numpy.float64] = data.astype(numpy.float64)
 
         where_gain: list[tuple[NDArray[numpy.int_], ...]] = [
             numpy.where(data & 2**14 == 0),

src/om/algorithms/crystallography.py

@@ -24,20 +24,168 @@
 """
 
 import random
-from typing import cast
+from typing import Any, Literal, cast
 
 import numpy
 from numpy.typing import NDArray
 
 from om.algorithms.common import PeakList
 from om.lib.files import load_hdf5_data
 from om.lib.geometry import DetectorLayoutInformation
-from om.lib.parameters import Peakfinder8PeakDetectionParameters
-from om.lib.protocols import OmPeakDetectionProtocol
+from om.lib.parameters import (
+    DataCompressionParameters,
+    Peakfinder8PeakDetectionParameters,
+    RoiBinSzCompressorParameters,
+)
+from om.lib.protocols import OmCompressionProtocol, OmPeakDetectionProtocol
 
 from ._crystallography_cython import peakfinder_8  # type: ignore
 
 
+class RoiBinSzCompression(OmCompressionProtocol):
+    def __init__(self, parameters: DataCompressionParameters) -> None:
+        assert parameters.backend == "roibinsz"
+        assert parameters.compression_parameters is not None
+        self._compression_parameters: RoiBinSzCompressorParameters = (
+            parameters.compression_parameters
+        )
+        self._load_mask_from_data: bool = False
+        self._mask: NDArray[numpy.int_] | None = None
+        self._setup_lp_json()
+
+    def _setup_lp_json(self) -> None:
+        compressor: Literal["qoz", "sz3"] = self._compression_parameters.compressor
+        abs_error: float = self._compression_parameters.abs_error
+        bin_size: int = self._compression_parameters.bin_size
+        roi_window_size: int = self._compression_parameters.roi_window_size
+        if compressor == "qoz":
+            pressio_opts: dict[str, Any] = {
+                "pressio:abs": abs_error,
+                "qoz": {"qoz:stride": 8},
+            }
+        elif compressor == "sz3":
+            pressio_opts = {"pressio:abs": abs_error}
+
+        lp_json: dict[str, Any] = {
+            "compressor_id": "pressio",
+            "early_config": {
+                "pressio": {
+                    "pressio:compressor": "roibin",
+                    "roibin": {
+                        "roibin:metric": "composite",
+                        "roibin:background": "mask_binning",
+                        "roibin:roi": "fpzip",
+                        "background": {
+                            "binning:compressor": "pressio",
+                            "mask_binning:compressor": "pressio",
+                            "pressio": {"pressio:compressor": compressor},
+                        },
+                        "composite": {
+                            "composite:plugins": [
+                                "size",
+                                "time",
+                                "input_stats",
+                                "error_stat",
+                            ]
+                        },
+                    },
+                }
+            },
+            "compressor_config": {
+                "pressio": {
+                    "roibin": {
+                        # If ever get to deslabbing, this array size needs to match
+                        # the array dimensions of the raw data (e.g. 3, or 4...)
+                        "roibin:roi_size": [roi_window_size, roi_window_size],
+                        "roibin:centers": None,  # "roibin:roi_strategy": "coordinates",
+                        "roibin:nthreads": 4,
+                        "roi": {"fpzip:prec": 0},
+                        "background": {
+                            "mask_binning:mask": None,
+                            # If ever get to deslabbing, this array size needs to match
+                            # the array dimensions of the raw data (e.g. 3, or 4...)
+                            "mask_binning:shape": [bin_size, bin_size],
+                            "mask_binning:nthreads": 4,
+                            "pressio": pressio_opts,
+                        },
+                    }
+                }
+            },
+            "name": "pressio",
+        }
+
+        # Setup mask
+        placeholder_mask: bool | None = None
+        if isinstance(self._compression_parameters.mask, str):
+            # Try to load
+            # libpressio_mask = 1
+            # lp_json["compressor_config"]["pressio"]["roibin"]["background"][
+            #     "mask_binning:mask"
+            # ] = (1 - libpressio_mask)
+            ...
+        elif self._compression_parameters.mask:
+            # If bool and True, will use a mask provided by the data layer
+            self._load_mask_from_data = True
+            placeholder_mask = None
+        else:
+            # If None, or False, no mask
+            placeholder_mask = None
+        lp_json["compressor_config"]["pressio"]["roibin"]["background"][
+            "mask_binning:mask"
+        ] = placeholder_mask  # Placeholder
+        self._lp_config_base = lp_json
+        self._update_lp_config_mask(placeholder_mask)
+
+    def compress(
+        self,
+        *,
+        data: NDArray[numpy.int_ | numpy.float64],
+        special_data: Any | None = None,
+    ) -> bytes:
+        if not isinstance(special_data, PeakList):
+            raise ValueError("ROIBinSz requires a PeakList!")
+
+        if self._mask is None:
+            self._mask = numpy.ones(data.shape).astype(numpy.uint16)
+            self._update_lp_config_mask(self._mask)
+        return self._compress(data=data, peaks=special_data)
+
+    def _update_lp_config_mask(self, mask: NDArray[numpy.uint16] | None) -> None:
+        self._lp_config_base["compressor_config"]["pressio"]["roibin"]["background"][
+            "mask_binning:mask"
+        ] = mask
+
+    def _compress(
+        self, *, data: NDArray[numpy.int_ | numpy.float64], peaks: PeakList
+    ) -> bytes:
+        from libpressio import PressioCompressor
+
+        lp_config_with_peaks = self._add_peaks_to_libpressio_configuration(
+            config=self._lp_config_base, peaks=peaks
+        )
+        self._compressor = PressioCompressor.from_config(lp_config_with_peaks)
+        compressed_img: bytes = self._compressor.encode(data)
+        return compressed_img
+
+    def uncompress(
+        self, *, compressed_data: bytes, data_shape: tuple[int, ...]
+    ) -> NDArray[numpy.int_ | numpy.float64]:
+        decompressed_img = numpy.zeros(data_shape,dtype=numpy.float32)
+        _ = self._compressor.decode(compressed_data, decompressed_img)
+        return decompressed_img
+
+    def _add_peaks_to_libpressio_configuration(
+        self, *, config: dict[str, Any], peaks: PeakList
+    ) -> dict[str, Any]:
+        peaks_rotated: NDArray[numpy.uint64] = numpy.zeros((len(peaks.fs), 2))
+        peaks_rotated[:, 0] = numpy.array(peaks.fs).astype(numpy.uint64)
+        peaks_rotated[:, 1] = numpy.array(peaks.ss).astype(numpy.uint64)
+        config["compressor_config"]["pressio"]["roibin"][
+            "roibin:centers"
+        ] = peaks_rotated
+        return config
+
+
 class Peakfinder8PeakDetection(OmPeakDetectionProtocol):
     """
     See documentation of the `__init__` function.
@@ -46,7 +194,7 @@ class Peakfinder8PeakDetection(OmPeakDetectionProtocol):
     def __init__(
         self,
         *,
-        radius_pixel_map: NDArray[numpy.float_],
+        radius_pixel_map: NDArray[numpy.float64],
         layout_info: DetectorLayoutInformation,
         parameters: Peakfinder8PeakDetectionParameters,
     ) -> None:
@@ -161,7 +309,7 @@ def __init__(
             self._bad_pixel_map = None
 
         self._mask: NDArray[numpy.int_] | None = None
-        self._radius_pixel_map: NDArray[numpy.float_] = radius_pixel_map
+        self._radius_pixel_map: NDArray[numpy.float64] = radius_pixel_map
 
         self._radial_stats_pixel_index: NDArray[numpy.int_] | None = None
         self._radial_stats_radius: NDArray[numpy.int_] | None = None
@@ -208,7 +356,7 @@ def set_bad_pixel_map(self, bad_pixel_map: NDArray[numpy.int_] | None) -> None:
         self._bad_pixel_map = bad_pixel_map
         self._mask = None
 
-    def set_radius_pixel_map(self, radius_pixel_map: NDArray[numpy.float_]) -> None:
+    def set_radius_pixel_map(self, radius_pixel_map: NDArray[numpy.float64]) -> None:
         self._radius_pixel_map = radius_pixel_map.astype(numpy.float32)
         if self._peakfinder8_parameters.fast_mode is True:
             self._compute_radial_stats_pixels(
@@ -422,7 +570,7 @@ def set_max_res(self, max_res: int) -> None:
         self._mask = None
 
     def find_peaks(
-        self, *, data: NDArray[numpy.int_] | NDArray[numpy.float_]
+        self, *, data: NDArray[numpy.int_] | NDArray[numpy.float64]
     ) -> PeakList:
         """
         Finds peaks in a detector data frame.

src/om/algorithms/generic.py

@@ -36,7 +36,7 @@
 
 from ._generic_cython import bin_detector_data  # type: ignore
 
-A = TypeVar("A", numpy.float_, numpy.int_)
+A = TypeVar("A", numpy.float64, numpy.int_)
 
 
 class RadialProfile:
@@ -47,7 +47,7 @@ class RadialProfile:
     def __init__(
         self,
         *,
-        radius_pixel_map: NDArray[numpy.float_],
+        radius_pixel_map: NDArray[numpy.float64],
         parameters: RadialProfileParameters,
     ) -> None:
         """
@@ -122,7 +122,7 @@ def __init__(
         # Calculates the radial bins
         self._num_bins: int = int(radius_pixel_map.max() / parameters.radius_bin_size)
 
-        radial_bins: NDArray[numpy.float_] = numpy.linspace(
+        radial_bins: NDArray[numpy.float64] = numpy.linspace(
             0,
             self._num_bins * parameters.radius_bin_size,
             self._num_bins + 1,
@@ -170,8 +170,8 @@ def get_bad_pixel_map(self) -> NDArray[numpy.bool_] | None:
 
     def calculate_profile(
         self,
-        data: NDArray[numpy.float_ | numpy.int_],
-    ) -> NDArray[numpy.float_]:
+        data: NDArray[numpy.float64] | NDArray[numpy.int_],
+    ) -> NDArray[numpy.float64]:
         """
         Calculates the radial profile for a detector data frame.
 
@@ -195,7 +195,7 @@ def calculate_profile(
         )
         with numpy.errstate(divide="ignore", invalid="ignore"):
             # numpy.errstate allows to ignore the divide by zero warning
-            radial_average: NDArray[numpy.float_] = numpy.nan_to_num(
+            radial_average: NDArray[numpy.float64] = numpy.nan_to_num(
                 radius_sum / radius_count
             )
 
@@ -322,10 +322,10 @@ def __init__(
         # # Binned mask = num good pixels per bin
         self._binned_mask: NDArray[numpy.int_] = self._bin_data_array(data=self._mask)
 
-        self._float_data_array: NDArray[numpy.float_] = numpy.zeros(
+        self._float_data_array: NDArray[numpy.float64] = numpy.zeros(
             (self._original_nx, self._original_ny), dtype=numpy.float64
         )
-        self._binned_data_array: NDArray[numpy.float_] = numpy.zeros(
+        self._binned_data_array: NDArray[numpy.float64] = numpy.zeros(
             (self._binned_nx, self._binned_ny), dtype=numpy.float64
         )
         self._bad_pixel_value: int | float | None = parameters.bad_pixel_value
@@ -420,8 +420,8 @@ def get_binned_layout_info(self) -> DetectorLayoutInformation:
         )
 
     def bin_detector_data(
-        self, *, data: NDArray[numpy.float_ | numpy.int_]
-    ) -> NDArray[numpy.float_]:
+        self, *, data: NDArray[numpy.float64 | numpy.int_]
+    ) -> NDArray[numpy.float64]:
         """
         Computes a binned version of the detector data frame.
 
@@ -455,7 +455,7 @@ def bin_detector_data(
         if numpy.issubdtype(data_type, numpy.integer):
             self._saturation_value = float(numpy.iinfo(data_type).max)
 
-        self._float_data_array[:] = data.astype(numpy.float_)
+        self._float_data_array[:] = data.astype(numpy.float64)
         bin_detector_data(
             self._float_data_array,
             self._binned_data_array,
@@ -655,8 +655,8 @@ def get_binned_layout_info(self) -> DetectorLayoutInformation:
         return self._layout_info
 
     def bin_detector_data(
-        self, *, data: NDArray[numpy.float_ | numpy.int_]
-    ) -> NDArray[numpy.float_]:
+        self, *, data: NDArray[numpy.float64 | numpy.int_]
+    ) -> NDArray[numpy.float64]:
         """
         Computes a binned version of the detector data frame.
 
@@ -673,7 +673,7 @@ def bin_detector_data(
 
             A binned version of the detector data frame.
         """
-        return data.astype(numpy.float_)
+        return data.astype(numpy.float64)
 
     def bin_bad_pixel_map(
         self, *, mask: NDArray[numpy.int_] | None