diff --git a/pyproject.toml b/pyproject.toml index 7e8035f0..4ac4ec2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ classifiers = [ "Natural Language :: English", "Intended Audience :: Science/Research", ] -dependencies = [ "h5py", "numpy<2", "pillow", "pyzmq", "scipy", "cython>=3.1.2,<4", "fabio>=2024.9.0,<2025", "hdf5plugin>=5.1.0,<6", "msgpack>=1.1.1,<2", "msgpack-numpy>=0.4.8,<0.5", "pydantic>=2.11.7,<3", "pyyaml>=6.0.2,<7", "rich>=14.0.0,<15", "ruamel-yaml>=0.18.14,<0.19", "typer>=0.16.0,<0.17"] +dependencies = [ "h5py", "numpy", "pillow", "pyzmq", "scipy", "cython>=3.1.2,<4", "fabio>=2024.9.0,<2025", "hdf5plugin>=5.1.0,<6", "msgpack>=1.1.1,<2", "msgpack-numpy>=0.4.8,<0.5", "pydantic>=2.10.0,<3", "pyyaml>=6.0.2,<7", "rich>=14.0.0,<15", "ruamel-yaml>=0.18.14,<0.19", "typer>=0.16.0,<0.17"] [project.urls] Homepage = "https://www.ondamonitor.com" diff --git a/src/cython/peakfinder8.cpp b/src/cython/peakfinder8.cpp index 2cb98ab7..f5c5afb4 100644 --- a/src/cython/peakfinder8.cpp +++ b/src/cython/peakfinder8.cpp @@ -972,7 +972,6 @@ int peakfinder8(tPeakList *peaklist, float *data, char *mask, float *pix_r, struct peakfinder_peak_data *pkdata; int iterations; int num_pix_fs, num_pix_ss; - int num_pix_tot; int max_num_peaks; int num_found_peaks; int ret; @@ -984,7 +983,6 @@ int peakfinder8(tPeakList *peaklist, float *data, char *mask, float *pix_r, // Derived values num_pix_fs = asic_nx * nasics_x; num_pix_ss = asic_ny * nasics_y; - num_pix_tot = num_pix_fs * num_pix_ss; // Compute radial statistics as 1 function (O.Y.) iterations = 5; diff --git a/src/om/algorithms/calibration.py b/src/om/algorithms/calibration.py index 6a80ca4f..5ab811fe 100644 --- a/src/om/algorithms/calibration.py +++ b/src/om/algorithms/calibration.py @@ -61,10 +61,10 @@ def __init__( """ num_panels: int = len(dark_filenames) - self._dark: NDArray[numpy.float_] = numpy.ndarray( + self._dark: NDArray[numpy.float64] = numpy.ndarray( (3, 512 * num_panels, 1024), dtype=numpy.float32 ) - self._gain: NDArray[numpy.float_] = numpy.ndarray( + self._gain: NDArray[numpy.float64] = numpy.ndarray( (3, 512 * num_panels, 1024), dtype=numpy.float32 ) @@ -87,7 +87,7 @@ def __init__( self._photon_energy_kev: float = photon_energy_kev - def apply_calibration(self, *, data: NDArray[numpy.int_]) -> NDArray[numpy.float_]: + def apply_calibration(self, *, data: NDArray[numpy.int_]) -> NDArray[numpy.float64]: """ Applies the calibration to a detector data frame. @@ -103,7 +103,7 @@ def apply_calibration(self, *, data: NDArray[numpy.int_]) -> NDArray[numpy.float The calibrated data frame. """ - calibrated_data: NDArray[numpy.float_] = data.astype(numpy.float_) + calibrated_data: NDArray[numpy.float64] = data.astype(numpy.float64) where_gain: list[tuple[NDArray[numpy.int_], ...]] = [ numpy.where(data & 2**14 == 0), diff --git a/src/om/algorithms/crystallography.py b/src/om/algorithms/crystallography.py index 2ef085d8..fae693b7 100644 --- a/src/om/algorithms/crystallography.py +++ b/src/om/algorithms/crystallography.py @@ -24,7 +24,7 @@ """ import random -from typing import cast +from typing import Any, Literal, cast import numpy from numpy.typing import NDArray @@ -32,12 +32,160 @@ from om.algorithms.common import PeakList from om.lib.files import load_hdf5_data from om.lib.geometry import DetectorLayoutInformation -from om.lib.parameters import Peakfinder8PeakDetectionParameters -from om.lib.protocols import OmPeakDetectionProtocol +from om.lib.parameters import ( + DataCompressionParameters, + Peakfinder8PeakDetectionParameters, + RoiBinSzCompressorParameters, +) +from om.lib.protocols import OmCompressionProtocol, OmPeakDetectionProtocol from ._crystallography_cython import peakfinder_8 # type: ignore +class RoiBinSzCompression(OmCompressionProtocol): + def __init__(self, parameters: DataCompressionParameters) -> None: + assert parameters.backend == "roibinsz" + assert parameters.compression_parameters is not None + self._compression_parameters: RoiBinSzCompressorParameters = ( + parameters.compression_parameters + ) + self._load_mask_from_data: bool = False + self._mask: NDArray[numpy.int_] | None = None + self._setup_lp_json() + + def _setup_lp_json(self) -> None: + compressor: Literal["qoz", "sz3"] = self._compression_parameters.compressor + abs_error: float = self._compression_parameters.abs_error + bin_size: int = self._compression_parameters.bin_size + roi_window_size: int = self._compression_parameters.roi_window_size + if compressor == "qoz": + pressio_opts: dict[str, Any] = { + "pressio:abs": abs_error, + "qoz": {"qoz:stride": 8}, + } + elif compressor == "sz3": + pressio_opts = {"pressio:abs": abs_error} + + lp_json: dict[str, Any] = { + "compressor_id": "pressio", + "early_config": { + "pressio": { + "pressio:compressor": "roibin", + "roibin": { + "roibin:metric": "composite", + "roibin:background": "mask_binning", + "roibin:roi": "fpzip", + "background": { + "binning:compressor": "pressio", + "mask_binning:compressor": "pressio", + "pressio": {"pressio:compressor": compressor}, + }, + "composite": { + "composite:plugins": [ + "size", + "time", + "input_stats", + "error_stat", + ] + }, + }, + } + }, + "compressor_config": { + "pressio": { + "roibin": { + # If ever get to deslabbing, this array size needs to match + # the array dimensions of the raw data (e.g. 3, or 4...) + "roibin:roi_size": [roi_window_size, roi_window_size], + "roibin:centers": None, # "roibin:roi_strategy": "coordinates", + "roibin:nthreads": 4, + "roi": {"fpzip:prec": 0}, + "background": { + "mask_binning:mask": None, + # If ever get to deslabbing, this array size needs to match + # the array dimensions of the raw data (e.g. 3, or 4...) + "mask_binning:shape": [bin_size, bin_size], + "mask_binning:nthreads": 4, + "pressio": pressio_opts, + }, + } + } + }, + "name": "pressio", + } + + # Setup mask + placeholder_mask: bool | None = None + if isinstance(self._compression_parameters.mask, str): + # Try to load + # libpressio_mask = 1 + # lp_json["compressor_config"]["pressio"]["roibin"]["background"][ + # "mask_binning:mask" + # ] = (1 - libpressio_mask) + ... + elif self._compression_parameters.mask: + # If bool and True, will use a mask provided by the data layer + self._load_mask_from_data = True + placeholder_mask = None + else: + # If None, or False, no mask + placeholder_mask = None + lp_json["compressor_config"]["pressio"]["roibin"]["background"][ + "mask_binning:mask" + ] = placeholder_mask # Placeholder + self._lp_config_base = lp_json + self._update_lp_config_mask(placeholder_mask) + + def compress( + self, + *, + data: NDArray[numpy.int_ | numpy.float64], + special_data: Any | None = None, + ) -> bytes: + if not isinstance(special_data, PeakList): + raise ValueError("ROIBinSz requires a PeakList!") + + if self._mask is None: + self._mask = numpy.ones(data.shape).astype(numpy.uint16) + self._update_lp_config_mask(self._mask) + return self._compress(data=data, peaks=special_data) + + def _update_lp_config_mask(self, mask: NDArray[numpy.uint16] | None) -> None: + self._lp_config_base["compressor_config"]["pressio"]["roibin"]["background"][ + "mask_binning:mask" + ] = mask + + def _compress( + self, *, data: NDArray[numpy.int_ | numpy.float64], peaks: PeakList + ) -> bytes: + from libpressio import PressioCompressor + + lp_config_with_peaks = self._add_peaks_to_libpressio_configuration( + config=self._lp_config_base, peaks=peaks + ) + self._compressor = PressioCompressor.from_config(lp_config_with_peaks) + compressed_img: bytes = self._compressor.encode(data) + return compressed_img + + def uncompress( + self, *, compressed_data: bytes, data_shape: tuple[int, ...] + ) -> NDArray[numpy.int_ | numpy.float64]: + decompressed_img = numpy.zeros(data_shape,dtype=numpy.float32) + _ = self._compressor.decode(compressed_data, decompressed_img) + return decompressed_img + + def _add_peaks_to_libpressio_configuration( + self, *, config: dict[str, Any], peaks: PeakList + ) -> dict[str, Any]: + peaks_rotated: NDArray[numpy.uint64] = numpy.zeros((len(peaks.fs), 2)) + peaks_rotated[:, 0] = numpy.array(peaks.fs).astype(numpy.uint64) + peaks_rotated[:, 1] = numpy.array(peaks.ss).astype(numpy.uint64) + config["compressor_config"]["pressio"]["roibin"][ + "roibin:centers" + ] = peaks_rotated + return config + + class Peakfinder8PeakDetection(OmPeakDetectionProtocol): """ See documentation of the `__init__` function. @@ -46,7 +194,7 @@ class Peakfinder8PeakDetection(OmPeakDetectionProtocol): def __init__( self, *, - radius_pixel_map: NDArray[numpy.float_], + radius_pixel_map: NDArray[numpy.float64], layout_info: DetectorLayoutInformation, parameters: Peakfinder8PeakDetectionParameters, ) -> None: @@ -161,7 +309,7 @@ def __init__( self._bad_pixel_map = None self._mask: NDArray[numpy.int_] | None = None - self._radius_pixel_map: NDArray[numpy.float_] = radius_pixel_map + self._radius_pixel_map: NDArray[numpy.float64] = radius_pixel_map self._radial_stats_pixel_index: NDArray[numpy.int_] | None = None self._radial_stats_radius: NDArray[numpy.int_] | None = None @@ -208,7 +356,7 @@ def set_bad_pixel_map(self, bad_pixel_map: NDArray[numpy.int_] | None) -> None: self._bad_pixel_map = bad_pixel_map self._mask = None - def set_radius_pixel_map(self, radius_pixel_map: NDArray[numpy.float_]) -> None: + def set_radius_pixel_map(self, radius_pixel_map: NDArray[numpy.float64]) -> None: self._radius_pixel_map = radius_pixel_map.astype(numpy.float32) if self._peakfinder8_parameters.fast_mode is True: self._compute_radial_stats_pixels( @@ -422,7 +570,7 @@ def set_max_res(self, max_res: int) -> None: self._mask = None def find_peaks( - self, *, data: NDArray[numpy.int_] | NDArray[numpy.float_] + self, *, data: NDArray[numpy.int_] | NDArray[numpy.float64] ) -> PeakList: """ Finds peaks in a detector data frame. diff --git a/src/om/algorithms/generic.py b/src/om/algorithms/generic.py index f4360468..bb03a3a0 100644 --- a/src/om/algorithms/generic.py +++ b/src/om/algorithms/generic.py @@ -36,7 +36,7 @@ from ._generic_cython import bin_detector_data # type: ignore -A = TypeVar("A", numpy.float_, numpy.int_) +A = TypeVar("A", numpy.float64, numpy.int_) class RadialProfile: @@ -47,7 +47,7 @@ class RadialProfile: def __init__( self, *, - radius_pixel_map: NDArray[numpy.float_], + radius_pixel_map: NDArray[numpy.float64], parameters: RadialProfileParameters, ) -> None: """ @@ -122,7 +122,7 @@ def __init__( # Calculates the radial bins self._num_bins: int = int(radius_pixel_map.max() / parameters.radius_bin_size) - radial_bins: NDArray[numpy.float_] = numpy.linspace( + radial_bins: NDArray[numpy.float64] = numpy.linspace( 0, self._num_bins * parameters.radius_bin_size, self._num_bins + 1, @@ -170,8 +170,8 @@ def get_bad_pixel_map(self) -> NDArray[numpy.bool_] | None: def calculate_profile( self, - data: NDArray[numpy.float_ | numpy.int_], - ) -> NDArray[numpy.float_]: + data: NDArray[numpy.float64] | NDArray[numpy.int_], + ) -> NDArray[numpy.float64]: """ Calculates the radial profile for a detector data frame. @@ -195,7 +195,7 @@ def calculate_profile( ) with numpy.errstate(divide="ignore", invalid="ignore"): # numpy.errstate allows to ignore the divide by zero warning - radial_average: NDArray[numpy.float_] = numpy.nan_to_num( + radial_average: NDArray[numpy.float64] = numpy.nan_to_num( radius_sum / radius_count ) @@ -322,10 +322,10 @@ def __init__( # # Binned mask = num good pixels per bin self._binned_mask: NDArray[numpy.int_] = self._bin_data_array(data=self._mask) - self._float_data_array: NDArray[numpy.float_] = numpy.zeros( + self._float_data_array: NDArray[numpy.float64] = numpy.zeros( (self._original_nx, self._original_ny), dtype=numpy.float64 ) - self._binned_data_array: NDArray[numpy.float_] = numpy.zeros( + self._binned_data_array: NDArray[numpy.float64] = numpy.zeros( (self._binned_nx, self._binned_ny), dtype=numpy.float64 ) self._bad_pixel_value: int | float | None = parameters.bad_pixel_value @@ -420,8 +420,8 @@ def get_binned_layout_info(self) -> DetectorLayoutInformation: ) def bin_detector_data( - self, *, data: NDArray[numpy.float_ | numpy.int_] - ) -> NDArray[numpy.float_]: + self, *, data: NDArray[numpy.float64 | numpy.int_] + ) -> NDArray[numpy.float64]: """ Computes a binned version of the detector data frame. @@ -455,7 +455,7 @@ def bin_detector_data( if numpy.issubdtype(data_type, numpy.integer): self._saturation_value = float(numpy.iinfo(data_type).max) - self._float_data_array[:] = data.astype(numpy.float_) + self._float_data_array[:] = data.astype(numpy.float64) bin_detector_data( self._float_data_array, self._binned_data_array, @@ -655,8 +655,8 @@ def get_binned_layout_info(self) -> DetectorLayoutInformation: return self._layout_info def bin_detector_data( - self, *, data: NDArray[numpy.float_ | numpy.int_] - ) -> NDArray[numpy.float_]: + self, *, data: NDArray[numpy.float64 | numpy.int_] + ) -> NDArray[numpy.float64]: """ Computes a binned version of the detector data frame. @@ -673,7 +673,7 @@ def bin_detector_data( A binned version of the detector data frame. """ - return data.astype(numpy.float_) + return data.astype(numpy.float64) def bin_bad_pixel_map( self, *, mask: NDArray[numpy.int_] | None diff --git a/src/om/algorithms/xes.py b/src/om/algorithms/xes.py index 1683c52a..92e06463 100644 --- a/src/om/algorithms/xes.py +++ b/src/om/algorithms/xes.py @@ -80,8 +80,8 @@ def __init__( # TODO: Enforce return dict content for the function below def calculate_spectrum( - self, *, data: NDArray[numpy.float_ | numpy.int_] - ) -> dict[str, NDArray[numpy.float_]]: + self, *, data: NDArray[numpy.float64 | numpy.int_] + ) -> dict[str, NDArray[numpy.float64]]: """ Calculates beam energy spectrum information from a camera data frame. @@ -120,8 +120,8 @@ def calculate_spectrum( # TODO: Perhaps better type hints can be found for this if self._xes_parameters.intensity_threshold is not None: data[data < self._xes_parameters.intensity_threshold] = 0 - imr: NDArray[numpy.float_ | numpy.int_] = cast( - DArray[numpy.float_ | numpy.int_], + imr: NDArray[numpy.float64 | numpy.int_] = cast( + DArray[numpy.float64 | numpy.int_], ndimage.rotate( data, self._xes_parameters.rotation_in_degrees, @@ -130,7 +130,7 @@ def calculate_spectrum( ) min_row: int = self._xes_parameters.min_row_in_pix_for_integration max_row: int = self._xes_parameters.max_row_in_pix_for_integration - spectrum: NDArray[numpy.float_] = numpy.mean( + spectrum: NDArray[numpy.float64] = numpy.mean( imr[ :, min_row:max_row, @@ -138,7 +138,7 @@ def calculate_spectrum( axis=1, ) - spectrum_smoothed: NDArray[numpy.float_] = gaussian_filter1d(spectrum, 2) + spectrum_smoothed: NDArray[numpy.float64] = gaussian_filter1d(spectrum, 2) return { "spectrum": spectrum, "spectrum_smoothed": spectrum_smoothed, diff --git a/src/om/data_retrieval_layer/data_event_handlers_asapo.py b/src/om/data_retrieval_layer/data_event_handlers_asapo.py index c62e81ee..31dfad82 100644 --- a/src/om/data_retrieval_layer/data_event_handlers_asapo.py +++ b/src/om/data_retrieval_layer/data_event_handlers_asapo.py @@ -48,7 +48,7 @@ class _AsapoEvent: # This named tuple is used internally to store ASAP::O event data, metadata and # corresponding ASAP::O stream information. - event_data: NDArray[numpy.float_ | numpy.int_] + event_data: NDArray[numpy.float64 | numpy.int_] event_metadata: dict[str, Any] stream_name: str stream_metadata: dict[str, Any] @@ -158,7 +158,7 @@ def _online_event_generator( time.sleep(1) last_stream = consumer.get_last_stream()["name"] stream_metadata: dict[str, Any] = consumer.get_stream_meta(last_stream) - event_data: NDArray[numpy.float_ | numpy.int_] + event_data: NDArray[numpy.float64 | numpy.int_] event_metadata: dict[str, Any] while True: try: @@ -382,7 +382,7 @@ def retrieve_event_data(self, event_id: str) -> dict[str, Any]: stream: str = event_id_parts[0].strip() asapo_event_id: int = int(event_id_parts[1]) - event_data: NDArray[numpy.float_ | numpy.int_] + event_data: NDArray[numpy.float64 | numpy.int_] event_metadata: dict[str, Any] event_data, event_metadata = self._consumer.get_by_id( asapo_event_id, diff --git a/src/om/data_retrieval_layer/data_sources_asapo.py b/src/om/data_retrieval_layer/data_sources_asapo.py index 4e757743..961b2cab 100644 --- a/src/om/data_retrieval_layer/data_sources_asapo.py +++ b/src/om/data_retrieval_layer/data_sources_asapo.py @@ -101,7 +101,7 @@ class DetectorDataAsapo(OmBaseAsapoDataSourceMixin, OmDataSourceProtocol): def get_data( self, *, event: dict[str, Any] - ) -> NDArray[numpy.float_ | numpy.int_]: + ) -> NDArray[numpy.float64 | numpy.int_]: """ Retrieves a detector data frame from ASAP::O. @@ -122,7 +122,7 @@ def get_data( """ # TODO: Fix type hinting return cast( - NDArray[numpy.float_ | numpy.int_], + NDArray[numpy.float64 | numpy.int_], seedee.deserialize( event["data"], event["metadata"]["meta"]["_data_format"] ), diff --git a/src/om/data_retrieval_layer/data_sources_common.py b/src/om/data_retrieval_layer/data_sources_common.py index 81e48351..c6c47933 100644 --- a/src/om/data_retrieval_layer/data_sources_common.py +++ b/src/om/data_retrieval_layer/data_sources_common.py @@ -442,12 +442,12 @@ def initialize_data_source(self) -> None: it raises an exception if the parameter is not available), and requires its value to be a float number. """ - self._array: NDArray[numpy.float_ | numpy.int_] = load_hdf5_data( + self._array: NDArray[numpy.float64 | numpy.int_] = load_hdf5_data( hdf5_filename=self._hdf5_filename, hdf5_path=self._hdf5_path, ) - def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_ | numpy.int_]: + def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float64 | numpy.int_]: """ Retrieves the numerical value of an OM's configuration parameter diff --git a/src/om/data_retrieval_layer/data_sources_files.py b/src/om/data_retrieval_layer/data_sources_files.py index d587656e..33574bef 100644 --- a/src/om/data_retrieval_layer/data_sources_files.py +++ b/src/om/data_retrieval_layer/data_sources_files.py @@ -100,7 +100,7 @@ class PilatusSingleFrameFiles(OmBaseFileDataSourceMixin, OmDataSourceProtocol): See documentation of the `__init__` function. """ - def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_]: + def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float64]: """ Retrieves an Eiger 16M detector data frame from files. @@ -119,7 +119,7 @@ def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_]: A detector data frame. """ - return cast(NDArray[numpy.float_], event["data"].data) + return cast(NDArray[numpy.float64], event["data"].data) class Eiger16MFiles(OmBaseFileDataSourceMixin, OmDataSourceProtocol): @@ -191,7 +191,7 @@ class Lambda1M5Files(OmBaseFileDataSourceMixin, OmDataSourceProtocol): def get_data( self, *, event: dict[str, Any] - ) -> NDArray[numpy.float_ | numpy.int_]: + ) -> NDArray[numpy.float64 | numpy.int_]: """ Retrieves a Lambda 1.5M detector data frame from files. @@ -423,7 +423,7 @@ class Jungfrau1MFiles(OmJungfrau1MDataSourceMixin, OmDataSourceProtocol): def get_data( self, *, event: dict[str, Any] - ) -> NDArray[numpy.float_ | numpy.int_]: + ) -> NDArray[numpy.float64 | numpy.int_]: """ Retrieves a Jungfrau 1M detector data frame from a file-based event. diff --git a/src/om/data_retrieval_layer/data_sources_http.py b/src/om/data_retrieval_layer/data_sources_http.py index a3dc63f0..1571d67a 100644 --- a/src/om/data_retrieval_layer/data_sources_http.py +++ b/src/om/data_retrieval_layer/data_sources_http.py @@ -96,7 +96,7 @@ class Eiger16MHttp(OmBaseGenericDataSourceMixin, OmDataSourceProtocol): def get_data( self, *, event: dict[str, Any] - ) -> NDArray[numpy.float_ | numpy.int_]: + ) -> NDArray[numpy.float64 | numpy.int_]: """ Retrieves an Eiger 16M detector data frame. diff --git a/src/om/data_retrieval_layer/data_sources_psana.py b/src/om/data_retrieval_layer/data_sources_psana.py index 73a0b65f..9f3ecaee 100644 --- a/src/om/data_retrieval_layer/data_sources_psana.py +++ b/src/om/data_retrieval_layer/data_sources_psana.py @@ -131,7 +131,7 @@ class RayonixPsana(OmDetectorInterfacePsanaDataSourceMixin, OmDataSourceProtocol See documentation of the `__init__` function. """ - def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_]: + def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float64]: """ Retrieves a Rayonix detector data frame from psana. @@ -153,7 +153,7 @@ def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_]: OmDataExtractionError: Raised when data cannot be retrieved from psana. """ - rayonix_psana: NDArray[numpy.float_] | None = self._detector_interface.calib( + rayonix_psana: NDArray[numpy.float64] | None = self._detector_interface.calib( event["data"] ) if rayonix_psana is None: @@ -169,7 +169,7 @@ class OpalPsana(OmDetectorInterfacePsanaDataSourceMixin, OmDataSourceProtocol): See documentation of the `__init__` function. """ - def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_]: + def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float64]: """ Retrieves an Opal camera data frame from psana. @@ -191,7 +191,7 @@ def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_]: OmDataExtractionError: Raised when data cannot be retrieved from psana. """ - opal_psana: NDArray[numpy.float_] | None = self._detector_interface( + opal_psana: NDArray[numpy.float64] | None = self._detector_interface( event["data"] ) if opal_psana is None: @@ -207,7 +207,7 @@ class Epix100aPsana(OmDetectorInterfacePsanaDataSourceMixin, OmDataSourceProtoco See documentation of the `__init__` function. """ - def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_]: + def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float64]: """ Retrieves an Opal camera data frame from psana. @@ -229,7 +229,7 @@ def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_]: OmDataExtractionError: Raised when data cannot be retrieved from psana. """ - opal_psana: NDArray[numpy.float_] | None = self._detector_interface( + opal_psana: NDArray[numpy.float64] | None = self._detector_interface( event["data"] ) if opal_psana is None: @@ -247,7 +247,7 @@ class AcqirisPsana(OmDetectorInterfacePsanaDataSourceMixin, OmDataSourceProtocol def get_data( self, *, event: dict[str, Any] - ) -> tuple[NDArray[numpy.float_], NDArray[numpy.float_]]: + ) -> tuple[NDArray[numpy.float64], NDArray[numpy.float64]]: """ Retrieves Acqiris waveform data from psana. @@ -275,10 +275,10 @@ def get_data( A tuple, with two entries, storing the digitized waveform data from the Acqiris detector. """ - wftime: NDArray[numpy.float_] | None = self._detector_interface.wftime( + wftime: NDArray[numpy.float64] | None = self._detector_interface.wftime( event["data"] ) - waveform: NDArray[numpy.float_] | None = self._detector_interface.waveform( + waveform: NDArray[numpy.float64] | None = self._detector_interface.waveform( event["data"] ) @@ -298,7 +298,7 @@ class AssembledDetectorPsana( See documentation of the `__init__` function. """ - def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_]: + def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float64]: """ Retrieves an assembled detector data frame from psana. @@ -321,7 +321,7 @@ def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_]: OmDataExtractionError: Raised when data cannot be retrieved from psana. """ - assembled_data: NDArray[numpy.float_] | None = self._detector_interface.image( + assembled_data: NDArray[numpy.float64] | None = self._detector_interface.image( event["data"] ) if assembled_data is None: @@ -601,8 +601,8 @@ def initialize_data_source(self) -> None: self._data_retrieval_function = detector_interface.raw if self._gain_map_filename != Path("") and self._gain_map_hdf5_path != "": - self._gain_map: NDArray[numpy.float_] | None = cast( - NDArray[numpy.float_] | None, + self._gain_map: NDArray[numpy.float64] | None = cast( + NDArray[numpy.float64] | None, load_hdf5_data( hdf5_filename=self._gain_map_filename, hdf5_path=self._gain_map_hdf5_path, @@ -611,7 +611,7 @@ def initialize_data_source(self) -> None: else: self._gain_map = None - def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_ | numpy.int_]: + def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float64 | numpy.int_]: """ Retrieves a Jungfrau 4M detector data frame from psana. @@ -636,7 +636,7 @@ def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_ | numpy.int OmDataExtractionError: Raised when data cannot be retrieved from psana. """ - psana_data: NDArray[numpy.float_ | numpy.int_] | None = ( + psana_data: NDArray[numpy.float64 | numpy.int_] | None = ( self._data_retrieval_function(event["data"]) ) if psana_data is None: @@ -648,7 +648,7 @@ def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_ | numpy.int # Rearranges the data into 'slab' format. psana_data_shape: tuple[int, ...] = psana_data.shape if len(psana_data_shape) == 2: - psana_data_reshaped: NDArray[numpy.float_ | numpy.int_] = psana_data + psana_data_reshaped: NDArray[numpy.float64 | numpy.int_] = psana_data else: psana_data_reshaped = psana_data.reshape( psana_data_shape[0] * psana_data_shape[1], psana_data_shape[2] @@ -735,7 +735,7 @@ def initialize_data_source(self) -> None: else: self._data_retrieval_function = detector_interface.raw - def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_ | numpy.int_]: + def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float64 | numpy.int_]: """ Retrieves a CSPAD detector data frame from psana. @@ -760,7 +760,7 @@ def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_ | numpy.int OmDataExtractionError: Raised when data cannot be retrieved from psana. """ - cspad_psana: NDArray[numpy.float_ | numpy.int_] | None = ( + cspad_psana: NDArray[numpy.float64 | numpy.int_] | None = ( self._data_retrieval_function(event["data"]) ) if cspad_psana is None: @@ -770,11 +770,11 @@ def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_ | numpy.int ) # Rearranges the data into 'slab' format. - cspad_reshaped: NDArray[numpy.float_ | numpy.int_] = cspad_psana.reshape( + cspad_reshaped: NDArray[numpy.float64 | numpy.int_] = cspad_psana.reshape( (4, 8, 185, 388) ) - cspad_slab: NDArray[numpy.float_ | numpy.int_] = cast( - NDArray[numpy.float_ | numpy.int_], + cspad_slab: NDArray[numpy.float64 | numpy.int_] = cast( + NDArray[numpy.float64 | numpy.int_], numpy.zeros(shape=(1480, 1552), dtype=cspad_reshaped.dtype), ) index: int diff --git a/src/om/data_retrieval_layer/data_sources_psana2.py b/src/om/data_retrieval_layer/data_sources_psana2.py index faa487d3..fdfcae5f 100644 --- a/src/om/data_retrieval_layer/data_sources_psana2.py +++ b/src/om/data_retrieval_layer/data_sources_psana2.py @@ -130,7 +130,7 @@ class AssembledDetectorPsana2( See documentation of the `__init__` function. """ - def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_]: + def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float64]: """ Retrieves an assembled detector data frame from psana. @@ -153,7 +153,7 @@ def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_]: OmDataExtractionError: Raised when data cannot be retrieved from psana. """ - assembled_data: NDArray[numpy.float_] | None = self._detector_interface.image( + assembled_data: NDArray[numpy.float64] | None = self._detector_interface.image( event["data"] ) if assembled_data is None: @@ -337,10 +337,19 @@ def __init__( f"{data_source_name}, but entry 'gain_map_hdf5_path' is not" ) sys.exit(1) - self._gain_map_filename = extra_parameters["gain_map_filename"] self._gain_map_hdf5_path = extra_parameters["gain_map_hdf5_path"] + self._psana_algorithm: str + if "psana_algorithm" not in extra_parameters: + log.warning( + f"Entry 'algorithm' is not defined for data source {data_source_name}. " + "We will default to using the 'raw' algorithm." + ) + self._psana_algorithm = "raw" + else: + self._psana_algorithm = extra_parameters["psana_algorithm"] + self._psana_name: str = extra_parameters["psana_name"] self._calibration: bool = extra_parameters["calibration"] @@ -354,11 +363,16 @@ def initialize_data_source(self) -> None: No initialization is required to retrieve event identifiers for psana-based data events, so this function actually does nothing. """ - self._detector_interface: Any = self._run.Detector(self._psana_name) + detector_interface: Any = self._run.Detector(self._psana_name) + algorithm: Any = getattr(detector_interface, self._psana_algorithm) + if self._calibration: + self._data_retrieval_function: Callable[[Any], Any] = getattr(algorithm, "calib") + else: + self._data_retrieval_function = getattr(algorithm, "raw") if self._gain_map_filename != Path("") and self._gain_map_hdf5_path != "": - self._gain_map: NDArray[numpy.float_] | None = cast( - NDArray[numpy.float_] | None, + self._gain_map: NDArray[numpy.float64] | None = cast( + NDArray[numpy.float64] | None, load_hdf5_data( hdf5_filename=self._gain_map_filename, hdf5_path=self._gain_map_hdf5_path, @@ -367,7 +381,7 @@ def initialize_data_source(self) -> None: else: self._gain_map = None - def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_ | numpy.int_]: + def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float64 | numpy.int_]: """ Retrieves a Jungfrau 4M detector data frame from psana. Please see the documentation of the base Protocol class for additional @@ -391,16 +405,9 @@ def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_ | numpy.int OmDataExtractionError: Raised when data cannot be retrieved from psana. """ - - if self._calibration: - psana_data: NDArray[numpy.float_ | numpy.int_] | None = ( - self._detector_interface.raw.calib(event["data"]) - ) - else: - psana_data = ( - self._detector_interface.raw.raw(event["data"]) - ) - + psana_data: NDArray[numpy.float64 | numpy.int_] | None = ( + self._data_retrieval_function(event["data"]) + ) if psana_data is None: raise OmDataExtractionError( "Could not retrieve data from psana for the following data source: " @@ -410,7 +417,7 @@ def get_data(self, *, event: dict[str, Any]) -> NDArray[numpy.float_ | numpy.int # Rearranges the data into 'slab' format. psana_data_shape: tuple[int, ...] = psana_data.shape if len(psana_data_shape) == 2: - psana_data_reshaped: NDArray[numpy.float_ | numpy.int_] = psana_data + psana_data_reshaped: NDArray[numpy.float64 | numpy.int_] = psana_data else: psana_data_reshaped = psana_data.reshape( psana_data_shape[0] * psana_data_shape[1], psana_data_shape[2] @@ -559,7 +566,7 @@ def get_data(self, *, event: dict[str, Any]) -> str: A unique event identifier. """ - return f"{event["additional_info"]["timestamp"]}" + return f'{event["additional_info"]["timestamp"]}' class BeamEnergyPsana2(OmDataSourceProtocol): diff --git a/src/om/data_retrieval_layer/data_sources_zmq.py b/src/om/data_retrieval_layer/data_sources_zmq.py index b920afaf..0a022113 100644 --- a/src/om/data_retrieval_layer/data_sources_zmq.py +++ b/src/om/data_retrieval_layer/data_sources_zmq.py @@ -149,7 +149,7 @@ class Jungfrau1MZmq(OmJungfrau1MDataSourceMixin, OmDataSourceProtocol): def get_data( self, *, event: dict[str, Any] - ) -> NDArray[numpy.float_ | numpy.int_]: + ) -> NDArray[numpy.float64 | numpy.int_]: """ Retrieves a Jungfrau 1M detector data frame from a ZMQ data stream. diff --git a/src/om/graphical_interfaces/crystallography_gui.py b/src/om/graphical_interfaces/crystallography_gui.py index 3e2e3f08..a887f131 100644 --- a/src/om/graphical_interfaces/crystallography_gui.py +++ b/src/om/graphical_interfaces/crystallography_gui.py @@ -375,7 +375,7 @@ def update_gui(self) -> None: QtWidgets.QApplication.processEvents() - peakogram: NDArray[numpy.float_] = local_data["peakogram"] + peakogram: NDArray[numpy.float64] = local_data["peakogram"] peakogram[numpy.where(peakogram == 0)] = numpy.nan self._peakogram_plot_image_view.setImage( numpy.log(peakogram), diff --git a/src/om/graphical_interfaces/crystallography_parameter_tweaker.py b/src/om/graphical_interfaces/crystallography_parameter_tweaker.py index defb1fe9..ece5bfb3 100644 --- a/src/om/graphical_interfaces/crystallography_parameter_tweaker.py +++ b/src/om/graphical_interfaces/crystallography_parameter_tweaker.py @@ -111,7 +111,7 @@ def __init__(self, *, url: str, parameters: dict[str, Any]): _ParameterTweakerParameters.model_validate(parameters["crystallography"]) ) - self._img: NDArray[numpy.float_] | None = None + self._img: NDArray[numpy.float64] | None = None self._frame_list: deque[dict[str, Any]] = deque(maxlen=20) self._current_frame_index: int = -1 @@ -137,7 +137,7 @@ def __init__(self, *, url: str, parameters: dict[str, Any]): self._data_visualizer.get_visualization_pixel_maps().y.ravel() ) - self._assembled_img: NDArray[numpy.float_] = numpy.zeros( + self._assembled_img: NDArray[numpy.float64] = numpy.zeros( shape=self._data_visualizer.get_min_array_shape_for_visualization(), dtype=numpy.float32, ) diff --git a/src/om/graphical_interfaces/frame_viewer.py b/src/om/graphical_interfaces/frame_viewer.py index 65d3dd4a..99ec5b89 100644 --- a/src/om/graphical_interfaces/frame_viewer.py +++ b/src/om/graphical_interfaces/frame_viewer.py @@ -80,7 +80,7 @@ def __init__(self, *, url: str): tag="omframedata", ) - self._img: NDArray[numpy.float_] | None = None + self._img: NDArray[numpy.float64] | None = None self._frame_list: deque[dict[str, Any]] = deque(maxlen=20) self._current_frame_index: int = -1 @@ -147,8 +147,8 @@ def __init__(self, *, url: str): def _update_peaks( self, *, - peak_list_x_in_frame: NDArray[numpy.float_], - peak_list_y_in_frame: NDArray[numpy.float_], + peak_list_x_in_frame: NDArray[numpy.float64], + peak_list_y_in_frame: NDArray[numpy.float64], ) -> None: # Updates the Bragg peaks shown by the viewer. QtWidgets.QApplication.processEvents() diff --git a/src/om/lib/cheetah.py b/src/om/lib/cheetah.py index 8406bc7b..c0824b27 100644 --- a/src/om/lib/cheetah.py +++ b/src/om/lib/cheetah.py @@ -24,6 +24,7 @@ import pathlib import time +import sys from dataclasses import dataclass from typing import Any, TextIO, cast @@ -31,6 +32,8 @@ import hdf5plugin # type: ignore import numpy from numpy.typing import NDArray +from typing import Set + from om.algorithms.common import PeakList from om.lib.exceptions import OmHdf5UnsupportedDataFormat @@ -57,8 +60,8 @@ class ClassSumData: """ num_frames: int - sum_frames: NDArray[numpy.float_] - peak_powder: NDArray[numpy.float_] + sum_frames: NDArray[numpy.float64] + peak_powder: NDArray[numpy.float64] @dataclass(order=True) @@ -403,7 +406,7 @@ def add_frame( self, *, class_number: int, - frame_data: NDArray[numpy.float_ | numpy.int_], + frame_data: NDArray[numpy.float64 | numpy.int_], peak_list: PeakList, ) -> None: """ @@ -467,7 +470,9 @@ def get_sums_for_sending( The sum and virtual powder plot stored by the accumulator, or None. """ - if ( + if self._cheetah_parameters.class_sums_sending_interval == -1: + return None + elif ( self._sum_sending_counter >= self._cheetah_parameters.class_sums_sending_interval ) or (self._sum_sending_counter > 0 and disregard_counter): @@ -860,7 +865,7 @@ def _create_extra_datasets( ) elif ( numpy.issubdtype(type(value), numpy.int_) - or numpy.issubdtype(type(value), numpy.float_) + or numpy.issubdtype(type(value), numpy.float64) or numpy.issubdtype(type(value), numpy.bool_) ): self._resizable_datasets[group_name + "/" + key] = self._extra_groups[ diff --git a/src/om/lib/crystallography.py b/src/om/lib/crystallography.py index 48dff13a..a843feea 100644 --- a/src/om/lib/crystallography.py +++ b/src/om/lib/crystallography.py @@ -114,7 +114,7 @@ def __init__( layout_info=geometry_information.get_layout_info(), ) - def find_peaks(self, detector_data: NDArray[numpy.int_ | numpy.float_]) -> PeakList: + def find_peaks(self, detector_data: NDArray[numpy.int_ | numpy.float64]) -> PeakList: """ Finds peaks in a detector data frame. @@ -210,7 +210,7 @@ def __init__( / parameters.crystallography.peakogram_radius_bin_size ) - self._peakogram: NDArray[numpy.float_] = numpy.zeros( + self._peakogram: NDArray[numpy.float64] = numpy.zeros( (peakogram_num_bins_radius, peakogram_num_bins_intensity) ) self._running_average_window_size: int = ( @@ -264,7 +264,7 @@ def update_plots( deque[float], deque[float], NDArray[numpy.int_], - NDArray[numpy.float_], + NDArray[numpy.float64], float, float, list[float], diff --git a/src/om/lib/files.py b/src/om/lib/files.py index 79f4b719..1a48ee26 100644 --- a/src/om/lib/files.py +++ b/src/om/lib/files.py @@ -43,7 +43,7 @@ def load_hdf5_data( *, hdf5_filename: Path, hdf5_path: str, -) -> NDArray[numpy.int_ | numpy.float_]: +) -> NDArray[numpy.int_ | numpy.float64]: """ Loads data from an HDF5 file. @@ -71,7 +71,7 @@ def load_hdf5_data( try: hdf5_file_handle: Any with h5py.File(hdf5_filename_path, "r") as hdf5_file_handle: - data: NDArray[numpy.float_ | numpy.int_] = hdf5_file_handle[hdf5_path][:] + data: NDArray[numpy.float64 | numpy.int_] = hdf5_file_handle[hdf5_path][:] except (IOError, OSError, KeyError) as exc: exc_type, exc_value = sys.exc_info()[:2] raise OmHdf5FileReadingError( diff --git a/src/om/lib/geometry.py b/src/om/lib/geometry.py index 3f5af6f2..24c5813b 100644 --- a/src/om/lib/geometry.py +++ b/src/om/lib/geometry.py @@ -381,11 +381,11 @@ class PixelMaps: the pixel, the center of the reference system, and the x axis. """ - x: NDArray[numpy.float_] - y: NDArray[numpy.float_] - z: NDArray[numpy.float_] - radius: NDArray[numpy.float_] - phi: NDArray[numpy.float_] + x: NDArray[numpy.float64] + y: NDArray[numpy.float64] + z: NDArray[numpy.float64] + radius: NDArray[numpy.float64] + phi: NDArray[numpy.float64] @dataclass @@ -1006,13 +1006,13 @@ def _compute_pix_maps(*, geometry: Detector) -> PixelMaps: [geometry.panels[k].orig_max_ss for k in geometry.panels] ).max() - x_map: NDArray[numpy.float_] = numpy.zeros( + x_map: NDArray[numpy.float64] = numpy.zeros( shape=(max_ss_in_slab + 1, max_fs_in_slab + 1), dtype=numpy.float32 ) - y_map: NDArray[numpy.float_] = numpy.zeros( + y_map: NDArray[numpy.float64] = numpy.zeros( shape=(max_ss_in_slab + 1, max_fs_in_slab + 1), dtype=numpy.float32 ) - z_map: NDArray[numpy.float_] = numpy.zeros( + z_map: NDArray[numpy.float64] = numpy.zeros( shape=(max_ss_in_slab + 1, max_fs_in_slab + 1), dtype=numpy.float32 ) @@ -1038,12 +1038,12 @@ def _compute_pix_maps(*, geometry: Detector) -> PixelMaps: ), indexing="ij", ) - y_panel: NDArray[numpy.float_] = ( + y_panel: NDArray[numpy.float64] = ( ss_grid * geometry.panels[panel_name].ssy + fs_grid * geometry.panels[panel_name].fsy + geometry.panels[panel_name].cny ) - x_panel: NDArray[numpy.float_] = ( + x_panel: NDArray[numpy.float64] = ( ss_grid * geometry.panels[panel_name].ssx + fs_grid * geometry.panels[panel_name].fsx + geometry.panels[panel_name].cnx @@ -1079,8 +1079,8 @@ def _compute_pix_maps(*, geometry: Detector) -> PixelMaps: + 1, ] = first_panel_camera_length - r_map: NDArray[numpy.float_] = numpy.sqrt(numpy.square(x_map) + numpy.square(y_map)) - phi_map: NDArray[numpy.float_] = numpy.arctan2(y_map, x_map) + r_map: NDArray[numpy.float64] = numpy.sqrt(numpy.square(x_map) + numpy.square(y_map)) + phi_map: NDArray[numpy.float64] = numpy.arctan2(y_map, x_map) return PixelMaps( x=x_map, @@ -1406,9 +1406,9 @@ def get_min_array_shape_for_visualization(self) -> tuple[int, int]: def visualize_data( self, *, - data: NDArray[numpy.int_ | numpy.float_], - array_for_visualization: NDArray[numpy.float_] | None = None, - ) -> NDArray[numpy.float_]: + data: NDArray[numpy.int_ | numpy.float64], + array_for_visualization: NDArray[numpy.float64] | None = None, + ) -> NDArray[numpy.float64]: """ Applies geometry information to a detector data frame. @@ -1443,7 +1443,7 @@ def visualize_data( cannot be used to store the pixel information. """ if array_for_visualization is None: - visualization_array: NDArray[numpy.float_] = numpy.zeros( + visualization_array: NDArray[numpy.float64] = numpy.zeros( self._min_array_shape, dtype=numpy.float32 ) else: diff --git a/src/om/lib/parameters.py b/src/om/lib/parameters.py index a30ff74f..53b2feac 100644 --- a/src/om/lib/parameters.py +++ b/src/om/lib/parameters.py @@ -1,8 +1,9 @@ from enum import Enum from pathlib import Path +from typing import Literal +from typing_extensions import Self -from pydantic import BaseModel, ConfigDict, field_validator, model_validator -from typing_extensions import Literal, Self +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator class Hdf5Compression(Enum): @@ -25,6 +26,19 @@ class OmParameters(CustomBaseModel): configuration_file: Path = Path("") +class RoiBinSzCompressorParameters(CustomBaseModel): + compressor: Literal["qoz", "sz3"] = Field( + "qoz", description='Compression algorithm ("qoz" or "sz3")' + ) + abs_error: float = Field(10.0, description="Absolute error bound") + bin_size: int = Field(2, description="Bin size") + roi_window_size: int = Field( + 9, + description="Default window size", + ) + mask: str | bool | None = None + + class DataSourceParameters(CustomBaseModel): type: str model_config = ConfigDict(extra="allow") @@ -267,6 +281,25 @@ class CrystallographyParameters(CustomBaseModel): non_hit_frame_sending_interval: int = 0 +class DataCompressionParameters(CustomBaseModel): + run_compression: bool = False + backend: Literal["roibinsz"] | None = None + compression_parameters: RoiBinSzCompressorParameters | None = None + + @model_validator(mode="after") + def check_backend_matches_parameters(self) -> Self: + if self.run_compression: + if self.backend == "roibinsz": + if not isinstance( + self.compression_parameters, RoiBinSzCompressorParameters + ): + raise ValueError( + "For the libpressio compression backend you must use a " + "SZCompressorParameters for `compression_parameters`." + ) + return self + + class MonitorParameters(CustomBaseModel): om: OmParameters data_retrieval_layer: DataRetrievalLayerParameters @@ -276,6 +309,7 @@ class MonitorParameters(CustomBaseModel): crystallography: CrystallographyParameters | None = None xes: XesParameters | None = None cheetah: CheetahParameters | None = None + compression: DataCompressionParameters | None = None @model_validator(mode="after") def check_peakfinder8_peak_detection_parameters(self) -> Self: diff --git a/src/om/lib/protocols.py b/src/om/lib/protocols.py index 67b8f3f2..4ead1bbd 100644 --- a/src/om/lib/protocols.py +++ b/src/om/lib/protocols.py @@ -636,6 +636,34 @@ def __init__( """ """ ... - def find_peaks(self, *, data: NDArray[numpy.int_ | numpy.float_]) -> PeakList: + def find_peaks(self, *, data: NDArray[numpy.int_ | numpy.float64]) -> PeakList: + """ """ + ... + + +class OmCompressionProtocol(Protocol): + """ + See documentation of the `__init__` function. + """ + + def __init__( + self, + parameters: dict[str, Any], + ) -> None: + """ """ + ... + + def compress( + self, + *, + data: NDArray[numpy.int_ | numpy.float64], + special_data: Any | None = None, + ) -> bytes: + """ """ + ... + + def uncompress( + self, *, compressed_data: bytes + ) -> NDArray[numpy.int_ | numpy.float64]: """ """ ... diff --git a/src/om/lib/radial_profile.py b/src/om/lib/radial_profile.py index 4aace314..679e760a 100644 --- a/src/om/lib/radial_profile.py +++ b/src/om/lib/radial_profile.py @@ -40,11 +40,11 @@ def _fit_by_least_squares( *, - radial_profile: NDArray[numpy.float_], - vectors: NDArray[numpy.float_], + radial_profile: NDArray[numpy.float64], + vectors: NDArray[numpy.float64], start_bin: int | None = None, stop_bin: int | None = None, -) -> NDArray[numpy.float_]: +) -> NDArray[numpy.float64]: # This function fits a set of linearly combined vectors to a radial profile, # using a least-squares-based approach. The fit only takes into account the # range of radial bins defined by the xmin and xmax arguments. @@ -52,26 +52,26 @@ def _fit_by_least_squares( start_bin = 0 if stop_bin is None: stop_bin = len(radial_profile) - a: NDArray[numpy.float_] = numpy.nan_to_num(numpy.atleast_2d(vectors).T) - b: NDArray[numpy.float_] = numpy.nan_to_num(radial_profile) + a: NDArray[numpy.float64] = numpy.nan_to_num(numpy.atleast_2d(vectors).T) + b: NDArray[numpy.float64] = numpy.nan_to_num(radial_profile) a = a[start_bin:stop_bin] b = b[start_bin:stop_bin] - coefficients: NDArray[numpy.float_] + coefficients: NDArray[numpy.float64] coefficients, _, _, _ = numpy.linalg.lstsq(a, b, rcond=None) return coefficients def _cumulative_moving_average( - new_radial: NDArray[numpy.float_], - previous_cumulative_avg: NDArray[numpy.float_], + new_radial: NDArray[numpy.float64], + previous_cumulative_avg: NDArray[numpy.float64], num_events: int, -) -> NDArray[numpy.float_]: +) -> NDArray[numpy.float64]: return ((previous_cumulative_avg * num_events) + new_radial) / (num_events + 1) def _calc_rg_by_guinier( - q: NDArray[numpy.float_], - radial: NDArray[numpy.float_], + q: NDArray[numpy.float64], + radial: NDArray[numpy.float64], nb: int | None = None, ne: int | None = None, ) -> float: @@ -107,8 +107,8 @@ def _calc_rg_by_guinier( def _calc_rg_by_guinier_peak( - q: NDArray[numpy.float_], - radial: NDArray[numpy.float_], + q: NDArray[numpy.float64], + radial: NDArray[numpy.float64], exp: int = 1, nb: int | None = None, ne: int | None = None, @@ -121,9 +121,9 @@ def _calc_rg_by_guinier_peak( nb = 0 if ne is None: ne = len(q) - qs: NDArray[numpy.float_] = q[nb:ne] - Is: NDArray[numpy.float_] = radial[nb:ne] - qdI: NDArray[numpy.float_] = qs**d * Is + qs: NDArray[numpy.float64] = q[nb:ne] + Is: NDArray[numpy.float64] = radial[nb:ne] + qdI: NDArray[numpy.float64] = qs**d * Is try: # fit a quick quadratic for smoothness, ax^2 + bx + c a: float @@ -141,8 +141,8 @@ def _calc_rg_by_guinier_peak( def _sphere_form_factor( - radius: float, q_mags: NDArray[numpy.float_], check_divide_by_zero: bool = True -) -> NDArray[numpy.float_]: + radius: float, q_mags: NDArray[numpy.float64], check_divide_by_zero: bool = True +) -> NDArray[numpy.float64]: # By Rick Kirian and Joe Chen # Copied from reborn.simulate.form_factors with permission. # Form factor :math:`f(q)` for a sphere of radius :math:`r`, at given :math:`q` @@ -163,9 +163,9 @@ def _sphere_form_factor( # E.g., water molecules have 10 electrons, a molecular weight of 18 g/mol and a # density of 1 g/ml, so you can google search the electron density of water, which # is 10*(1 g/cm^3)/(18 g/6.022e23) = 3.346e29 per m^3 . - qr: NDArray[numpy.float_] = q_mags * radius + qr: NDArray[numpy.float64] = q_mags * radius if check_divide_by_zero is True: - amp: NDArray[numpy.float_] = numpy.zeros_like(qr) + amp: NDArray[numpy.float64] = numpy.zeros_like(qr) amp[qr == 0] = (4 * numpy.pi * radius**3) / 3 w: NDArray[numpy.bool_] = qr != 0 amp[w] = ( @@ -185,8 +185,8 @@ class _SphericalDroplets: # Copied from reborn.analysis.optimize with permission. def __init__( self, - q: NDArray[numpy.float_] | None = None, - r: NDArray[numpy.float_] | None = None, + q: NDArray[numpy.float64] | None = None, + r: NDArray[numpy.float64] | None = None, ): if q is None: q = numpy.linspace(0, 1e10, 517) @@ -194,13 +194,13 @@ def __init__( r = numpy.linspace( 50, 3000, 20 ) # set of spherical radii to test in angstroms - self.q: NDArray[numpy.float_] = q.copy() - self.r: NDArray[numpy.float_] = ( + self.q: NDArray[numpy.float64] = q.copy() + self.r: NDArray[numpy.float64] = ( r.copy() ) # radius range of sphere to scan through self.N: int = len(self.r) - self.I_R_precompute: NDArray[numpy.float_] = numpy.zeros((self.N, len(self.q))) + self.I_R_precompute: NDArray[numpy.float64] = numpy.zeros((self.N, len(self.q))) for i in range(self.N): self.I_R_precompute[i, :] = ( _sphere_form_factor( @@ -209,18 +209,18 @@ def __init__( ) ** 2 def fit_profile( - self, I_D: NDArray[numpy.float_], mask: NDArray[numpy.float_] | None = None + self, I_D: NDArray[numpy.float64], mask: NDArray[numpy.float64] | None = None ): if mask is None: mask = numpy.ones_like(I_D) w: NDArray[numpy.bool_] = mask > 0 - A_save: NDArray[numpy.float_] = numpy.zeros(self.N) - error_vec: NDArray[numpy.float_] = numpy.zeros(self.N) + A_save: NDArray[numpy.float64] = numpy.zeros(self.N) + error_vec: NDArray[numpy.float64] = numpy.zeros(self.N) for i in range(self.N): I_R = self.I_R_precompute[i, :] - A: numpy.float_ = numpy.sum(I_D[w] * I_R[w]) / numpy.sum(I_R[w] ** 2) + A: numpy.float64 = numpy.sum(I_D[w] * I_R[w]) / numpy.sum(I_R[w] ** 2) diff_sq = (A * I_R[w] - I_D[w]) ** 2 error_vec[i] = numpy.sum(diff_sq) A_save[i] = A @@ -230,7 +230,7 @@ def fit_profile( A_min: float = A_save[ind_min] r_min: float = self.r[ind_min] e_min: float = error_vec[ind_min] - I_R_min: NDArray[numpy.float_] = self.I_R_precompute[ind_min, :] + I_R_min: NDArray[numpy.float64] = self.I_R_precompute[ind_min, :] r_dic = dict( A_min=A_min, e_min=e_min, error_vec=error_vec, I_R_min=I_R_min.copy() @@ -382,8 +382,8 @@ class includes optional subtraction of a background profile, detection of a parameters.radial_profile.background_subtraction ) if self._background_subtraction is True: - self._background_profile_vectors: NDArray[numpy.float_] = cast( - NDArray[numpy.float_], + self._background_profile_vectors: NDArray[numpy.float64] = cast( + NDArray[numpy.float64], load_hdf5_data( hdf5_filename=Path( parameters.radial_profile.background_profile_filename @@ -450,14 +450,14 @@ class includes optional subtraction of a background profile, detection of a def analyze_radial_profile( self, *, - data: NDArray[numpy.float_ | numpy.int_], + data: NDArray[numpy.float64 | numpy.int_], beam_energy: float, detector_distance: float, downstream_intensity: float, ) -> tuple[ - NDArray[numpy.float_], - NDArray[numpy.float_], - NDArray[numpy.float_], + NDArray[numpy.float64], + NDArray[numpy.float64], + NDArray[numpy.float64], bool, float, float, @@ -482,12 +482,12 @@ def analyze_radial_profile( from the data frame. """ - radial_profile: NDArray[numpy.float_] = self._radial_profile.calculate_profile( + radial_profile: NDArray[numpy.float64] = self._radial_profile.calculate_profile( data=data ) - errors: NDArray[numpy.float_] = cast( - NDArray[numpy.float_], + errors: NDArray[numpy.float64] = cast( + NDArray[numpy.float64], stats.binned_statistic( self._radial_bin_labels[self._radial_profile_bad_pixel_map].ravel(), data[self._radial_profile_bad_pixel_map].ravel(), @@ -496,13 +496,13 @@ def analyze_radial_profile( ) if self._background_subtraction is True: - coefficients: NDArray[numpy.float_] = _fit_by_least_squares( + coefficients: NDArray[numpy.float64] = _fit_by_least_squares( radial_profile=radial_profile, vectors=self._background_profile_vectors, start_bin=self._background_subtraction_min_bin, stop_bin=self._background_subtraction_max_bin, ) - background_fit: NDArray[numpy.float_] = radial_profile * 0 + background_fit: NDArray[numpy.float64] = radial_profile * 0 index: int for index in range(len(coefficients)): background_fit += ( @@ -515,13 +515,13 @@ def analyze_radial_profile( constants.c * constants.h / (beam_energy * constants.electron_volt) ) real_detector_distance: float = detector_distance * 1e-3 + self._coffset - theta: NDArray[numpy.float_] = ( + theta: NDArray[numpy.float64] = ( numpy.arctan( self._pixel_size * self._radial_bin_centers / real_detector_distance ) * 0.5 ) - q: NDArray[numpy.float_] = ( + q: NDArray[numpy.float64] = ( numpy.sin(theta) * 4 * numpy.pi / wavelength ) * 1e-10 @@ -666,36 +666,36 @@ def __init__( # self._hit_rate_history: Deque[float] = deque(5000 * [0.0], maxlen=5000) self._hit_rate_history: deque[float] = deque([]) - self._q_history: deque[NDArray[numpy.float_]] = deque([]) - self._radials_history: deque[NDArray[numpy.float_]] = deque([]) + self._q_history: deque[NDArray[numpy.float64]] = deque([]) + self._radials_history: deque[NDArray[numpy.float64]] = deque([]) self._image_sum_history: deque[float] = deque([]) self._downstream_intensity_history: deque[float] = deque([]) self._roi1_intensity_history: deque[float] = deque([]) self._roi2_intensity_history: deque[float] = deque([]) self._rg_history: deque[float] = deque([]) - self._cumulative_hits_radial: NDArray[numpy.float_] = numpy.array([]) + self._cumulative_hits_radial: NDArray[numpy.float64] = numpy.array([]) def update_plots( self, *, - radial_profile: NDArray[numpy.float_], + radial_profile: NDArray[numpy.float64], detector_data_sum: float, - q: NDArray[numpy.float_], + q: NDArray[numpy.float64], downstream_intensity: float, roi1_intensity: float, roi2_intensity: float, sample_detected: bool, rg: float, ) -> tuple[ - deque[NDArray[numpy.float_]], - deque[NDArray[numpy.float_]], + deque[NDArray[numpy.float64]], + deque[NDArray[numpy.float64]], deque[float], deque[float], deque[float], deque[float], deque[float], deque[float], - NDArray[numpy.float_], + NDArray[numpy.float64], ]: """ #TODO: Documentation. diff --git a/src/om/lib/xes.py b/src/om/lib/xes.py index 1026abde..6f0566b7 100644 --- a/src/om/lib/xes.py +++ b/src/om/lib/xes.py @@ -61,13 +61,13 @@ def __init__(self, *, parameters: XesParameters, time_resolved: bool) -> None: """ self._time_resolved: bool = time_resolved - self._spectra_cumulative_sum: NDArray[numpy.float_ | numpy.int_] | None = None - self._spectra_cumulative_sum_smoothed: NDArray[numpy.float_] | None = None + self._spectra_cumulative_sum: NDArray[numpy.float64 | numpy.int_] | None = None + self._spectra_cumulative_sum_smoothed: NDArray[numpy.float64] | None = None - self._cumulative_2d: NDArray[numpy.float_ | numpy.int_] | None = None - self._cumulative_2d_pumped: NDArray[numpy.float_ | numpy.int_] | None = None + self._cumulative_2d: NDArray[numpy.float64 | numpy.int_] | None = None + self._cumulative_2d_pumped: NDArray[numpy.float64 | numpy.int_] | None = None - self._cumulative_2d_dark: NDArray[numpy.float_ | numpy.int_] | None = None + self._cumulative_2d_dark: NDArray[numpy.float64 | numpy.int_] | None = None self._num_events_pumped: int = 0 self._num_events_dark: int = 0 @@ -78,15 +78,15 @@ def __init__(self, *, parameters: XesParameters, time_resolved: bool) -> None: def update_plots( self, *, - detector_data: NDArray[numpy.float_ | numpy.int_], + detector_data: NDArray[numpy.float64 | numpy.int_], optical_laser_active: bool, ) -> tuple[ - NDArray[numpy.float_ | numpy.int_] | None, - NDArray[numpy.float_] | None, - NDArray[numpy.float_ | numpy.int_] | None, - NDArray[numpy.float_] | None, - NDArray[numpy.float_] | None, - NDArray[numpy.float_] | None, + NDArray[numpy.float64 | numpy.int_] | None, + NDArray[numpy.float64] | None, + NDArray[numpy.float64 | numpy.int_] | None, + NDArray[numpy.float64] | None, + NDArray[numpy.float64] | None, + NDArray[numpy.float64] | None, ]: """ Updates and recovers the X-ray Emission Spectroscopy data plots. @@ -148,16 +148,16 @@ def update_plots( ) # Calculate normalized spectrum from cumulative 2D images. - cumulative_xes: dict[str, NDArray[numpy.float_]] = ( + cumulative_xes: dict[str, NDArray[numpy.float64]] = ( self._energy_spectrum_retrieval.calculate_spectrum(data=self._cumulative_2d) ) self._spectra_cumulative_sum = cumulative_xes["spectrum"] self._spectra_cumulative_sum_smoothed = cumulative_xes["spectrum_smoothed"] - spectra_cumulative_sum_pumped: NDArray[numpy.float_] | None = None - spectra_cumulative_sum_dark: NDArray[numpy.float_] | None = None - spectra_cumulative_sum_difference: NDArray[numpy.float_] | None = None + spectra_cumulative_sum_pumped: NDArray[numpy.float64] | None = None + spectra_cumulative_sum_dark: NDArray[numpy.float64] | None = None + spectra_cumulative_sum_difference: NDArray[numpy.float64] | None = None if numpy.mean(numpy.abs(self._spectra_cumulative_sum)) > 0: self._spectra_cumulative_sum /= numpy.mean( @@ -190,7 +190,7 @@ def update_plots( ) # Calculate spectrum from cumulative 2D images - cumulative_xes_pumped: dict[str, NDArray[numpy.float_]] = ( + cumulative_xes_pumped: dict[str, NDArray[numpy.float64]] = ( self._energy_spectrum_retrieval.calculate_spectrum( data=self._cumulative_2d_pumped ) @@ -198,7 +198,7 @@ def update_plots( spectra_cumulative_sum_pumped = cumulative_xes_pumped["spectrum"] # calculate spectrum from cumulative 2D images - cumulative_xes_dark: dict[str, NDArray[numpy.float_]] = ( + cumulative_xes_dark: dict[str, NDArray[numpy.float64]] = ( self._energy_spectrum_retrieval.calculate_spectrum( data=self._cumulative_2d_dark ) diff --git a/src/om/processing_layer/cheetah.py b/src/om/processing_layer/cheetah.py index f3e98c03..7ef80a4b 100644 --- a/src/om/processing_layer/cheetah.py +++ b/src/om/processing_layer/cheetah.py @@ -34,6 +34,7 @@ from numpy.typing import NDArray from om.algorithms.common import PeakList +from om.algorithms.crystallography import RoiBinSzCompression from om.algorithms.generic import Binning, BinningPassthrough from om.lib.cheetah import ( CheetahClassSumsAccumulator, @@ -50,9 +51,10 @@ from om.lib.parameters import ( CheetahParameters, CrystallographyParameters, + DataCompressionParameters, MonitorParameters, ) -from om.lib.protocols import OmProcessingProtocol +from om.lib.protocols import OmCompressionProtocol, OmProcessingProtocol from om.lib.zmq import ZmqResponder T = TypeVar("T") @@ -98,6 +100,9 @@ def __init__(self, *, parameters: MonitorParameters) -> None: sys.exit(1) self._cheetah_parameters: CheetahParameters = parameters.cheetah + self._compression_parameters: DataCompressionParameters | None = ( + parameters.compression + ) self._crystallography_parameters: CrystallographyParameters = ( parameters.crystallography ) @@ -141,6 +146,17 @@ def _common_initialize_processing_node( geometry_information=self._geometry_information, ) + # Optional compression + self._compressor: OmCompressionProtocol | None = None + if ( + self._compression_parameters is not None + and self._compression_parameters.run_compression + ): + if self._compression_parameters.backend == "roibinsz": + self._compressor = RoiBinSzCompression( + parameters=self._compression_parameters + ) + # Post-processing binning if self._post_processing_binning_enabled: if self._monitor_parameters.binning is None: @@ -165,7 +181,7 @@ def _common_initialize_processing_node( ) # An array to store processed data converted to float32 (required by CrystFEL) - self._float_detector_data: NDArray[numpy.float_] = numpy.zeros( + self._float_detector_data: NDArray[numpy.float64] = numpy.zeros( self._processed_data_shape, dtype=numpy.float32 ) @@ -226,7 +242,7 @@ def _common_initialize_collecting_node( def common_process_data( # noqa: C901 self, *, node_rank: int, node_pool_size: int, data: dict[str, Any] - ) -> tuple[NDArray[numpy.float_ | numpy.int_], PeakList, bool]: + ) -> tuple[NDArray[numpy.float64 | numpy.int_], PeakList, bool]: """ Processes a detector data frame. @@ -265,6 +281,14 @@ def common_process_data( # noqa: C901 peak_list: PeakList = self._peak_detection.find_peaks( detector_data=data["detector_data"] ) + if self._compressor is not None: + compressed_data: bytes = self._compressor.compress( + data=data["detector_data"], special_data=peak_list + ) + data["detector_data"] = self._compressor.uncompress( + compressed_data=compressed_data, data_shape=data["detector_data"].shape + ) + frame_is_hit: bool = ( self._crystallography_parameters.min_num_peaks_for_hit < peak_list.num_peaks @@ -275,7 +299,7 @@ def common_process_data( # noqa: C901 peak_list = self._post_processing_binning.bin_peak_positions( peak_list=peak_list ) - binned_detector_data: NDArray[numpy.float_ | numpy.int_] = ( + binned_detector_data: NDArray[numpy.float64 | numpy.int_] = ( self._post_processing_binning.bin_detector_data(data=data["detector_data"]) ) diff --git a/src/om/processing_layer/crystallography.py b/src/om/processing_layer/crystallography.py index e6733b09..c774397d 100644 --- a/src/om/processing_layer/crystallography.py +++ b/src/om/processing_layer/crystallography.py @@ -314,7 +314,7 @@ def process_data( ) if send_detector_data: - data_to_send: NDArray[numpy.int_ | numpy.float_] = data["detector_data"] + data_to_send: NDArray[numpy.int_ | numpy.float64] = data["detector_data"] data_to_send = self._post_processing_binning.bin_detector_data( data=data_to_send @@ -430,7 +430,7 @@ def collect_data( curr_hit_rate_timestamp_history_dark: deque[float] | None curr_hit_rate_history_dark: deque[float] | None curr_virt_powd_plot_img: NDArray[numpy.int_] - curr_peakogram: NDArray[numpy.float_] + curr_peakogram: NDArray[numpy.float64] peakogram_radius_bin_size: float peakogram_intensity_bin_size: float peak_list_x_in_frame: list[float] diff --git a/src/om/processing_layer/xes.py b/src/om/processing_layer/xes.py index 78bd2a7f..af96d9c1 100644 --- a/src/om/processing_layer/xes.py +++ b/src/om/processing_layer/xes.py @@ -195,7 +195,7 @@ def process_data( entry is the OM rank number of the node that processed the information. """ processed_data: dict[str, Any] = {} - camera_data: NDArray[numpy.float_] = data["detector_data"] + camera_data: NDArray[numpy.float64] = data["detector_data"] # Mask the camera edges camera_data[camera_data.shape[0] // 2 - 1 : camera_data.shape[0] // 2 + 1] = 0 @@ -204,7 +204,7 @@ def process_data( camera_data.shape[1] // 2 - 1 : camera_data.shape[1] // 2 + 1, ] = 0 - xes: dict[str, NDArray[numpy.float_]] = ( + xes: dict[str, NDArray[numpy.float64]] = ( self._energy_spectrum_retrieval.calculate_spectrum(data=camera_data) ) @@ -288,12 +288,12 @@ def collect_data( spectrum_for_gui = received_data["spectrum"] - spectra_cumulative_sum: NDArray[numpy.float_ | numpy.int_] | None - spectra_cumulative_sum_smoothed: NDArray[numpy.float_] | None - cumulative_2d: NDArray[numpy.float_ | numpy.int_] | None - spectra_cumulative_sum_pumped: NDArray[numpy.float_] | None - spectra_cumulative_sum_dark: NDArray[numpy.float_] | None - spectra_cumulative_sum_difference: NDArray[numpy.float_] | None + spectra_cumulative_sum: NDArray[numpy.float64 | numpy.int_] | None + spectra_cumulative_sum_smoothed: NDArray[numpy.float64] | None + cumulative_2d: NDArray[numpy.float64 | numpy.int_] | None + spectra_cumulative_sum_pumped: NDArray[numpy.float64] | None + spectra_cumulative_sum_dark: NDArray[numpy.float64] | None + spectra_cumulative_sum_difference: NDArray[numpy.float64] | None ( spectra_cumulative_sum, spectra_cumulative_sum_smoothed, diff --git a/src/om/tools/jungfrau_dark.py b/src/om/tools/jungfrau_dark.py index d831574f..c7629687 100755 --- a/src/om/tools/jungfrau_dark.py +++ b/src/om/tools/jungfrau_dark.py @@ -47,8 +47,8 @@ def main( raise OmInvalidSourceError(f"Error reading the {input} source file.") from exc n: int = 1024 * 512 - sd: NDArray[numpy.float_] = numpy.zeros((3, n), dtype=numpy.float64) - nd: NDArray[numpy.float_] = numpy.zeros((3, n)) + sd: NDArray[numpy.float64] = numpy.zeros((3, n), dtype=numpy.float64) + nd: NDArray[numpy.float64] = numpy.zeros((3, n)) for fn in filelist: i: int = int(re.findall("_f(\\d+)_", fn)[0]) h5_data_path: str = "/data_" + f"f{i:012d}" @@ -69,7 +69,7 @@ def main( nd[i][where_gain[i]] += 1 with numpy.errstate(divide="ignore", invalid="ignore"): - dark: NDArray[numpy.float_] = (sd / nd).astype(numpy.float32) + dark: NDArray[numpy.float64] = (sd / nd).astype(numpy.float32) if numpy.any(nd == 0): log.warning("Some pixels don't have data in all gains")