ENH Add support for RoiBinSz compression

[gadorlhiac]

Description

This PR adds support for ROIBinSz compression. Currently implemented in a "dummy" format which automatically uncompresses the data afterwards. This is for validation of compression algorithms. In the future this can be easily extended to save compressed data if there is a use case for that.

Checklist

• OmCompressionProtocol and RoiBinSzCompression
• DataCompressionParameters and RoiBinSzCompressorParameters
• Addition to cheetah processing layer.

PR Type:

• New feature/Enhancement

Address issues:

Testing

This has been tested using LUTE as part of a larger set of PRs for various workflows. The generated OM configuration file is as follows:

om:  
  parallelization_layer: "MpiParallelization"
  data_retrieval_layer: "Psana2DataEventHandler"
  processing_layer: "CheetahProcessing"

data_retrieval_layer:  
  asapo_url: null
  asapo_path: null
  asapo_data_source: null
  asapo_has_filesystem: null
  asapo_token: null
  asapo_group_id: "default_om_group"
  buffer_size: null
  psana_calibration_directory: null
  data_sources:  
    timestamp:  
      type: "TimestampPsana2"
      psana_name: "EventID"
    event_id:  
      type: "EventIdPsana2"
    detector_data:  
      type: "AreaDetectorPsana2"
      psana_name: "epix10k2M"
      calibration: true
      psana_algorithm: "xtc1dump"
    detector_distance:  
      type: "FloatValueFromConfiguration"
      psana_name: null
      value: 100.0
    beam_energy:  
      type: "FloatValueFromConfiguration"
      value: 9000.0
  node_pool_size: 0

cheetah:  
  processed_directory: "/sdf/data/lcls/ds/mfx/mfx100903824/results"
  processed_filename_prefix: "mfx100903824-27"
  processed_filename_extension: "cxi"
  write_class_sums: true
  class_sums_update_interval: 5
  class_sums_sending_interval: -1
  class_sums_filename_prefix: "27"
  status_file_update_interval: 100
  hdf5_file_data_type: "float32"
  hdf5_file_compression: null
  hdf5_file_gzip_compression_level: 4
  hdf5_file_zstd_compression_level: 3
  hdf5_file_compression_shuffle: false
  hdf5_file_max_num_peaks: 2048
  hdf5_fields:  
    detector_data: "/entry_1/data_1/data"
    event_id: "/LCLS/fiducial"
    beam_energy: "/LCLS/photon_energy_eV"
    detector_distance: "/LCLS/detector_1/EncoderValue"
    timestamp: "/LCLS/timestamp"
    peak_list: "/entry_1/result_1"
  external_data_request_list_size: 20
  responding_url: null

crystallography:  
  geometry_file: "/sdf/data/lcls/ds/mfx/mfx100903824/scratch/dorlhiac/test.geom"
  min_num_peaks_for_hit: 10
  max_num_peaks_for_hit: 5000
  running_average_window_size: 200
  geometry_is_optimized: false
  speed_report_interval: 100
  data_broadcast_interval: 0

peakfinder8_peak_detection:  
  max_num_peaks: 2048
  adc_threshold: 100.0
  minimum_snr: 5.0
  min_pixel_count: 1
  max_pixel_count: 30
  local_bg_radius: 4
  min_res: 80
  max_res: 800
  fast_mode: false
  num_pixel_per_bin_in_radial_statistics: 100
  bad_pixel_map_filename: null
  bad_pixel_map_hdf5_path: "/data/data"

compression:  
  run_compression: true
  backend: "roibinsz"
  compression_parameters:  
    compressor: "qoz"
    abs_error: 42.0
    bin_size: 2
    roi_window_size: 9

Screenshots