COO-140 Added preprocessing tasks and initial pipeline engine

eregion/configs/config.py

@@ -0,0 +1,126 @@
+from abc import ABC, abstractmethod
+import yaml
+import logging
+
+# A yaml constructor for slice objects
+def slice_constructor(loader, node):
+    values = loader.construct_sequence(node)
+    # slice will be created from a list, e.g., [start, stop, step]
+    start, stop, step = None, None, None
+    if len(values) == 1:
+        stop = values[0]
+    elif len(values) == 2:
+        start, stop = values
+    elif len(values) == 3:
+        start, stop, step = values
+    else:
+        raise ValueError("Invalid number of arguments for slice.")
+    return slice(start, stop, step)
+
+yaml.add_constructor('!slice', slice_constructor)
+
+
+### Generic Config Loader Base Class ###
+class ConfigLoader(ABC):
+    required_keys = []
+
+    def __init__(self, config_input):
+        """
+        Initialize the ConfigLoader either from a YAML config file path, config data in string or dict format.
+        :param config_input: str or dict
+            Path to a YAML config file or config data as a string or dictionary.
+        """
+        self.config = None
+        self.logger = logging.getLogger(__name__)
+
+        if isinstance(config_input, str) and config_input.endswith(('.yaml', '.yml')):
+            self.set_from_file(config_input)
+        elif isinstance(config_input, str) and not config_input.endswith(('.yaml', '.yml')):
+            self.set_from_string(config_input)
+        elif isinstance(config_input, dict):
+            self.set_from_dict(config_input)
+        else:
+            raise ValueError("config_input must be either a file path, string or a dictionary.")
+
+    def set_from_file(self, input: str):
+        try:
+            with open(input, 'r') as stream:
+                self.load_config(stream)
+            self.logger.info("Config loaded from file '{}'".format(input))
+        except Exception as e:
+            raise ValueError(f"Error reading config file {input}: {e}")
+        self.validate_config()
+
+    def set_from_string(self, input: str):
+        try:
+            self.load_config(input)
+            self.logger.info("Config loaded from string input")
+        except Exception as e:
+            raise ValueError(f"Error parsing input string into yaml: {e}")
+        self.validate_config()
+
+    def set_from_dict(self, input: dict):
+        self.config = input
+        self.validate_config()
+        self.logger.info("Config loaded from dict input")
+
+    def load_config(self, stream):
+        try:
+            self.config = yaml.load(stream, Loader=yaml.FullLoader)
+        except Exception as e:
+            raise ValueError(f"Error loading configuration: {e}")
+
+    @abstractmethod
+    def validate_config(self):
+        for key in self.required_keys:
+            if key not in self.config:
+                raise ValueError(f"Missing required config key: {key}")
+
+
+### Detector Configuration Class ###
+class DetectorConfig(ConfigLoader):
+    required_keys = ['detector_type', 'detector_output_class', 'objects']
+    required_objects_keys = ['name', 'class', 'properties', 'outputs']
+    required_properties_keys = ['x_size', 'y_size', 'pixel_size']
+
+    def __init__(self, config_input):
+        """
+        Initialize the DetectorConfig either from a YAML config file path, config data in string or json format,
+        or generate from a FITS file.
+        :param config_input: str or dict
+            Path to a YAML config file or config data as a string or dictionary.
+        """
+        super().__init__(config_input)
+
+    def validate_config(self):
+        for key in self.required_keys:
+            if key not in self.config:
+                raise ValueError(f"Missing required config key: {key}")
+
+        for obj in self.config['objects']:
+            for key in self.required_objects_keys:
+                if key not in obj:
+                    raise ValueError(f"Missing required object key: {key} in object {obj.get('name', 'unknown')}")
+
+            for prop_key in self.required_properties_keys:
+                if prop_key not in obj['properties']:
+                    raise ValueError(f"Missing required property key: {prop_key} in object {obj.get('name', 'unknown')}")
+
+
+
+### Pipeline Configuration Class ###
+class PipelineConfig(ConfigLoader):
+    required_keys = ['pipeline']
+
+    def __init__(self, config_input):
+        """
+        Initialize the PipelineConfig either from a YAML config file path or config data in string or dict format.
+        :param config_input: str or dict
+            Path to a YAML config file or config data as a string or dictionary.
+        """
+        super().__init__(config_input)
+
+    def validate_config(self):
+        for key in self.required_keys:
+            if key not in self.config:
+                raise ValueError(f"Missing required config key: {key}")

eregion/configs/detectors/basic_ccd.yaml

@@ -0,0 +1,55 @@
+## Config file for creating instances of an image class (here DetImage) for a basic CCD detector
+## The DetImage instance contains the outputs from a "single" detector.
+## The "image" can be stored in one FITS file with one extension that has all the detectors/outputs,
+## OR in multiple FITS extensions, one per detector/output, OR in multiple FITS files, one per detector/output.
+## The config structure should describe how to load one FITS file.
+---
+description: Basic single CCD detector, one channel
+detector_type: CCD # (specify the type of detector, e.g., CCD, CMOS, H2RG, etc.)
+detector_output_class: CCDOutput
+
+# List of class objects, each containing a list of outputs.
+# An output is described by the FITS file it is in, the extension in that file, and the slices that define its data.
+objects:
+  - name: 'det_1'
+    class: DetImage
+    filename_format: '*.fits*'  # Use wildcard to indicate the filename
+    properties:
+      x_size: 2048
+      y_size: 4096
+      saturation_level: 65535  # ADU
+      pixel_size: 0.015  # mm
+    outputs:
+      - id: 'chan_1'
+        ext_id: 1 # FITS extension ID
+        ext_slice: [!slice [0, 4096], !slice [0, 1024]]  # Slice of the ext_id that has the data for this output
+        data_slice: [!slice [0, 4096], !slice [0, 1024]] # Slice of the full DetImage data array where this output's data will go
+        serial_prescan: !slice [0, 0]
+        serial_overscan: !slice [1024, 1024]
+        parallel_prescan: !slice [0, 0]
+        parallel_overscan: !slice [4096, 4096]
+        parallel_axis: 'y' # First axis in the data array (rows) represent parallel readout direction
+        readout_pixel: [0, 0]  # Readout starts at (0,0)
+        gain: 1.0  # electrons/ADU
+        read_noise: 5.0  # electrons
+        bias_level: 1000  # ADU
+      - id: 'chan_2'
+        ext_id: 1 # FITS extension ID
+        ext_slice: [!slice [0, 4096], !slice [1024, 2048]] # Slice of the ext_id that has the data for this output
+        data_slice: [!slice [0, 4096], !slice [1024, 2048]] # Slice of the full DetImage data array where this output's data will go
+        serial_prescan: !slice [0, 0]
+        serial_overscan: !slice [1024, 1024]
+        parallel_prescan: !slice [0, 0]
+        parallel_overscan: !slice [4096, 4096]
+        parallel_axis: 'y' # First axis in the data array (rows) represent parallel readout direction
+        readout_pixel: [0, 2047]  # Readout starts at (0,2047)
+        gain: 1.0  # electrons/ADU
+        read_noise: 5.0  # electrons
+        bias_level: 1000  # ADU
+    focal_plane_position:
+      x_cen: 0.0  # mm
+      y_cen: 0.0  # mm
+      angle: 0.0  # degrees
+
+
+

eregion/configs/detectors/deimos_singledet.yaml

@@ -0,0 +1,54 @@
+## Example detector definition file for the DEIMOS instrument CCD array
+## A single FITS file contains 8 extensions, one per detector, so 8 DetImage instances are created from this file.
+## Each DetImage instance has two outputs (two channels per detector).
+---
+description: DEIMOS CCD detector single
+detector_type: CCD # (specify the type of detector, e.g., CCD, CMOS, H2RG, etc.)
+detector_output_class: CCDOutput # (use the correct class here for the detector output type)
+
+objects:
+  - name: 'det_1'
+    class: DetImage
+    filename_format: '*.fits*'  # Use wildcard to indicate the filename
+    properties:
+      x_size: 2188
+      y_size: 4125
+      saturation_level: 65535  # ADU
+      pixel_size: 0.015  # mm
+    outputs:
+      - id: 'chan_1'
+        ext_id: 1 # FITS extension ID
+        ext_slice: [!slice [0, 4125], !slice [0, 1094]]  # Slice of the ext_id that has the data for this output
+        data_slice: [!slice [0, 4125], !slice [0, 1094]] # Slice of the full DetImage data array where this output's data will go
+        serial_prescan: !slice [0, 0]
+        serial_overscan: !slice [1094, 1094]
+        parallel_prescan: !slice [0, 0]
+        parallel_overscan: !slice [4125, 4125]
+        parallel_axis: 'y' # First axis in the data array (rows) represent parallel readout direction
+        readout_pixel: [0, 0]  # top left pixel
+        gain: 1.0  # electrons/ADU
+        read_noise: 5.0  # electrons
+        bias_level: 1000  # ADU
+      - id: 'chan_2'
+        ext_id: 2 # FITS extension ID
+        ext_slice: [!slice [0, 4125], !slice [0, 1094]] # Slice of the ext_id that has the data for this output
+        data_slice: [!slice [0, 4125], !slice [1094, 2188]] # Slice of the full DetImage data array where this output's data will go
+        serial_prescan: !slice [0, 0]
+        serial_overscan: !slice [1094, 1094]
+        parallel_prescan: !slice [0, 0]
+        parallel_overscan: !slice [4125, 4125]
+        parallel_axis: 'y' # First axis in the data array (rows) represent parallel readout direction
+        readout_pixel: [0, 1094]  # top right pixel
+        gain: 1.0  # electrons/ADU
+        read_noise: 5.0  # electrons
+        bias_level: 1000  # ADU
+    focal_plane_position:
+      x_cen: -46.08  # mm
+      y_cen: 30.72  # mm
+      angle: 0.0  # degrees
+
+
+
+
+
+

eregion/configs/pipeline_flows/masterbias_example.yaml

@@ -0,0 +1,32 @@
+# This is an example pipeline flow configuration for creating a master bias frame
+# Example
+
+debug: false                    # Optional: set to true to enable debug mode (more verbose logging, etc.)
+pipeline:
+  - name: image_creator
+    task: tasks.imagegen.ImageCreator
+    lazy: false
+    init:                                                                                                     # Initialization parameters for the image creator
+      detector_config: "/Users/yashvi/Desktop/Detector Characterization Tools/eregion/configs/detectors/deimos_singledet.yaml"      # For required params, see the task documentation
+                                                                                                              # Any additional params are passed as kwargs that get set in the task's meta dict
+                                                                                                              # See task documentation for which kwargs are used
+
+    run:                                                                                                      # Run-time inputs and parameters for the image creator
+      params:                                                                                                 # Check which params go here in the task documentation
+        input_source: "/Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/*_bias_*.fits"
+        identifier_func: tasks.custom.guess_image_type_from_filename_DEIMOS
+
+
+  - name: master_bias
+    task: tasks.calibration.MasterBias
+    lazy: false
+    init:
+      method: "median"                                      # Optional param (kwarg) method to combine bias frames; see task documentation for options
+    run:
+      inputs:                                               # use inputs to specify data input coming in from output of other tasks
+        bias_images: image_creator.data.images              # E.g. Input images from the image creator task's output
+                                                            # Each task's outputs are objects of class TaskResult
+                                                            # TaskResult.data is a dict containing the actual data produced by the task
+                                                            # Check which outputs are available in the task documentation
+
+    depends_on: [image_creator]