From 6130ec7a0911bcd7805ec56bb8d8fc061fce19cc Mon Sep 17 00:00:00 2001 From: Luisa Date: Fri, 19 Dec 2025 12:58:20 -0700 Subject: [PATCH 1/2] mask out invalid times --- imap_processing/ultra/l1b/de.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/imap_processing/ultra/l1b/de.py b/imap_processing/ultra/l1b/de.py index 219100a15..a0261540d 100644 --- a/imap_processing/ultra/l1b/de.py +++ b/imap_processing/ultra/l1b/de.py @@ -133,6 +133,8 @@ def calculate_de( e_bin_l1a = np.full(len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8) species_bin = np.full(len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8) t2 = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) + event_times = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) + spin_starts = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) shape = (len(de_dataset["epoch"]), 3) sc_velocity = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) sc_dps_velocity = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) @@ -162,13 +164,12 @@ def calculate_de( start_type[valid_indices] = de_dataset["start_type"].data[valid_indices] spin_ds = get_spin_info(aux_dataset, de_dataset["shcoarse"].data) - (event_times, spin_starts) = get_event_times( + (event_times[valid_mask], spin_starts[valid_mask]) = get_event_times( aux_dataset, - de_dataset["shcoarse"].data, - de_dataset["phase_angle"].data, - spin_ds, + de_dataset["shcoarse"].data[valid_mask], + de_dataset["phase_angle"].data[valid_mask], + spin_ds.isel(epoch=valid_mask), ) - de_dict["spin"] = spin_ds.spin_number.data de_dict["event_times"] = event_times.astype(np.float64) # Pulse height @@ -323,15 +324,18 @@ def calculate_de( # Annotated Events. ultra_frame = getattr(SpiceFrame, f"IMAP_ULTRA_{sensor}") - valid_events = np.ones(event_times.shape, bool) - + # Account for counts=0 (event times have FILL value) + valid_events = (event_times != FILLVAL_FLOAT32).copy() if repoint_id is not None: - in_pointing = calculate_events_in_pointing(repoint_id, et_to_met(event_times)) - events_to_flag = ~in_pointing + in_pointing = calculate_events_in_pointing( + repoint_id, et_to_met(event_times[valid_events]) + ) + events_to_flag = np.zeros(len(quality_flags), dtype=bool) + events_to_flag[valid_events] = ~in_pointing # Update quality flags for valid events that are not in the pointing quality_flags[events_to_flag] |= ImapDEOutliersUltraFlags.DURINGREPOINT.value # Update valid_events to only include times within a pointing - valid_events &= in_pointing + valid_events[valid_events] &= in_pointing ( sc_velocity[valid_events], @@ -424,5 +428,4 @@ def calculate_events_in_pointing( in_pointing = (event_times >= pointing_start_met) & ( event_times <= pointing_end_met ) - return in_pointing From f67d022bdf5256e831246e37dcd7a9b10189a7c0 Mon Sep 17 00:00:00 2001 From: Luisa Date: Mon, 22 Dec 2025 09:31:34 -0500 Subject: [PATCH 2/2] update dtype for start times and event times --- imap_processing/ultra/l1b/de.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/imap_processing/ultra/l1b/de.py b/imap_processing/ultra/l1b/de.py index a0261540d..d9b4466cc 100644 --- a/imap_processing/ultra/l1b/de.py +++ b/imap_processing/ultra/l1b/de.py @@ -133,8 +133,8 @@ def calculate_de( e_bin_l1a = np.full(len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8) species_bin = np.full(len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8) t2 = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - event_times = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - spin_starts = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) + event_times = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64) + spin_starts = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64) shape = (len(de_dataset["epoch"]), 3) sc_velocity = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) sc_dps_velocity = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) @@ -170,6 +170,7 @@ def calculate_de( de_dataset["phase_angle"].data[valid_mask], spin_ds.isel(epoch=valid_mask), ) + de_dict["spin"] = spin_ds.spin_number.data de_dict["event_times"] = event_times.astype(np.float64) # Pulse height @@ -327,10 +328,13 @@ def calculate_de( # Account for counts=0 (event times have FILL value) valid_events = (event_times != FILLVAL_FLOAT32).copy() if repoint_id is not None: + # Check all valid event times to see which are in the pointing in_pointing = calculate_events_in_pointing( repoint_id, et_to_met(event_times[valid_events]) ) + # Initialize an array of all events as False events_to_flag = np.zeros(len(quality_flags), dtype=bool) + # Identify valid events that are outside the pointing events_to_flag[valid_events] = ~in_pointing # Update quality flags for valid events that are not in the pointing quality_flags[events_to_flag] |= ImapDEOutliersUltraFlags.DURINGREPOINT.value @@ -423,7 +427,6 @@ def calculate_events_in_pointing( combined with the valid_events mask. """ pointing_start_met, pointing_end_met = get_pointing_times_from_id(repoint_id) - # Check which events are within the pointing in_pointing = (event_times >= pointing_start_met) & ( event_times <= pointing_end_met