From a96138921298ae900039e0cf536651efe0118d9d Mon Sep 17 00:00:00 2001 From: Luisa Date: Wed, 17 Dec 2025 12:19:37 -0700 Subject: [PATCH 1/8] use aux packet to get spins --- .../tests/ultra/unit/test_ultra_l1b.py | 16 ++-- .../ultra/unit/test_ultra_l1b_culling.py | 26 +++-- .../ultra/unit/test_ultra_l1b_extended.py | 28 +++++- imap_processing/ultra/l1b/badtimes.py | 2 +- imap_processing/ultra/l1b/de.py | 5 +- imap_processing/ultra/l1b/extendedspin.py | 2 +- imap_processing/ultra/l1b/goodtimes.py | 2 +- .../ultra/l1b/ultra_l1b_culling.py | 81 +--------------- .../ultra/l1b/ultra_l1b_extended.py | 94 ++++++++++++++----- imap_processing/ultra/l1c/helio_pset.py | 4 + imap_processing/ultra/l1c/spacecraft_pset.py | 4 + imap_processing/ultra/l1c/ultra_l1c.py | 8 ++ .../ultra/l1c/ultra_l1c_pset_bins.py | 9 +- 13 files changed, 156 insertions(+), 125 deletions(-) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b.py b/imap_processing/tests/ultra/unit/test_ultra_l1b.py index fe6e08f59..70f55e576 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b.py @@ -190,7 +190,7 @@ def test_cdf_de_flags( @pytest.mark.external_test_data def test_ultra_l1b_extendedspin( - use_fake_spin_data_for_time, faux_aux_dataset, rates_dataset + use_fake_spin_data_for_time, aux_dataset, rates_dataset ): """Tests that L1b data is created.""" use_fake_spin_data_for_time(0, 141 * 15) @@ -205,7 +205,7 @@ def test_ultra_l1b_extendedspin( "imap_ultra_l1a_45sensor-params", ] } - data_dict["imap_ultra_l1a_45sensor-aux"] = faux_aux_dataset + data_dict["imap_ultra_l1a_45sensor-aux"] = aux_dataset data_dict["imap_ultra_l1a_45sensor-rates"] = rates_dataset ancillary_files = {} @@ -219,7 +219,7 @@ def test_ultra_l1b_extendedspin( @pytest.mark.external_test_data -def test_cdf_extendedspin(use_fake_spin_data_for_time, faux_aux_dataset, rates_dataset): +def test_cdf_extendedspin(use_fake_spin_data_for_time, aux_dataset, rates_dataset): use_fake_spin_data_for_time(0, 141 * 15) l1b_de_dataset_path = ( TEST_PATH / "imap_ultra_l1b_45sensor-de_20240207-repoint99999_v999.cdf" @@ -233,7 +233,7 @@ def test_cdf_extendedspin(use_fake_spin_data_for_time, faux_aux_dataset, rates_d "imap_ultra_l1a_45sensor-params", ] } - data_dict["imap_ultra_l1a_45sensor-aux"] = faux_aux_dataset + data_dict["imap_ultra_l1a_45sensor-aux"] = aux_dataset data_dict["imap_ultra_l1a_45sensor-rates"] = rates_dataset ancillary_files = {} @@ -251,7 +251,7 @@ def test_cdf_extendedspin(use_fake_spin_data_for_time, faux_aux_dataset, rates_d @pytest.mark.external_test_data -def test_cdf_goodtimes(use_fake_spin_data_for_time, faux_aux_dataset, rates_dataset): +def test_cdf_goodtimes(use_fake_spin_data_for_time, aux_dataset, rates_dataset): """Tests that CDF file is created and contains same attributes as xarray.""" use_fake_spin_data_for_time(0, 141 * 15) l1b_de_dataset_path = ( @@ -266,7 +266,7 @@ def test_cdf_goodtimes(use_fake_spin_data_for_time, faux_aux_dataset, rates_data "imap_ultra_l1a_45sensor-params", ] } - data_dict["imap_ultra_l1a_45sensor-aux"] = faux_aux_dataset + data_dict["imap_ultra_l1a_45sensor-aux"] = aux_dataset data_dict["imap_ultra_l1a_45sensor-rates"] = rates_dataset ancillary_files = {} @@ -288,7 +288,7 @@ def test_cdf_goodtimes(use_fake_spin_data_for_time, faux_aux_dataset, rates_data @pytest.mark.external_test_data -def test_cdf_badtimes(use_fake_spin_data_for_time, faux_aux_dataset, rates_dataset): +def test_cdf_badtimes(use_fake_spin_data_for_time, aux_dataset, rates_dataset): """Tests that CDF file is created and contains same attributes as xarray.""" use_fake_spin_data_for_time(0, 141 * 15) l1b_de_dataset_path = ( @@ -303,7 +303,7 @@ def test_cdf_badtimes(use_fake_spin_data_for_time, faux_aux_dataset, rates_datas "imap_ultra_l1a_45sensor-params", ] } - data_dict["imap_ultra_l1a_45sensor-aux"] = faux_aux_dataset + data_dict["imap_ultra_l1a_45sensor-aux"] = aux_dataset data_dict["imap_ultra_l1a_45sensor-rates"] = rates_dataset ancillary_files = {} diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py index ff80abaec..7daaa0279 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py @@ -3,6 +3,7 @@ import numpy as np import pandas as pd import pytest +import xarray as xr from imap_processing import imap_module_directory from imap_processing.quality_flags import ( @@ -170,21 +171,34 @@ def test_compare_aux_univ_spin_table(use_fake_spin_data_for_time, faux_aux_datas assert np.all(result == expected) -def test_get_duration(rates_l1_test_path, use_fake_spin_data_for_time): +def test_get_duration(rates_l1_test_path, use_fake_spin_data_for_time, aux_test_path): """Tests get_duration function.""" use_fake_spin_data_for_time(start_met=0, end_met=141 * 15) - df = pd.read_csv(rates_l1_test_path) + # Should be evenly spaced spins of 15 seconds each except the first one has 14. + num_spins = 15 + spin_start_times = np.concatenate([[0], np.arange(14, 222, num_spins)]) + spin_numbers = np.arange(127, 142) + num_spins = len(spin_numbers) + + aux_ds = xr.Dataset( + data_vars={ + "timespinstart": ("epoch", spin_start_times), + "duration": ("epoch", np.full(num_spins, 15)), + "spinnumber": ("epoch", spin_numbers), + }, + coords={"epoch": ("epoch", np.arange(num_spins))}, + ) + met = df["TimeTag"] - df["TimeTag"].values[0] spin = df["Spin"] - spin_number, duration = get_spin_and_duration(met, spin) - + spin_number, duration = get_spin_and_duration(aux_ds, met) assert np.array_equal(spin, spin_number) assert np.all(duration == 15) -def test_get_pulses(rates_l1_test_path, use_fake_spin_data_for_time): +def test_get_pulses(rates_l1_test_path, use_fake_spin_data_for_time, aux_dataset): """Tests get_pulses_per_spin function.""" df = pd.read_csv(rates_l1_test_path) @@ -214,7 +228,7 @@ def test_get_pulses(rates_l1_test_path, use_fake_spin_data_for_time): "spin": df["Spin"], } - pulses = get_pulses_per_spin(pulse_dict) + pulses = get_pulses_per_spin(aux_dataset, pulse_dict) unique_spins = np.unique(pulse_dict["spin"]) start_pulses_total = pulse_dict["start_rf"] + pulse_dict["start_lf"] diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py index 08a5a8c6a..5cc1fdc79 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py @@ -31,6 +31,7 @@ get_path_length, get_ph_tof_and_back_positions, get_phi_theta, + get_spin_and_duration, get_ssd_back_position_and_tof_offset, get_ssd_tof, interpolate_fwhm, @@ -521,7 +522,7 @@ def test_get_eventtimes(test_fixture, aux_dataset): """Tests get_eventtimes function.""" df_filt, _, _, de_dataset = test_fixture - event_times, spin_start_times, spin_numbers = get_event_times( + event_times, spin_start_times = get_event_times( aux_dataset, de_dataset["phase_angle"].values, de_dataset["shcoarse"].values, @@ -529,10 +530,7 @@ def test_get_eventtimes(test_fixture, aux_dataset): # Check shapes assert ( - event_times.shape - == spin_start_times.shape - == spin_numbers.shape - == de_dataset["phase_angle"].shape + event_times.shape == spin_start_times.shape == de_dataset["phase_angle"].shape ) t1_start_sec = aux_dataset["timespinstart"].values[0] @@ -561,6 +559,26 @@ def test_get_eventtimes(test_fixture, aux_dataset): assert start_time <= int(event_times[i]) <= end_time +@pytest.mark.external_test_data +def test_get_spin_and_duration(test_fixture, aux_dataset): + """Tests get_eventtimes function.""" + df_filt, _, _, de_dataset = test_fixture + + spin_number, spin_duration = get_spin_and_duration( + aux_dataset, + de_dataset["shcoarse"].values, + ) + + # Check shapes + assert spin_number.shape == spin_duration.shape == de_dataset["shcoarse"].shape + + t1_spin_number = aux_dataset["spinnumber"].values[0] + t1_start_dur = aux_dataset["duration"].values[0] + # Check the first event spin number and duration + assert spin_number[0] == t1_spin_number + assert spin_duration[0] == t1_start_dur + + @pytest.mark.external_test_data def test_get_event_times_out_of_range(test_fixture, aux_dataset): """Tests get_event_times with out of range values.""" diff --git a/imap_processing/ultra/l1b/badtimes.py b/imap_processing/ultra/l1b/badtimes.py index a3f82ef40..c78520a49 100644 --- a/imap_processing/ultra/l1b/badtimes.py +++ b/imap_processing/ultra/l1b/badtimes.py @@ -34,7 +34,7 @@ def calculate_badtimes( badtimes_dataset : xarray.Dataset Dataset containing the extendedspin data that has been culled. """ - n_bins = extendedspin_dataset.dims["energy_bin_geometric_mean"] + n_bins = extendedspin_dataset.sizes["energy_bin_geometric_mean"] culled_spins = np.setdiff1d( extendedspin_dataset["spin_number"].values, goodtimes_spins ) diff --git a/imap_processing/ultra/l1b/de.py b/imap_processing/ultra/l1b/de.py index acc0f65de..d8f6e266b 100644 --- a/imap_processing/ultra/l1b/de.py +++ b/imap_processing/ultra/l1b/de.py @@ -37,6 +37,7 @@ get_path_length, get_ph_tof_and_back_positions, get_phi_theta, + get_spin_and_duration, get_ssd_back_position_and_tof_offset, get_ssd_tof, is_back_tof_valid, @@ -158,11 +159,13 @@ def calculate_de( ancillary_files, ) start_type[valid_indices] = de_dataset["start_type"].data[valid_indices] - (event_times, spin_starts, spin_number) = get_event_times( + (event_times, spin_starts) = get_event_times( aux_dataset, de_dataset["phase_angle"].data, de_dataset["shcoarse"].data, ) + spin_number, _ = get_spin_and_duration(aux_dataset, de_dataset["shcoarse"].data) + de_dict["spin"] = spin_number de_dict["event_times"] = event_times.astype(np.float64) # Pulse height diff --git a/imap_processing/ultra/l1b/extendedspin.py b/imap_processing/ultra/l1b/extendedspin.py index 93f3faf1c..b930d1107 100644 --- a/imap_processing/ultra/l1b/extendedspin.py +++ b/imap_processing/ultra/l1b/extendedspin.py @@ -61,7 +61,7 @@ def calculate_extendedspin( inst_qf = flag_imap_instruments(de_dataset["spin"].values) # Get the number of pulses per spin. - pulses = get_pulses_per_spin(rates_dataset) + pulses = get_pulses_per_spin(aux_dataset, rates_dataset) # Track rejected events in each spin based on # quality flags in de l1b data. diff --git a/imap_processing/ultra/l1b/goodtimes.py b/imap_processing/ultra/l1b/goodtimes.py index 1271d6adb..915c770b5 100644 --- a/imap_processing/ultra/l1b/goodtimes.py +++ b/imap_processing/ultra/l1b/goodtimes.py @@ -28,7 +28,7 @@ def calculate_goodtimes(extendedspin_dataset: xr.Dataset, name: str) -> xr.Datas goodtimes_dataset : xarray.Dataset Dataset containing the extendedspin data that remains after culling. """ - n_bins = extendedspin_dataset.dims["energy_bin_geometric_mean"] + n_bins = extendedspin_dataset.sizes["energy_bin_geometric_mean"] # If the spin rate was too high or low then the spin should be thrown out. # If the rates at any energy level are too high then throw out the entire spin. good_mask = ( diff --git a/imap_processing/ultra/l1b/ultra_l1b_culling.py b/imap_processing/ultra/l1b/ultra_l1b_culling.py index d95af415e..9e429e93c 100644 --- a/imap_processing/ultra/l1b/ultra_l1b_culling.py +++ b/imap_processing/ultra/l1b/ultra_l1b_culling.py @@ -22,6 +22,7 @@ get_scattering_thresholds, ) from imap_processing.ultra.l1b.quality_flag_filters import DE_QUALITY_FLAG_FILTERS +from imap_processing.ultra.l1b.ultra_l1b_extended import get_spin_and_duration from imap_processing.ultra.l1c.l1c_lookup_utils import build_energy_bins logger = logging.getLogger(__name__) @@ -339,86 +340,14 @@ def compare_aux_univ_spin_table( return mismatch_indices -# TODO: Make this a common util since it is being used for the de and rates packets. -def get_spin_and_duration(met: NDArray, spin: NDArray) -> tuple[NDArray, NDArray]: - """ - Get the spin number and duration. - - Parameters - ---------- - met : NDArray - Mission elapsed time. - spin : NDArray - Spin number 0-255. - - Returns - ------- - assigned_spin_number : NDArray - Spin number for packet data product. - """ - # Packet data. - # Since the spin number in the direct events packet - # is only 8 bits it goes from 0-255. - # Within a pointing that means we will always have duplicate spin numbers. - # In other words, different spins will be represented by the same spin number. - # Just to make certain that we won't accidentally combine - # multiple spins we need to sort by time here. - sort_idx = np.argsort(met) - packet_met_sorted = met[sort_idx] - packet_spin_sorted = spin[sort_idx] - # Here we are finding the start and end indices of each spin in the sorted array. - is_new_spin = np.concatenate( - [[True], packet_spin_sorted.values[1:] != packet_spin_sorted.values[:-1]] - ) - spin_start_indices = np.where(is_new_spin)[0] - spin_end_indices = np.append(spin_start_indices[1:], len(packet_met_sorted)) - - # Universal Spin Table. - spin_df = get_spin_data() - # Retrieve the met values of the start of the spin. - spin_start_mets = spin_df["spin_start_met"].values - # Retrieve the corresponding spin numbers. - spin_numbers = spin_df["spin_number"].values - spin_period_sec = spin_df["spin_period_sec"].values - assigned_spin_number_sorted = np.empty(packet_spin_sorted.shape, dtype=np.uint32) - assigned_spin_duration_sorted = np.empty(packet_spin_sorted.shape, dtype=np.float32) - # These last 8 bits are the same as the spin number in the DE packet. - # So this will give us choices of which spins are - # available to assign to the packet data. - possible_spins = spin_numbers & 0xFF - - # Assign each group based on time. - for start, end in zip(spin_start_indices, spin_end_indices, strict=False): - # Now that we have the possible spins from the Universal Spin Table, - # we match the times of those spins to the nearest times in the DE data. - possible_times = spin_start_mets[ - possible_spins == packet_spin_sorted.values[start] - ] - # Get nearest time for matching spins. - nearest_idx = np.abs(possible_times - packet_met_sorted.values[start]).argmin() - nearest_value = possible_times[nearest_idx] - assigned_spin_number_sorted[start:end] = spin_numbers[ - spin_start_mets == nearest_value - ] - assigned_spin_duration_sorted[start:end] = spin_period_sec[ - spin_start_mets == nearest_value - ] - - # Undo the sort to match original order. - assigned_spin_number = np.empty_like(assigned_spin_number_sorted) - assigned_spin_number[sort_idx] = assigned_spin_number_sorted - assigned_duration = np.empty_like(assigned_spin_duration_sorted) - assigned_duration[sort_idx] = assigned_spin_duration_sorted - - return assigned_spin_number, assigned_duration - - -def get_pulses_per_spin(rates: xr.Dataset) -> RateResult: +def get_pulses_per_spin(aux: xr.Dataset, rates: xr.Dataset) -> RateResult: """ Get the total number of pulses per spin. Parameters ---------- + aux : xr.Dataset + Auxiliary dataset. rates : xr.Dataset Rates dataset. @@ -439,7 +368,7 @@ def get_pulses_per_spin(rates: xr.Dataset) -> RateResult: coin_pulses : NDArray Total coincidence pulses. """ - spin_number, _duration = get_spin_and_duration(rates["shcoarse"], rates["spin"]) + spin_number, _ = get_spin_and_duration(aux, rates["shcoarse"].values) # Top coin pulses top_coin_pulses = np.stack( diff --git a/imap_processing/ultra/l1b/ultra_l1b_extended.py b/imap_processing/ultra/l1b/ultra_l1b_extended.py index 4618d7cf8..3d61de725 100644 --- a/imap_processing/ultra/l1b/ultra_l1b_extended.py +++ b/imap_processing/ultra/l1b/ultra_l1b_extended.py @@ -917,40 +917,24 @@ def get_phi_theta( return np.degrees(phi), np.degrees(theta) -def get_event_times( - aux_dataset: xr.Dataset, phase_angle: NDArray, de_event_met: NDArray -) -> tuple[NDArray, NDArray, NDArray]: +def get_spin_start_indices(aux_dataset: xr.Dataset, de_event_met: NDArray) -> NDArray: """ - Get the event times, spin start times, and spin numbers. - - Use formula from section 3.3.1 of the ULTRA algorithm document. - t_e = t_spin_start + (t_start_sub / 1000) + - (t_spin_duration * theta_event) / (1000 * 720) + Get the spin start indices in the aux dataset for each event. Parameters ---------- aux_dataset : numpy.ndarray Auxiliary dataset containing spin information. - phase_angle : numpy.ndarray - Phase angle. de_event_met : numpy.ndarray Direct event MET. Returns ------- - event_times : numpy.ndarray - Event times in et. - spin_start_times: numpy.ndarray - Spin start times in et. - spin_numbers: numpy.ndarray - Spin numbers for each event. + start_inds : numpy.ndarray + Spin start indices for each event. """ # Get Spin Start Time in seconds spin_start_sec = aux_dataset["timespinstart"].values - # Get Spin Start Subsecond in milliseconds - spin_start_subsec = aux_dataset["timespinstartsub"].values - # Get spin duration in milliseconds - spin_duration = aux_dataset["duration"].values # Check that all events fall within the aux dataset time range. # The time window spans from the first spin start to the end of the last spin. @@ -968,9 +952,44 @@ def get_event_times( start_inds = np.searchsorted(spin_start_sec, de_event_met, side="right") - 1 # Clip to valid range of indices start_inds = np.clip(start_inds, 0, len(spin_start_sec) - 1) - # Get the spin numbers for each event - spin_numbers = aux_dataset["spinnumber"].values[start_inds] + return start_inds + + +def get_event_times( + aux_dataset: xr.Dataset, phase_angle: NDArray, de_event_met: NDArray +) -> tuple[NDArray, NDArray]: + """ + Get the event times, spin start times. + + Use formula from section 3.3.1 of the ULTRA algorithm document. + t_e = t_spin_start + (t_start_sub / 1000) + + (t_spin_duration * theta_event) / (1000 * 720) + + Parameters + ---------- + aux_dataset : numpy.ndarray + Auxiliary dataset containing spin information. + phase_angle : numpy.ndarray + Phase angle. + de_event_met : numpy.ndarray + Direct event MET. + + Returns + ------- + event_times : numpy.ndarray + Event times in et. + spin_start_times: numpy.ndarray + Spin start times in et. + """ + # Get Spin Start Time in seconds + spin_start_sec = aux_dataset["timespinstart"].values + # Get Spin Start Subsecond in milliseconds + spin_start_subsec = aux_dataset["timespinstartsub"].values + # Get spin duration in milliseconds + spin_duration = aux_dataset["duration"].values + + start_inds = get_spin_start_indices(aux_dataset, de_event_met) # Get the relevant spin parameters for each event evt_spin_starts = spin_start_sec[start_inds] evt_spin_start_subs = spin_start_subsec[start_inds] @@ -986,10 +1005,39 @@ def get_event_times( return ( ttj2000ns_to_et(met_to_ttj2000ns(event_times)), ttj2000ns_to_et(met_to_ttj2000ns(spin_start_times)), - spin_numbers, ) +def get_spin_and_duration( + aux_dataset: xr.Dataset, de_event_met: NDArray +) -> tuple[NDArray, NDArray]: + """ + Get the spin numbers and durations. + + Parameters + ---------- + aux_dataset : numpy.ndarray + Auxiliary dataset containing spin information. + de_event_met : numpy.ndarray + Direct event MET. + + Returns + ------- + spin_numbers: numpy.ndarray + Spin numbers for each event. + spin_durations: numpy.ndarray + Spin durations for each event. + """ + # Get the spin start indices for each event + start_inds = get_spin_start_indices(aux_dataset, de_event_met) + # Get the spin numbers for each event + spin_numbers = aux_dataset["spinnumber"].values[start_inds] + # Get the spin duration for each event + spin_durations = aux_dataset["duration"].values[start_inds] + + return spin_numbers, spin_durations + + def interpolate_fwhm( lookup_table: pandas.DataFrame, energy: NDArray, diff --git a/imap_processing/ultra/l1c/helio_pset.py b/imap_processing/ultra/l1c/helio_pset.py index 228db8f3e..64c0d8862 100644 --- a/imap_processing/ultra/l1c/helio_pset.py +++ b/imap_processing/ultra/l1c/helio_pset.py @@ -39,6 +39,7 @@ def calculate_helio_pset( de_dataset: xr.Dataset, goodtimes_dataset: xr.Dataset, rates_dataset: xr.Dataset, + aux_dataset: xr.Dataset,, name: str, ancillary_files: dict, instrument_id: int, @@ -55,6 +56,8 @@ def calculate_helio_pset( Dataset containing goodtimes data. rates_dataset : xarray.Dataset Dataset containing image rates data. + aux_dataset : xarray.Dataset + Dataset containing auxiliary data. name : str Name of the dataset. ancillary_files : dict @@ -176,6 +179,7 @@ def calculate_helio_pset( # Calculate background rates background_rates = get_spacecraft_background_rates( rates_dataset, + aux_dataset, sensor_id, ancillary_files, intervals, diff --git a/imap_processing/ultra/l1c/spacecraft_pset.py b/imap_processing/ultra/l1c/spacecraft_pset.py index 9cf2aaae2..40a2e401c 100644 --- a/imap_processing/ultra/l1c/spacecraft_pset.py +++ b/imap_processing/ultra/l1c/spacecraft_pset.py @@ -37,6 +37,7 @@ def calculate_spacecraft_pset( de_dataset: xr.Dataset, goodtimes_dataset: xr.Dataset, rates_dataset: xr.Dataset, + aux_dataset: xr.Dataset, name: str, ancillary_files: dict, instrument_id: int, @@ -53,6 +54,8 @@ def calculate_spacecraft_pset( Dataset containing goodtimes data. rates_dataset : xarray.Dataset Dataset containing image rates data. + aux_dataset : xarray.Dataset + Dataset containing auxiliary data. name : str Name of the dataset. ancillary_files : dict @@ -163,6 +166,7 @@ def calculate_spacecraft_pset( # Calculate background rates background_rates = get_spacecraft_background_rates( rates_dataset, + aux_dataset, sensor_id, ancillary_files, intervals, diff --git a/imap_processing/ultra/l1c/ultra_l1c.py b/imap_processing/ultra/l1c/ultra_l1c.py index 75981180f..1456d1d66 100644 --- a/imap_processing/ultra/l1c/ultra_l1c.py +++ b/imap_processing/ultra/l1c/ultra_l1c.py @@ -35,12 +35,14 @@ def ultra_l1c( f"imap_ultra_l1b_{instrument_id}sensor-goodtimes" in data_dict and f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict and f"imap_ultra_l1a_{instrument_id}sensor-rates" in data_dict + and f"imap_ultra_l1a_{instrument_id}sensor-aux" in data_dict and create_helio_pset ): helio_pset = calculate_helio_pset( data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"], data_dict[f"imap_ultra_l1b_{instrument_id}sensor-goodtimes"], data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"], + data_dict[f"imap_ultra_l1a_{instrument_id}sensor-aux"], f"imap_ultra_l1c_{instrument_id}sensor-heliopset", ancillary_files, instrument_id, @@ -51,11 +53,15 @@ def ultra_l1c( f"imap_ultra_l1b_{instrument_id}sensor-goodtimes" in data_dict and f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict and f"imap_ultra_l1a_{instrument_id}sensor-rates" in data_dict + and f"imap_ultra_l1a_{instrument_id}sensor-aux" in data_dict + and f"imap_ultra_l1a_{instrument_id}sensor-params" in data_dict ): spacecraft_pset = calculate_spacecraft_pset( data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"], data_dict[f"imap_ultra_l1b_{instrument_id}sensor-goodtimes"], data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"], + data_dict[f"imap_ultra_l1a_{instrument_id}sensor-aux"], + data_dict[f"imap_ultra_l1a_{instrument_id}sensor-params"], f"imap_ultra_l1c_{instrument_id}sensor-spacecraftpset", ancillary_files, instrument_id, @@ -66,6 +72,8 @@ def ultra_l1c( data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"], data_dict[f"imap_ultra_l1b_{instrument_id}sensor-goodtimes"], data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"], + data_dict[f"imap_ultra_l1a_{instrument_id}sensor-aux"], + data_dict[f"imap_ultra_l1a_{instrument_id}sensor-params"], f"imap_ultra_l1c_{instrument_id}sensor-spacecraftpset-nonproton", ancillary_files, instrument_id, diff --git a/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py b/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py index 44c265499..b46aba032 100644 --- a/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +++ b/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py @@ -23,11 +23,11 @@ ) from imap_processing.ultra.l1b.ultra_l1b_culling import ( get_pulses_per_spin, - get_spin_and_duration, ) from imap_processing.ultra.l1b.ultra_l1b_extended import ( get_efficiency, get_efficiency_interpolator, + get_spin_and_duration, ) from imap_processing.ultra.l1c.l1c_lookup_utils import ( build_energy_bins, @@ -645,6 +645,7 @@ def get_efficiencies_and_geometric_function( def get_spacecraft_background_rates( rates_dataset: xr.Dataset, + aux_dataset: xr.Dataset, sensor_id: int, ancillary_files: dict, energy_bin_edges: list[tuple[float, float]], @@ -658,6 +659,8 @@ def get_spacecraft_background_rates( ---------- rates_dataset : xr.Dataset Rates dataset. + aux_dataset : xr.Dataset + Auxiliary dataset. sensor_id : int Sensor ID: either 45 or 90. ancillary_files : dict[Path] @@ -680,12 +683,12 @@ def get_spacecraft_background_rates( ----- See Eqn. 3, 8, and 20 in the Algorithm Document for the equation. """ - pulses = get_pulses_per_spin(rates_dataset) + pulses = get_pulses_per_spin(aux_dataset, rates_dataset) # Pulses for the pointing. etof_min = get_image_params("eTOFMin", f"ultra{sensor_id}", ancillary_files) etof_max = get_image_params("eTOFMax", f"ultra{sensor_id}", ancillary_files) spin_number, _ = get_spin_and_duration( - rates_dataset["shcoarse"], rates_dataset["spin"] + aux_dataset, rates_dataset["shcoarse"].values ) # Get dmin for PH (mm). From 13607dfaae9d4e20d2963fc4dd3515114bc39c1c Mon Sep 17 00:00:00 2001 From: Luisa Date: Wed, 17 Dec 2025 12:20:54 -0700 Subject: [PATCH 2/8] name unused var' --- imap_processing/ultra/l1b/ultra_l1b_culling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/ultra/l1b/ultra_l1b_culling.py b/imap_processing/ultra/l1b/ultra_l1b_culling.py index 9e429e93c..518e71a2b 100644 --- a/imap_processing/ultra/l1b/ultra_l1b_culling.py +++ b/imap_processing/ultra/l1b/ultra_l1b_culling.py @@ -368,7 +368,7 @@ def get_pulses_per_spin(aux: xr.Dataset, rates: xr.Dataset) -> RateResult: coin_pulses : NDArray Total coincidence pulses. """ - spin_number, _ = get_spin_and_duration(aux, rates["shcoarse"].values) + spin_number, _duration = get_spin_and_duration(aux, rates["shcoarse"].values) # Top coin pulses top_coin_pulses = np.stack( From 346ed06712c271e520b283b569390f39423091d3 Mon Sep 17 00:00:00 2001 From: Luisa Date: Wed, 17 Dec 2025 13:00:25 -0700 Subject: [PATCH 3/8] fix test --- .../ultra/unit/test_ultra_l1c_pset_bins.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py b/imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py index 9cd191cec..8aab16049 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py @@ -436,6 +436,22 @@ def test_get_spacecraft_background_rates( "Tests calculate_background_rates function." # Simulate a spin table from MET = 0 to MET = 141 * 15 seconds use_fake_spin_data_for_time(start_met=0, end_met=141 * 15) + + # Should be evenly spaced spins of 15 seconds each except the first one has 14. + num_spins = 15 + spin_start_times = np.concatenate([[0], np.arange(14, 222, num_spins)]) + 445015651 + spin_numbers = np.arange(127, 142) + num_spins = len(spin_numbers) + + aux_ds = xr.Dataset( + data_vars={ + "timespinstart": ("epoch", spin_start_times), + "duration": ("epoch", np.full(num_spins, 15)), + "spinnumber": ("epoch", spin_numbers), + }, + coords={"epoch": ("epoch", np.arange(num_spins))}, + ) + df = pd.read_csv(rates_l1_test_path) rates = { @@ -464,7 +480,12 @@ def test_get_spacecraft_background_rates( goodtimes_spin_number = np.array([130, 131]) background_rates = get_spacecraft_background_rates( - rates, "ultra45", ancillary_files, energy_bin_edges, goodtimes_spin_number + rates, + aux_ds, + "ultra45", + ancillary_files, + energy_bin_edges, + goodtimes_spin_number, ) assert background_rates.shape == (len(energy_bin_edges), hp.nside2npix(128)) From 87169b9d1c79f574fb7ea40c583361db0058413f Mon Sep 17 00:00:00 2001 From: Luisa Date: Wed, 17 Dec 2025 13:43:54 -0700 Subject: [PATCH 4/8] test fixes --- imap_processing/tests/ultra/unit/test_ultra_l1c.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1c.py b/imap_processing/tests/ultra/unit/test_ultra_l1c.py index 2f1614e1c..1889fdc9d 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1c.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1c.py @@ -123,6 +123,8 @@ def test_ultra_l1c_error(mock_data_l1b_dict): def test_calculate_spacecraft_pset_with_cdf( ancillary_files, rates_dataset, + deadtime_datasets, + aux_dataset, imap_ena_sim_metakernel, use_fake_spin_data_for_time, mock_spacecraft_pointing_lookups, @@ -186,7 +188,9 @@ def test_calculate_spacecraft_pset_with_cdf( "imap_ultra_l1b_45sensor-de": dataset, "imap_ultra_l1b_45sensor-extendedspin": dataset, # placeholder "imap_ultra_l1b_45sensor-goodtimes": dataset, # placeholder - "imap_ultra_l1a_45sensor-rates": rates_dataset, + "imap_ultra_l1a_45sensor-rates": deadtime_datasets["rates"], + "imap_ultra_l1a_45sensor-aux": aux_dataset, + "imap_ultra_l1a_45sensor-params": deadtime_datasets["params"], } with ( mock.patch( @@ -215,7 +219,8 @@ def test_calculate_helio_pset_with_cdf( ancillary_files, imap_ena_sim_metakernel, mock_helio_pointing_lookups, - rates_dataset, + deadtime_datasets, + aux_dataset, use_fake_spin_data_for_time, ): """Tests ultra_l1c function with imported test data.""" @@ -281,6 +286,8 @@ def test_calculate_helio_pset_with_cdf( } ), # placeholder "imap_ultra_l1a_45sensor-rates": rates_dataset, + "imap_ultra_l1a_45sensor-aux": aux_dataset, + "imap_ultra_l1a_45sensor-params": deadtime_datasets["params"], } n_pix = hp.nside2npix(32) mock_eff = np.ones((46, n_pix)) From 799589ef2707db69a57558080982b887bb8ef9db Mon Sep 17 00:00:00 2001 From: Luisa Date: Wed, 17 Dec 2025 14:13:39 -0700 Subject: [PATCH 5/8] use rates fixture --- imap_processing/tests/ultra/unit/test_ultra_l1c.py | 1 + 1 file changed, 1 insertion(+) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1c.py b/imap_processing/tests/ultra/unit/test_ultra_l1c.py index 1889fdc9d..5450aadfa 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1c.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1c.py @@ -221,6 +221,7 @@ def test_calculate_helio_pset_with_cdf( mock_helio_pointing_lookups, deadtime_datasets, aux_dataset, + rates_dataset, use_fake_spin_data_for_time, ): """Tests ultra_l1c function with imported test data.""" From 39384684e6f58de287c416843ac04336d7e94ca7 Mon Sep 17 00:00:00 2001 From: Luisa Date: Wed, 17 Dec 2025 14:50:06 -0700 Subject: [PATCH 6/8] fix tests --- imap_processing/tests/ultra/unit/test_spacecraft_pset.py | 9 +++++++++ imap_processing/tests/ultra/unit/test_ultra_l1c.py | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/imap_processing/tests/ultra/unit/test_spacecraft_pset.py b/imap_processing/tests/ultra/unit/test_spacecraft_pset.py index 22f3b72be..d2d7e7fae 100644 --- a/imap_processing/tests/ultra/unit/test_spacecraft_pset.py +++ b/imap_processing/tests/ultra/unit/test_spacecraft_pset.py @@ -31,6 +31,8 @@ @pytest.mark.external_test_data @pytest.mark.external_kernel def test_calculate_spacecraft_pset( + deadtime_datasets, + aux_dataset, rates_dataset, imap_ena_sim_metakernel, use_fake_spin_data_for_time, @@ -102,6 +104,8 @@ def test_calculate_spacecraft_pset( test_l1b_de_dataset, test_l1b_de_dataset, # placeholder for goodtimes_dataset rates_dataset, + aux_dataset, + deadtime_datasets["params"], "imap_ultra_l1c_45sensor-spacecraftpset", ancillary_files, 45, @@ -116,6 +120,8 @@ def test_calculate_spacecraft_pset( @pytest.mark.external_kernel def test_calculate_spacecraft_pset_with_cdf( ancillary_files, + deadtime_datasets, + aux_dataset, rates_dataset, imap_ena_sim_metakernel, use_fake_spin_data_for_time, @@ -186,6 +192,9 @@ def test_calculate_spacecraft_pset_with_cdf( dataset, dataset, # placeholder for goodtimes_dataset rates_dataset, + rates_dataset, + aux_dataset, + deadtime_datasets["params"], "imap_ultra_l1c_45sensor-spacecraftpset", ancillary_files, 45, diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1c.py b/imap_processing/tests/ultra/unit/test_ultra_l1c.py index 5450aadfa..7fd8e3693 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1c.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1c.py @@ -124,6 +124,7 @@ def test_calculate_spacecraft_pset_with_cdf( ancillary_files, rates_dataset, deadtime_datasets, + rates_dataset, aux_dataset, imap_ena_sim_metakernel, use_fake_spin_data_for_time, @@ -188,7 +189,7 @@ def test_calculate_spacecraft_pset_with_cdf( "imap_ultra_l1b_45sensor-de": dataset, "imap_ultra_l1b_45sensor-extendedspin": dataset, # placeholder "imap_ultra_l1b_45sensor-goodtimes": dataset, # placeholder - "imap_ultra_l1a_45sensor-rates": deadtime_datasets["rates"], + "imap_ultra_l1a_45sensor-rates": rates_dataset, "imap_ultra_l1a_45sensor-aux": aux_dataset, "imap_ultra_l1a_45sensor-params": deadtime_datasets["params"], } From 1961b3189210b81235b710acdefd8c1b2209d2c2 Mon Sep 17 00:00:00 2001 From: Luisa Date: Thu, 18 Dec 2025 10:43:59 -0700 Subject: [PATCH 7/8] pr comments --- .../tests/ultra/unit/test_spacecraft_pset.py | 13 +++++------- .../ultra/unit/test_ultra_l1b_culling.py | 2 +- .../ultra/unit/test_ultra_l1b_extended.py | 2 +- .../tests/ultra/unit/test_ultra_l1c.py | 20 +++++++++---------- .../ultra/l1b/ultra_l1b_extended.py | 6 +++--- imap_processing/ultra/l1c/helio_pset.py | 2 +- imap_processing/ultra/l1c/ultra_l1c.py | 3 --- 7 files changed, 20 insertions(+), 28 deletions(-) diff --git a/imap_processing/tests/ultra/unit/test_spacecraft_pset.py b/imap_processing/tests/ultra/unit/test_spacecraft_pset.py index d2d7e7fae..a3624e264 100644 --- a/imap_processing/tests/ultra/unit/test_spacecraft_pset.py +++ b/imap_processing/tests/ultra/unit/test_spacecraft_pset.py @@ -31,7 +31,6 @@ @pytest.mark.external_test_data @pytest.mark.external_kernel def test_calculate_spacecraft_pset( - deadtime_datasets, aux_dataset, rates_dataset, imap_ena_sim_metakernel, @@ -42,10 +41,11 @@ def test_calculate_spacecraft_pset( """Tests calculate_spacecraft_pset function.""" # Simulate a spin table from MET = 0 to MET = 141 * 15 seconds use_fake_spin_data_for_time(start_met=0, end_met=141 * 15) - # Ensure rate dataset has correct time range - rates_dataset.shcoarse.data = np.linspace( - 0, 141 * 15, len(rates_dataset.shcoarse.data) - ) + # Ensure rate and aux data have the correct time range + t_rates = np.linspace(0, 141 * 15, len(rates_dataset.shcoarse.data)) + rates_dataset.shcoarse.data = t_rates + aux_dataset.timespinstart.data = t_rates[: len(aux_dataset.timespinstart.data)] + aux_dataset.timespinstart.data[-1] = t_rates[-1] # This is just setting up the data so that it is in the format of l1b_de_dataset. test_path = TEST_PATH / "ultra-90_raw_event_data_shortened.csv" df = pd.read_csv(test_path) @@ -105,7 +105,6 @@ def test_calculate_spacecraft_pset( test_l1b_de_dataset, # placeholder for goodtimes_dataset rates_dataset, aux_dataset, - deadtime_datasets["params"], "imap_ultra_l1c_45sensor-spacecraftpset", ancillary_files, 45, @@ -120,7 +119,6 @@ def test_calculate_spacecraft_pset( @pytest.mark.external_kernel def test_calculate_spacecraft_pset_with_cdf( ancillary_files, - deadtime_datasets, aux_dataset, rates_dataset, imap_ena_sim_metakernel, @@ -194,7 +192,6 @@ def test_calculate_spacecraft_pset_with_cdf( rates_dataset, rates_dataset, aux_dataset, - deadtime_datasets["params"], "imap_ultra_l1c_45sensor-spacecraftpset", ancillary_files, 45, diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py index 7daaa0279..0c5cbac2d 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py @@ -171,7 +171,7 @@ def test_compare_aux_univ_spin_table(use_fake_spin_data_for_time, faux_aux_datas assert np.all(result == expected) -def test_get_duration(rates_l1_test_path, use_fake_spin_data_for_time, aux_test_path): +def test_get_duration(rates_l1_test_path, use_fake_spin_data_for_time): """Tests get_duration function.""" use_fake_spin_data_for_time(start_met=0, end_met=141 * 15) df = pd.read_csv(rates_l1_test_path) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py index 5cc1fdc79..25c79ce20 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py @@ -561,7 +561,7 @@ def test_get_eventtimes(test_fixture, aux_dataset): @pytest.mark.external_test_data def test_get_spin_and_duration(test_fixture, aux_dataset): - """Tests get_eventtimes function.""" + """Tests get_spin_and_duration function.""" df_filt, _, _, de_dataset = test_fixture spin_number, spin_duration = get_spin_and_duration( diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1c.py b/imap_processing/tests/ultra/unit/test_ultra_l1c.py index 7fd8e3693..295613f9e 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1c.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1c.py @@ -123,8 +123,6 @@ def test_ultra_l1c_error(mock_data_l1b_dict): def test_calculate_spacecraft_pset_with_cdf( ancillary_files, rates_dataset, - deadtime_datasets, - rates_dataset, aux_dataset, imap_ena_sim_metakernel, use_fake_spin_data_for_time, @@ -140,10 +138,11 @@ def test_calculate_spacecraft_pset_with_cdf( df_subset = df[df["pointing_number"] == pointing].copy() de_dict = {} - # Ensure rate dataset has correct time range - rates_dataset.shcoarse.data = np.linspace( - 0, 141 * 15, len(rates_dataset.shcoarse.data) - ) + # Ensure rate and aux data are in the correct time window + t_rates = np.linspace(0, 141 * 15, len(rates_dataset.shcoarse.data)) + rates_dataset.shcoarse.data = t_rates + aux_dataset.timespinstart.data = t_rates[: len(aux_dataset.timespinstart.data)] + aux_dataset.timespinstart.data[-1] = t_rates[-1] de_dict["epoch"] = df_subset["epoch"].values species_bin = np.full(len(df_subset), 1, dtype=np.uint8) @@ -191,7 +190,6 @@ def test_calculate_spacecraft_pset_with_cdf( "imap_ultra_l1b_45sensor-goodtimes": dataset, # placeholder "imap_ultra_l1a_45sensor-rates": rates_dataset, "imap_ultra_l1a_45sensor-aux": aux_dataset, - "imap_ultra_l1a_45sensor-params": deadtime_datasets["params"], } with ( mock.patch( @@ -220,7 +218,6 @@ def test_calculate_helio_pset_with_cdf( ancillary_files, imap_ena_sim_metakernel, mock_helio_pointing_lookups, - deadtime_datasets, aux_dataset, rates_dataset, use_fake_spin_data_for_time, @@ -233,8 +230,10 @@ def test_calculate_helio_pset_with_cdf( ) df = pd.read_csv(TEST_PATH / "IMAP-Ultra45_r1_L1_V0_shortened.csv") - # Ensure rate dataset has correct time range - rates_dataset.shcoarse.data += 56970124 + # Ensure rate and aux data are in the correct time window + t_off = 56970125 + rates_dataset.shcoarse.data += t_off + aux_dataset.timespinstart.data += t_off # Select a single pointing number pointing = 0 df_subset = df[df["pointing_number"] == pointing].copy() @@ -289,7 +288,6 @@ def test_calculate_helio_pset_with_cdf( ), # placeholder "imap_ultra_l1a_45sensor-rates": rates_dataset, "imap_ultra_l1a_45sensor-aux": aux_dataset, - "imap_ultra_l1a_45sensor-params": deadtime_datasets["params"], } n_pix = hp.nside2npix(32) mock_eff = np.ones((46, n_pix)) diff --git a/imap_processing/ultra/l1b/ultra_l1b_extended.py b/imap_processing/ultra/l1b/ultra_l1b_extended.py index 3d61de725..251940a39 100644 --- a/imap_processing/ultra/l1b/ultra_l1b_extended.py +++ b/imap_processing/ultra/l1b/ultra_l1b_extended.py @@ -923,7 +923,7 @@ def get_spin_start_indices(aux_dataset: xr.Dataset, de_event_met: NDArray) -> ND Parameters ---------- - aux_dataset : numpy.ndarray + aux_dataset : xarray.Dataset Auxiliary dataset containing spin information. de_event_met : numpy.ndarray Direct event MET. @@ -968,7 +968,7 @@ def get_event_times( Parameters ---------- - aux_dataset : numpy.ndarray + aux_dataset : xarray.Dataset Auxiliary dataset containing spin information. phase_angle : numpy.ndarray Phase angle. @@ -1016,7 +1016,7 @@ def get_spin_and_duration( Parameters ---------- - aux_dataset : numpy.ndarray + aux_dataset : xarray.Dataset Auxiliary dataset containing spin information. de_event_met : numpy.ndarray Direct event MET. diff --git a/imap_processing/ultra/l1c/helio_pset.py b/imap_processing/ultra/l1c/helio_pset.py index 64c0d8862..5f699076a 100644 --- a/imap_processing/ultra/l1c/helio_pset.py +++ b/imap_processing/ultra/l1c/helio_pset.py @@ -39,7 +39,7 @@ def calculate_helio_pset( de_dataset: xr.Dataset, goodtimes_dataset: xr.Dataset, rates_dataset: xr.Dataset, - aux_dataset: xr.Dataset,, + aux_dataset: xr.Dataset, name: str, ancillary_files: dict, instrument_id: int, diff --git a/imap_processing/ultra/l1c/ultra_l1c.py b/imap_processing/ultra/l1c/ultra_l1c.py index 1456d1d66..daccfaf37 100644 --- a/imap_processing/ultra/l1c/ultra_l1c.py +++ b/imap_processing/ultra/l1c/ultra_l1c.py @@ -54,14 +54,12 @@ def ultra_l1c( and f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict and f"imap_ultra_l1a_{instrument_id}sensor-rates" in data_dict and f"imap_ultra_l1a_{instrument_id}sensor-aux" in data_dict - and f"imap_ultra_l1a_{instrument_id}sensor-params" in data_dict ): spacecraft_pset = calculate_spacecraft_pset( data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"], data_dict[f"imap_ultra_l1b_{instrument_id}sensor-goodtimes"], data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"], data_dict[f"imap_ultra_l1a_{instrument_id}sensor-aux"], - data_dict[f"imap_ultra_l1a_{instrument_id}sensor-params"], f"imap_ultra_l1c_{instrument_id}sensor-spacecraftpset", ancillary_files, instrument_id, @@ -73,7 +71,6 @@ def ultra_l1c( data_dict[f"imap_ultra_l1b_{instrument_id}sensor-goodtimes"], data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"], data_dict[f"imap_ultra_l1a_{instrument_id}sensor-aux"], - data_dict[f"imap_ultra_l1a_{instrument_id}sensor-params"], f"imap_ultra_l1c_{instrument_id}sensor-spacecraftpset-nonproton", ancillary_files, instrument_id, From fb84bcfe00f85d61065f05042c89b09f88ddc570 Mon Sep 17 00:00:00 2001 From: Luisa Date: Thu, 18 Dec 2025 10:53:34 -0700 Subject: [PATCH 8/8] fix aux data time --- .../tests/ultra/unit/test_helio_pset.py | 3 ++- .../tests/ultra/unit/test_spacecraft_pset.py | 15 +++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/imap_processing/tests/ultra/unit/test_helio_pset.py b/imap_processing/tests/ultra/unit/test_helio_pset.py index fdeca17c6..6a4390cc2 100644 --- a/imap_processing/tests/ultra/unit/test_helio_pset.py +++ b/imap_processing/tests/ultra/unit/test_helio_pset.py @@ -16,7 +16,7 @@ @pytest.mark.skip(reason="Long running test for validation purposes.") def test_validate_exposure_time_and_sensitivities( - ancillary_files, rates_dataset, imap_ena_sim_metakernel + ancillary_files, rates_dataset, imap_ena_sim_metakernel, aux_dataset ): """Validates exposure time and sensitivities for ebin 0.""" sens_filename = "SENS-IMAP_ULTRA_90-IMAP_DPS-HELIO-nside32-ebin0.csv" @@ -91,6 +91,7 @@ def test_validate_exposure_time_and_sensitivities( l1b_de, dataset, rates_dataset, + aux_dataset, "imap_ultra_l1c_90sensor-heliopset", ancillary_files, 90, diff --git a/imap_processing/tests/ultra/unit/test_spacecraft_pset.py b/imap_processing/tests/ultra/unit/test_spacecraft_pset.py index a3624e264..ac72830e6 100644 --- a/imap_processing/tests/ultra/unit/test_spacecraft_pset.py +++ b/imap_processing/tests/ultra/unit/test_spacecraft_pset.py @@ -138,10 +138,11 @@ def test_calculate_spacecraft_pset_with_cdf( de_dict["epoch"] = df_subset["epoch"].values species_bin = np.full(len(df_subset), 1, dtype=np.uint8) - # Ensure rate dataset has correct time range - rates_dataset.shcoarse.data = np.linspace( - 0, 141 * 15, len(rates_dataset.shcoarse.data) - ) + # Ensure rate and aux data have the correct time range + t_rates = np.linspace(0, 141 * 15, len(rates_dataset.shcoarse.data)) + rates_dataset.shcoarse.data = t_rates + aux_dataset.timespinstart.data = t_rates[: len(aux_dataset.timespinstart.data)] + aux_dataset.timespinstart.data[-1] = t_rates[-1] # PosYSlit is True for left (start_type = 1) # PosYSlit is False for right (start_type = 2) start_type = np.where(df_subset["PosYSlit"].values, 1, 2) @@ -190,7 +191,6 @@ def test_calculate_spacecraft_pset_with_cdf( dataset, dataset, # placeholder for goodtimes_dataset rates_dataset, - rates_dataset, aux_dataset, "imap_ultra_l1c_45sensor-spacecraftpset", ancillary_files, @@ -205,7 +205,9 @@ def test_calculate_spacecraft_pset_with_cdf( @pytest.mark.skip(reason="Long running test for validation purposes.") -def test_validate_exposure_time_and_sensitivities(ancillary_files, rates_dataset): +def test_validate_exposure_time_and_sensitivities( + ancillary_files, rates_dataset, aux_dataset +): """Validates exposure time and sensitivities for ebin 0.""" test_data = [ ( @@ -302,6 +304,7 @@ def test_validate_exposure_time_and_sensitivities(ancillary_files, rates_dataset l1b_de, dataset, rates_dataset, + aux_dataset, "imap_ultra_l1c_90sensor-spacecraftpset", ancillary_files, 90,