From 70ead3165705748042fe0e34332718ab132ffe6c Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Thu, 29 Jan 2026 14:36:39 +0100 Subject: [PATCH 01/15] added v.1.X.11 config file --- config/calibration/mask_v1.X.10.yaml | 2 +- config/calibration/mask_v1.X.11.yaml | 110 +++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 config/calibration/mask_v1.X.11.yaml diff --git a/config/calibration/mask_v1.X.10.yaml b/config/calibration/mask_v1.X.10.yaml index 4729bfdc..9118bd93 100644 --- a/config/calibration/mask_v1.X.10.yaml +++ b/config/calibration/mask_v1.X.10.yaml @@ -1,5 +1,5 @@ # Config file for masking and calibration. -# Standard cuts without halo masks, type v1.X.10 +# Standard cuts without halo masks, including deblended objects, type v1.X.10 # General parameters (can also given on command line) params: diff --git a/config/calibration/mask_v1.X.11.yaml b/config/calibration/mask_v1.X.11.yaml new file mode 100644 index 00000000..13e9187f --- /dev/null +++ b/config/calibration/mask_v1.X.11.yaml @@ -0,0 +1,110 @@ +# Config file for masking and calibration. +# Standard cuts without halo masks, including deblended objects, type v1.X.11 + +# General parameters (can also given on command line) +params: + input_path: unions_shapepipe_comprehensive_struc_2024_v1.X.c.hdf5 + cmatrices: False + sky_regions: False + verbose: True + +# Masks +## Using columns in 'dat' group (ShapePipe flags) +dat: + # SExtractor flags + - col_name: FLAGS + label: SE FLAGS + kind: smaller_equal + value: 3 + + # Duplicate objects + - col_name: overlap + label: tile overlap + kind: equal + value: True + + # ShapePipe flags + - col_name: IMAFLAGS_ISO + label: SP mask + kind: equal + value: 0 + + # Number of epochs + - col_name: N_EPOCH + label: r"$n_{\rm epoch}$" + kind: greater_equal + value: 2 + + # Magnitude range + - col_name: mag + label: mag range + kind: range + value: [15, 30] + + # ngmix flags + - col_name: NGMIX_MOM_FAIL + label: "ngmix moments failure" + kind: equal + value: 0 + + # invalid PSF ellipticities + - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + label: "bad PSF ellipticity comp 1" + kind: not_equal + value: -10 + - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + label: "bad PSF ellipticity comp 2" + kind: not_equal + value: -10 + +## Using columns in 'dat_ext' group (post-processing flags) +dat_ext: + + # Stars + - col_name: 4_Stars + label: "Stars" + kind: equal + value: False + + # Manual mask + - col_name: 8_Manual + label: "manual mask" + kind: equal + value: False + + # r-band footprint + - col_name: 64_r + label: "r-band imaging" + kind: equal + value: False + + # Maximask + - col_name: 1024_Maximask + label: "maximask" + kind: equal + value: False + + # Rough pointing coverage + - col_name: npoint3 + label: r"$n_{\rm pointing}$" + kind: greater_equal + value: 3 + +# Metacal parameters +metacal: + # Ellipticity dispersion + sigma_eps_prior: 0.34 + + # Signal-to-noise range + gal_snr_min: 10 + gal_snr_max: 500 + + # Relative-size (hlr / hlr_psf) range + gal_rel_size_min: 0.707 + gal_rel_size_max: 3 + + # Correct relative size for ellipticity? + gal_size_corr_ell: False + + # Weight for global response matrix, None for unweighted mean + global_R_weight: w From 99c0fee77c6b25836f3a36a030324b9b2684e6cb Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 30 Jan 2026 08:21:53 +0100 Subject: [PATCH 02/15] Fixed missing propagation of min and max SNR and size ratio for DES weights --- src/sp_validation/calibration.py | 43 +++++++++++++++++++++++++----- src/sp_validation/run_joint_cat.py | 13 ++++++++- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index a4368f99..ee3e2734 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -244,7 +244,14 @@ def build_df(cat_gal): return pd.DataFrame(arr, columns=cat_gal.keys()) -def get_w_des(cat_gal, num_bins): +def get_w_des( + cat_gal, + num_bins, + snr_min=None, + snr_max=None, + size_ratio_min=None, + size_ratio=None, +): """ Get DES weights. (Gatti et al. 2021) Return an array of DES weights obtained by binning in SNR and size and computing the ratio between @@ -253,9 +260,18 @@ def get_w_des(cat_gal, num_bins): Parameters ---------- cat_gal: dict - A catalog of galaxies containing the response matrix and the uncalibrated ellipticities + A catalog of galaxies containing the response matrix and the + uncalibrated ellipticities num_bins : int Number of bins to use for the binning of the SNR and size. + snr_min : float, optional + Minimum SNR, default (`None`): determined by the data + snr_max : float, optional + Maximum SNR, default (`None`): determined by the data + size_ratio_min : float, optional + Minimum size ratio, default (`None`): determined by the data + size_ratio_max : float, optional + Maximum size ratio, default (`None`): determined by the data Returns ------- @@ -266,11 +282,24 @@ def get_w_des(cat_gal, num_bins): df_gal = build_df(cat_gal) #Create logarithmic bins in size and SNR - cut_to_bins(df_gal, "snr", num_bins, type="log") - cut_to_bins(df_gal, "size_ratio", num_bins, type="log") - - #Compute shape noise and the shear response in each bin + cut_to_bins( + df_gal, + "snr", + num_bins, + type="log", + x_min=snr_min, + x_max=snr_max, + ) + cut_to_bins( + df_gal, + "size_ratio", + num_bins, + type="log", + x_min=size_ratio_min, + x_max=size_ratio_max, + ) + # Compute shape noise and the shear response in each bin for i in range(num_bins): for j in range(num_bins): bin_mask = ( @@ -496,7 +525,7 @@ def get_quantities_binned(cat_gal, num_bins_x, num_bins_y=None, which=["response # Create logarithmic bins in size and SNR bin_edges = {} - bin_edges["snr"] = cut_to_bins(df_gal, "snr", num_bins_x, type="log", x_min=3, x_max=700) + bin_edges["snr"] = cut_to_bins(df_gal, "snr", num_bins_x, type="log", x_min=2, x_max=700) bin_edges["size_ratio"] = cut_to_bins(df_gal, "size_ratio", num_bins_y, type="log", x_min=0.3, x_max=10) # Initialize output dict diff --git a/src/sp_validation/run_joint_cat.py b/src/sp_validation/run_joint_cat.py index ef404a58..576c971d 100644 --- a/src/sp_validation/run_joint_cat.py +++ b/src/sp_validation/run_joint_cat.py @@ -1641,6 +1641,10 @@ def compute_weights_gatti( mask_combined, mask_metacal, num_bins=20, + snr_min=5, + snr_max=10, + size_ratio_min=0.3, + size_ratio_max=3, ): """Compute Weights Gatti. @@ -1657,7 +1661,14 @@ def compute_weights_gatti( purpose="weights" ) - cat_gal["w_des"] = calibration.get_w_des(cat_gal, num_bins) + cat_gal["w_des"] = calibration.get_w_des( + cat_gal, + num_bins, + snr_min=snr_min, + snr_max=snr_max, + size_ratio_min=size_ratio_min, + size_ratio_max=size_ratio_max, + ) def compute_PSF_leakage( From ea0480a2d786dbc8d5ea682f49dff24897496cd4 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 30 Jan 2026 08:22:44 +0100 Subject: [PATCH 03/15] Fixed missing propagation of min and max SNR and size ratio for DES weights --- src/sp_validation/run_joint_cat.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sp_validation/run_joint_cat.py b/src/sp_validation/run_joint_cat.py index 576c971d..d82d1250 100644 --- a/src/sp_validation/run_joint_cat.py +++ b/src/sp_validation/run_joint_cat.py @@ -1641,9 +1641,9 @@ def compute_weights_gatti( mask_combined, mask_metacal, num_bins=20, - snr_min=5, - snr_max=10, - size_ratio_min=0.3, + snr_min=10, + snr_max=500, + size_ratio_min=0.707, size_ratio_max=3, ): """Compute Weights Gatti. From 4fd3893374e5179fbcaf64bbc0405c5f9fc46449 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 30 Jan 2026 08:23:52 +0100 Subject: [PATCH 04/15] Fixed missing propagation of min and max SNR and size ratio for DES weights --- notebooks/calibrate_comprehensive_cat.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/notebooks/calibrate_comprehensive_cat.py b/notebooks/calibrate_comprehensive_cat.py index 168d608c..e62288ae 100644 --- a/notebooks/calibrate_comprehensive_cat.py +++ b/notebooks/calibrate_comprehensive_cat.py @@ -125,6 +125,10 @@ mask_combined, mask_metacal, num_bins=20, + snr_min=cm["gal_snr_min"], + snr_max=cm["gal_snr_max"], + size_ratio_min=cm["size_ratio_min"], + size_ratio_max=cm["size_ratio_max"], ) # %% From c16434fee097f4b612f49089b9922e4979beaa74 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Tue, 3 Feb 2026 14:54:00 +0100 Subject: [PATCH 05/15] added footprint mask output to all mask config files --- config/calibration/mask_v1.X.2.yaml | 11 +++++++++++ config/calibration/mask_v1.X.3.yaml | 11 +++++++++++ config/calibration/mask_v1.X.4.yaml | 11 +++++++++++ config/calibration/mask_v1.X.5.yaml | 11 +++++++++++ config/calibration/mask_v1.X.6.yaml | 11 +++++++++++ config/calibration/mask_v1.X.7.yaml | 11 +++++++++++ config/calibration/mask_v1.X.8.yaml | 11 +++++++++++ 7 files changed, 77 insertions(+) diff --git a/config/calibration/mask_v1.X.2.yaml b/config/calibration/mask_v1.X.2.yaml index 46672c04..7d93a57a 100644 --- a/config/calibration/mask_v1.X.2.yaml +++ b/config/calibration/mask_v1.X.2.yaml @@ -116,3 +116,14 @@ metacal: # Correct relative size for ellipticity? gal_size_corr_ell: False + +# Binned footprint mask +binned_mask: + # Resolution + nside: 8192 + + # Mask value for "good region (observed) + good: False + + # Output file path + output_path: "footprint_binned.fits" diff --git a/config/calibration/mask_v1.X.3.yaml b/config/calibration/mask_v1.X.3.yaml index 96aeecd5..242031a8 100644 --- a/config/calibration/mask_v1.X.3.yaml +++ b/config/calibration/mask_v1.X.3.yaml @@ -99,3 +99,14 @@ metacal: # Correct relative size for ellipticity? gal_size_corr_ell: False + +# Binned footprint mask +binned_mask: + # Resolution + nside: 8192 + + # Mask value for "good region (observed) + good: False + + # Output file path + output_path: "footprint_binned.fits" diff --git a/config/calibration/mask_v1.X.4.yaml b/config/calibration/mask_v1.X.4.yaml index 6db36f6b..59080851 100644 --- a/config/calibration/mask_v1.X.4.yaml +++ b/config/calibration/mask_v1.X.4.yaml @@ -110,3 +110,14 @@ metacal: # Correct relative size for ellipticity? gal_size_corr_ell: False + +# Binned footprint mask +binned_mask: + # Resolution + nside: 8192 + + # Mask value for "good region (observed) + good: False + + # Output file path + output_path: "footprint_binned.fits" diff --git a/config/calibration/mask_v1.X.5.yaml b/config/calibration/mask_v1.X.5.yaml index 6fa04aff..7fb14835 100644 --- a/config/calibration/mask_v1.X.5.yaml +++ b/config/calibration/mask_v1.X.5.yaml @@ -105,3 +105,14 @@ metacal: # Correct relative size for ellipticity? gal_size_corr_ell: False + +# Binned footprint mask +binned_mask: + # Resolution + nside: 8192 + + # Mask value for "good region (observed) + good: False + + # Output file path + output_path: "footprint_binned.fits" diff --git a/config/calibration/mask_v1.X.6.yaml b/config/calibration/mask_v1.X.6.yaml index c3c7b254..1e82211d 100644 --- a/config/calibration/mask_v1.X.6.yaml +++ b/config/calibration/mask_v1.X.6.yaml @@ -108,3 +108,14 @@ metacal: # Weight for global response matrix, None for unweighted mean global_R_weight: w + +# Binned footprint mask +binned_mask: + # Resolution + nside: 8192 + + # Mask value for "good region (observed) + good: False + + # Output file path + output_path: "footprint_binned.fits" diff --git a/config/calibration/mask_v1.X.7.yaml b/config/calibration/mask_v1.X.7.yaml index 49325e36..830d7a28 100644 --- a/config/calibration/mask_v1.X.7.yaml +++ b/config/calibration/mask_v1.X.7.yaml @@ -113,3 +113,14 @@ metacal: # Weight for global response matrix, None for unweighted mean global_R_weight: w + +# Binned footprint mask +binned_mask: + # Resolution + nside: 8192 + + # Mask value for "good region (observed) + good: False + + # Output file path + output_path: "footprint_binned.fits" diff --git a/config/calibration/mask_v1.X.8.yaml b/config/calibration/mask_v1.X.8.yaml index c346b339..bd74e54b 100644 --- a/config/calibration/mask_v1.X.8.yaml +++ b/config/calibration/mask_v1.X.8.yaml @@ -119,3 +119,14 @@ metacal: # Weight for global response matrix, None for unweighted mean global_R_weight: w + +# Binned footprint mask +binned_mask: + # Resolution + nside: 8192 + + # Mask value for "good region (observed) + good: False + + # Output file path + output_path: "footprint_binned.fits" From eb00974aeb32bc20f7b73b15ffb85fd6b5694560 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Wed, 4 Feb 2026 11:33:48 +0100 Subject: [PATCH 06/15] Chasing size cut bug --- notebooks/calibrate_comprehensive_cat.py | 16 +++++-- notebooks/cosmo_val/cat_config.yaml | 53 ++++++++++++++++++++++-- notebooks/extract_info.py | 1 - src/sp_validation/basic.py | 26 ++++++++++-- src/sp_validation/calibration.py | 3 +- src/sp_validation/cosmo_val.py | 4 +- 6 files changed, 87 insertions(+), 16 deletions(-) diff --git a/notebooks/calibrate_comprehensive_cat.py b/notebooks/calibrate_comprehensive_cat.py index e62288ae..c4aa9eb6 100644 --- a/notebooks/calibrate_comprehensive_cat.py +++ b/notebooks/calibrate_comprehensive_cat.py @@ -127,8 +127,8 @@ num_bins=20, snr_min=cm["gal_snr_min"], snr_max=cm["gal_snr_max"], - size_ratio_min=cm["size_ratio_min"], - size_ratio_max=cm["size_ratio_max"], + size_ratio_min=cm["gal_rel_size_min"], + size_ratio_max=cm["gal_rel_size_max"], ) # %% @@ -199,7 +199,6 @@ mask_combined._mask, mask_metacal ) - #add_cols_data[key] = dat[key][mask_combined._mask][mask_metacal] # %% # Additional post-processing columns to write to output cat @@ -235,7 +234,16 @@ # %% -header +# MKDEBUG check bug of minimal size objects +print("MKDEBUG calibration min size checks") +T = add_cols_data["NGMIX_T_NOSHEAR"] +Tpsf = add_cols_data["NGMIX_Tpsf_NOSHEAR"] +x = T / Tpsf +ind_low = np.argsort(x)[:5] +print("ratio", x[ind_low]) +print("gal size", T[ind_low]) +print("PSF size", Tpsf[ind_low]) +print("rel_size_min", cm["gal_rel_size_min"]) # %% output_shape_cat_path = obj._params["input_path"].replace( diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 5e6b824e..758f6ae9 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -161,7 +161,7 @@ SP_v1.3: SP_v1.3.6: subdir: /n17data/UNIONS/WL/v1.3.x pipeline: SP - colour: violet + colour: coral getdist_colour: 0.0, 0.5, 1.0 ls: dashed marker: h @@ -550,7 +550,7 @@ SP_v1.4.6.1: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits -SSP_v1.4.7: +SP_v1.4.7: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: darkorchid @@ -643,7 +643,7 @@ SP_v1.4.8: SP_v1.4.10.1: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP - colour: blue + colour: orange getdist_colour: 0.0, 0.5, 1.0 ls: dashed marker: s @@ -685,7 +685,52 @@ SP_v1.4.10.1: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits -SSP_v1.4_LFmask_8k: +SP_v1.4.11.2: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: blue + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: s + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.11.2/unions_shapepipe_cut_struc_2024_v1.4.11.2.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4_LFmask_8k: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask pipeline: SP colour: black diff --git a/notebooks/extract_info.py b/notebooks/extract_info.py index 0bddceab..a4e33451 100644 --- a/notebooks/extract_info.py +++ b/notebooks/extract_info.py @@ -175,7 +175,6 @@ g_star_psf[1], ) - #### Refine: Match to SPREAD_CLASS samples if "SPREAD_CLASS" in dd.dtype.names: spv_cat.match_spread_class(dd, ind_star, m_star, stats_file, len(ra_star), verbose=verbose) diff --git a/src/sp_validation/basic.py b/src/sp_validation/basic.py index 7ce29fb2..4c761dc7 100644 --- a/src/sp_validation/basic.py +++ b/src/sp_validation/basic.py @@ -194,6 +194,10 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): masked_data[f'{self._prefix}_Tpsf_{name_shear}'] ) + # Hack to fix incorrect propagation of noshear PSF size + print("FHP hack using p1 PSF for ns in cuts") + ns["Tpsf"] = p1["Tpsf"] + ns["C11"], ns["C22"], ns["w"] = self.get_variance_ivweights( masked_data, self._sigma_eps, @@ -372,14 +376,15 @@ def _masking_gal(self): else: snr_flux = data['flux'] / data['flux_err'] - if name == 'ns': + #if name == 'ns': # This is a FHP hack, the ns PSF measured in shapepipe is not correct, # fortunately it is the same dilated PSF as the other branches, # thus we can simply use p1 print("FHP using p1 PSF for ns in cuts") - Tpsf = self.p1['Tpsf'] - else: - Tpsf = data['Tpsf'] + #Tpsf = self.p1['Tpsf'] + #else: + #Tpsf = data['Tpsf'] + Tpsf = data["Tpsf"] mask_tmp = ( (data['flag'] == 0) @@ -394,6 +399,19 @@ def _masking_gal(self): self.mask_dict[name] = ind_masked + # MKDBEUG: Check why not all small objects are cut + if name == "ns": + ma = self.mask_dict[name] + Tr_tmp_ma = Tr_tmp[ma] + Tpsf_ma = Tpsf[ma] + x = Tr_tmp_ma / Tpsf_ma + ind_low = np.argsort(x)[:5] + print("MKDEBUG lowest objects:") + print("ratio", x[ind_low]) + print("gal size", Tr_tmp[ind_low]) + print("PSF size", Tpsf[ind_low]) + print("rel_size_min", self._rel_size_max) + def _masking_gal_mom(self): """Add docstring. diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index ee3e2734..9e6a656c 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -143,6 +143,7 @@ def create_bins(x, num_bins, type="log", x_min=None, x_max=None): if type == "log": xmin = x.min() if not x_min else x_min xmax = x.max() if not x_max else x_max + print(f"MKDEBUG create bins {x_min} ... {x_max}") return np.logspace(np.log10(xmin), np.log10(xmax), num_bins + 1) else: raise ValueError("Type not supported") @@ -250,7 +251,7 @@ def get_w_des( snr_min=None, snr_max=None, size_ratio_min=None, - size_ratio=None, + size_ratio_max=None, ): """ Get DES weights. (Gatti et al. 2021) diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index e2e15c84..ec8781ee 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -1967,7 +1967,7 @@ def plot_aperture_mass_dispersion(self): labels=labels, xlog=True, xlim=[self.theta_min_plot, self.theta_max_plot], - ylim=[-1e-6, 2e-5], + ylim=[-2e-6, 5e-6], colors=colors, linestyles=linestyles, shift_x=True, @@ -2000,7 +2000,7 @@ def plot_aperture_mass_dispersion(self): xlog=True, ylog=True, xlim=[self.theta_min_plot, self.theta_max_plot], - ylim=[1e-9, 3e-5], + ylim=[1e-8, 1e-5], colors=colors, linestyles=linestyles, shift_x=True, From 76781c4ab85329f30642efe3325b538c17f8185b Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Thu, 5 Feb 2026 09:47:04 +0100 Subject: [PATCH 07/15] Still fixing PSF size bug --- notebooks/demo_apply_hsp_masks.py | 2 +- src/sp_validation/basic.py | 24 ++++++------------------ 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/notebooks/demo_apply_hsp_masks.py b/notebooks/demo_apply_hsp_masks.py index d8f9897d..c19fdf94 100644 --- a/notebooks/demo_apply_hsp_masks.py +++ b/notebooks/demo_apply_hsp_masks.py @@ -57,7 +57,7 @@ # + # Set parameters base = "unions_shapepipe_comprehensive" -ver = "v1.6.c" +ver = "v1.3.c" if ver == "v1.3.c": year = 2022 diff --git a/src/sp_validation/basic.py b/src/sp_validation/basic.py index 4c761dc7..659e690a 100644 --- a/src/sp_validation/basic.py +++ b/src/sp_validation/basic.py @@ -194,9 +194,9 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): masked_data[f'{self._prefix}_Tpsf_{name_shear}'] ) - # Hack to fix incorrect propagation of noshear PSF size - print("FHP hack using p1 PSF for ns in cuts") - ns["Tpsf"] = p1["Tpsf"] + + print("FHP using p1 PSF for ns in read_data_ngmix") + dict_tmp["Tpsf"] = masked_data[f"{self._prefix}_Tpsf_1P"] ns["C11"], ns["C22"], ns["w"] = self.get_variance_ivweights( masked_data, @@ -380,11 +380,12 @@ def _masking_gal(self): # This is a FHP hack, the ns PSF measured in shapepipe is not correct, # fortunately it is the same dilated PSF as the other branches, # thus we can simply use p1 - print("FHP using p1 PSF for ns in cuts") + #print("FHP using p1 PSF for ns in cuts") #Tpsf = self.p1['Tpsf'] #else: #Tpsf = data['Tpsf'] - Tpsf = data["Tpsf"] + + Tpsf = data['Tpsf'] mask_tmp = ( (data['flag'] == 0) @@ -399,19 +400,6 @@ def _masking_gal(self): self.mask_dict[name] = ind_masked - # MKDBEUG: Check why not all small objects are cut - if name == "ns": - ma = self.mask_dict[name] - Tr_tmp_ma = Tr_tmp[ma] - Tpsf_ma = Tpsf[ma] - x = Tr_tmp_ma / Tpsf_ma - ind_low = np.argsort(x)[:5] - print("MKDEBUG lowest objects:") - print("ratio", x[ind_low]) - print("gal size", Tr_tmp[ind_low]) - print("PSF size", Tpsf[ind_low]) - print("rel_size_min", self._rel_size_max) - def _masking_gal_mom(self): """Add docstring. From 8fc6cca82f30f5ba349fcaf90bd47927156f24c9 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Thu, 5 Feb 2026 17:49:05 +0100 Subject: [PATCH 08/15] Fixed PSF size hack --- src/sp_validation/basic.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/sp_validation/basic.py b/src/sp_validation/basic.py index 659e690a..826334f5 100644 --- a/src/sp_validation/basic.py +++ b/src/sp_validation/basic.py @@ -147,6 +147,12 @@ def _read_data(self, data, mask): ns, ) + print("FHP/MK hack using p1 PSF for ns in cuts") + indices = np.where(mask)[0] + new_psf = data[indices][f"{self._prefix}_Tpsf_1P"] + ns["Tpsf"] = new_psf + data[f"{self._prefix}_Tpsf_NOSHEAR"][mask] = new_psf + self.m1 = m1 self.p1 = p1 self.m2 = m2 @@ -159,7 +165,7 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): Read data from ngmix catalogue. """ - + for name_shear, dict_tmp in zip( ['1M', '1P', '2M', '2P', 'NOSHEAR'], [m1, p1, m2, p2, ns] @@ -194,10 +200,6 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): masked_data[f'{self._prefix}_Tpsf_{name_shear}'] ) - - print("FHP using p1 PSF for ns in read_data_ngmix") - dict_tmp["Tpsf"] = masked_data[f"{self._prefix}_Tpsf_1P"] - ns["C11"], ns["C22"], ns["w"] = self.get_variance_ivweights( masked_data, self._sigma_eps, @@ -376,16 +378,8 @@ def _masking_gal(self): else: snr_flux = data['flux'] / data['flux_err'] - #if name == 'ns': - # This is a FHP hack, the ns PSF measured in shapepipe is not correct, - # fortunately it is the same dilated PSF as the other branches, - # thus we can simply use p1 - #print("FHP using p1 PSF for ns in cuts") - #Tpsf = self.p1['Tpsf'] - #else: - #Tpsf = data['Tpsf'] - Tpsf = data['Tpsf'] + print("MKDEBUG cuts on Tpsf", name, Tpsf[:5]) mask_tmp = ( (data['flag'] == 0) From 1d7557e36f4a73bf62ac2c3bd98dbf708c6af2c8 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Thu, 5 Feb 2026 17:49:47 +0100 Subject: [PATCH 09/15] Tesing PSF size hack --- notebooks/calibrate_comprehensive_cat.py | 37 ++++---- notebooks/config_mask.yaml | 110 ----------------------- src/sp_validation/run_joint_cat.py | 2 +- 3 files changed, 22 insertions(+), 127 deletions(-) delete mode 100644 notebooks/config_mask.yaml diff --git a/notebooks/calibrate_comprehensive_cat.py b/notebooks/calibrate_comprehensive_cat.py index c4aa9eb6..5ecdb75b 100644 --- a/notebooks/calibrate_comprehensive_cat.py +++ b/notebooks/calibrate_comprehensive_cat.py @@ -8,9 +8,10 @@ # enable autoreload for interactive sessions ipython = get_ipython() if ipython is not None: - ipython.run_line_magic("load_ext", "autoreload") + ipython.run_line_magic("reload_ext", "autoreload") ipython.run_line_magic("autoreload", "2") - ipython.run_line_magic("load_ext", "log_cell_time") + ipython.run_line_magic("reload_ext", "log_cell_time") + # %% import sys @@ -36,6 +37,11 @@ # Read configuration file and set parameters config = obj.read_config_set_params("config_mask.yaml") + + +# %% +obj._params + # %% # Get data. Set load_into_memory to False for very large files dat, dat_ext = obj.read_cat(load_into_memory=False) @@ -49,7 +55,6 @@ dat_ext = dat_ext[:n_test] - # %% # ## Masking @@ -232,19 +237,6 @@ for my_mask in masks: my_mask.add_summary_to_FITS_header(header) - -# %% -# MKDEBUG check bug of minimal size objects -print("MKDEBUG calibration min size checks") -T = add_cols_data["NGMIX_T_NOSHEAR"] -Tpsf = add_cols_data["NGMIX_Tpsf_NOSHEAR"] -x = T / Tpsf -ind_low = np.argsort(x)[:5] -print("ratio", x[ind_low]) -print("gal size", T[ind_low]) -print("PSF size", Tpsf[ind_low]) -print("rel_size_min", cm["gal_rel_size_min"]) - # %% output_shape_cat_path = obj._params["input_path"].replace( "comprehensive", "cut" @@ -278,6 +270,19 @@ my_mask.print_summary(f_out) +# %% +# MKDEBUG check bug of minimal size objects +print("MKDEBUG calibration min size checks") +T = add_cols_data["NGMIX_T_NOSHEAR"] +Tpsf = add_cols_data["NGMIX_Tpsf_NOSHEAR"] +x = T / Tpsf +ind_low = np.argsort(x)[:5] +print("ratio", x[ind_low]) +print("gal size", T[ind_low]) +print("PSF size", Tpsf[ind_low]) +print("rel_size_min", cm["gal_rel_size_min"]) + + # %% if not obj._params["cmatrices"]: diff --git a/notebooks/config_mask.yaml b/notebooks/config_mask.yaml deleted file mode 100644 index 4af95017..00000000 --- a/notebooks/config_mask.yaml +++ /dev/null @@ -1,110 +0,0 @@ -params: - input_path: unions_shapepipe_comprehensive_struc_2024_v1.4.2.hdf5 - cmatrices: True - verbose: True - -dat: - # SExtractor flags - - col_name: FLAGS - label: SE FLAGS - kind: equal - value: 0 - - # Duplicate objects - - col_name: overlap - label: tile overlap - kind: equal - value: True - - # ShapePipe flags - - col_name: IMAFLAGS_ISO - label: SP mask - kind: equal - value: 0 - - # Number of epochs - - col_name: N_EPOCH - label: r"$n_{\rm epoch}$" - kind: greater_equal - value: 2 - - # Magnitude range - - col_name: mag - label: mag range - kind: range - value: [15, 30] - - # ngmix flags - - col_name: NGMIX_MOM_FAIL - label: "ngmix moments failure" - kind: equal - value: 0 - - # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 - label: "bad PSF ellipticity comp 1" - kind: not_equal - value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 - label: "bad PSF ellipticity comp 2" - kind: not_equal - value: -10 - -dat_ext: - # Faint star halos - - col_name: 1_Faint_star_halos - label: "Faint star halos" - kind: equal - value: False - - # Bright star halos - - col_name: 2_Bright_star_halos - label: "Bright star halos" - kind: equal - value: False - - # Stars - - col_name: 4_Stars - label: "Stars" - kind: equal - value: False - - # Manual mask - - col_name: 8_Manual - label: "manual mask" - kind: equal - value: False - - # r-band footprint - - col_name: 64_r - label: "r-band imaging" - kind: equal - value: False - - # Maximask - - col_name: 1024_Maximask - label: "maximask" - kind: equal - value: False - - # Rought pointing coverage - - col_name: npoint3 - label: r"$n_{\rm pointing}$" - kind: greater_equal - value: 3 - -# Metacal parameters -metacal: - # Ellipticity dispersion - sigma_eps_prior: 0.34 - - # Signal-to-noise range - gal_snr_min: 10 - gal_snr_max: 500 - - # Relative-size (hlr / hlr_psf) range - gal_rel_size_min: 0.5 - gal_rel_size_max: 3 - - # Correct relative size for ellipticity? - gal_size_corr_ell: False \ No newline at end of file diff --git a/src/sp_validation/run_joint_cat.py b/src/sp_validation/run_joint_cat.py index d82d1250..7aabb2e5 100644 --- a/src/sp_validation/run_joint_cat.py +++ b/src/sp_validation/run_joint_cat.py @@ -1143,7 +1143,7 @@ def read_cat(self, load_into_memory=False): except: print(f"Error while reading file {fpath}") raise - + if verbose: print( f"Found {len(dat)} (~{util.millify(len(dat))}) objects" From 93a32275fe1fe1e778a463e5ac78228766cab770 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 6 Feb 2026 10:17:42 +0100 Subject: [PATCH 10/15] remove conflict marker --- notebooks/cosmo_val/cat_config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 6e2e09fe..dce68d0a 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -836,7 +836,6 @@ SP_v1.4.8_uncal: hdu: 1 patch_number: 100 ->>>>>>> upstream/develop SP_v1.4_LFmask_8k: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask pipeline: SP From 247e8c5980cb583757fcb768a34aa1a149024f7d Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 6 Feb 2026 10:18:52 +0100 Subject: [PATCH 11/15] remove debug msg --- src/sp_validation/basic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sp_validation/basic.py b/src/sp_validation/basic.py index 5ed36cc9..4e5afc2c 100644 --- a/src/sp_validation/basic.py +++ b/src/sp_validation/basic.py @@ -377,7 +377,6 @@ def _masking_gal(self): snr_flux = data['flux'] / data['flux_err'] Tpsf = data['Tpsf'] - print("MKDEBUG cuts on Tpsf", name, Tpsf[:5]) mask_tmp = ( (data['flag'] == 0) From e3a05cbc8731bcf5734c4176d96c4aaa1be5df20 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 6 Feb 2026 10:19:17 +0100 Subject: [PATCH 12/15] remove debug msg --- src/sp_validation/calibration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index 9e6a656c..0272822a 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -143,7 +143,6 @@ def create_bins(x, num_bins, type="log", x_min=None, x_max=None): if type == "log": xmin = x.min() if not x_min else x_min xmax = x.max() if not x_max else x_max - print(f"MKDEBUG create bins {x_min} ... {x_max}") return np.logspace(np.log10(xmin), np.log10(xmax), num_bins + 1) else: raise ValueError("Type not supported") From 678db73d00debb6b45bc22b8f5379a34021fc337 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Mon, 9 Feb 2026 08:24:56 +0100 Subject: [PATCH 13/15] debug msg for size ratio bug fix --- notebooks/create_shear_mb_empty.py | 2 +- src/sp_validation/basic.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/notebooks/create_shear_mb_empty.py b/notebooks/create_shear_mb_empty.py index 9c3d1760..6d29bfc2 100644 --- a/notebooks/create_shear_mb_empty.py +++ b/notebooks/create_shear_mb_empty.py @@ -39,7 +39,7 @@ # Set parameters base = "unions_shapepipe_comprehensive_struc" year = 2024 -ver = "v1.5.c" +ver = "v1.6.c" obj._params = {} diff --git a/src/sp_validation/basic.py b/src/sp_validation/basic.py index 4e5afc2c..36458656 100644 --- a/src/sp_validation/basic.py +++ b/src/sp_validation/basic.py @@ -146,11 +146,14 @@ def _read_data(self, data, mask): ) print("FHP/MK hack using p1 PSF for ns in cuts") + print("MKDEBUG before ", data[f"{self._prefix}_Tpsf_NOSHEAR"][mask][:5]) indices = np.where(mask)[0] new_psf = data[indices][f"{self._prefix}_Tpsf_1P"] ns["Tpsf"] = new_psf data[f"{self._prefix}_Tpsf_NOSHEAR"][mask] = new_psf + print("MKDEBUG after ", data[f"{self._prefix}_Tpsf_NOSHEAR"][mask][:5]) + self.m1 = m1 self.p1 = p1 self.m2 = m2 From 5d2283645ae891545f3e4bf60948d1b45c7235c8 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Tue, 10 Feb 2026 09:18:39 +0100 Subject: [PATCH 14/15] hack to fix PSF size propa is finally fixed --- notebooks/calibrate_comprehensive_cat.py | 21 ++++++++------------- src/sp_validation/basic.py | 10 +++++----- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/notebooks/calibrate_comprehensive_cat.py b/notebooks/calibrate_comprehensive_cat.py index 5ecdb75b..5d5a8be0 100644 --- a/notebooks/calibrate_comprehensive_cat.py +++ b/notebooks/calibrate_comprehensive_cat.py @@ -205,6 +205,14 @@ mask_metacal ) +# Keep original NOSHEAR column, override with 1P PSF values (FHP/MK hack) +print( + "FHP/MK hack: explicit copying of the metacal no-shear (updated from 1p)" + + f" PSF size" +) +add_cols_data["NGMIX_Tpsf_NOSHEAR_orig"] = add_cols_data["NGMIX_Tpsf_NOSHEAR"] +add_cols_data["NGMIX_Tpsf_NOSHEAR"] = gal_metacal.ns["Tpsf"][mask_metacal] + # %% # Additional post-processing columns to write to output cat add_cols_post = [ @@ -270,19 +278,6 @@ my_mask.print_summary(f_out) -# %% -# MKDEBUG check bug of minimal size objects -print("MKDEBUG calibration min size checks") -T = add_cols_data["NGMIX_T_NOSHEAR"] -Tpsf = add_cols_data["NGMIX_Tpsf_NOSHEAR"] -x = T / Tpsf -ind_low = np.argsort(x)[:5] -print("ratio", x[ind_low]) -print("gal size", T[ind_low]) -print("PSF size", Tpsf[ind_low]) -print("rel_size_min", cm["gal_rel_size_min"]) - - # %% if not obj._params["cmatrices"]: diff --git a/src/sp_validation/basic.py b/src/sp_validation/basic.py index 36458656..8eed0eca 100644 --- a/src/sp_validation/basic.py +++ b/src/sp_validation/basic.py @@ -146,13 +146,13 @@ def _read_data(self, data, mask): ) print("FHP/MK hack using p1 PSF for ns in cuts") - print("MKDEBUG before ", data[f"{self._prefix}_Tpsf_NOSHEAR"][mask][:5]) indices = np.where(mask)[0] - new_psf = data[indices][f"{self._prefix}_Tpsf_1P"] - ns["Tpsf"] = new_psf - data[f"{self._prefix}_Tpsf_NOSHEAR"][mask] = new_psf + col_noshear = f"{self._prefix}_Tpsf_NOSHEAR" + col_1p = f"{self._prefix}_Tpsf_1P" + new_psf = data[col_1p][indices] - print("MKDEBUG after ", data[f"{self._prefix}_Tpsf_NOSHEAR"][mask][:5]) + # Overwriting incorrect no-shear PSF size to the one from 1p + ns["Tpsf"] = new_psf self.m1 = m1 self.p1 = p1 From 9e01c7934835d75dd65ec4a33d337be1e1a18fa1 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Tue, 10 Feb 2026 09:19:19 +0100 Subject: [PATCH 15/15] v1.4.11.3 (fixed PSF size hack) --- notebooks/cosmo_val/cat_config.yaml | 10 +++++----- pyproject.toml | 1 - src/sp_validation/cosmo_val.py | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index dce68d0a..8a8e22eb 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -688,13 +688,13 @@ SP_v1.4.11.2: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits -SP_v1.4.11.2: +SP_v1.4.11.3: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP - colour: blue + colour: red getdist_colour: 0.0, 0.5, 1.0 - ls: dashed - marker: s + ls: dashdot + marker: v cov_th: A: 2405.3892055695346 n_e: 6.128201234871523 @@ -718,7 +718,7 @@ SP_v1.4.11.2: shear: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - path: v1.4.11.2/unions_shapepipe_cut_struc_2024_v1.4.11.2.fits + path: v1.4.11.3/unions_shapepipe_cut_struc_2024_v1.4.11.3.fits redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 diff --git a/pyproject.toml b/pyproject.toml index 15912c95..5329a120 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,6 @@ dependencies = [ "camb", "clmm", "colorama", - "cosmo-numba @ git+https://github.com/aguinot/cosmo-numba", "cs_util>=0.1.5", "emcee", "getdist @ git+https://github.com/benabed/getdist.git@upper_triangle_whisker", diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index 9bbc0b2c..b19b67de 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -1798,8 +1798,8 @@ def plot_ratio_xi_sys_xi(self, threshold=0.1, offset=0.02): fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(10, 7)) - for idx, ver in enumerate(self.versions): + self.calculate_2pcf(ver) xi_psf_sys = self.xi_psf_sys[ver] gg = self.cat_ggs[ver]