diff --git a/config/calibration/mask_v1.X.10.yaml b/config/calibration/mask_v1.X.10.yaml index 4729bfdc..9118bd93 100644 --- a/config/calibration/mask_v1.X.10.yaml +++ b/config/calibration/mask_v1.X.10.yaml @@ -1,5 +1,5 @@ # Config file for masking and calibration. -# Standard cuts without halo masks, type v1.X.10 +# Standard cuts without halo masks, including deblended objects, type v1.X.10 # General parameters (can also given on command line) params: diff --git a/notebooks/config_mask.yaml b/config/calibration/mask_v1.X.11.yaml similarity index 72% rename from notebooks/config_mask.yaml rename to config/calibration/mask_v1.X.11.yaml index 4af95017..13e9187f 100644 --- a/notebooks/config_mask.yaml +++ b/config/calibration/mask_v1.X.11.yaml @@ -1,14 +1,21 @@ +# Config file for masking and calibration. +# Standard cuts without halo masks, including deblended objects, type v1.X.11 + +# General parameters (can also given on command line) params: - input_path: unions_shapepipe_comprehensive_struc_2024_v1.4.2.hdf5 - cmatrices: True + input_path: unions_shapepipe_comprehensive_struc_2024_v1.X.c.hdf5 + cmatrices: False + sky_regions: False verbose: True +# Masks +## Using columns in 'dat' group (ShapePipe flags) dat: # SExtractor flags - col_name: FLAGS label: SE FLAGS - kind: equal - value: 0 + kind: smaller_equal + value: 3 # Duplicate objects - col_name: overlap @@ -50,18 +57,8 @@ dat: kind: not_equal value: -10 +## Using columns in 'dat_ext' group (post-processing flags) dat_ext: - # Faint star halos - - col_name: 1_Faint_star_halos - label: "Faint star halos" - kind: equal - value: False - - # Bright star halos - - col_name: 2_Bright_star_halos - label: "Bright star halos" - kind: equal - value: False # Stars - col_name: 4_Stars @@ -87,7 +84,7 @@ dat_ext: kind: equal value: False - # Rought pointing coverage + # Rough pointing coverage - col_name: npoint3 label: r"$n_{\rm pointing}$" kind: greater_equal @@ -103,8 +100,11 @@ metacal: gal_snr_max: 500 # Relative-size (hlr / hlr_psf) range - gal_rel_size_min: 0.5 + gal_rel_size_min: 0.707 gal_rel_size_max: 3 # Correct relative size for ellipticity? - gal_size_corr_ell: False \ No newline at end of file + gal_size_corr_ell: False + + # Weight for global response matrix, None for unweighted mean + global_R_weight: w diff --git a/config/calibration/mask_v1.X.2.yaml b/config/calibration/mask_v1.X.2.yaml index 46672c04..7d93a57a 100644 --- a/config/calibration/mask_v1.X.2.yaml +++ b/config/calibration/mask_v1.X.2.yaml @@ -116,3 +116,14 @@ metacal: # Correct relative size for ellipticity? gal_size_corr_ell: False + +# Binned footprint mask +binned_mask: + # Resolution + nside: 8192 + + # Mask value for "good region (observed) + good: False + + # Output file path + output_path: "footprint_binned.fits" diff --git a/config/calibration/mask_v1.X.3.yaml b/config/calibration/mask_v1.X.3.yaml index 96aeecd5..242031a8 100644 --- a/config/calibration/mask_v1.X.3.yaml +++ b/config/calibration/mask_v1.X.3.yaml @@ -99,3 +99,14 @@ metacal: # Correct relative size for ellipticity? gal_size_corr_ell: False + +# Binned footprint mask +binned_mask: + # Resolution + nside: 8192 + + # Mask value for "good region (observed) + good: False + + # Output file path + output_path: "footprint_binned.fits" diff --git a/config/calibration/mask_v1.X.4.yaml b/config/calibration/mask_v1.X.4.yaml index 6db36f6b..59080851 100644 --- a/config/calibration/mask_v1.X.4.yaml +++ b/config/calibration/mask_v1.X.4.yaml @@ -110,3 +110,14 @@ metacal: # Correct relative size for ellipticity? gal_size_corr_ell: False + +# Binned footprint mask +binned_mask: + # Resolution + nside: 8192 + + # Mask value for "good region (observed) + good: False + + # Output file path + output_path: "footprint_binned.fits" diff --git a/config/calibration/mask_v1.X.5.yaml b/config/calibration/mask_v1.X.5.yaml index 6fa04aff..7fb14835 100644 --- a/config/calibration/mask_v1.X.5.yaml +++ b/config/calibration/mask_v1.X.5.yaml @@ -105,3 +105,14 @@ metacal: # Correct relative size for ellipticity? gal_size_corr_ell: False + +# Binned footprint mask +binned_mask: + # Resolution + nside: 8192 + + # Mask value for "good region (observed) + good: False + + # Output file path + output_path: "footprint_binned.fits" diff --git a/config/calibration/mask_v1.X.6.yaml b/config/calibration/mask_v1.X.6.yaml index 9dee1944..0ea23f47 100644 --- a/config/calibration/mask_v1.X.6.yaml +++ b/config/calibration/mask_v1.X.6.yaml @@ -108,3 +108,14 @@ metacal: # Weight for global response matrix, None for unweighted mean global_R_weight: w + +# Binned footprint mask +binned_mask: + # Resolution + nside: 8192 + + # Mask value for "good region (observed) + good: False + + # Output file path + output_path: "footprint_binned.fits" diff --git a/config/calibration/mask_v1.X.7.yaml b/config/calibration/mask_v1.X.7.yaml index 49325e36..830d7a28 100644 --- a/config/calibration/mask_v1.X.7.yaml +++ b/config/calibration/mask_v1.X.7.yaml @@ -113,3 +113,14 @@ metacal: # Weight for global response matrix, None for unweighted mean global_R_weight: w + +# Binned footprint mask +binned_mask: + # Resolution + nside: 8192 + + # Mask value for "good region (observed) + good: False + + # Output file path + output_path: "footprint_binned.fits" diff --git a/config/calibration/mask_v1.X.8.yaml b/config/calibration/mask_v1.X.8.yaml index c346b339..bd74e54b 100644 --- a/config/calibration/mask_v1.X.8.yaml +++ b/config/calibration/mask_v1.X.8.yaml @@ -119,3 +119,14 @@ metacal: # Weight for global response matrix, None for unweighted mean global_R_weight: w + +# Binned footprint mask +binned_mask: + # Resolution + nside: 8192 + + # Mask value for "good region (observed) + good: False + + # Output file path + output_path: "footprint_binned.fits" diff --git a/notebooks/calibrate_comprehensive_cat.py b/notebooks/calibrate_comprehensive_cat.py index 168d608c..5d5a8be0 100644 --- a/notebooks/calibrate_comprehensive_cat.py +++ b/notebooks/calibrate_comprehensive_cat.py @@ -8,9 +8,10 @@ # enable autoreload for interactive sessions ipython = get_ipython() if ipython is not None: - ipython.run_line_magic("load_ext", "autoreload") + ipython.run_line_magic("reload_ext", "autoreload") ipython.run_line_magic("autoreload", "2") - ipython.run_line_magic("load_ext", "log_cell_time") + ipython.run_line_magic("reload_ext", "log_cell_time") + # %% import sys @@ -36,6 +37,11 @@ # Read configuration file and set parameters config = obj.read_config_set_params("config_mask.yaml") + + +# %% +obj._params + # %% # Get data. Set load_into_memory to False for very large files dat, dat_ext = obj.read_cat(load_into_memory=False) @@ -49,7 +55,6 @@ dat_ext = dat_ext[:n_test] - # %% # ## Masking @@ -125,6 +130,10 @@ mask_combined, mask_metacal, num_bins=20, + snr_min=cm["gal_snr_min"], + snr_max=cm["gal_snr_max"], + size_ratio_min=cm["gal_rel_size_min"], + size_ratio_max=cm["gal_rel_size_max"], ) # %% @@ -195,7 +204,14 @@ mask_combined._mask, mask_metacal ) - #add_cols_data[key] = dat[key][mask_combined._mask][mask_metacal] + +# Keep original NOSHEAR column, override with 1P PSF values (FHP/MK hack) +print( + "FHP/MK hack: explicit copying of the metacal no-shear (updated from 1p)" + + f" PSF size" +) +add_cols_data["NGMIX_Tpsf_NOSHEAR_orig"] = add_cols_data["NGMIX_Tpsf_NOSHEAR"] +add_cols_data["NGMIX_Tpsf_NOSHEAR"] = gal_metacal.ns["Tpsf"][mask_metacal] # %% # Additional post-processing columns to write to output cat @@ -229,10 +245,6 @@ for my_mask in masks: my_mask.add_summary_to_FITS_header(header) - -# %% -header - # %% output_shape_cat_path = obj._params["input_path"].replace( "comprehensive", "cut" diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 83c7a357..8a8e22eb 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -161,7 +161,7 @@ SP_v1.3: SP_v1.3.6: subdir: /n17data/UNIONS/WL/v1.3.x pipeline: SP - colour: violet + colour: coral getdist_colour: 0.0, 0.5, 1.0 ls: dashed marker: h @@ -646,7 +646,7 @@ SP_v1.4.8: SP_v1.4.11.2: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP - colour: blue + colour: orange getdist_colour: 0.0, 0.5, 1.0 ls: dashed marker: s @@ -688,18 +688,18 @@ SP_v1.4.11.2: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits -SP_v1.4.11.2_w_iv: +SP_v1.4.11.3: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP - colour: teal - getdist_colour: 0.0, 0.7, 0.7 - ls: dotted - marker: s + colour: red + getdist_colour: 0.0, 0.5, 1.0 + ls: dashdot + marker: v cov_th: A: 2405.3892055695346 - n_e: 7.529428768078532 + n_e: 6.128201234871523 n_psf: 0.752316232272063 - sigma_e: 0.380499698062307 + sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits psf: PSF_flag: FLAG_PSF_HSM @@ -718,9 +718,9 @@ SP_v1.4.11.2_w_iv: shear: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - path: v1.4.11.2/unions_shapepipe_cut_struc_2024_v1.4.11.2.fits + path: v1.4.11.3/unions_shapepipe_cut_struc_2024_v1.4.11.3.fits redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - w_col: w_iv + w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected e1_PSF_col: e1_PSF diff --git a/notebooks/create_shear_mb_empty.py b/notebooks/create_shear_mb_empty.py index 9c3d1760..6d29bfc2 100644 --- a/notebooks/create_shear_mb_empty.py +++ b/notebooks/create_shear_mb_empty.py @@ -39,7 +39,7 @@ # Set parameters base = "unions_shapepipe_comprehensive_struc" year = 2024 -ver = "v1.5.c" +ver = "v1.6.c" obj._params = {} diff --git a/notebooks/demo_apply_hsp_masks.py b/notebooks/demo_apply_hsp_masks.py index 1760cd7f..a264bb1d 100644 --- a/notebooks/demo_apply_hsp_masks.py +++ b/notebooks/demo_apply_hsp_masks.py @@ -57,7 +57,8 @@ # + # Set parameters base = "unions_shapepipe_comprehensive" -year = 2024 +ver = "v1.3.c" + if ver == "v1.3.c": year = 2022 else: diff --git a/notebooks/extract_info.py b/notebooks/extract_info.py index 0bddceab..a4e33451 100644 --- a/notebooks/extract_info.py +++ b/notebooks/extract_info.py @@ -175,7 +175,6 @@ g_star_psf[1], ) - #### Refine: Match to SPREAD_CLASS samples if "SPREAD_CLASS" in dd.dtype.names: spv_cat.match_spread_class(dd, ind_star, m_star, stats_file, len(ra_star), verbose=verbose) diff --git a/pyproject.toml b/pyproject.toml index 15912c95..5329a120 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,6 @@ dependencies = [ "camb", "clmm", "colorama", - "cosmo-numba @ git+https://github.com/aguinot/cosmo-numba", "cs_util>=0.1.5", "emcee", "getdist @ git+https://github.com/benabed/getdist.git@upper_triangle_whisker", diff --git a/src/sp_validation/basic.py b/src/sp_validation/basic.py index 9767ad79..8eed0eca 100644 --- a/src/sp_validation/basic.py +++ b/src/sp_validation/basic.py @@ -145,6 +145,15 @@ def _read_data(self, data, mask): ns, ) + print("FHP/MK hack using p1 PSF for ns in cuts") + indices = np.where(mask)[0] + col_noshear = f"{self._prefix}_Tpsf_NOSHEAR" + col_1p = f"{self._prefix}_Tpsf_1P" + new_psf = data[col_1p][indices] + + # Overwriting incorrect no-shear PSF size to the one from 1p + ns["Tpsf"] = new_psf + self.m1 = m1 self.p1 = p1 self.m2 = m2 @@ -157,7 +166,7 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): Read data from ngmix catalogue. """ - + for name_shear, dict_tmp in zip( ['1M', '1P', '2M', '2P', 'NOSHEAR'], [m1, p1, m2, p2, ns] @@ -370,14 +379,7 @@ def _masking_gal(self): else: snr_flux = data['flux'] / data['flux_err'] - if name == 'ns': - # This is a FHP hack, the ns PSF measured in shapepipe is not correct, - # fortunately it is the same dilated PSF as the other branches, - # thus we can simply use p1 - print("FHP using p1 PSF for ns in cuts") - Tpsf = self.p1['Tpsf'] - else: - Tpsf = data['Tpsf'] + Tpsf = data['Tpsf'] mask_tmp = ( (data['flag'] == 0) diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index a4368f99..0272822a 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -244,7 +244,14 @@ def build_df(cat_gal): return pd.DataFrame(arr, columns=cat_gal.keys()) -def get_w_des(cat_gal, num_bins): +def get_w_des( + cat_gal, + num_bins, + snr_min=None, + snr_max=None, + size_ratio_min=None, + size_ratio_max=None, +): """ Get DES weights. (Gatti et al. 2021) Return an array of DES weights obtained by binning in SNR and size and computing the ratio between @@ -253,9 +260,18 @@ def get_w_des(cat_gal, num_bins): Parameters ---------- cat_gal: dict - A catalog of galaxies containing the response matrix and the uncalibrated ellipticities + A catalog of galaxies containing the response matrix and the + uncalibrated ellipticities num_bins : int Number of bins to use for the binning of the SNR and size. + snr_min : float, optional + Minimum SNR, default (`None`): determined by the data + snr_max : float, optional + Maximum SNR, default (`None`): determined by the data + size_ratio_min : float, optional + Minimum size ratio, default (`None`): determined by the data + size_ratio_max : float, optional + Maximum size ratio, default (`None`): determined by the data Returns ------- @@ -266,11 +282,24 @@ def get_w_des(cat_gal, num_bins): df_gal = build_df(cat_gal) #Create logarithmic bins in size and SNR - cut_to_bins(df_gal, "snr", num_bins, type="log") - cut_to_bins(df_gal, "size_ratio", num_bins, type="log") - - #Compute shape noise and the shear response in each bin + cut_to_bins( + df_gal, + "snr", + num_bins, + type="log", + x_min=snr_min, + x_max=snr_max, + ) + cut_to_bins( + df_gal, + "size_ratio", + num_bins, + type="log", + x_min=size_ratio_min, + x_max=size_ratio_max, + ) + # Compute shape noise and the shear response in each bin for i in range(num_bins): for j in range(num_bins): bin_mask = ( @@ -496,7 +525,7 @@ def get_quantities_binned(cat_gal, num_bins_x, num_bins_y=None, which=["response # Create logarithmic bins in size and SNR bin_edges = {} - bin_edges["snr"] = cut_to_bins(df_gal, "snr", num_bins_x, type="log", x_min=3, x_max=700) + bin_edges["snr"] = cut_to_bins(df_gal, "snr", num_bins_x, type="log", x_min=2, x_max=700) bin_edges["size_ratio"] = cut_to_bins(df_gal, "size_ratio", num_bins_y, type="log", x_min=0.3, x_max=10) # Initialize output dict diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index f314c2fe..b19b67de 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -1798,8 +1798,8 @@ def plot_ratio_xi_sys_xi(self, threshold=0.1, offset=0.02): fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(10, 7)) - for idx, ver in enumerate(self.versions): + self.calculate_2pcf(ver) xi_psf_sys = self.xi_psf_sys[ver] gg = self.cat_ggs[ver] @@ -1967,7 +1967,7 @@ def plot_aperture_mass_dispersion(self): labels=labels, xlog=True, xlim=[self.theta_min_plot, self.theta_max_plot], - ylim=[-1e-6, 2e-5], + ylim=[-2e-6, 5e-6], colors=colors, linestyles=linestyles, shift_x=True, @@ -2000,7 +2000,7 @@ def plot_aperture_mass_dispersion(self): xlog=True, ylog=True, xlim=[self.theta_min_plot, self.theta_max_plot], - ylim=[1e-9, 3e-5], + ylim=[1e-8, 1e-5], colors=colors, linestyles=linestyles, shift_x=True, diff --git a/src/sp_validation/run_joint_cat.py b/src/sp_validation/run_joint_cat.py index 0b17d274..51a4d84e 100644 --- a/src/sp_validation/run_joint_cat.py +++ b/src/sp_validation/run_joint_cat.py @@ -1143,7 +1143,7 @@ def read_cat(self, load_into_memory=False): except: print(f"Error while reading file {fpath}") raise - + if verbose: print( f"Found {len(dat)} (~{util.millify(len(dat))}) objects" @@ -1565,6 +1565,10 @@ def compute_weights_gatti( mask_combined, mask_metacal, num_bins=20, + snr_min=10, + snr_max=500, + size_ratio_min=0.707, + size_ratio_max=3, ): """Compute Weights Gatti. @@ -1581,7 +1585,14 @@ def compute_weights_gatti( purpose="weights" ) - cat_gal["w_des"] = calibration.get_w_des(cat_gal, num_bins) + cat_gal["w_des"] = calibration.get_w_des( + cat_gal, + num_bins, + snr_min=snr_min, + snr_max=snr_max, + size_ratio_min=size_ratio_min, + size_ratio_max=size_ratio_max, + ) def compute_PSF_leakage(