diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index 48349043d..6bcbfed9b 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -385,7 +385,37 @@ def covariance_label(npatch): return "analytic" if npatch is None else f"jackknife ({npatch} patches)" -def calculate_cosebis(gg, nmodes=10, scale_cuts=None, cov_path=None): +def cosebis_operator(theta, left_edges, right_edges, scale_cut, nmodes): + """COSEBIs map from full-grid ``[ξ+; ξ−]`` to ``[En; Bn]``. + + The covariance convention sets all E×B blocks to zero after propagation, + matching cosmo_numba's single-pair covariance convention. + """ + from cosmo_numba.B_modes.cosebis import COSEBIS + from scipy.integrate import simpson + + theta = np.asarray(theta) + start, stop = bins_from_edges(left_edges, right_edges, *scale_cut) + inds = np.arange(start, stop) + cut = theta[inds] + kernel = COSEBIS( + theta_min=cut.min(), theta_max=cut.max(), N_max=nmodes, precision=120 + ) + rad = np.deg2rad(cut / 60) + weights = simpson(np.eye(len(cut)), x=rad, axis=1) * rad / 2 + plus = kernel.get_Tp_log(cut) * weights + minus = kernel.get_Tm_log(cut) * weights + operator = np.zeros((2 * nmodes, 2 * len(theta))) + operator[:nmodes, inds] = plus + operator[nmodes:, inds] = plus + operator[:nmodes, inds + len(theta)] = minus + operator[nmodes:, inds + len(theta)] = -minus + return operator + + +def calculate_cosebis( + gg, nmodes=10, scale_cuts=None, cov_path=None, *, cov_xi=None, npatch=None +): """ Calculate COSEBIs modes from a correlation function for multiple scale cuts. @@ -409,7 +439,15 @@ def calculate_cosebis(gg, nmodes=10, scale_cuts=None, cov_path=None): Each results dictionary contains 'En', 'Bn', 'cov', 'chi2_E', 'chi2_B', 'pte_B', 'scale_cut', and 'mask' entries. """ - cov_xipm = np.loadtxt(cov_path) if cov_path is not None else gg.cov + if cov_xi is not None and cov_path is not None: + raise ValueError("give cov_xi or cov_path, not both") + cov_xipm = ( + cov_xi + if cov_xi is not None + else np.loadtxt(cov_path) + if cov_path is not None + else gg.cov + ) return cosebis_scan_from_xi( gg.meanr, gg.xip, @@ -420,7 +458,9 @@ def calculate_cosebis(gg, nmodes=10, scale_cuts=None, cov_path=None): nmodes=nmodes, scale_cuts=scale_cuts, # A theory covariance has no jackknife realisations to debias. - npatch=None if cov_path is not None else gg.npatch1, + npatch=npatch + if cov_xi is not None + else (None if cov_path is not None else gg.npatch1), ) diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 5e5576ca0..67541f922 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -611,24 +611,6 @@ def blind(self, version): catalogues = self._config_as_read return blinding.open_blind(blinding.blind_of(catalogues, entry), catalogues) - def _refuse_blinded_tomography(self, version): - """Refuse to measure tomographic shear signal on a blinded catalogue. - - A statistic is blinded by sealing its SACC part, whose tracers carry - the n(z) the blind's theory shifts it by. ξ± (and ⟨M_ap²⟩, derived from - it) is born as such a part for every tomographic bin pair; the - tomographic pseudo-Cℓ are still FITS tables with no per-bin tracer, so - on a blinded catalogue they would hold true values. - """ - blind = self.blind(version) - if blind.name != blinding.NONE.name: - raise blinding.BlindingError( - f"{version} is blinded under {blind.name}, and blinding covers " - "its tomographic ξ± and non-tomographic statistics only; other " - "tomographic shear statistics are measured on public " - "(`blind: none`) catalogues" - ) - def basename( self, version, diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index efce2cfc0..b898c2b2e 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -16,6 +16,7 @@ plot_cosebis_scale_cut_heatmap, save_cosebis_results, ) +from .derived_covariance import xi_covariance class CosebisMixin: @@ -61,9 +62,8 @@ def calculate_cosebis( nmodes : int, optional Number of COSEBIs modes to compute. Defaults to 10. cov_path : str, optional - Path to theoretical covariance matrix. When provided, enables analytic - covariance calculation for a single pair. Cannot be combined with - ``compute_tomography=True``. + Path to the OneCovariance integration-grid matrix, including all + pairs for tomography. scale_cuts : list of tuples, optional Explicit list of (min_theta, max_theta) scale cuts to evaluate. Overrides evaluate_all_scale_cuts when provided. @@ -93,12 +93,6 @@ def calculate_cosebis( """ self.print_start(f"Computing {version} COSEBIs") - if cov_path is not None and compute_tomography: - raise ValueError( - "cov_path holds a single ξ± covariance; it cannot serve every " - "tomographic bin pair." - ) - # Calculate single fine-binned correlation function for COSEBIs, as the # sealed part: on a blinded catalogue the COSEBIs are concealed with it. print( @@ -115,12 +109,15 @@ def calculate_cosebis( max_sep=max_sep_int, nbins=nbins_int, ) - results = { - bin_key: None for bin_key in ggs.keys() - } # Initialize results dictionary + parts = self.xi_tomo_parts if compute_tomography else self.xi_parts + part = parts[version, "integration"] + covariance, cov_npatch = xi_covariance(part, cov_path) + results = {bin_key: None for bin_key in ggs} for bin_key in ggs.keys(): - # LG TO-DO: account for tomographic scale-cuts + rows = ggs[bin_key]._rows + pair_cov = covariance[np.ix_(rows, rows)] + # The same scale cuts apply to each pair. if scale_cuts is not None: # Explicit scale cuts provided print(f"Evaluating {len(scale_cuts)} explicit scale cuts") @@ -128,7 +125,8 @@ def calculate_cosebis( gg=ggs[bin_key], nmodes=nmodes, scale_cuts=scale_cuts, - cov_path=cov_path, + cov_xi=pair_cov, + npatch=cov_npatch, ) elif evaluate_all_scale_cuts: # Use reporting binning parameters or inherit from class config @@ -154,12 +152,17 @@ def calculate_cosebis( gg=ggs[bin_key], nmodes=nmodes, scale_cuts=generated_cuts, - cov_path=cov_path, + cov_xi=pair_cov, + npatch=cov_npatch, ) else: # Single scale cut behavior: use full range results[bin_key] = calculate_cosebis( - gg=ggs[bin_key], nmodes=nmodes, scale_cuts=None, cov_path=cov_path + gg=ggs[bin_key], + nmodes=nmodes, + scale_cuts=None, + cov_xi=pair_cov, + npatch=cov_npatch, ) # Extract single results dict from scale_cuts dictionary results[bin_key] = next(iter(results[bin_key].values())) @@ -314,6 +317,10 @@ def plot_cosebis( bin_stub + "_data.npz", fiducial_scale_cut, edges, - derived_from=[self.xi_parts[version, "integration"]], + derived_from=[ + (self.xi_tomo_parts if compute_tomography else self.xi_parts)[ + version, "integration" + ] + ], ) self._cosebis_results[version] = results_tomo diff --git a/src/sp_validation/cosmo_val/derived_covariance.py b/src/sp_validation/cosmo_val/derived_covariance.py new file mode 100644 index 000000000..b04b57bef --- /dev/null +++ b/src/sp_validation/cosmo_val/derived_covariance.py @@ -0,0 +1,43 @@ +"""Covariance propagation for derived tomographic ξ± statistics.""" + +import numpy as np +from scipy.linalg import block_diag + +from .. import sacc_io +from .sacc_writers import onecov_xi_covariance + + +def xi_pairs(part): + """Source-bin pairs in the order in which the ξ± part holds them.""" + pairs = [] + for dp in part.data: + if dp.data_type == sacc_io.XI_PLUS: + pair = tuple(int(t.removeprefix("source_")) for t in dp.tracers) + if pair not in pairs: + pairs.append(pair) + return pairs + + +def xi_covariance(part, path=None): + """Pair-major covariance and jackknife patch count, or analytic covariance.""" + if path is not None: + return onecov_xi_covariance(part, np.loadtxt(path)), None + if part.covariance is None: + raise ValueError("the integration-grid ξ± part has no covariance") + return part.covariance.dense, int(part.metadata.get("npatch", 1)) + + +def propagate(operators, covariance, *, cosebis=False): + """Propagate all cross-pair blocks, with the COSEBIs E×B-zero convention.""" + operator = block_diag(*operators) + result = operator @ covariance @ operator.T + if cosebis: + e, b, offset = [], [], 0 + for op in operators: + n = op.shape[0] // 2 + e.extend(range(offset, offset + n)) + b.extend(range(offset + n, offset + 2 * n)) + offset += 2 * n + result[np.ix_(e, b)] = 0 + result[np.ix_(b, e)] = 0 + return result diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 051eefd23..d624e9b1e 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -24,7 +24,6 @@ from ..io import open_entry from ..rho_tau import get_params_rho_tau from ..statistics import cov_from_one_covariance -from .sacc_writers import BIN as SACC_BIN from .sacc_writers import pseudo_cl_to_sacc @@ -92,9 +91,9 @@ def plot_pseudo_cl_spectrum(datasets, spectrum, output_path): plt.close(fig) -def _spectra(part): - """The ELL/EE/EB/BE/BB dict of a single-field pseudo-Cl part, whose BE is EB.""" - ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(part, SACC_BIN) +def _spectra(part, bins=(0, 0)): + """The ELL/EE/EB/BE/BB view of a sealed part, whose BE is EB.""" + ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(part, bins) return {"ELL": ell, "EE": ee, "EB": eb, "BE": eb.copy(), "BB": bb} @@ -132,17 +131,10 @@ def calculate_pseudo_cl(self, compute_tomography=True, out_path=None): by default to ``pseudo_cl_{ver}.sacc``; ``out_path`` overrides that destination (one version only). An existing part is reused only if it was sealed under the blind declared now. With - ``compute_tomography=True`` every tomographic bin pair is written to its - own FITS file and ``out_path`` is not accepted; tomographic pairs are - measured only on a public catalogue - (:meth:`_refuse_blinded_tomography`). + ``compute_tomography=True`` all pairs are written to one sealed SACC + part; ``out_path`` overrides its destination for a single version. """ if out_path is not None: - if compute_tomography: - raise ValueError( - "calculate_pseudo_cl(out_path=...) names the non-tomographic " - "SACC part; call it with compute_tomography=False" - ) if len(self.versions) != 1: raise ValueError( "calculate_pseudo_cl(out_path=...) writes one part to one " @@ -163,66 +155,65 @@ def calculate_pseudo_cl(self, compute_tomography=True, out_path=None): for ver in self.versions: self.print_magenta(ver) - if ver not in self.pseudo_cls.keys(): - self._pseudo_cls[ver] = {} - + self._pseudo_cls.setdefault(ver, {}) if compute_tomography: - self._refuse_blinded_tomography(ver) tomo_bin_ids, tomo_bin_pairs = self._get_tomo_bins(ver) - if tomo_bin_ids is None or tomo_bin_pairs is None: raise ValueError( f"Version {ver} does not have tomography information." ) - + index = {b: i for i, b in enumerate(sorted(tomo_bin_ids))} else: tomo_bin_pairs = [("all", "all")] + index = {"all": 0} - # Loop on the different tomographic bin pairs - for bin_key1, bin_key2 in tomo_bin_pairs: - self.print_cyan(f"Tomo Bin Pair: ({bin_key1}, {bin_key2})") - - if ( - f"tomo_bin_{bin_key1}_tomo_bin_{bin_key2}" - not in self._pseudo_cls[ver].keys() - ): - self._pseudo_cls[ver][ - f"tomo_bin_{bin_key1}_tomo_bin_{bin_key2}" - ] = {} - - pair_out_path = out_path or self._output_path_pseudo_cl( - ver, tomo_bin_pair=(bin_key1, bin_key2) - ) - held = ( - self._load_pseudo_cl(ver, pair_out_path, (bin_key1, bin_key2)) - if os.path.exists(pair_out_path) and not self.force_run - else None - ) - if held is not None: - self.print_done( - f"Skipping Pseudo-Cl's calculation, {pair_out_path} exists" - ) - self._pseudo_cls[ver][f"tomo_bin_{bin_key1}_tomo_bin_{bin_key2}"][ - "pseudo_cl" - ] = held - continue - - if self.cell_method == "map": - self.calculate_pseudo_cl_map( - ver, self.nside, pair_out_path, bin_key1, bin_key2 + path = out_path or self._output_path_pseudo_cl( + ver, tomography=compute_tomography + ) + part = None + if os.path.exists(path) and not self.force_run: + candidate = sacc_io.load(path) + if sacc_io.stamp(candidate) == self.blind(ver).name: + part = candidate + if part is None: + nz = self.sacc_nz(ver, tomography=compute_tomography) + if len(nz) != len(index): + raise ValueError( + f"{ver}: the n(z) file has {len(nz)} distribution column(s) " + f"but the catalogue has {len(index)} tomographic bin(s); " + "every bin needs its own n(z)" ) - elif self.cell_method == "catalog": - self.calculate_pseudo_cl_catalog( - ver, pair_out_path, bin_key1, bin_key2 + measured = [] + for a, b in tomo_bin_pairs: + self.print_cyan(f"Tomo Bin Pair: ({a}, {b})") + if self.cell_method == "map": + ell, cl, wsp, nside = self._measure_pseudo_cl_map( + ver, self.nside, a, b + ) + elif self.cell_method == "catalog": + ell, cl, wsp, nside = self._measure_pseudo_cl_catalog(ver, a, b) + else: + raise ValueError(f"Unknown cell method: {self.cell_method}") + measured.append( + ( + (index[a], index[b]), + {"ell_eff": ell, "cl_all": cl, "wsp": wsp, "nside": nside}, + ) ) - else: - raise ValueError(f"Unknown cell method: {self.cell_method}") + s = pseudo_cl_to_sacc(nz, self.sacc_metadata(ver), measured) + part = sacc_io.save(s, path, blind=self.blind(ver)) + else: + self.print_done(f"Skipping Pseudo-Cl's calculation, {path} exists") + + for a, b in tomo_bin_pairs: + key = f"tomo_bin_{a}_tomo_bin_{b}" + self._pseudo_cls[ver].setdefault(key, {})["pseudo_cl"] = _spectra( + part, (index[a], index[b]) + ) self.print_done("Done pseudo-Cl's") - def calculate_pseudo_cl_map(self, ver, nside, out_path, tomo_bin_a, tomo_bin_b): - if (tomo_bin_a, tomo_bin_b) != ("all", "all"): - self._refuse_blinded_tomography(ver) + def _measure_pseudo_cl_map(self, ver, nside, tomo_bin_a, tomo_bin_b): assert (tomo_bin_a == "all" and tomo_bin_b == "all") or ( isinstance(tomo_bin_a, (int, np.integer)) and isinstance(tomo_bin_b, (int, np.integer)) @@ -313,17 +304,9 @@ def calculate_pseudo_cl_map(self, ver, nside, out_path, tomo_bin_a, tomo_bin_b): # Subtract the noise bias from the pseudo-Cl's cl_shear = cl_shear - cl_noise - self.print_cyan("Saving pseudo-Cl's...") - tomo_bin_pair = (tomo_bin_a, tomo_bin_b) - self._pseudo_cls[ver][f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}"][ - "pseudo_cl" - ] = self._save_pseudo_cl( - ver, out_path, tomo_bin_pair, ell_eff, cl_shear, wsp, nside=nside - ) + return ell_eff, cl_shear, wsp, nside - def calculate_pseudo_cl_catalog(self, ver, out_path, tomo_bin_a, tomo_bin_b): - if (tomo_bin_a, tomo_bin_b) != ("all", "all"): - self._refuse_blinded_tomography(ver) + def _measure_pseudo_cl_catalog(self, ver, tomo_bin_a, tomo_bin_b): assert (tomo_bin_a == "all" and tomo_bin_b == "all") or ( isinstance(tomo_bin_a, (int, np.integer)) and isinstance(tomo_bin_b, (int, np.integer)) @@ -338,11 +321,7 @@ def calculate_pseudo_cl_catalog(self, ver, out_path, tomo_bin_a, tomo_bin_b): catalog=cat_gal, params=params, tomo_bin_a=tomo_bin_a, tomo_bin_b=tomo_bin_b ) - self.print_cyan("Saving pseudo-Cl's...") - tomo_bin_pair = (tomo_bin_a, tomo_bin_b) - self._pseudo_cls[ver][f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}"][ - "pseudo_cl" - ] = self._save_pseudo_cl(ver, out_path, tomo_bin_pair, ell_eff, cl_shear, wsp) + return ell_eff, cl_shear, wsp, None def calculate_pseudo_cl_inka_cov( self, compute_tomography=True, load_all_block=False @@ -1268,19 +1247,10 @@ def _get_tomographic_bin(self, params, cat_gal, tomo_bin): mask = tomo_bin_id == tomo_bin return cat_gal[mask] - def _output_path_pseudo_cl(self, ver, tomo_bin_pair): - """Default pseudo-Cl product path of one bin pair. - - The ``("all", "all")`` pair is the SACC part ``pseudo_cl_{ver}.sacc``; - a tomographic pair is a FITS table under ``pseudo_cl/``. - """ - if tuple(tomo_bin_pair) == ("all", "all"): - return self._output_path(f"pseudo_cl_{ver}.sacc") - bin_key1, bin_key2 = tomo_bin_pair - return self._output_path( - "pseudo_cl", - f"pseudo_cl_from_{self.cell_method}_tomo_bin_{bin_key1}_tomo_bin_{bin_key2}_{ver}_binning_{self.binning}_nbins_{self.n_ell_bins}.fits", - ) + def _output_path_pseudo_cl(self, ver, tomography=False): + """Default path of the non-tomographic or tomographic SACC part.""" + suffix = "_tomo" if tomography else "" + return self._output_path(f"pseudo_cl_{ver}{suffix}.sacc") def _output_path_pseudo_cl_cov(self, ver, method, tomography): is_tomo = "tomo" if tomography else "non_tomo" @@ -1297,36 +1267,6 @@ def _output_path_iNKA_block_cov(self, ver, tomo_bin_quad): f"pseudo_cl_cov_from_iNKA_tomo_bin_{bin_key_a1}_tomo_bin_{bin_key_a2}_tomo_bin_{bin_key_b1}_tomo_bin_{bin_key_b2}_{ver}_binning_{self.binning}_nbins_{self.n_ell_bins}.fits", ) - def _save_pseudo_cl( - self, ver, out_path, tomo_bin_pair, ell_eff, cl_all, wsp, nside=None - ): - """Write one pair as SACC or FITS and return its spectra as written. - - Forwards ``nside`` for map spectra. The ``("all", "all")`` spectra are - those of the part as sealed under the version's blind. - """ - if tuple(tomo_bin_pair) == ("all", "all"): - return _spectra( - self.pseudo_cl_to_sacc_part( - ver, out_path, ell_eff, cl_all, wsp, nside=nside - ) - ) - self.save_pseudo_cl(ell_eff, cl_all, out_path) - return fits.getdata(out_path) - - def _load_pseudo_cl(self, ver, out_path, tomo_bin_pair): - """One bin pair's pseudo-Cl product written by ``_save_pseudo_cl``. - - ``None`` for an ``("all", "all")`` part sealed under a blind other than - the one ``ver`` declares now, which must be measured again. - """ - if tuple(tomo_bin_pair) != ("all", "all"): - return fits.getdata(out_path) - part = sacc_io.load(out_path) - if sacc_io.stamp(part) != self.blind(ver).name: - return None - return _spectra(part) - def pseudo_cl_to_sacc_part( self, version, out_path, ell_eff, cl_all, wsp, nside=None ): @@ -1341,35 +1281,15 @@ def pseudo_cl_to_sacc_part( s = pseudo_cl_to_sacc( self.sacc_nz(version), self.sacc_metadata(version), - ell_eff, - cl_all, - wsp, - nside=nside, + [ + ( + (0, 0), + {"ell_eff": ell_eff, "cl_all": cl_all, "wsp": wsp, "nside": nside}, + ) + ], ) return sacc_io.save(s, out_path, blind=self.blind(version)) - def save_pseudo_cl(self, ell_eff, pseudo_cl, out_path): - """ - Save a tomographic bin pair's pseudo-Cl's to a FITS file. - - Parameters - ---------- - pseudo_cl : np.array - Pseudo-Cl's to save. - out_path : str - Path to save the pseudo-Cl's to. - """ - # Create columns of the fits file - col1 = fits.Column(name="ELL", format="D", array=ell_eff) - col2 = fits.Column(name="EE", format="D", array=pseudo_cl[0]) - col3 = fits.Column(name="EB", format="D", array=pseudo_cl[1]) - col4 = fits.Column(name="BE", format="D", array=pseudo_cl[2]) - col5 = fits.Column(name="BB", format="D", array=pseudo_cl[3]) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - cell_hdu = fits.BinTableHDU.from_columns(coldefs, name="PSEUDO_CELL") - - cell_hdu.writeto(out_path, overwrite=True) - def _save_iNKA_covariance(self, covar, out_path): # covar_22_22 is indexed [ell, pol_a, ell, pol_b]; store each of the # 16 EE/EB/BE/BB cross-blocks as a named HDU (row-major pol order). @@ -1397,18 +1317,19 @@ def _merge_iNKA_covariance(self, ver, tomography): # Merge the tomographic covariance matrices tomo_bin_ids, tomo_bin_pairs = self._get_tomo_bins(ver) - # Get the number of bins from the pseudo_cls attribute - # The non-tomographic pseudo-cl are computed from the call - # to this attribute if not already computed. - n_ell = self._pseudo_cls[ver]["tomo_bin_all_tomo_bin_all"]["pseudo_cl"][ - "ELL" - ].shape[0] - if tomo_bin_ids is None or tomo_bin_pairs is None: raise AssertionError( f"Tomographic bin IDs of version {ver} is not available." ) + # Covariance blocks exist independently of measured spectra. + a, b = tomo_bin_pairs[0] + first_block = self._output_path_iNKA_block_cov( + ver, tomo_bin_quad=(a, b, a, b) + ) + with fits.open(first_block) as block_cov: + n_ell = block_cov["COVAR_EE_EE"].data.shape[0] + n_spectra = len(tomo_bin_pairs) block_indices = list( itertools.combinations_with_replacement(range(n_spectra), 2) diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index ef0e004b6..f9634d65e 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -76,8 +76,7 @@ def xi_psf_sys(self): def calculate_rho_tau_stats(self, tomography=True): """Measure ρ/τ statistics per version and tomographic bin. - Without ``tomography`` the single bin is ``"all"``, and its SACC part is - written per version (``rho_tau_to_sacc_part``). + Write one SACC part per version, with ρ once and τ for each source bin. """ out_dir = f"{self.cc['paths']['output']}/rho_tau_stats" if not os.path.exists(out_dir): @@ -100,7 +99,9 @@ def calculate_rho_tau_stats(self, tomography=True): # Get the selection for stars mask_star = self._get_star_mask(ver) - for tomo_bin_id in tomo_bin_ids: + tau_tables = [] + tau_covariances = [] + for index, tomo_bin_id in enumerate(sorted(tomo_bin_ids)): self.print_cyan(f"Computing for the tomographic bin: {tomo_bin_id}") base_rho = self.basename(ver) base_tau = self.basename(ver, tomo_bin_a=tomo_bin_id) @@ -122,41 +123,57 @@ def calculate_rho_tau_stats(self, tomography=True): cov_rho=self.compute_cov_rho, npatch=self.npatch, ) - if not tomography: - self.rho_tau_to_sacc_part( - ver, out_dir, base_tau, rho_stat_handler, tau_stat_handler - ) + tau_tables.append((index, tau_stat_handler.tau_stats)) + tau_covariances.append((index, base_tau)) + + self.rho_tau_to_sacc_part( + ver, + out_dir, + self.basename(ver), + rho_stat_handler, + tau_tables, + tau_covariances, + tomography=tomography, + ) self.print_done("Rho stats finished") self._rho_stat_handler = rho_stat_handler self._tau_stat_handler = tau_stat_handler def rho_tau_to_sacc_part( - self, version, out_dir, base, rho_stat_handler, tau_stat_handler + self, + version, + out_dir, + base, + rho_stat_handler, + tau_tables, + tau_bases, + tomography=False, ): """Write the ρ/τ SACC part for one version, under the version's blind. - ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage from the handler tables. The - ``CovTauTh`` theory covariance ``cov_tau_{base}_th.npy`` is passed as - ``tau_cov_th`` when it exists; without it the τ block falls back — - loudly — to a variance diagonal. + The part contains ρ_0…ρ_5 and τ_0/τ_2/τ_5 per source bin. + Each available ``CovTauTh`` block is attached to its bin; otherwise + its τ covariance falls back to a variance diagonal. """ - tau_cov_path = os.path.join(out_dir, f"cov_tau_{base}_th.npy") - tau_cov_th = np.load(tau_cov_path) if os.path.exists(tau_cov_path) else None - if tau_cov_th is None: - self.print_magenta( - f"No τ theory covariance at {tau_cov_path}; writing ρ/τ SACC part " - "with a diagonal placeholder covariance (τ inference block is a " - "variance diagonal, not CovTauTh)." - ) + covariances = [] + for index, tau_base in tau_bases: + tau_cov_path = os.path.join(out_dir, f"cov_tau_{tau_base}_th.npy") + if os.path.exists(tau_cov_path): + covariances.append((index, np.load(tau_cov_path))) + else: + self.print_magenta( + f"No τ theory covariance at {tau_cov_path}; using a variance diagonal." + ) s = rho_tau_to_sacc( - self.sacc_nz(version), + self.sacc_nz(version, tomography=tomography), self.sacc_metadata(version), rho_stat_handler.rho_stats, - tau_stat_handler.tau_stats, - tau_cov_th=tau_cov_th, + tau_tables, + tau_cov_th_by_bin=covariances or None, ) - out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") + suffix = "_tomo" if tomography else "" + out_path = os.path.join(out_dir, f"rho_tau_{base}{suffix}.sacc") sacc_io.save(s, out_path, blind=self.blind(version), theory=blinding.no_signal) def calculate_rho_tau_fits(self, tomography=True, track_result=True): diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 96a80faa9..7a22ed1f3 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -17,6 +17,7 @@ plot_pure_eb_correlations, save_pure_eb_results, ) +from .derived_covariance import xi_covariance class PureEBMixin: @@ -59,9 +60,8 @@ def calculate_pure_eb( instead of the non-tomographic pair. Defaults to False. cov_path_int : str, optional Analytic ξ± covariance on the integration grid. Without it the - covariance is the jackknife the integration-grid part carries. One - file holds one pair's covariance, so it cannot be combined with - ``compute_tomography``. + covariance is the jackknife the integration-grid part carries. The + OneCovariance file includes all pairs for tomography. Returns ------- @@ -75,12 +75,6 @@ def calculate_pure_eb( """ self.print_start(f"Computing {version} pure E/B") - if cov_path_int is not None and compute_tomography: - raise ValueError( - "cov_path_int holds a single ξ± covariance; it cannot serve every " - "tomographic bin pair." - ) - integration = self._binning(min_sep_int, max_sep_int, nbins_int) integration_edges = np.geomspace( integration["min_sep"], integration["max_sep"], integration["nbins"] + 1 @@ -99,12 +93,13 @@ def calculate_pure_eb( nbins=integration["nbins"], ) + parts = self.xi_tomo_parts if compute_tomography else self.xi_parts + covariance, cov_npatch = xi_covariance( + parts[version, "integration"], cov_path_int + ) results = {} for bin_key, gg_int in ggs_int.items(): - if cov_path_int is not None: - cov_xi, cov_npatch = np.loadtxt(cov_path_int), None - else: - cov_xi, cov_npatch = gg_int.cov, gg_int.npatch1 + cov_xi = covariance[np.ix_(gg_int._rows, gg_int._rows)] results[bin_key] = calculate_pure_eb_correlation( gg_int.meanr, gg_int.xip, @@ -290,6 +285,10 @@ def plot_pure_eb( save_pure_eb_results( version_results, out_stub + "_data.npz", - derived_from=[self.xi_parts[version, "integration"]], + derived_from=[ + (self.xi_tomo_parts if compute_tomography else self.xi_parts)[ + version, "integration" + ] + ], ) self._pure_eb_results[version][bin_key] = version_results diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index f6c860449..fccf288ea 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -1,17 +1,10 @@ -"""Born-as-SACC writers for the cosmo_val data products. - -A thin, pure layer between the ``cosmo_val`` mixins (which compute statistics as -TreeCorr / NaMaster / b_modes arrays) and :mod:`sp_validation.sacc_io` (which -knows the file layout). Each ``*_to_sacc`` function turns one already-computed -statistic into a single-statistic SACC — a *part* — carrying that statistic's -own covariance as its one block. :func:`assemble_analysis_sacc` rebuilds the -single ``{version}.sacc`` analysis file from these parts. - -ξ± is written over any set of bin pairs (:func:`xi_pairs_to_sacc`), and -:func:`onecov_xi_covariance` puts OneCovariance's ξ± matrix in a part's -order. The other statistics are single-bin today (``bins=(0, 0)``); their -interface is tomography-native so a future round supplies real bin pairs -unchanged. +"""Pair-native SACC writers and covariance helpers for cosmo_val products. + +Each writer turns a computed statistic into a single-statistic SACC part. +Pairs are inserted in the supplied order, typically (0, 0), (0, 1), …, with +points and covariances pair-major. Covariance helpers map OneCovariance ξ± and +NaMaster iNKA matrices to the part's point order. :func:`assemble_analysis_sacc` +combines the parts and their covariance blocks into an analysis file. """ import numpy as np @@ -27,8 +20,6 @@ # NaMaster spin-2 × spin-2 decoupled-spectrum row order (EE, EB, BE, BB). _NMT_EE, _NMT_EB, _NMT_BB = 0, 1, 3 -BIN = (0, 0) - def xi_pairs_to_sacc(nz, metadata, pairs, *, grid, variances=None, covariance=None): """One ξ± part over one or more bin pairs, on a named angular grid. @@ -93,7 +84,12 @@ def xi_to_sacc( "weight": weight, } return xi_pairs_to_sacc( - nz, metadata, [(BIN, xi)], grid=grid, variances=variances, covariance=covariance + nz, + metadata, + [((0, 0), xi)], + grid=grid, + variances=variances, + covariance=covariance, ) @@ -149,68 +145,125 @@ def bin_index(tracer): return cov[np.ix_(order, order)] -def pseudo_cl_to_sacc(nz, metadata, ell_eff, cl_all, wsp, covariance=None, nside=None): - """One pseudo-Cℓ part: EE/BB/EB with the shared bandpower window. +def namaster_cl_covariance(s, cov_fits): + """Map iNKA pair-major covariance HDUs to a pseudo-Cℓ part's order. - ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); the - window comes from :func:`bandpower_window_from_workspace`. Supply ``nside`` - only for HEALPix-map spectra, whose measured spectrum includes ``pw²(ℓ)``; - catalogue-based spectra have no pixel window. ``covariance``, when given, - is the dense ``[EE; BB; EB]``-ordered block matching insertion. + Each same-polarisation HDU is ordered by source pair, then increasing ℓ. + Cross-polarisation blocks are zero, including across source pairs. + """ + from astropy.io import fits + + hdus = { + sio.CL_EE: "COVAR_EE_EE", + sio.CL_BB: "COVAR_BB_BB", + sio.CL_EB: "COVAR_EB_EB", + } + if not s.data or any(dp.data_type not in hdus for dp in s.data): + raise ValueError("namaster_cl_covariance takes a pseudo-Cℓ part only") + + def pair(dp): + return tuple(sorted(int(t.removeprefix("source_")) for t in dp.tracers)) + + pairs = sorted({pair(dp) for dp in s.data}) + ells = {} + for dp in s.data: + ells.setdefault((dp.data_type, pair(dp)), []).append(dp.tags["ell"]) + if set(ells) != {(dtype, p) for dtype in hdus for p in pairs}: + raise ValueError("every pair must carry EE, BB and EB") + grids = [sorted(values) for values in ells.values()] + if any(grid != grids[0] for grid in grids) or len(set(grids[0])) != len(grids[0]): + raise ValueError("every spectrum must have the same distinct ℓ points") + nell = len(grids[0]) + size = len(pairs) * nell + with fits.open(cov_fits) as hdul: + missing = [name for name in hdus.values() if name not in hdul] + if missing: + raise ValueError(f"{cov_fits} lacks the pseudo-Cℓ cov HDUs {missing}") + blocks = { + dtype: np.asarray(hdul[name].data, float) for dtype, name in hdus.items() + } + if any(block.shape != (size, size) for block in blocks.values()): + raise ValueError(f"iNKA covariance blocks must have shape {(size, size)}") + result = np.zeros((len(s.data), len(s.data))) + for dtype, block in blocks.items(): + positions = [i for i, dp in enumerate(s.data) if dp.data_type == dtype] + order = [ + pairs.index(pair(s.data[i])) * nell + grids[0].index(s.data[i].tags["ell"]) + for i in positions + ] + result[np.ix_(positions, positions)] = block[np.ix_(order, order)] + return result + + +def pseudo_cl_to_sacc(nz, metadata, pairs, covariance=None): + """Write pair-major EE/BB/EB spectra and their bandpower windows. + + Each payload contains ``ell_eff``, NaMaster's ``cl_all`` in row order + [EE, EB, BE, BB], and ``wsp``. Supply ``nside`` only for HEALPix-map + spectra, to include the squared pixel window pw²(ℓ); catalogue spectra + have no pixel window. Covariance follows the pair-major insertion order. """ - window_ells, window_weights = bandpower_window_from_workspace(wsp, nside) s = sio.new_sacc(nz, metadata) - sio.add_pseudo_cl( - s, - BIN, - ell_eff, - cl_all[_NMT_EE], - cl_all[_NMT_BB], - cl_all[_NMT_EB], - window_ells=window_ells, - window_weights=window_weights, - ) + for bins, payload in pairs: + cl_all = payload["cl_all"] + window_ells, window_weights = bandpower_window_from_workspace( + payload["wsp"], payload.get("nside") + ) + sio.add_pseudo_cl( + s, + bins, + payload["ell_eff"], + cl_all[_NMT_EE], + cl_all[_NMT_BB], + cl_all[_NMT_EB], + window_ells=window_ells, + window_weights=window_weights, + ) if covariance is not None: s.add_covariance(np.asarray(covariance)) return s -def cosebis_to_sacc(nz, metadata, result, scale_cut): - """One COSEBIs part at the fiducial scale cut. +def cosebis_to_sacc(nz, metadata, pairs, scale_cut, covariance=None): + """Write each pair's COSEBIs in [En; Bn] order at the scale cut. - ``result`` is a single scale-cut result dict from - ``b_modes.calculate_cosebis`` — ``{"En", "Bn", "cov", ...}`` — where ``cov`` - is the ``[En; Bn]``-ordered COSEBIs covariance. + Each payload contains ``En`` and ``Bn``. The supplied covariance follows + that pair-major order and has its E×B and B×E blocks zeroed, including + across pairs. """ s = sio.new_sacc(nz, metadata) - sio.add_cosebis(s, BIN, result["En"], scale_cut, Bn=result["Bn"]) - s.add_covariance(np.asarray(result["cov"])) + for bins, result in pairs: + sio.add_cosebis(s, bins, result["En"], scale_cut, Bn=result["Bn"]) + if covariance is not None: + s.add_covariance(np.asarray(covariance)) return s -def pure_eb_to_sacc(nz, metadata, theta, eb, covariance=None): - """One pure-E/B part: the six ``sacc_io.PURE_KEYS`` blocks. +def pure_eb_to_sacc(nz, metadata, pairs, covariance=None): + """Write each pair's ``theta`` and six arrays in ``PURE_KEYS`` order. - ``eb`` is a mapping with the six keys (``xip_E`` … ``xim_amb``); each array - is sampled at ``theta``. ``covariance``, when given, is the dense block in - ``PURE_KEYS`` order (matching ``b_modes._EB_KEYS`` and the insertion order). + The order is ``xip_E``, ``xim_E``, ``xip_B``, ``xim_B``, ``xip_amb``, + ``xim_amb``; the covariance follows this order within each pair. """ s = sio.new_sacc(nz, metadata) - sio.add_pure_eb(s, BIN, theta, **{key: eb[key] for key in sio.PURE_KEYS}) + for bins, eb in pairs: + sio.add_pure_eb(s, bins, eb["theta"], **{key: eb[key] for key in sio.PURE_KEYS}) if covariance is not None: s.add_covariance(np.asarray(covariance)) return s -def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov_th=None): +def rho_tau_to_sacc(nz, metadata, rho_stats, tau_by_bin, tau_cov_th_by_bin=None): """One ρ/τ part: ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage. - ``rho_stats`` / ``tau_stats`` are the ``shear_psf_leakage`` handler tables - (columns ``theta``, ``rho_{k}_p``, ``varrho_{k}_p``, … and the τ analogue). - ρ carries a ``varrho`` diagonal; τ carries a ``vartau`` diagonal, with - ``tau_cov_th`` — a ``(3·nbin, 3·nbin)`` k-major matrix over the τ-plus points - only — scattered into the τ-plus rows/columns when given. ``tau_cov_th=None`` - leaves the τ block fully diagonal. + ``rho_stats`` contains ``theta``, ``rho_{k}_p``, ``rho_{k}_m``, + ``varrho_{k}_p`` and ``varrho_{k}_m`` for k=0,…,5. Each table in + ``tau_by_bin`` contains ``theta``, ``tau_{k}_p``, ``tau_{k}_m``, + ``vartau_{k}_p`` and ``vartau_{k}_m`` for k=0,2,5. + ρ carries a variance diagonal. Each τ block has a variance diagonal, + with its optional CovTauTh matrix scattered into its plus rows. CovTauTh + has shape (3·nθ, 3·nθ), ordered k-major (0, 2, 5), plus only. + Cross-bin covariances are approximated as zero. """ s = sio.new_sacc(nz, metadata) theta_rho = np.asarray(rho_stats["theta"]) @@ -222,51 +275,57 @@ def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov_th=None): np.asarray(rho_stats[f"rho_{k}_p"]), np.asarray(rho_stats[f"rho_{k}_m"]), ) - theta_tau = np.asarray(tau_stats["theta"]) - for k in TAU_K: - sio.add_tau( - s, - BIN, - k, - theta_tau, - np.asarray(tau_stats[f"tau_{k}_p"]), - np.asarray(tau_stats[f"tau_{k}_m"]), - ) - nbin = len(theta_tau) rho_var = np.concatenate( [ np.concatenate([rho_stats[f"varrho_{k}_p"], rho_stats[f"varrho_{k}_m"]]) for k in RHO_K ] ) - tau_var = np.concatenate( - [ - np.concatenate([tau_stats[f"vartau_{k}_p"], tau_stats[f"vartau_{k}_m"]]) - for k in TAU_K - ] - ) - if tau_cov_th is None: - s.add_covariance(np.concatenate([rho_var, tau_var])) - return s - tau_cov_th = np.asarray(tau_cov_th) - n_plus = len(TAU_K) * nbin - if tau_cov_th.shape != (n_plus, n_plus): - raise ValueError( - f"tau_cov_th shape {tau_cov_th.shape} does not match the " - f"{n_plus} τ-plus points ({len(TAU_K)} indices × {nbin} bins) — " - "CovTauTh.build_cov returns one (plus-folded) component per τ index" + theory = dict(tau_cov_th_by_bin or []) + blocks = [np.diag(rho_var)] + diagonals = [rho_var] + for bin_index, tau_stats in tau_by_bin: + theta = np.asarray(tau_stats["theta"]) + for k in TAU_K: + sio.add_tau( + s, + (bin_index, bin_index), + k, + theta, + np.asarray(tau_stats[f"tau_{k}_p"]), + np.asarray(tau_stats[f"tau_{k}_m"]), + ) + tau_var = np.concatenate( + [ + np.concatenate([tau_stats[f"vartau_{k}_p"], tau_stats[f"vartau_{k}_m"]]) + for k in TAU_K + ] ) - n_rho, n_tau = len(rho_var), len(tau_var) - tau_block = np.diag(tau_var) - # τ-plus local positions in the τ block, k-major (per-k layout is [+; −]). - plus = np.concatenate( - [np.arange(2 * i * nbin, 2 * i * nbin + nbin) for i in range(len(TAU_K))] - ) - tau_block[np.ix_(plus, plus)] = tau_cov_th - full = np.zeros((n_rho + n_tau, n_rho + n_tau)) - full[:n_rho, :n_rho] = np.diag(rho_var) - full[n_rho:, n_rho:] = tau_block - s.add_covariance(full) + diagonals.append(tau_var) + block = np.diag(tau_var) + cov = theory.get(bin_index) + if cov is not None: + cov = np.asarray(cov) + nbin = len(theta) + n_plus = len(TAU_K) * nbin + if cov.shape != (n_plus, n_plus): + raise ValueError( + f"tau_cov_th shape {cov.shape} does not match {n_plus} τ-plus points" + ) + plus = np.concatenate( + [ + np.arange(2 * i * nbin, 2 * i * nbin + nbin) + for i in range(len(TAU_K)) + ] + ) + block[np.ix_(plus, plus)] = cov + blocks.append(block) + if not theory: + s.add_covariance(np.concatenate(diagonals)) + else: + from scipy.linalg import block_diag + + s.add_covariance(block_diag(*blocks)) return s diff --git a/src/sp_validation/rho_tau.py b/src/sp_validation/rho_tau.py index 454c08917..2da5ccb3b 100644 --- a/src/sp_validation/rho_tau.py +++ b/src/sp_validation/rho_tau.py @@ -457,6 +457,20 @@ def get_theory_cov( return +def rho_cache_valid(rho_path, cov_rho_path, treecorr_config, force_run=False): + """Whether a cached ρ measurement and its jackknife covariance can be reused. + + The ρ FITS must record the current means and patched configurations + (:func:`~sp_validation.correlation.measurement_matches`); the PSF-only + result is then shared by every source bin of a tomographic run. + """ + return ( + not force_run + and Path(cov_rho_path).exists() + and measurement_matches(rho_path, treecorr_config) + ) + + def get_jackknife_cov( config, version, @@ -474,7 +488,6 @@ def get_jackknife_cov( Compute the covariance matrix of rho and tau-statistics using the jackknife method. Also compute rho and tau-statistics. """ - # TODO: Reorganise this to avoid recomputing unnecessarily the rho-stats multiple times. rho_filename = f"rho_stats_{base_rho}.fits" tau_filename = f"tau_stats_{base_tau}.fits" tau_cov_path = Path(outdir) / f"cov_tau_{base_tau}_jk.npy" @@ -524,6 +537,13 @@ def get_jackknife_cov( tau_stat_handler.catalogs.set_params(params, outdir) + rho_path = Path(outdir) / rho_filename + cov_rho_path = Path(outdir) / f"cov_rho_{base_rho}_jk.npy" + reuse_rho = rho_cache_valid(rho_path, cov_rho_path, treecorr_config, force_run) + if reuse_rho: + rho_stat_handler.load_rho_stats(rho_filename) + cached_rho_cov = np.load(cov_rho_path) + # shear_psf_leakage keys its catalogues by catalog_id and writes each # draw's covariance to cov_{rho,tau}_{catalog_id}.npy; base_tau names the # draws so that the bins of a tomographic run never share them. @@ -582,15 +602,18 @@ def chunks(i): mask=mask_gal, ) - # Compute and save rho stats - rho_means = _compute_stats_with_patches( - rho_stat_handler, - "rho", - catalog_id(i), - rho_filename, - save_cov=True, - means=rho_means, - ) + # The PSF-only result is shared by every source bin. + if reuse_rho: + np.save(chunks(i)[0], cached_rho_cov) + else: + rho_means = _compute_stats_with_patches( + rho_stat_handler, + "rho", + catalog_id(i), + rho_filename, + save_cov=True, + means=rho_means, + ) tau_means = _compute_stats_with_patches( tau_stat_handler, "tau", @@ -625,9 +648,9 @@ def chunks(i): tau_cov_path.parent.mkdir(parents=True, exist_ok=True) np.save(tau_cov_path, cov_tau) - cov_rho_path = Path(outdir) / f"cov_rho_{base_rho}_jk.npy" cov_rho_path.parent.mkdir(parents=True, exist_ok=True) - np.save(cov_rho_path, cov_rho) + if not reuse_rho: + np.save(cov_rho_path, cov_rho) return rho_stat_handler, tau_stat_handler diff --git a/src/sp_validation/tests/data/generate_test_cl_catalog_reference.py b/src/sp_validation/tests/data/generate_test_cl_catalog_reference.py index dbed4a606..4a4f753cb 100644 --- a/src/sp_validation/tests/data/generate_test_cl_catalog_reference.py +++ b/src/sp_validation/tests/data/generate_test_cl_catalog_reference.py @@ -123,13 +123,7 @@ "tomo_bin_2_tomo_bin_2": {}, } } -out_path = cv._output_path(f"pseudo_cl_cat_{ver}.fits") -tomo_bin_ids, tomo_bin_pairs = cv._get_tomo_bins(ver) -for tomo_bin_a, tomo_bin_b in tomo_bin_pairs: - out_path = cv._output_path(f"pseudo_cl_cat_{ver}_{tomo_bin_a}_{tomo_bin_b}.fits") - cv.calculate_pseudo_cl_catalog( - ver, out_path, tomo_bin_a=tomo_bin_a, tomo_bin_b=tomo_bin_b - ) +cv.calculate_pseudo_cl(compute_tomography=True) # %% diff --git a/src/sp_validation/tests/regression/test_tomo_inka_merge_crosspol_transpose.py b/src/sp_validation/tests/regression/test_tomo_inka_merge_crosspol_transpose.py index 4d7be5dbd..c3c910abc 100644 --- a/src/sp_validation/tests/regression/test_tomo_inka_merge_crosspol_transpose.py +++ b/src/sp_validation/tests/regression/test_tomo_inka_merge_crosspol_transpose.py @@ -57,9 +57,6 @@ def _output_path_iNKA_block_cov(self, ver, tomo_bin_quad): return str(tmp_path / ("_".join(map(str, tomo_bin_quad)) + ".fits")) instance = Tiny() - instance._pseudo_cls = { - "v": {"tomo_bin_all_tomo_bin_all": {"pseudo_cl": {"ELL": np.arange(n_ell)}}} - } for ia, ib in itertools.combinations_with_replacement(range(n_s), 2): # [ell_a, pol_a, ell_b, pol_b] block = np.transpose(covariance[:, ia, :, :, ib, :], (1, 0, 3, 2)) @@ -94,3 +91,66 @@ def _output_path_iNKA_block_cov(self, ver, tomo_bin_quad): f"iNKA joint covariance: symmetry error={np.max(np.abs(joint - joint.T)):.3g}, " f"min eigenvalue={eigenvalues.min():.12g}" ) + + +def test_fresh_cv_merges_tomographic_covariance(tmp_path, monkeypatch): + from sp_validation.cosmo_val import CosmologyValidation + + cv = CosmologyValidation.__new__(CosmologyValidation) + cv.versions = ["v"] + cv.nside = 4 + cv.cell_method = "catalog" + cv.fiducial_input_inka = "uncoupled" + cv.noise_bias_method = "analytic" + cv.force_run = False + cv.cc = {"v": {"shear": "unused"}} + pairs = [(1, 1), (1, 2), (2, 2)] + monkeypatch.setattr(cv, "_get_tomo_bins", lambda ver: ([1, 2], pairs)) + monkeypatch.setattr(cv, "_output_path", lambda *a: str(tmp_path)) + monkeypatch.setattr( + cv, "_output_path_pseudo_cl_cov", lambda *a, **k: str(tmp_path / "merged.fits") + ) + monkeypatch.setattr( + cv, + "_output_path_iNKA_block_cov", + lambda ver, tomo_bin_quad: str( + tmp_path / ("_".join(map(str, tomo_bin_quad)) + ".fits") + ), + ) + monkeypatch.setattr( + cv, "get_fiducial_cl", lambda *a: {f"W{x}xW{y}": np.zeros(8) for x, y in pairs} + ) + monkeypatch.setattr(cv, "_get_tomographic_bin", lambda params, cat, b: cat) + monkeypatch.setattr(cv, "get_namaster_bin", lambda *a: None) + monkeypatch.setattr( + pseudo_cl, + "get_params_rho_tau", + lambda entry: { + k: k for k in ("ra_col", "dec_col", "e1_col", "e2_col", "w_col") + }, + ) + monkeypatch.setattr( + pseudo_cl, + "open_entry", + lambda *a: { + k: np.zeros(2) for k in ("ra_col", "dec_col", "e1_col", "e2_col", "w_col") + }, + ) + monkeypatch.setattr( + pseudo_cl.spv_pseudo_cl, + "get_field_and_workspace_from_catalog", + lambda *a, **k: (None, None, None), + ) + monkeypatch.setattr( + pseudo_cl.spv_pseudo_cl, + "get_pseudo_cl_iNKA_covariance", + lambda *a, **k: np.zeros((3, 4, 3, 4)), + ) + cv.pol_factor = 1 + cv.calculate_pseudo_cl_inka_cov(compute_tomography=True) + with pseudo_cl.fits.open(tmp_path / "merged.fits") as merged: + assert merged["COVAR_EE_EE"].data.shape == (9, 9) + assert all( + "pseudo_cl" not in cv._pseudo_cls["v"][f"tomo_bin_{a}_tomo_bin_{b}"] + for a, b in pairs + ) diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py index 5c340e242..e1aed2cb0 100644 --- a/src/sp_validation/tests/test_assemble_sacc.py +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -110,27 +110,24 @@ def get_bandpower_windows(self): cl = sw.pseudo_cl_to_sacc( nz, META, - ell, - cl_all, - _Wsp(), + [((0, 0), {"ell_eff": ell, "cl_all": cl_all, "wsp": _Wsp()})], covariance=None if "pseudo_cl" in cov_less else _spd(9, 2), ) co = sw.cosebis_to_sacc( nz, META, - { - "En": np.arange(1, 6) * 1e-6, - "Bn": np.arange(1, 6) * 1e-7, - "cov": _spd(10, 3), - }, + [((0, 0), {"En": np.arange(1, 6) * 1e-6, "Bn": np.arange(1, 6) * 1e-7})], (1.0, 100.0), + covariance=_spd(10, 3), ) eb_arrays = { key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS) } - eb = sw.pure_eb_to_sacc(nz, META, theta, eb_arrays, covariance=_spd(36, 4)) + eb = sw.pure_eb_to_sacc( + nz, META, [((0, 0), {"theta": theta, **eb_arrays})], covariance=_spd(36, 4) + ) rho = {"theta": theta} tau = {"theta": theta} @@ -143,7 +140,7 @@ def get_bandpower_windows(self): for suffix in ("p", "m"): tau[f"tau_{k}_{suffix}"] = rng.normal(size=6) * 1e-6 tau[f"vartau_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, 6) - rt = sw.rho_tau_to_sacc(nz, META, rho, tau) + rt = sw.rho_tau_to_sacc(nz, META, rho, [(0, tau)]) parts = { "xi_reporting": xi, diff --git a/src/sp_validation/tests/test_blind_leak_gate.py b/src/sp_validation/tests/test_blind_leak_gate.py index dee57e71b..34d592dc3 100644 --- a/src/sp_validation/tests/test_blind_leak_gate.py +++ b/src/sp_validation/tests/test_blind_leak_gate.py @@ -148,6 +148,35 @@ def produce(config_path, version, blind, output): ), outputs={"pseudo_cl": str(paths["cl"])}, ) + paths["tomo_integration"] = output / "tomo_integration.sacc" + script( + "run_2pcf.py", + params=dict( + ver=version, + **INTEGRATION, + npatch=1, + grid="integration", + cat_config=str(config_path), + output_dir=str(output), + compute_tomography=True, + ), + outputs={"sacc": str(paths["tomo_integration"])}, + ) + paths["tomo_cl"] = output / "tomo_cl.sacc" + script( + "generate_pseudo_cl.py", + params=dict( + version=version, + cat_config=str(config_path), + nside=32, + npatch=1, + binning="powspace", + nbins=4, + power=0.5, + compute_tomography=True, + ), + outputs={"pseudo_cl": str(paths["tomo_cl"])}, + ) # Covariance is exempt from blinding. A positive synthetic analytic matrix # avoids OneCovariance/external survey masks and keeps this a small gate. covariance = output / "cov.txt" @@ -197,6 +226,53 @@ def produce(config_path, version, blind, output): ), }, ) + tomo_cov = output / "tomo_integration_cov.dat" + np.savetxt(tomo_cov, np.eye(6 * INTEGRATION["nbins"]) * 1e-10) + paths["tomo_pure"] = output / "tomo_pure.sacc" + paths["tomo_cosebis"] = output / "tomo_cosebis.sacc" + script( + "cv_pure_eb.py", + params=dict( + version=version, + blind=blind, + fiducial_scale_cut=CUT, + integration=INTEGRATION, + **REPORTING, + ), + inputs={ + "xi_integration": str(paths["tomo_integration"]), + "cov_integration": str(tomo_cov), + }, + outputs={"npz": str(output / "tomo_pure.npz"), "sacc": str(paths["tomo_pure"])}, + ) + script( + "cv_cosebis.py", + params=dict( + version=version, + blind=blind, + fiducial_scale_cut=CUT, + nmodes=2, + scale_cuts=CUTS, + **INTEGRATION, + ), + inputs={"xi": str(paths["tomo_integration"]), "cov": str(tomo_cov)}, + outputs={ + "npz": str(output / "tomo_cosebis.npz"), + "sacc": str(paths["tomo_cosebis"]), + }, + ) + script( + "run_rho_tau.py", + params=dict( + ver=version, + **REPORTING, + npatch=1, + cat_config=str(config_path), + output_dir=str(output), + compute_tomography=True, + ), + ) + paths["tomo_rho_tau"] = next(output.rglob("rho_tau_*_tomo.sacc")) return paths, pure["results"], cosebis["results"] @@ -277,6 +353,45 @@ def reference(label, raw, sealed): derived.covariance.dense, np.eye(len(part.mean)) * 1e-10 ) + for name in ("tomo_integration", "tomo_cl", "tomo_pure", "tomo_cosebis"): + raw_part = sacc_io.load(raw_paths[name], blind="none") + sealed_part = sacc_io.load(sealed_paths[name], blind=blind) + if name in ("tomo_integration", "tomo_cl"): + np.testing.assert_allclose( + sealed_part.mean, + raw_part.mean + shift_for(raw_part, blind), + rtol=1e-9, + atol=1e-20, + ) + for pair in ((0, 0), (0, 1), (1, 1)): + tracers = tuple(f"source_{i}" for i in pair) + found = False + for kind in set(dp.data_type for dp in raw_part.data): + rows = raw_part.indices(kind, tracers=tracers) + if len(rows) and np.any( + ~np.isclose( + raw_part.mean[rows], + sealed_part.mean[rows], + rtol=5e-6, + atol=1e-25, + ) + ): + reference( + f"{name}/{pair}/{kind}", + raw_part.mean[rows], + sealed_part.mean[rows], + ) + found = True + assert found, f"No shifted {name} values for {pair}" + raw_tau = sacc_io.load(raw_paths["tomo_rho_tau"], blind="none") + sealed_tau = sacc_io.load(sealed_paths["tomo_rho_tau"], blind=blind) + np.testing.assert_allclose(raw_tau.mean, sealed_tau.mean, rtol=1e-10, atol=1e-20) + for i in range(2): + for k in (0, 2, 5): + theta, plus, minus = sacc_io.get_tau(sealed_tau, (i, i), k) + assert len(theta) == REPORTING["nbins"] + assert np.all(np.isfinite(plus)) and np.all(np.isfinite(minus)) + raw = sacc_io.load(raw_paths["integration"]) shifted = raw.copy() for dp, delta in zip(shifted.data, shift_for(raw, blind), strict=True): @@ -364,8 +479,8 @@ def reference(label, raw, sealed): # Check SACC numerical values too: a stamp and nonempty output aren't enough. nz = {0: sacc_io.get_nz(raw, 0)} expected_parts = { - "pure": pure_eb_to_sacc(nz, {}, expected_pure["theta"], expected_pure), - "cosebis": cosebis_to_sacc(nz, {}, expected_cosebis[CUT], CUT), + "pure": pure_eb_to_sacc(nz, {}, [((0, 0), expected_pure)]), + "cosebis": cosebis_to_sacc(nz, {}, [((0, 0), expected_cosebis[CUT])], CUT), } for name, expected in expected_parts.items(): part = sacc_io.load(sealed_paths["cl"].parent / f"{name}.sacc", blind=blind) @@ -462,6 +577,41 @@ def render(paths, version, blind, cat_config): ), outputs={"sacc": str(assembled)}, ) + tomo_cov_cl = out / "tomo_cl_cov.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + *[ + fits.ImageHDU(np.eye(12) * 1e-12, name=f"COVAR_{k}_{k}") + for k in ("EE", "BB", "EB") + ], + ] + ).writeto(tomo_cov_cl) + tomo_cov_xi = out / "tomo_reporting_cov.txt" + np.savetxt(tomo_cov_xi, np.eye(6 * REPORTING["nbins"]) * 1e-10) + tomo_assembled = out / "assembled_tomo.sacc" + tomo_names = ("xi_reporting", "pseudo_cl", "cosebis", "pure_eb", "rho_tau") + tomo_parts = ( + paths["tomo"], + paths["tomo_cl"], + paths["tomo_cosebis"], + paths["tomo_pure"], + paths["tomo_rho_tau"], + ) + script( + "assemble_sacc.py", + params=dict(version=version, blind=blind, expected=list(tomo_names)), + inputs={ + **dict(zip(tomo_names, map(str, tomo_parts))), + "xi_cov": str(tomo_cov_xi), + "pseudo_cl_cov": str(tomo_cov_cl), + }, + outputs={"sacc": str(tomo_assembled)}, + ) + np.testing.assert_array_equal( + sacc_io.load(tomo_assembled, blind=blind).mean, + np.concatenate([sacc_io.load(p, blind=blind).mean for p in tomo_parts]), + ) n_figures = 10 # Four pure-E/B, three COSEBIs, three Cℓ on both APIs. if (SCRIPTS / "cv_plot_diagnostic.py").exists(): script( @@ -603,7 +753,7 @@ def test_workflow_emits_no_unblinded_signal(tmp_path, monkeypatch, capfd, caplog observed = [] observe_renderers(observed, monkeypatch) # Observe high-level returns and retained state without replacing computation. - for name in ("calculate_2pcf_version", "calculate_pseudo_cl_catalog"): + for name in ("calculate_2pcf_version", "calculate_pseudo_cl"): original = getattr(CosmologyValidation, name) def watch(self, *args, _original=original, **kwargs): @@ -619,14 +769,6 @@ def watch(self, *args, _original=original, **kwargs): ) render(sealed_paths, version, "gate", config_path) assert_dashboard_refuses_stale_public(raw_paths, config_path, version) - cv = CosmologyValidation( - versions=[version], - catalog_config=str(config_path), - output_dir=str(tmp_path / "sealed"), - npatch=1, - ) - with pytest.raises(blinding.BlindingError, match="tomographic"): - cv.calculate_pseudo_cl(compute_tomography=True) captured = capfd.readouterr() logs = captured.out + captured.err + caplog.text references, concealed = references_and_shifts( diff --git a/src/sp_validation/tests/test_cosebis_tomography.py b/src/sp_validation/tests/test_cosebis_tomography.py index d58c66967..2b3671981 100644 --- a/src/sp_validation/tests/test_cosebis_tomography.py +++ b/src/sp_validation/tests/test_cosebis_tomography.py @@ -1,30 +1,320 @@ -"""Covariance selection at the tomographic COSEBIs boundary.""" +"""Tomographic derived covariances and blind inheritance.""" +import runpy +from pathlib import Path from types import SimpleNamespace +import numpy as np import pytest -from sp_validation.cosmo_val.cosebis import CosebisMixin +from sp_validation import b_modes, blinding, sacc_io +from sp_validation.cosmo_val import sacc_writers +from sp_validation.cosmo_val.derived_covariance import propagate +SCRIPTS = Path(__file__).resolve().parents[3] / "workflow" / "scripts" -@pytest.mark.parametrize( - "options", - [ - {}, - {"scale_cuts": [(1, 10)]}, - {"evaluate_all_scale_cuts": True}, - ], -) -def test_tomographic_cosebis_rejects_single_covariance(options): - calls = [] + +@pytest.fixture +def synthetic(tmp_path): + n = 64 + edges = np.geomspace(0.1, 300, n + 1) + theta = np.sqrt(edges[:-1] * edges[1:]) + pairs = [(i, j) for i in range(3) for j in range(i, 3)] + z = np.linspace(0.01, 2, 10) + nz = {i: (z, np.exp(-(((z - 0.5 - i * 0.1) / 0.3) ** 2))) for i in range(3)} + rng = np.random.default_rng(42) + a = rng.normal(size=(len(pairs) * 2 * n, 30)) + cov = a @ a.T + np.eye(len(a)) * 4 + measurements = [ + ( + pair, + dict( + theta=theta, + theta_nom=theta, + xip=theta * 1e-5, + xim=theta * 0.3e-5, + weight=(1 + k * theta / 300), + npairs=np.ones(n), + ), + ) + for k, pair in enumerate(pairs) + ] + xi = sacc_writers.xi_pairs_to_sacc( + nz, {"npatch": 100}, measurements, grid="integration", covariance=cov + ) + blind = blinding.init("toy", {"paths": {"blinds": str(tmp_path / "blinds")}}) + path = tmp_path / "xi.sacc" + sacc_io.save( + xi, + path, + blind=blind, + theory=lambda params, s: np.full(len(s.mean), params["S8"]), + ) + return path, pairs, theta, edges, cov + + +def test_single_pair_operators_match_existing(synthetic): + path, pairs, theta, edges, cov = synthetic + part = sacc_io.load(path, blind="toy") + gg = sacc_io.xi_correlation(part, pairs[0], grid="integration") + c = cov[:128, :128] + reporting = np.geomspace(2, 60, 4) + result = b_modes.calculate_pure_eb_correlation( + theta, gg.xip, gg.xim, gg.weight, edges, c, reporting + ) + op = b_modes.pure_eb_operator(gg.weight, edges, reporting)[0] + np.testing.assert_array_equal(result["cov"], op @ c @ op.T) + cuts = [(edges[4], edges[-5])] + result = b_modes.cosebis_scan_from_xi( + theta, gg.xip, gg.xim, c, edges[:-1], edges[1:], nmodes=2, scale_cuts=cuts + )[cuts[0]] + op = b_modes.cosebis_operator(theta, edges[:-1], edges[1:], cuts[0], 2) + np.testing.assert_allclose( + propagate([op], c, cosebis=True), result["cov"], rtol=1e-12, atol=1e-25 + ) + + +@pytest.mark.parametrize("kind", ["pure", "cosebis"]) +@pytest.mark.parametrize("analytic", [False, True]) +def test_workflow_full_pair_covariance_and_stamp( + synthetic, tmp_path, monkeypatch, kind, analytic +): + monkeypatch.syspath_prepend(str(SCRIPTS)) + import cv_runner + + monkeypatch.setattr(cv_runner, "_unbuffer_streams", lambda: None) + monkeypatch.setattr(cv_runner, "verify_outputs", lambda _: None) + path, pairs, theta, edges, cov = synthetic + part = sacc_io.load(path, blind="toy") + n = len(theta) + # Construct OneCovariance's type-major layout independently. + perm = [ + k * 2 * n + t * n + q + for t in range(2) + for k in range(len(pairs)) + for q in range(n) + ] + cov_path = tmp_path / "cov.txt" + np.savetxt(cov_path, cov[np.ix_(perm, perm)]) + inputs = { + "xi": path, + "xi_integration": path, + "cov": str(cov_path) if analytic else None, + "cov_integration": str(cov_path) if analytic else None, + } + params = dict( + version="v", + blind="toy", + fiducial_scale_cut=(edges[4], edges[-5]), + min_sep=2.0, + max_sep=60.0, + nbins=3, + nmodes=2, + scale_cuts=[(edges[4], edges[-5])], + integration=dict(min_sep=edges[0], max_sep=edges[-1], nbins=n), + ) + if kind == "cosebis": + params.update(min_sep=edges[0], max_sep=edges[-1], nbins=n) + output = { + "npz": str(tmp_path / "derived.npz"), + "sacc": str(tmp_path / "derived.sacc"), + } + smk = SimpleNamespace(input=inputs, output=output, params=params) + monkeypatch.syspath_prepend(str(SCRIPTS)) + script = "cv_pure_eb.py" if kind == "pure" else "cv_cosebis.py" + runpy.run_path(str(SCRIPTS / script), init_globals={"snakemake": smk}) + out = sacc_io.load(output["sacc"], blind="toy") + assert sacc_io.stamp(out) == sacc_io.stamp(part) == "toy" + operators = [] + for pair in pairs: + gg = sacc_io.xi_correlation(part, pair, grid="integration") + if kind == "pure": + op = b_modes.pure_eb_operator(gg.weight, edges, np.geomspace(2, 60, 4))[0] + else: + op = b_modes.cosebis_operator( + theta, edges[:-1], edges[1:], params["scale_cuts"][0], 2 + ) + operators.append(op) + expected = np.zeros_like(out.covariance.dense) + for a, pa in enumerate(pairs): + ra = [ + i + for i, dp in enumerate(out.data) + if tuple(int(t.removeprefix("source_")) for t in dp.tracers) == pa + ] + for b, pb in enumerate(pairs): + rb = [ + i + for i, dp in enumerate(out.data) + if tuple(int(t.removeprefix("source_")) for t in dp.tracers) == pb + ] + source = cov[a * 2 * n : (a + 1) * 2 * n, b * 2 * n : (b + 1) * 2 * n] + block = operators[a] @ source @ operators[b].T + if kind == "cosebis": + block[:2, 2:] = 0 + block[2:, :2] = 0 + expected[np.ix_(ra, rb)] = block + # Block-wise and full-matrix products round differently across BLAS builds; + # a misplaced block would differ at order unity. + np.testing.assert_allclose(out.covariance.dense, expected, rtol=1e-10, atol=1e-23) + + +@pytest.mark.parametrize("kind", ["pure", "cosebis"]) +def test_single_pair_workflow_preserves_existing_results( + synthetic, tmp_path, monkeypatch, kind +): + monkeypatch.syspath_prepend(str(SCRIPTS)) + import cv_runner + + monkeypatch.setattr(cv_runner, "_unbuffer_streams", lambda: None) + monkeypatch.setattr(cv_runner, "verify_outputs", lambda _: None) + path, _, theta, edges, cov = synthetic + part = sacc_io.load(path, blind="toy") + gg = sacc_io.xi_correlation(part, (0, 0), grid="integration") + n = len(theta) + single_cov = cov[: 2 * n, : 2 * n] + single = sacc_writers.xi_pairs_to_sacc( + {0: sacc_io.get_nz(part, 0)}, + part.metadata, + [ + ( + (0, 0), + dict( + theta=gg.meanr, + theta_nom=gg.rnom, + xip=gg.xip, + xim=gg.xim, + weight=gg.weight, + npairs=gg.npairs, + ), + ) + ], + grid="integration", + covariance=single_cov, + ) + single_path = tmp_path / "single.sacc" + sacc_io.save(single, single_path, derived_from=[part]) + cov_path = tmp_path / "single.txt" + np.savetxt(cov_path, single_cov) + cuts = [(edges[4], edges[-5])] + params = dict( + version="v", + blind="toy", + fiducial_scale_cut=cuts[0], + min_sep=2.0, + max_sep=60.0, + nbins=3, + nmodes=2, + scale_cuts=cuts, + integration=dict(min_sep=edges[0], max_sep=edges[-1], nbins=n), + ) + if kind == "cosebis": + params.update(min_sep=edges[0], max_sep=edges[-1], nbins=n) + result = b_modes.cosebis_scan_from_xi( + gg.meanr, + gg.xip, + gg.xim, + single_cov, + *b_modes.log_bin_edges(edges[0], edges[-1], n), + nmodes=2, + scale_cuts=cuts, + )[cuts[0]] + expected_mean = np.r_[result["En"], result["Bn"]] + script = "cv_cosebis.py" + else: + result = b_modes.calculate_pure_eb_correlation( + gg.meanr, + gg.xip, + gg.xim, + gg.weight, + edges, + single_cov, + np.geomspace(2, 60, 4), + ) + expected_mean = np.concatenate([result[key] for key in sacc_io.PURE_KEYS]) + script = "cv_pure_eb.py" + smk = SimpleNamespace( + params=params, + input=dict( + xi=single_path, + xi_integration=single_path, + cov=cov_path, + cov_integration=cov_path, + ), + output=dict( + npz=str(tmp_path / "single.npz"), sacc=str(tmp_path / "single-derived.sacc") + ), + ) + runpy.run_path(str(SCRIPTS / script), init_globals={"snakemake": smk}) + out = sacc_io.load(smk.output["sacc"], blind="toy") + np.testing.assert_array_equal(out.mean, expected_mean) + np.testing.assert_array_equal(out.covariance.dense, result["cov"]) + + +@pytest.mark.parametrize("kind", ["pure", "cosebis"]) +@pytest.mark.parametrize("analytic", [False, True]) +def test_tomographic_mixins_take_each_pair_covariance( + synthetic, tmp_path, monkeypatch, kind, analytic +): + from sp_validation.cosmo_val import cosebis, pure_eb + + path, pairs, theta, edges, cov = synthetic + part = sacc_io.load(path, blind="toy") + n = len(theta) + order = [ + k * 2 * n + t * n + q + for t in range(2) + for k in range(len(pairs)) + for q in range(n) + ] + cov_path = tmp_path / "tomo_cov.txt" + np.savetxt(cov_path, cov[np.ix_(order, order)]) + views = { + str(pair): sacc_io.xi_correlation(part, pair, grid="integration") + for pair in pairs + } cv = SimpleNamespace( - npatch=7, print_start=lambda *args: None, - _binning=lambda *args: {}, - calculate_2pcf_version=lambda *args, **kwargs: calls.append(kwargs) or {}, + calculate_2pcf_version=lambda *args, **kwargs: views, + xi_tomo_parts={("v", "integration"): part}, + _binning=lambda a, b, c: dict(min_sep=a, max_sep=b, nbins=c), ) - with pytest.raises(ValueError, match="cov_path.*single.*tomographic bin pair"): - CosebisMixin.calculate_cosebis( - cv, "v", cov_path="xi_cov.txt", compute_tomography=True, **options + seen = [] + if kind == "cosebis": + + def fake(gg, *, cov_xi, npatch, **kwargs): + seen.append((cov_xi, npatch)) + return {(1, 2): {}} + + monkeypatch.setattr(cosebis, "calculate_cosebis", fake) + cosebis.CosebisMixin.calculate_cosebis( + cv, + "v", + compute_tomography=True, + cov_path=str(cov_path) if analytic else None, + ) + else: + + def fake(*args, npatch): + seen.append((args[5], npatch)) + return {} + + monkeypatch.setattr(pure_eb, "calculate_pure_eb_correlation", fake) + pure_eb.PureEBMixin.calculate_pure_eb( + cv, + "v", + min_sep=2, + max_sep=60, + nbins=3, + min_sep_int=edges[0], + max_sep_int=edges[-1], + nbins_int=n, + compute_tomography=True, + cov_path_int=str(cov_path) if analytic else None, + ) + assert len(seen) == len(pairs) + for i, (block, npatch) in enumerate(seen): + np.testing.assert_array_equal( + block, cov[i * 2 * n : (i + 1) * 2 * n, i * 2 * n : (i + 1) * 2 * n] ) - assert calls == [] + assert npatch == (None if analytic else 100) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 6247bc6e5..373d717bc 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -823,10 +823,6 @@ def test_aperture_mass_means_are_layout_independent(self, tmp_path, monkeypatch) np.testing.assert_array_equal(measured[0][key], measured[1][key]) assert not np.array_equal(measured[0]["varmapsq"], measured[1]["varmapsq"]) - def test_a_blinded_catalogue_refuses_tomographic_signal(self, tmp_path): - """Blinding shifts the ("all", "all") parts only, so a blinded - catalogue's tomographic ξ± and pseudo-Cℓ are refused before measuring.""" - @staticmethod def _use_tomographic_nz(params, version, tmp_path, nbin=2): """Point ``version`` at an n(z) file with one column per tomographic bin.""" @@ -914,32 +910,92 @@ def test_a_blinded_catalogue_conceals_tomographic_xi(self, tmp_path): for key in public: assert not np.allclose(blinded[key].xip, public[key].xip) - def test_a_blinded_catalogue_refuses_tomographic_pseudo_cl(self, tmp_path): - """Blinding covers the tomographic ξ± parts but not the pseudo-Cℓ FITS - tables, so a blinded catalogue's tomographic pseudo-Cℓ are refused.""" + def test_a_blinded_catalogue_conceals_tomographic_pseudo_cl( + self, tmp_path, monkeypatch + ): + """Every pair is sealed with its own tracers, including on stale reuse.""" + pytest.importorskip("smokescreen") from sp_validation import blinding params, version = self._write_synthetic_catalogs( tmp_path, n_gal=400, with_tomography=True ) + self._use_tomographic_nz(params, version, tmp_path) + + def theory(p, s): + values = [] + for dp in s.data: + a, b = (s.tracers[name] for name in dp.tracers) + za = np.sum(a.z * a.nz) / np.sum(a.nz) + zb = np.sum(b.z * b.nz) / np.sum(b.nz) + values.append( + p["S8"] * (za + zb) if dp.data_type == sacc_io.CL_EE else 0 + ) + return np.array(values) + + monkeypatch.setattr(blinding, "shear", theory) + + class Workspace: + def get_bandpower_windows(self): + return np.ones((4, 2, 4, 8)) + + calls = [] + + def measure(ver, a, b): + calls.append((a, b)) + return np.array([3.0, 6.0]), np.full((4, 2), a + b), Workspace(), None + + public_cv = CosmologyValidation(versions=[version], npatch=1, **params) + public_cv.cell_method = "catalog" + monkeypatch.setattr(public_cv, "_measure_pseudo_cl_catalog", measure) + public_cv.calculate_pseudo_cl(compute_tomography=True) + path = public_cv._output_path_pseudo_cl(version, tomography=True) + public = sacc_io.load(path) + assert sacc_io.stamp(public) == "none" + config = yaml.safe_load(open(params["catalog_config"])) config["paths"]["blinds"] = str(tmp_path / "blinds") config[version]["blind"] = "toy" with open(params["catalog_config"], "w") as f: yaml.dump(config, f, sort_keys=False) - blinding.init("toy", config) + blind = blinding.init("toy", config) cv = CosmologyValidation(versions=[version], npatch=1, **params) - - with pytest.raises(blinding.BlindingError, match="tomographic"): - cv.calculate_pseudo_cl(compute_tomography=True) - # Direct producers must refuse before opening even a missing catalogue. - cv.cc[version]["shear"]["path"] = str(tmp_path / "missing.fits") - out = tmp_path / "tomographic.fits" - with pytest.raises(blinding.BlindingError, match="tomographic"): - cv.calculate_pseudo_cl_map(version, 16, out, 1, 2) - with pytest.raises(blinding.BlindingError, match="tomographic"): - cv.calculate_pseudo_cl_catalog(version, out, 1, 2) - assert not out.exists() + cv.cell_method = "catalog" + monkeypatch.setattr(cv, "_measure_pseudo_cl_catalog", measure) + assert cv.calculate_pseudo_cl(compute_tomography=True) is None + sealed = sacc_io.load(path) + assert sacc_io.stamp(sealed) == "toy" + public_methods = [ + name for name in dir(cv) if name.startswith("calculate_pseudo_cl") + ] + assert "calculate_pseudo_cl_map" not in public_methods + assert "calculate_pseudo_cl_catalog" not in public_methods + for bins, pair in zip(((0, 0), (0, 1), (1, 1)), ((1, 1), (1, 2), (2, 2))): + stored = cv._pseudo_cls[version][f"tomo_bin_{pair[0]}_tomo_bin_{pair[1]}"][ + "pseudo_cl" + ] + ell, ee, bb, eb, _ = sacc_io.get_pseudo_cl(sealed, bins) + for key, values in ( + ("ELL", ell), + ("EE", ee), + ("BB", bb), + ("EB", eb), + ("BE", eb), + ): + np.testing.assert_array_equal(stored[key], values) + assert len(calls) == 6 # the public part was stale under the new blind + raw = public.copy() + del raw.metadata["blind"] + expected = blinding.conceal(raw, blind, theory=theory) + np.testing.assert_allclose(sealed.mean, expected.mean, rtol=0, atol=1e-14) + shifts = [] + for bins in ((0, 0), (0, 1), (1, 1)): + shifted = sacc_io.get_pseudo_cl(sealed, bins)[1] + original = sacc_io.get_pseudo_cl(public, bins)[1] + shifts.append(shifted[0] - original[0]) + assert len(set(shifts)) == 3 + cv.calculate_pseudo_cl(compute_tomography=True) + assert len(calls) == 6 @pytest.mark.parametrize("stamp", ["none", "toy"]) def test_held_xi_matches_declared_blind(self, tmp_path, stamp): diff --git a/src/sp_validation/tests/test_namaster_covariance_order.py b/src/sp_validation/tests/test_namaster_covariance_order.py new file mode 100644 index 000000000..04bebf165 --- /dev/null +++ b/src/sp_validation/tests/test_namaster_covariance_order.py @@ -0,0 +1,54 @@ +"""Independent checks of NaMaster covariance ordering.""" + +import numpy as np +from astropy.io import fits + +from sp_validation import sacc_io as sio +from sp_validation.cosmo_val.sacc_writers import namaster_cl_covariance + + +def test_namaster_covariance_matches_written_point_labels(tmp_path): + pairs = [(i, j) for i in range(3) for j in range(i, 3)] + ells = [20.0, 50.0, 90.0] + types = {sio.CL_EE: "EE", sio.CL_BB: "BB", sio.CL_EB: "EB"} + nz = {i: (np.array([0.1, 0.5, 1.0]), np.array([0.1, 1.0, 0.1])) for i in range(3)} + part = sio.new_sacc(nz, {}) + # A noncanonical insertion order makes the lookup depend on point labels. + for i, j in reversed(pairs): + sio.add_pseudo_cl( + part, + (i, j), + ells, + np.zeros(3), + np.zeros(3), + np.zeros(3), + window_ells=ells, + window_weights=np.eye(3), + ) + n = len(pairs) * len(ells) + rng = np.random.default_rng(4) + blocks = {} + hdus = [fits.PrimaryHDU()] + for pol in types.values(): + a = rng.normal(size=(n, n)) + blocks[pol] = a + a.T + hdus.append(fits.ImageHDU(blocks[pol], name=f"COVAR_{pol}_{pol}")) + path = tmp_path / "cov.fits" + fits.HDUList(hdus).writeto(path) + + labels = [ + ( + types[dp.data_type], + tuple(int(t.split("_")[1]) for t in dp.tracers), + dp.tags["ell"], + ) + for dp in part.data + ] + source = [(p, pair, ell) for p in types.values() for pair in pairs for ell in ells] + matrix = np.zeros((len(source), len(source))) + for k, pol in enumerate(types.values()): + matrix[k * n : (k + 1) * n, k * n : (k + 1) * n] = blocks[pol] + perm = [source.index(label) for label in labels] + np.testing.assert_array_equal( + namaster_cl_covariance(part, path), matrix[np.ix_(perm, perm)] + ) diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 2c8bdcf8a..929d3fb46 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -21,7 +21,7 @@ * ``get_n_gal_map`` -- weighted galaxy number-density map * ``get_pseudo_cls_map`` -- map-based pseudo-Cl (decoupled EE/EB/BE/BB) * ``get_pseudo_cls_catalog`` -- catalog-based pseudo-Cl (NmtFieldCatalog) -* ``calculate_pseudo_cl_catalog`` -- the deterministic end-to-end catalog path +* ``_measure_pseudo_cl_catalog`` -- the deterministic end-to-end catalog path * ``apply_random_rotation`` -- pinned by INVARIANT + seed reproducibility (see below) REPRODUCIBILITY / TOLERANCE (measured over 3 independent container processes) @@ -32,7 +32,7 @@ float-noise floor of 0, yet a hair of margin so an exact-equality assertion doesn't make the suite brittle to a future NaMaster build. * ``get_pseudo_cls_catalog`` (the ``NmtFieldCatalog`` path) and the end-to-end - ``calculate_pseudo_cl_catalog`` are NOT bitwise-reproducible: their reduction + ``_measure_pseudo_cl_catalog`` are NOT bitwise-reproducible: their reduction order drifts at the ~2e-12 relative level (worst case measured 2.1e-12, absolute 2.7e-19). Pinned at ``rtol=1e-6 / atol=1e-12`` -- ~6 orders of magnitude above that float-noise floor (no flakiness margin consumed) yet @@ -61,7 +61,6 @@ from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation from sp_validation.cosmo_val.pseudo_cl import _apply_pixel_window_to_fiducial_cl -from sp_validation.cosmo_val.sacc_writers import BIN as SACC_BIN from sp_validation.pseudo_cl import ( apply_random_rotation, bandpower_window_from_workspace, @@ -458,7 +457,7 @@ def test_get_n_gal_map(cv, cat_and_params): # get_pseudo_cls_map -- map-based pseudo-Cl (bitwise-stable) # =========================================================================== def _build_shear_map(cv, cat_gal, params): - """Replicate calculate_pseudo_cl_map's weighted shear-map construction.""" + """Replicate _measure_pseudo_cl_map's weighted shear-map construction.""" unique_pix, _idx, idx_rep = cv.get_pixels(params, NSIDE, cat_gal) n_gal = cv.get_n_gal_map( params, NSIDE, cat_gal, unique_pix=unique_pix, idx=_idx, idx_rep=idx_rep @@ -785,34 +784,30 @@ def test_apply_random_rotation_reproducible_with_seed(cv, cat_and_params): # =========================================================================== -# calculate_pseudo_cl_catalog -- deterministic end-to-end catalog path +# _measure_pseudo_cl_catalog -- deterministic end-to-end catalog path # =========================================================================== -def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path, monkeypatch): +def test__measure_pseudo_cl_catalog_end_to_end(cv, tmp_path, monkeypatch): """Catalog SACC output retains the unwindowed workspace bandpower window.""" ver = cv._test_version cv.cell_method = "catalog" cv._pseudo_cls = {ver: {"tomo_bin_all_tomo_bin_all": {}}} saved = {} - save_pseudo_cl = cv._save_pseudo_cl - - def capture_workspace( - ver, out_path, tomo_bin_pair, ell_eff, cl_all, wsp, nside=None - ): - saved["workspace"] = wsp - saved["nside"] = nside - return save_pseudo_cl( - ver, out_path, tomo_bin_pair, ell_eff, cl_all, wsp, nside=nside - ) + measure = cv._measure_pseudo_cl_catalog + + def capture_workspace(*args): + result = measure(*args) + saved["workspace"], saved["nside"] = result[2:] + return result - monkeypatch.setattr(cv, "_save_pseudo_cl", capture_workspace) + monkeypatch.setattr(cv, "_measure_pseudo_cl_catalog", capture_workspace) out_path = cv._output_path(f"pseudo_cl_{ver}.sacc") - cv.calculate_pseudo_cl_catalog(ver, out_path, tomo_bin_a="all", tomo_bin_b="all") + cv.calculate_pseudo_cl(compute_tomography=False, out_path=out_path) assert os.path.exists(out_path) s = sacc_io.load(out_path) - ell, ee, bb, eb, window = sacc_io.get_pseudo_cl(s, SACC_BIN) + ell, ee, bb, eb, window = sacc_io.get_pseudo_cl(s, (0, 0)) assert window is not None # the shared BandpowerWindow rides the part - readback = cv._load_pseudo_cl(ver, out_path, ("all", "all")) + readback = cv._pseudo_cls[ver]["tomo_bin_all_tomo_bin_all"]["pseudo_cl"] npt.assert_array_equal(readback["BE"], eb) assert not np.shares_memory(readback["BE"], readback["EB"]) assert saved["nside"] is None @@ -889,30 +884,26 @@ def capture_workspace( npt.assert_allclose(ee, cl_prim[0], rtol=RTOL_CAT, atol=ATOL_CAT) -def test_calculate_pseudo_cl_map_sacc_pixel_window(cv, monkeypatch): +def test__measure_pseudo_cl_map_sacc_pixel_window(cv, monkeypatch): """Map SACC output multiplies its workspace window by pw²(ℓ).""" ver = cv._test_version cv.cell_method = "map" cv.noise_bias_method = "analytic" cv._pseudo_cls = {ver: {"tomo_bin_all_tomo_bin_all": {}}} saved = {} - save_pseudo_cl = cv._save_pseudo_cl - - def capture_workspace( - ver, out_path, tomo_bin_pair, ell_eff, cl_all, wsp, nside=None - ): - saved["workspace"] = wsp - saved["nside"] = nside - return save_pseudo_cl( - ver, out_path, tomo_bin_pair, ell_eff, cl_all, wsp, nside=nside - ) + measure = cv._measure_pseudo_cl_map - monkeypatch.setattr(cv, "_save_pseudo_cl", capture_workspace) + def capture_workspace(*args): + result = measure(*args) + saved["workspace"], saved["nside"] = result[2:] + return result + + monkeypatch.setattr(cv, "_measure_pseudo_cl_map", capture_workspace) out_path = cv._output_path(f"pseudo_cl_map_{ver}.sacc") - cv.calculate_pseudo_cl_map(ver, NSIDE, out_path, "all", "all") + cv.calculate_pseudo_cl(compute_tomography=False, out_path=out_path) s = sacc_io.load(out_path) - _ell, _ee, _bb, _eb, window = sacc_io.get_pseudo_cl(s, SACC_BIN) + _ell, _ee, _bb, _eb, window = sacc_io.get_pseudo_cl(s, (0, 0)) assert saved["nside"] == NSIDE window_ells, unwindowed_weights = bandpower_window_from_workspace( saved["workspace"] @@ -947,11 +938,18 @@ def test_fiducial_pixel_window_applies_only_to_map(): npt.assert_array_equal(fiducial[key], cl) -def test_calculate_pseudo_cl_catalog_end_to_end_tomo(cv, tmp_path): - """End-to-end catalog path: FITS round-trip of ell + EE/EB/BB. +def test_calculate_pseudo_cl_tomography_requires_one_nz_per_bin(cv): + """Missing n(z) columns are rejected before measuring any pair.""" + cv.cell_method = "catalog" + with pytest.raises(ValueError, match=r"n\(z\).*[2] tomographic"): + cv.calculate_pseudo_cl(compute_tomography=True) + + +def test__measure_pseudo_cl_catalog_end_to_end_tomo(cv, tmp_path): + """End-to-end catalog path: SACC round-trip of ell + EE/EB/BB. The catalog method has no random noise debiasing, so it is reproducible to - the same ~2e-12 catalog-path float noise. save_pseudo_cl stores ELL/EE/EB/BB + the same ~2e-12 catalog-path float noise. The part stores ELL/EE/EB/BB (it drops the BE row); we pin the round-tripped table. """ ver = cv._test_version @@ -962,26 +960,23 @@ def test_calculate_pseudo_cl_catalog_end_to_end_tomo(cv, tmp_path): "tomo_bin_2_tomo_bin_2": {}, } } - out_path = cv._output_path(f"pseudo_cl_cat_{ver}.fits") - tomo_bin_ids, tomo_bin_pairs = cv._get_tomo_bins(ver) + cv.cell_method = "catalog" + nz_path = tmp_path / "nz_tomo.txt" + z = np.linspace(0.1, 2, 20) + np.savetxt(nz_path, np.column_stack((z, np.exp(-z), np.exp(-2 * z)))) + cv.cc[ver]["shear"]["redshift_path"] = str(nz_path) + _, tomo_bin_pairs = cv._get_tomo_bins(ver) + tagged = cv._output_path(f"pseudo_cl_{ver}_tagged_tomo.sacc") + cv.calculate_pseudo_cl(compute_tomography=True, out_path=tagged) + assert os.path.exists(tagged) + assert not os.path.exists(cv._output_path_pseudo_cl(ver, tomography=True)) result_to_compare = np.load(REFERENCE_TOMO, allow_pickle=True)["arr_0"].item() for tomo_bin_a, tomo_bin_b in tomo_bin_pairs: - out_path = cv._output_path( - f"pseudo_cl_cat_{ver}_{tomo_bin_a}_{tomo_bin_b}.fits" - ) - cv.calculate_pseudo_cl_catalog( - ver, out_path, tomo_bin_a=tomo_bin_a, tomo_bin_b=tomo_bin_b - ) - - assert os.path.exists(out_path) - d = fits.getdata(out_path) - # FITS gives big-endian f8; normalize for value comparison. - ell = np.asarray(d["ELL"], dtype=np.float64) - ee = np.asarray(d["EE"], dtype=np.float64) - eb = np.asarray(d["EB"], dtype=np.float64) - be = np.asarray(d["BE"], dtype=np.float64) - bb = np.asarray(d["BB"], dtype=np.float64) + d = cv._pseudo_cls[ver][f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}"][ + "pseudo_cl" + ] + ell, ee, eb, bb = (d[key] for key in ("ELL", "EE", "EB", "BB")) npt.assert_allclose( ell, @@ -1007,14 +1002,6 @@ def test_calculate_pseudo_cl_catalog_end_to_end_tomo(cv, tmp_path): rtol=RTOL_CAT, atol=ATOL_CAT, ) - npt.assert_allclose( - be, - result_to_compare[f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}"][ - "pseudo_cl" - ]["BE"], - rtol=RTOL_CAT, - atol=ATOL_CAT, - ) npt.assert_allclose( bb, result_to_compare[f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}"][ @@ -1025,7 +1012,7 @@ def test_calculate_pseudo_cl_catalog_end_to_end_tomo(cv, tmp_path): ) # The end-to-end catalog EE matches the primitive get_pseudo_cls_catalog EE - # (same computation, FITS round-trip) -- consistency, not an independent pin. + # (same computation, SACC round-trip) -- consistency, not an independent pin. cat_gal = fits.getdata(cv.cc[ver]["shear"]["path"]) params = get_params_rho_tau(cv.cc[ver]) _, cl_prim, _ = cv.get_pseudo_cls_catalog( @@ -1056,12 +1043,3 @@ def test_calculate_pseudo_cl_out_path_rejects_multiversion(cv): cv.calculate_pseudo_cl( compute_tomography=False, out_path=cv._output_path("pseudo_cl_x.sacc") ) - - -def test_calculate_pseudo_cl_out_path_rejects_tomography(cv): - """out_path names the non-tomographic SACC part; tomographic pairs keep - their own per-pair paths, so asking for both must fail loudly.""" - with pytest.raises(ValueError, match="compute_tomography=False"): - cv.calculate_pseudo_cl( - compute_tomography=True, out_path=cv._output_path("pseudo_cl_x.sacc") - ) diff --git a/src/sp_validation/tests/test_rho_cache_reuse.py b/src/sp_validation/tests/test_rho_cache_reuse.py new file mode 100644 index 000000000..0cf836c80 --- /dev/null +++ b/src/sp_validation/tests/test_rho_cache_reuse.py @@ -0,0 +1,36 @@ +"""Tomographic ρ reuse honours the measurement metadata of the cache.""" + +import numpy as np + +from sp_validation.correlation import write_measurement_metadata +from sp_validation.rho_tau import rho_cache_valid + +CONFIG = {"min_sep": 1.0, "max_sep": 100.0, "nbins": 5, "sep_units": "arcmin"} + + +def _cache(tmp_path, config=None): + rho = tmp_path / "rho_stats.fits" + cov = tmp_path / "cov_rho_jk.npy" + rho.write_bytes(b"") + np.save(cov, np.eye(2)) + if config is not None: + write_measurement_metadata(rho, config) + return rho, cov + + +def test_rho_cache_without_metadata_is_remeasured(tmp_path): + rho, cov = _cache(tmp_path) + assert not rho_cache_valid(rho, cov, CONFIG) + + +def test_rho_cache_with_other_means_settings_is_remeasured(tmp_path): + rho, cov = _cache(tmp_path, {**CONFIG, "nbins": 6}) + assert not rho_cache_valid(rho, cov, CONFIG) + + +def test_rho_cache_matching_is_reused_unless_forced(tmp_path): + rho, cov = _cache(tmp_path, CONFIG) + assert rho_cache_valid(rho, cov, CONFIG) + assert not rho_cache_valid(rho, cov, CONFIG, force_run=True) + cov.unlink() + assert not rho_cache_valid(rho, cov, CONFIG) diff --git a/src/sp_validation/tests/test_rho_tau_tomographic_part.py b/src/sp_validation/tests/test_rho_tau_tomographic_part.py new file mode 100644 index 000000000..ac3999881 --- /dev/null +++ b/src/sp_validation/tests/test_rho_tau_tomographic_part.py @@ -0,0 +1,117 @@ +"""Tomographic ρ/τ parts from synthetic handler tables.""" + +from types import SimpleNamespace + +import numpy as np + +from sp_validation import blinding, sacc_io +from sp_validation.cosmo_val import psf_systematics as psf +from sp_validation.cosmo_val import sacc_writers as sw + + +def test_tomographic_rho_tau_part(tmp_path, monkeypatch): + theta = np.array([1.0, 2.0]) + rho = {"theta": theta} + for k in sw.RHO_K: + for sign in ("p", "m"): + rho[f"rho_{k}_{sign}"] = theta * (k + 1) + rho[f"varrho_{k}_{sign}"] = theta + calls = [] + + def measure(*args, **kwargs): + base = args[5] + label = base.split("_")[-1] + bin_id = 4 if label == "all" else int(label) + calls.append(bin_id) + tau = {"theta": theta} + for k in sw.TAU_K: + for sign in ("p", "m"): + tau[f"tau_{k}_{sign}"] = theta * bin_id + tau[f"vartau_{k}_{sign}"] = theta + np.save( + tmp_path / "rho_tau_stats" / f"cov_tau_{base}_th.npy", + np.eye(6) * bin_id, + ) + return SimpleNamespace(rho_stats=rho), SimpleNamespace(tau_stats=tau) + + monkeypatch.setattr(psf, "get_rho_tau_w_cov", measure) + saved = {} + monkeypatch.setattr( + sacc_io, "save", lambda s, path, **kw: saved.update(part=s, path=path, **kw) + ) + + class CV(psf.PSFSystematicsMixin): + cc = {"paths": {"output": str(tmp_path)}} + versions = ["v"] + treecorr_config = {} + cov_estimate_method = "th" + compute_cov_rho = False + npatch = 1 + + def _get_tomo_bins(self, v): + return [9, 2, 5], [(2, 2)] + + def _get_star_mask(self, v): + return None + + def _get_galaxy_mask(self, v, i): + return None + + def basename(self, v, tomo_bin_a="all"): + return f"bin_{tomo_bin_a}" + + def sacc_nz(self, v, tomography=False): + return { + i: (np.array([0.1, 0.5]), np.array([0.5, 1.0])) + for i in range(3 if tomography else 1) + } + + def sacc_metadata(self, v): + return {} + + def blind(self, v): + return "none" + + def print_start(self, *args): + pass + + print_done = print_cyan = print_magenta = print_start + + CV().calculate_rho_tau_stats(tomography=True) + part = saved["part"] + assert calls == [2, 5, 9] + assert saved["path"].endswith("rho_tau_bin_all_tomo.sacc") + assert saved["theory"] is blinding.no_signal + assert len([d for d in part.data if d.tracers == (sacc_io.PSF_TRACER,) * 2]) == 24 + for k in sw.RHO_K: + _, plus, minus = sacc_io.get_rho(part, k) + np.testing.assert_array_equal(plus, rho[f"rho_{k}_p"]) + np.testing.assert_array_equal(minus, rho[f"rho_{k}_m"]) + rows = [] + for i, value in enumerate(calls): + inds = np.concatenate( + [ + part.indices( + sacc_io.TAU_PLUS.format(k=k), (f"source_{i}", sacc_io.PSF_TRACER) + ) + for k in sw.TAU_K + ] + ) + rows.append(inds) + for k in sw.TAU_K: + _, plus, minus = sacc_io.get_tau(part, (i, 0), k) + np.testing.assert_array_equal(plus, theta * value) + np.testing.assert_array_equal(minus, theta * value) + np.testing.assert_array_equal( + part.covariance.dense[np.ix_(inds, inds)], np.eye(6) * value + ) + for i in range(3): + for j in range(i): + assert not np.any(part.covariance.dense[np.ix_(rows[i], rows[j])]) + + CV().calculate_rho_tau_stats(tomography=False) + for k in sw.RHO_K: + for tomo_values, all_values in zip( + sacc_io.get_rho(part, k), sacc_io.get_rho(saved["part"], k) + ): + np.testing.assert_array_equal(tomo_values, all_values) diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index 92460f1c7..c0a7b49c6 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -108,7 +108,11 @@ def __init__(self, nbp, nell): def get_bandpower_windows(self): return self._w - s = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell, cl_all, _Wsp(nbp, 24)) + s = sw.pseudo_cl_to_sacc( + {0: _nz()}, + META, + [((0, 0), {"ell_eff": ell, "cl_all": cl_all, "wsp": _Wsp(nbp, 24)})], + ) s2 = _roundtrip(s, tmp_path, "cl") ell_r, ee, bb, eb, window = sio.get_pseudo_cl(s2, (0, 0)) assert np.array_equal(ell_r, ell) @@ -142,7 +146,11 @@ def get_bandpower_windows(self): window[component, band, component, band * 4 : (band + 1) * 4] = 1.0 return window - part = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell, cl_all, _Workspace()) + part = sw.pseudo_cl_to_sacc( + {0: _nz()}, + META, + [((0, 0), {"ell_eff": ell, "cl_all": cl_all, "wsp": _Workspace()})], + ) path = tmp_path / "pseudo_cl.sacc" sio.save(part, str(path), blind=NONE) @@ -167,7 +175,9 @@ def test_pseudo_cl_to_sacc_real_namaster_pixel_window(tmp_path): rng.normal(size=12 * nside**2) + 1j * rng.normal(size=12 * nside**2) ) * 1e-2 ell_eff, cl_all, wsp = get_pseudo_cls_map(shear, mask, nside, "linear", ell_step=8) - s = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell_eff, cl_all, wsp) + s = sw.pseudo_cl_to_sacc( + {0: _nz()}, META, [((0, 0), {"ell_eff": ell_eff, "cl_all": cl_all, "wsp": wsp})] + ) s2 = _roundtrip(s, tmp_path, "clreal") ell_r, ee, bb, eb, window = sio.get_pseudo_cl(s2, (0, 0)) assert np.array_equal(ell_r, ell_eff) @@ -175,7 +185,11 @@ def test_pseudo_cl_to_sacc_real_namaster_pixel_window(tmp_path): # With nside unset, the workspace window carries no pixel window. assert window.weight.shape[1] == len(ell_eff) - mapped = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell_eff, cl_all, wsp, nside=nside) + mapped = sw.pseudo_cl_to_sacc( + {0: _nz()}, + META, + [((0, 0), {"ell_eff": ell_eff, "cl_all": cl_all, "wsp": wsp, "nside": nside})], + ) mapped_window = mapped.get_bandpower_windows(mapped.indices(sio.CL_EE)) pw2 = hp.pixwin(nside, lmax=window.weight.shape[0] - 1) ** 2 nonzero = np.abs(window.weight) > 1e-20 @@ -189,7 +203,9 @@ def test_pseudo_cl_to_sacc_real_namaster_pixel_window(tmp_path): def test_cosebis_to_sacc(tmp_path): En, Bn = np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7 result = {"En": En, "Bn": Bn, "cov": _spd(20, 7)} - s = sw.cosebis_to_sacc({0: _nz()}, META, result, (1.0, 100.0)) + s = sw.cosebis_to_sacc( + {0: _nz()}, META, [((0, 0), result)], (1.0, 100.0), covariance=result["cov"] + ) s2 = _roundtrip(s, tmp_path, "co") n, E, B = sio.get_cosebis(s2, (0, 0)) assert np.array_equal(n, np.arange(1, 11)) @@ -202,7 +218,9 @@ def test_pure_eb_to_sacc(tmp_path): theta = _theta() eb = {key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS)} cov = _spd(6 * len(theta), 9) - s = sw.pure_eb_to_sacc({0: _nz()}, META, theta, eb, covariance=cov) + s = sw.pure_eb_to_sacc( + {0: _nz()}, META, [((0, 0), {"theta": theta, **eb})], covariance=cov + ) s2 = _roundtrip(s, tmp_path, "eb") th, back = sio.get_pure_eb(s2, (0, 0)) assert np.array_equal(th, theta) @@ -229,7 +247,7 @@ def _rho_tau_tables(nth=6, seed=0): def test_rho_tau_to_sacc_diagonal(tmp_path): rho, tau, theta = _rho_tau_tables() - s = sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau) + s = sw.rho_tau_to_sacc({0: _nz()}, META, rho, [(0, tau)]) s2 = _roundtrip(s, tmp_path, "rt") for k in sw.RHO_K: th, p, m = sio.get_rho(s2, k) @@ -249,7 +267,9 @@ def test_rho_tau_to_sacc_tau_theory_block(tmp_path): nbin = len(theta) n_plus = len(sw.TAU_K) * nbin # τ-plus points (k-major, one component per k) tau_cov_th = _spd(n_plus, 11) - s = sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov_th=tau_cov_th) + s = sw.rho_tau_to_sacc( + {0: _nz()}, META, rho, [(0, tau)], tau_cov_th_by_bin=[(0, tau_cov_th)] + ) assert type(s.covariance).__name__ == "FullCovariance" tr = ("source_0", sio.PSF_TRACER) tau_plus = np.concatenate( @@ -271,7 +291,9 @@ def test_rho_tau_to_sacc_tau_theory_block(tmp_path): def test_rho_tau_to_sacc_tau_cov_shape_mismatch(): rho, tau, _ = _rho_tau_tables() with pytest.raises(ValueError, match="tau_cov_th shape"): - sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov_th=_spd(3, 1)) + sw.rho_tau_to_sacc( + {0: _nz()}, META, rho, [(0, tau)], tau_cov_th_by_bin=[(0, _spd(3, 1))] + ) # --------------------------------------------------------------------------- # @@ -296,16 +318,18 @@ def get_bandpower_windows(self): cl_all = np.vstack( [np.arange(3) * 1e-9, np.arange(3) * 2e-9, np.zeros(3), np.arange(3) * 3e-9] ) - cl = sw.pseudo_cl_to_sacc(nz, META, ell, cl_all, _Wsp(), covariance=_spd(9, 2)) + cl = sw.pseudo_cl_to_sacc( + nz, + META, + [((0, 0), {"ell_eff": ell, "cl_all": cl_all, "wsp": _Wsp()})], + covariance=_spd(9, 2), + ) co = sw.cosebis_to_sacc( nz, META, - { - "En": np.arange(1, 6) * 1e-6, - "Bn": np.arange(1, 6) * 1e-7, - "cov": _spd(10, 3), - }, + [((0, 0), {"En": np.arange(1, 6) * 1e-6, "Bn": np.arange(1, 6) * 1e-7})], (1.0, 100.0), + covariance=_spd(10, 3), ) return [xi, cl, co] diff --git a/src/sp_validation/tests/test_sigma_e_convention.py b/src/sp_validation/tests/test_sigma_e_convention.py index 61c8f766f..6982703f9 100644 --- a/src/sp_validation/tests/test_sigma_e_convention.py +++ b/src/sp_validation/tests/test_sigma_e_convention.py @@ -26,7 +26,7 @@ def test_config_preserves_covariance_inputs(version, old_sigma): namespace = {"catalogue_entry": lambda _: entry} exec(compile(ast.parse(function), "covariance.smk", "exec"), namespace) sigma, density, _ = namespace["get_onecov_cat_params"](version) - sigma = float(sigma) + sigma, density = float(sigma), float(density) assert sigma == pytest.approx(old_sigma / np.sqrt(2)) assert sigma**2 / density == pytest.approx(old_sigma**2 / (2 * density)) # CosmoCov's summed-component input, when constructed from this value. diff --git a/src/sp_validation/tests/test_tomographic_covariance_order.py b/src/sp_validation/tests/test_tomographic_covariance_order.py new file mode 100644 index 000000000..36ac605de --- /dev/null +++ b/src/sp_validation/tests/test_tomographic_covariance_order.py @@ -0,0 +1,240 @@ +"""Independent checks of tomographic covariance ordering.""" + +import runpy +from pathlib import Path + +import numpy as np + +from sp_validation import sacc_io as sio +from sp_validation.blinding import Blind +from sp_validation.cosmo_val.sacc_writers import xi_pairs_to_sacc + + +def test_onecov_assembly_matches_written_point_labels(tmp_path): + pairs = [(i, j) for i in range(3) for j in range(i, 3)] + theta = [2.0, 7.0, 20.0] + nz = {i: (np.array([0.1, 0.5, 1.0]), np.array([0.1, 1.0, 0.1])) for i in range(3)} + measurements = [ + (pair, dict(theta=theta, xip=np.zeros(3), xim=np.zeros(3))) + for pair in reversed(pairs) + ] + part = xi_pairs_to_sacc(nz, {}, measurements, grid="reporting") + part_path = tmp_path / "xi.sacc" + sio.save(part, part_path, blind=Blind("none")) + labels = [ + (dtype, pair, t) + for dtype in (sio.XI_PLUS, sio.XI_MINUS) + for pair in pairs + for t in theta + ] + rng = np.random.default_rng(376) + a = rng.normal(size=(len(labels), len(labels))) + source = a + a.T + cov_path = tmp_path / "onecov.dat" + np.savetxt(cov_path, source, header="OneCovariance covariance matrix") + script = Path(__file__).resolve().parents[3] / "workflow/scripts/assemble_sacc.py" + assemble = runpy.run_path(str(script))["assemble_sacc"] + output = tmp_path / "assembled.sacc" + assemble("v", {"xi_reporting": part_path}, output, blind="none", xi_cov=cov_path) + written = sio.load(output, blind="none") + perm = [ + labels.index( + ( + dp.data_type, + tuple(int(t.removeprefix("source_")) for t in dp.tracers), + dp.tags["theta"], + ) + ) + for dp in written.data + ] + np.testing.assert_array_equal(written.covariance.dense, source[np.ix_(perm, perm)]) + + +def test_tomographic_jackknife_matches_independent_pair_order(tmp_path): + import treecorr + from astropy.table import Table + + from sp_validation.cosmo_val import CosmologyValidation + from sp_validation.statistics import jackknife_patch_centers + from sp_validation.tests.test_cosmo_val import TestCosmologyValidation + + params, version = TestCosmologyValidation._write_synthetic_catalogs( + tmp_path, n_gal=3000, with_tomography=True + ) + path = tmp_path / "catalog/shear.fits" + table = Table.read(path) + table["tomo_bin_id"] = np.arange(len(table)) % 3 + 1 + table.write(path, overwrite=True) + z = np.linspace(0.05, 3, 31) + np.savetxt( + tmp_path / "nz/dndz_SP_A.txt", + np.column_stack([z] + [np.exp(-(((z - m) / 0.3) ** 2)) for m in (0.5, 1, 1.5)]), + header="z bin1 bin2 bin3", + ) + cv = CosmologyValidation( + versions=[version], npatch=4, theta_min=5, theta_max=100, nbins=5, **params + ) + cv.calculate_2pcf_version(version, compute_tomography=True, num_threads=1) + part = cv.xi_tomo_parts[version, "reporting"] + cols = cv._shear_columns(version, True) + full = treecorr.Catalog( + ra=cols["ra"], + dec=cols["dec"], + w=cols["w"], + ra_units=cv.treecorr_config["ra_units"], + dec_units=cv.treecorr_config["dec_units"], + ) + centers = jackknife_patch_centers(full, 4) + catalogs = {} + for b in range(3): + mask = cols["tomo_bin"] == b + 1 + catalogs[b] = treecorr.Catalog( + ra=cols["ra"][mask], + dec=cols["dec"][mask], + g1=cols["g1"][mask], + g2=cols["g2"][mask], + w=cols["w"][mask], + ra_units=cv.treecorr_config["ra_units"], + dec_units=cv.treecorr_config["dec_units"], + patch_centers=centers, + ) + pairs = [(i, j) for i in range(3) for j in range(i, 3)][::-1] + correlations = [] + patched_config = {**cv._binning(), "var_method": "jackknife"} + # The covariance pass uses TreeCorr's default tolerance, independently of + # the explicit bin_slop used for the full-sample means. + patched_config.pop("bin_slop", None) + for i, j in pairs: + gg = treecorr.GGCorrelation(patched_config) + gg.process(catalogs[i], cat2=catalogs[j] if i != j else None, num_threads=1) + correlations.append(gg) + independent = treecorr.estimate_multi_cov(correlations, "jackknife") + perm = [] + for dp in part.data: + pair = tuple(int(t.removeprefix("source_")) for t in dp.tracers) + same = [ + p.tags["theta"] + for p in part.data + if p.data_type == dp.data_type and p.tracers == dp.tracers + ] + rank = sorted(same).index(dp.tags["theta"]) + component = (sio.XI_PLUS, sio.XI_MINUS).index(dp.data_type) + perm.append(pairs.index(pair) * 10 + component * 5 + rank) + np.testing.assert_allclose( + part.covariance.dense, independent[np.ix_(perm, perm)], rtol=1e-12, atol=0 + ) + + +def test_full_tomographic_assembly_covariance_blocks(tmp_path): + from astropy.io import fits + + pairs = [(i, j) for i in range(3) for j in range(i, 3)] + theta = [2.0, 7.0, 20.0] + ell = [20.0, 50.0, 90.0] + nz = {i: (np.array([0.1, 0.5, 1.0]), np.array([0.1, 1.0, 0.1])) for i in range(3)} + xi = xi_pairs_to_sacc( + nz, + {}, + [ + (p, dict(theta=theta, xip=np.zeros(3), xim=np.zeros(3))) + for p in reversed(pairs) + ], + grid="reporting", + ) + cl, co, pure, rt = [sio.new_sacc(nz, {}) for _ in range(4)] + for p in reversed(pairs): + sio.add_pseudo_cl( + cl, + p, + ell, + np.zeros(3), + np.zeros(3), + np.zeros(3), + window_ells=ell, + window_weights=np.eye(3), + ) + sio.add_cosebis(co, p, np.zeros(3), (1.0, 100.0), Bn=np.zeros(3)) + sio.add_pure_eb(pure, p, theta, **{k: np.zeros(3) for k in sio.PURE_KEYS}) + for k in range(6): + sio.add_rho(rt, k, theta, np.zeros(3), np.zeros(3)) + for b in reversed(range(3)): + for k in (0, 2, 5): + sio.add_tau(rt, (b, b), k, theta, np.zeros(3), np.zeros(3)) + parts = dict(xi_reporting=xi, pseudo_cl=cl, cosebis=co, pure_eb=pure, rho_tau=rt) + rng = np.random.default_rng(376) + + def matrix(n, ident): + a = rng.normal(size=(n, n)) + return ident * 1000 + a + a.T + + xi_source = matrix(len(xi.data), 1) + xi_path = tmp_path / "onecov.dat" + np.savetxt(xi_path, xi_source, header="OneCovariance covariance matrix") + xi_labels = [ + (d, p, t) for d in (sio.XI_PLUS, sio.XI_MINUS) for p in pairs for t in theta + ] + cl_types = {sio.CL_EE: "EE", sio.CL_BB: "BB", sio.CL_EB: "EB"} + cl_blocks = { + d: matrix(len(pairs) * len(ell), 2 + i) for i, d in enumerate(cl_types) + } + cl_path = tmp_path / "inka.fits" + fits.HDUList( + [fits.PrimaryHDU()] + + [ + fits.ImageHDU(cl_blocks[d], name=f"COVAR_{pol}_{pol}") + for d, pol in cl_types.items() + ] + ).writeto(cl_path) + + expected = {} + + def label(dp, coordinate): + return ( + dp.data_type, + tuple(int(t.removeprefix("source_")) for t in dp.tracers), + dp.tags[coordinate], + ) + + perm = [xi_labels.index(label(dp, "theta")) for dp in xi.data] + expected["xi_reporting"] = xi_source[np.ix_(perm, perm)] + cl_expected = np.zeros((len(cl.data), len(cl.data))) + for dtype, block in cl_blocks.items(): + positions = [i for i, dp in enumerate(cl.data) if dp.data_type == dtype] + order = [ + pairs.index(label(cl.data[i], "ell")[1]) * len(ell) + + ell.index(cl.data[i].tags["ell"]) + for i in positions + ] + cl_expected[np.ix_(positions, positions)] = block[np.ix_(order, order)] + expected["pseudo_cl"] = cl_expected + paths = {} + for ident, (name, part) in enumerate(parts.items(), 5): + if name not in expected: + expected[name] = matrix(len(part.data), ident) + part.add_covariance(expected[name]) + paths[name] = tmp_path / f"{name}.sacc" + sio.save(part, paths[name], blind=Blind("none")) + script = Path(__file__).resolve().parents[3] / "workflow/scripts/assemble_sacc.py" + assemble = runpy.run_path(str(script))["assemble_sacc"] + out = tmp_path / "assembled.sacc" + assemble("v", paths, out, blind="none", xi_cov=xi_path, pseudo_cl_cov=cl_path) + s = sio.load(out, blind="none") + indices = {} + for name, part in parts.items(): + # Query each (type, tracers) separately, in the part's insertion order. + keys = list(dict.fromkeys((dp.data_type, dp.tracers) for dp in part.data)) + indices[name] = np.concatenate( + [s.indices(dtype, tracers) for dtype, tracers in keys] + ) + part_indices = np.concatenate( + [part.indices(dtype, tracers) for dtype, tracers in keys] + ) + np.testing.assert_array_equal( + s.covariance.dense[np.ix_(indices[name], indices[name])], + expected[name][np.ix_(part_indices, part_indices)], + ) + for i, name in enumerate(parts): + for other in list(parts)[i + 1 :]: + np.testing.assert_array_equal( + s.covariance.dense[np.ix_(indices[name], indices[other])], 0 + ) diff --git a/src/sp_validation/tests/test_tomographic_sacc_writers.py b/src/sp_validation/tests/test_tomographic_sacc_writers.py new file mode 100644 index 000000000..0d32da096 --- /dev/null +++ b/src/sp_validation/tests/test_tomographic_sacc_writers.py @@ -0,0 +1,63 @@ +"""Pair-major SACC writers on synthetic tomographic data.""" + +import numpy as np + +from sp_validation import sacc_io as sio +from sp_validation.cosmo_val import sacc_writers as sw + + +def test_pair_native_writers_and_tau_covariance(): + nz = {i: (np.array([0.1, 0.5]), np.array([0.5, 1.0])) for i in range(3)} + pairs = [(i, j) for i in range(3) for j in range(i, 3)] + theta = np.array([1.0, 2.0]) + + class Workspace: + def get_bandpower_windows(self): + w = np.zeros((4, 2, 4, 2)) + for i in range(4): + w[i, :, i, :] = np.eye(2) + return w + + cl = [ + ((i, j), {"ell_eff": theta, "cl_all": np.ones((4, 2)) * k, "wsp": Workspace()}) + for k, (i, j) in enumerate(pairs) + ] + co = [(p, {"En": np.ones(2) * k, "Bn": np.zeros(2)}) for k, p in enumerate(pairs)] + eb = [ + (p, {"theta": theta, **{key: np.ones(2) * k for key in sio.PURE_KEYS}}) + for k, p in enumerate(pairs) + ] + for part in ( + sw.pseudo_cl_to_sacc(nz, {}, cl, covariance=np.eye(36)), + sw.cosebis_to_sacc(nz, {}, co, (1.0, 2.0), covariance=np.eye(24)), + sw.pure_eb_to_sacc(nz, {}, eb, covariance=np.eye(72)), + ): + found = list(dict.fromkeys(dp.tracers for dp in part.data)) + assert found == [(f"source_{i}", f"source_{j}") for i, j in pairs] + assert np.array_equal(part.covariance.dense, np.eye(len(part.mean))) + + rho = {"theta": theta} + tau = {"theta": theta} + for k in sw.RHO_K: + for sign in ("p", "m"): + rho[f"rho_{k}_{sign}"] = theta + rho[f"varrho_{k}_{sign}"] = theta + for k in sw.TAU_K: + for sign in ("p", "m"): + tau[f"tau_{k}_{sign}"] = theta + tau[f"vartau_{k}_{sign}"] = theta + cov = np.ones((6, 6)) * 3 + part = sw.rho_tau_to_sacc( + nz, {}, rho, [(i, tau) for i in range(3)], tau_cov_th_by_bin=[(1, cov)] + ) + indices = [ + np.concatenate( + [ + part.indices(sio.TAU_PLUS.format(k=k), (f"source_{i}", sio.PSF_TRACER)) + for k in sw.TAU_K + ] + ) + for i in range(3) + ] + assert np.array_equal(part.covariance.dense[np.ix_(indices[1], indices[1])], cov) + assert not np.any(part.covariance.dense[np.ix_(indices[0], indices[1])]) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index fb05f7ef5..b2d3c1052 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -77,7 +77,7 @@ def cv_pure_eb_figures(version): } -def _grid_cov(version, grid, gaussian): +def _grid_cov(version, grid, gaussian, tomo_suffix=""): """OneCovariance ξ± covariance on a named grid's own binning.""" binning = XI_GRIDS[grid] return covariance_path( @@ -87,17 +87,17 @@ def _grid_cov(version, grid, gaussian): max_sep=binning["max_sep"], nbins=binning["nbins"], mask_suffix=DEFAULT_MASK_SUFFIX, + tomo_suffix=tomo_suffix, ) -def cv_xi_cov_integration(version): +def cv_xi_cov_integration(version, tomo_suffix=""): """OneCovariance Gaussian ξ± covariance on the integration grid. - The one analytic covariance both B-mode statistics take: the pure-E/B Monte - Carlo draws from it and COSEBIs carry it through their kernel. Gaussian - because B-modes only need the scatter a Gaussian field would give. + Both B-mode statistics propagate this Gaussian covariance through their + linear operators. """ - return _grid_cov(version, "integration", "g") + return _grid_cov(version, "integration", "g", tomo_suffix) def _cosebis_stub(version): @@ -137,46 +137,46 @@ def cv_cosebis_figures(version): _PSEUDO_CL_TAG = pseudo_cl_tag(config) -def cv_pseudo_cl_analysis_sacc(version): +def cv_pseudo_cl_analysis_sacc(version, tomo_suffix=""): """Tagged pseudo-Cl SACC part: the harmonic block of the analysis file.""" - return str(COSMO_VAL / f"pseudo_cl_{version}_{_PSEUDO_CL_TAG}.sacc") + return str(COSMO_VAL / f"pseudo_cl_{version}_{_PSEUDO_CL_TAG}{tomo_suffix}.sacc") -def cv_pseudo_cl_cov(version): +def cv_pseudo_cl_cov(version, tomo_suffix=""): """NaMaster pseudo-Cl covariance FITS (COVAR_EE_EE/BB_BB/EB_EB extensions).""" - return str(COSMO_VAL / f"pseudo_cl_cov_{version}_{_PSEUDO_CL_TAG}.fits") + return str(COSMO_VAL / f"pseudo_cl_cov_{version}_{_PSEUDO_CL_TAG}{tomo_suffix}.fits") -def cv_xi_cov(version): +def cv_xi_cov(version, tomo_suffix=""): """OneCovariance ξ± covariance on the reporting grid, the terminal file's.""" - return _grid_cov(version, "reporting", "ng") + return _grid_cov(version, "reporting", "ng", tomo_suffix) -def cv_cosebis_sacc(version): +def cv_cosebis_sacc(version, tomo_suffix=""): """COSEBIs SACC part, at the fiducial scale cut.""" - return str(COSMO_VAL / f"{version}_cosebis.sacc") + return str(COSMO_VAL / f"{version}_cosebis{tomo_suffix}.sacc") -def cv_pure_eb_sacc(version): +def cv_pure_eb_sacc(version, tomo_suffix=""): """Pure-E/B SACC part.""" - return str(COSMO_VAL / f"{version}_pure_eb.sacc") + return str(COSMO_VAL / f"{version}_pure_eb{tomo_suffix}.sacc") -def cv_rho_tau_sacc(version): +def cv_rho_tau_sacc(version, tomo_suffix=""): """ρ/τ SACC part.""" return str( - COSMO_VAL / "rho_tau_stats" / f"rho_tau_{cv_basename(version, CV_FIDUCIAL)}.sacc" + COSMO_VAL / "rho_tau_stats" / f"rho_tau_{cv_basename(version, CV_FIDUCIAL)}{tomo_suffix}.sacc" ) -def cv_xi_sacc(version, grid): +def cv_xi_sacc(version, grid, tomo_suffix=""): """ξ± SACC part for a version on a named grid, named by that grid's binning.""" - return str(COSMO_VAL / f"{version}_xi_{xi_binning(grid)}.sacc") + return str(COSMO_VAL / f"{version}_xi_{xi_binning(grid)}{tomo_suffix}.sacc") -def cv_analysis_sacc(version): +def cv_analysis_sacc(version, tomo_suffix=""): """Terminal assembled analysis file {version}.sacc.""" - return str(COSMO_VAL / f"{version}.sacc") + return str(COSMO_VAL / f"{version}{tomo_suffix}.sacc") # The CosmologyValidation constructor kwargs every cv_runner rule passes. @@ -483,11 +483,11 @@ rule cv_pure_eb: the integration-grid ξ± covariance pushed exactly through it. """ input: - xi_integration=lambda w: cv_xi_sacc(w.version, "integration"), - cov_integration=lambda w: cv_xi_cov_integration(w.version), + xi_integration=lambda w: cv_xi_sacc(w.version, "integration", w.tomo_suffix), + cov_integration=lambda w: cv_xi_cov_integration(w.version, w.tomo_suffix), output: - npz=cv_pure_eb_npz("{version}"), - sacc=cv_pure_eb_sacc("{version}"), + npz=cv_pure_eb_npz("{version}").replace("_data.npz", "{tomo_suffix}_data.npz"), + sacc=cv_pure_eb_sacc("{version}", "{tomo_suffix}"), params: version="{version}", blind=lambda w: blind_of(w.version), @@ -510,11 +510,11 @@ rule cv_cosebis: on the same grid through the same kernel as the modes. """ input: - xi=lambda w: cv_xi_sacc(w.version, "integration"), - cov=lambda w: cv_xi_cov_integration(w.version), + xi=lambda w: cv_xi_sacc(w.version, "integration", w.tomo_suffix), + cov=lambda w: cv_xi_cov_integration(w.version, w.tomo_suffix), output: - npz=cv_cosebis_npz("{version}"), - sacc=cv_cosebis_sacc("{version}"), + npz=cv_cosebis_npz("{version}").replace("_data.npz", "{tomo_suffix}_data.npz"), + sacc=cv_cosebis_sacc("{version}", "{tomo_suffix}"), params: version="{version}", blind=lambda w: blind_of(w.version), @@ -598,30 +598,30 @@ rule cv_summarize_bmodes: # the covariance inputs below. -def cv_assemble_inputs(version): +def cv_assemble_inputs(version, tomo_suffix=""): """The per-statistic SACC parts + covariance inputs assemble_sacc consumes. Each part's filename carries enough to bind its producing rule's wildcards. """ parts = dict( - xi_reporting=cv_xi_sacc(version, "reporting"), - xi_cov=cv_xi_cov(version), - cosebis=cv_cosebis_sacc(version), - pure_eb=cv_pure_eb_sacc(version), - rho_tau=cv_rho_tau_sacc(version), + xi_reporting=cv_xi_sacc(version, "reporting", tomo_suffix), + xi_cov=cv_xi_cov(version, tomo_suffix), + cosebis=cv_cosebis_sacc(version, tomo_suffix), + pure_eb=cv_pure_eb_sacc(version, tomo_suffix), + rho_tau=cv_rho_tau_sacc(version, tomo_suffix), ) if CV.get("include_pseudo_cl", False): - parts["pseudo_cl"] = cv_pseudo_cl_analysis_sacc(version) - parts["pseudo_cl_cov"] = cv_pseudo_cl_cov(version) + parts["pseudo_cl"] = cv_pseudo_cl_analysis_sacc(version, tomo_suffix) + parts["pseudo_cl_cov"] = cv_pseudo_cl_cov(version, tomo_suffix) return parts rule assemble_sacc: """Assemble the terminal {version}.sacc from the per-statistic SACC parts.""" input: - unpack(lambda w: cv_assemble_inputs(w.version)), + unpack(lambda w: cv_assemble_inputs(w.version, w.tomo_suffix)), output: - sacc=cv_analysis_sacc("{version}"), + sacc=cv_analysis_sacc("{version}", "{tomo_suffix}"), params: version="{version}", blind=lambda w: blind_of(w.version), @@ -629,7 +629,7 @@ rule assemble_sacc: # silently drop one. expected=lambda w: [ k - for k in cv_assemble_inputs(w.version) + for k in cv_assemble_inputs(w.version, w.tomo_suffix) if k not in ("xi_cov", "pseudo_cl_cov") ], resources: diff --git a/workflow/rules/covariance.smk b/workflow/rules/covariance.smk index a02bf338b..48fe88e97 100644 --- a/workflow/rules/covariance.smk +++ b/workflow/rules/covariance.smk @@ -15,14 +15,73 @@ def get_footprint_path(version): # tomographic. Named by covariance_base() in common.py. # --------------------------------------------------------------------------- -def get_onecov_cat_params(version): +def get_onecov_cat_params(version, tomo_suffix=""): """OneCovariance survey parameters (sigma_e, n_e, area) of ``version``'s - catalogue entry. Tomographic values are comma-separated strings, one entry - per bin.""" - cov_th = catalogue_entry(version)["cov_th"] + catalogue entry, as comma-separated strings with one entry per bin. + + A tomographic covariance reads ``cov_th.tomo`` (per-bin ``sigma_e`` and + ``n_e``) when the entry has one; a non-tomographic covariance reads the + whole-sample ``cov_th.sigma_e`` and ``cov_th.n_e``. A tomographic covariance + of an entry that declares ``shear.tomo_bin_col`` refuses single + whole-sample values, which would stand in for every bin.""" + entry = catalogue_entry(version) + cov_th = entry["cov_th"] + survey = cov_th.get("tomo", cov_th) if tomo_suffix else cov_th + + def per_bin(value): + values = value if isinstance(value, (list, tuple)) else str(value).split(",") + return [repr(float(v)) for v in values] + # Both cov_th.sigma_e and OneCovariance use per-component dispersion. - sigma_e = ", ".join(repr(float(s)) for s in str(cov_th["sigma_e"]).split(",")) - return sigma_e, cov_th["n_e"], cov_th["A"] + sigma_e, n_e = per_bin(survey["sigma_e"]), per_bin(survey["n_e"]) + if tomo_suffix and "tomo_bin_col" in entry["shear"] and len(n_e) == 1: + raise ValueError( + f"{version}: a tomographic covariance needs per-bin n_e and sigma_e " + "in cov_th.tomo" + ) + return ", ".join(sigma_e), ", ".join(n_e), cov_th["A"] + + +def onecov_nz_path(version, tomo_suffix=""): + """The n(z) file OneCovariance reads for one covariance. + + The catalogue's own ``redshift_path`` for a tomographic covariance or a + catalogue without tomographic bins; for a non-tomographic covariance of a + catalogue that declares ``shear.tomo_bin_col``, the bin sum written by + ``nz_bin_sum`` (the whole-sample n(z), as + ``CosmologyValidation.get_redshift`` reads it).""" + path = redshift_path(version) + if tomo_suffix or "tomo_bin_col" not in catalogue_entry(version)["shear"]: + return path + return str(COSMO_VAL / "nz" / f"{version}_nz_bin_sum.txt") + + +rule nz_bin_sum: + """Whole-sample n(z) of a multi-bin catalogue: z and the sum of its bins. + + The bins' columns must carry their relative weights (integrals summing to + one, as the shear-weight fractions); a file whose every column integrates + to one would weight the bins equally, and is refused.""" + input: + lambda w: redshift_path(w.version), + output: + str(COSMO_VAL / "nz" / "{version}_nz_bin_sum.txt"), + run: + import numpy as np + + table = np.loadtxt(input[0]) + trapezoid = getattr(np, "trapezoid", None) or np.trapz + integrals = trapezoid(table[:, 1:], table[:, 0], axis=0) + if np.allclose(integrals, 1.0, atol=0.02): + raise ValueError( + f"{input[0]}: every bin's n(z) integrates to 1, so their sum " + "would weight the bins equally; give each bin its weight fraction" + ) + np.savetxt( + output[0], + np.column_stack([table[:, 0], table[:, 1:].sum(axis=1)]), + header=f"z n(z): sum of the {table.shape[1] - 1} bins of {input[0]}", + ) def _onecov_args(w): @@ -114,7 +173,7 @@ rule generate_glass_mock_rhotau_samples: rule covariance_ini: input: - nz_file=lambda w: redshift_path(w.version), + nz_file=lambda w: onecov_nz_path(w.version, w.tomo_suffix), mask=lambda w: [] if w.mask_suffix != "_masked" else [get_footprint_path(w.version)], output: cov_output(".ini"), @@ -129,9 +188,9 @@ rule covariance_ini: h=PLANCK18["h"], omega_b=PLANCK18["Omega_b"], hmcode_logT_AGN=7.75, - sigma_e_param=lambda w: get_onecov_cat_params(w.version)[0], - n_e=lambda w: get_onecov_cat_params(w.version)[1], - area=lambda w: get_onecov_cat_params(w.version)[2], + sigma_e_param=lambda w: get_onecov_cat_params(w.version, w.tomo_suffix)[0], + n_e=lambda w: get_onecov_cat_params(w.version, w.tomo_suffix)[1], + area=lambda w: get_onecov_cat_params(w.version, w.tomo_suffix)[2], # OneCovariance takes directory and file separately. mask_dir=lambda w: ( os.path.dirname(get_footprint_path(w.version)) @@ -143,8 +202,8 @@ rule covariance_ini: if w.mask_suffix == "_masked" else "" ), - nz_dir=lambda w: os.path.dirname(redshift_path(w.version)), - nz_file=lambda w: os.path.basename(redshift_path(w.version)), + nz_dir=lambda w: os.path.dirname(onecov_nz_path(w.version, w.tomo_suffix)), + nz_file=lambda w: os.path.basename(onecov_nz_path(w.version, w.tomo_suffix)), num_cores=ONECOV_THREADS, threads: 1 shell: @@ -335,5 +394,6 @@ ruleorder: covariance_ini > covariance_run > covariance localrules: cosmology_params, + nz_bin_sum, covariance_ini, generate_glass_mock_rhotau_samples, diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index ae9bc17da..680767c13 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -107,6 +107,26 @@ rule rho_tau_stats: "../scripts/run_rho_tau.py" +rule rho_tau_stats_tomo: + """Measure τ per source bin, reusing the PSF-only ρ measurement.""" + input: + rho=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_tomo_bin_all_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + output: + rho_tau=str(COSMO_VAL / "rho_tau_stats/rho_tau_{version}_tomo_bin_all_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}_tomo.sacc"), + threads: 48 + params: + ver="{version}", + cov_estimate_method=config.get("cosmo_val", {}).get("cov_estimate_method", "th"), + min_sep="{min_sep}", max_sep="{max_sep}", nbins="{nbins}", npatch="{npatch}", + cat_config=CAT_CONFIG, output_dir=str(COSMO_VAL), + blind=lambda w: blind_of(w.version), + compute_tomography=True, + resources: + mem_mb=64000, disk_mb=20000, runtime=360, + script: + "../scripts/run_rho_tau.py" + + # Pseudo-Cl generation for harmonic-space data vectors and COSEBIS validation. wildcard_constraints: @@ -116,7 +136,7 @@ wildcard_constraints: rule pseudo_cl: """Generate pseudo-Cl data vector (born as SACC) with configurable binning.""" output: - pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_{binning}_nbins={nbins}.sacc"), + pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_{binning}_nbins={nbins}{tomo_suffix}.sacc"), params: version="{version}", cat_config=CAT_CONFIG, @@ -126,6 +146,7 @@ rule pseudo_cl: binning="{binning}", nbins=lambda w: int(w.nbins), power=0.5, + compute_tomography=lambda w: w.tomo_suffix == "_tomo", blind=lambda w: blind_of(w.version), resources: mem_mb=32000, @@ -138,7 +159,7 @@ rule pseudo_cl: rule pseudo_cl_cov: """Generate pseudo-Cl covariance with configurable binning.""" output: - pseudo_cl_cov=str(COSMO_VAL / "pseudo_cl_cov_{version}_{binning}_nbins={nbins}.fits"), + pseudo_cl_cov=str(COSMO_VAL / "pseudo_cl_cov_{version}_{binning}_nbins={nbins}{tomo_suffix}.fits"), params: version="{version}", cat_config=CAT_CONFIG, @@ -148,6 +169,7 @@ rule pseudo_cl_cov: binning="{binning}", nbins=lambda w: int(w.nbins), power=0.5, + compute_tomography=lambda w: w.tomo_suffix == "_tomo", resources: mem_mb=16000, runtime=180, diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 1f1a0968a..2144c44d5 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -15,39 +15,19 @@ import argparse -import numpy as np - from sp_validation import sacc_io from sp_validation.blinding import Blind from sp_validation.cosmo_val.sacc_writers import ( assemble_analysis_sacc, + namaster_cl_covariance, onecov_xi_covariance, ) -# NaMaster iNKA covariance FITS: per-spectrum HDU names, in SACC insertion order. -_CL_HDUS = ("COVAR_EE_EE", "COVAR_BB_BB", "COVAR_EB_EB") - # Canonical part order — the order points are inserted in, which must match the # covariance block order. Missing parts are simply skipped. CANONICAL = ("xi_reporting", "pseudo_cl", "cosebis", "pure_eb", "rho_tau") -def _pseudo_cl_cov_block(cov_fits): - """Block-diagonal ``[EE; BB; EB]`` from the NaMaster iNKA covariance FITS.""" - from astropy.io import fits - - with fits.open(cov_fits) as hdul: - missing = [name for name in _CL_HDUS if name not in {h.name for h in hdul}] - if missing: - raise ValueError(f"{cov_fits} lacks the pseudo-Cℓ cov HDUs {missing}") - blocks = [np.asarray(hdul[name].data, float) for name in _CL_HDUS] - n = blocks[0].shape[0] - full = np.zeros((3 * n, 3 * n)) - for i, block in enumerate(blocks): - full[i * n : (i + 1) * n, i * n : (i + 1) * n] = block - return full - - # The statistics whose analysis covariance is external, and the input each one # takes it from. A part of one of these types may be born with an estimate of # its own — the ξ± reporting part carries the jackknife it was measured with — @@ -79,7 +59,7 @@ def _attach_cov(part, name, xi_cov, pseudo_cl_cov): block = ( onecov_xi_covariance(part, sacc_io.read_onecov_matrix(supplied)) if name == "xi_reporting" - else _pseudo_cl_cov_block(supplied) + else namaster_cl_covariance(part, supplied) ) if part.covariance is not None: print( diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index 1f58a304a..913558e71 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -1,51 +1,74 @@ -"""Rule cv_cosebis: COSEBIs E/B decomposition for one version. +"""Derive COSEBIs from a concealed integration-grid ξ± part. -A consumer of the integration-grid ξ± part and the OneCovariance ξ± covariance on -the same grid — nothing here touches a catalogue. The covariance goes through -the same linear kernel as the modes to give the COSEBIs covariance; it is -analytic, so no Hartlap debiasing applies. The COSEBIs part is a derivation of -the ξ± part and carries its blind stamp. +All pairs use the same fiducial scale cut; tomographic cuts are not applied. +The COSEBIs convention sets E×B covariance blocks to zero across all pairs. """ -import numpy as np +from pathlib import Path + from cv_runner import _unbuffer_streams, verify_outputs from sp_validation import sacc_io from sp_validation.b_modes import ( + cosebis_operator, cosebis_scan_from_xi, find_conservative_scale_cut_key, log_bin_edges, save_cosebis_results, ) +from sp_validation.cosmo_val.derived_covariance import ( + propagate, + xi_covariance, + xi_pairs, +) from sp_validation.cosmo_val.sacc_writers import cosebis_to_sacc _unbuffer_streams() p = snakemake.params -version = p["version"] fiducial_scale_cut = tuple(p["fiducial_scale_cut"]) - part = sacc_io.load(snakemake.input["xi"], blind=p["blind"]) -theta, xip, xim = sacc_io.get_xi(part, (0, 0), grid="integration") +pairs = xi_pairs(part) +# Read only after the declared blind has been checked. +xis = [sacc_io.get_xi(part, pair, grid="integration") for pair in pairs] +covariance, npatch = xi_covariance(part, snakemake.input["cov"]) edges = log_bin_edges(p["min_sep"], p["max_sep"], p["nbins"]) +outputs, operators = [], [] +for pair, (theta, xip, xim) in zip(pairs, xis): + view = sacc_io.xi_correlation(part, pair, grid="integration") + cov = covariance[view._rows[:, None], view._rows] + results = cosebis_scan_from_xi( + theta, + xip, + xim, + cov, + *edges, + nmodes=p["nmodes"], + scale_cuts=[tuple(sc) for sc in p["scale_cuts"]], + npatch=npatch, + ) + key = find_conservative_scale_cut_key(results, fiducial_scale_cut) + outputs.append((pair, results[key])) + if len(pairs) > 1: + operators.append(cosebis_operator(theta, *edges, key, p["nmodes"])) + path = Path(snakemake.output["npz"]) + if len(pairs) > 1: + path = path.with_name(f"{path.stem}_{pair[0]}_{pair[1]}{path.suffix}") + save_cosebis_results(results, path, fiducial_scale_cut, edges, derived_from=[part]) + if pair == pairs[0] and len(pairs) > 1: + save_cosebis_results( + results, + snakemake.output["npz"], + fiducial_scale_cut, + edges, + derived_from=[part], + ) -results = cosebis_scan_from_xi( - theta, - xip, - xim, - np.loadtxt(snakemake.input["cov"]), - *edges, - nmodes=p["nmodes"], - scale_cuts=[tuple(sc) for sc in p["scale_cuts"]], -) - -fiducial_key = find_conservative_scale_cut_key(results, fiducial_scale_cut) -fiducial = results[fiducial_key] - -save_cosebis_results( - results, snakemake.output["npz"], fiducial_scale_cut, edges, derived_from=[part] +cov = ( + outputs[0][1]["cov"] + if len(pairs) == 1 + else propagate(operators, covariance, cosebis=True) ) - -s = cosebis_to_sacc({0: sacc_io.get_nz(part, 0)}, part.metadata, fiducial, fiducial_key) +nz = {i: sacc_io.get_nz(part, i) for pair in pairs for i in pair} +s = cosebis_to_sacc(nz, part.metadata, outputs, key, covariance=cov) sacc_io.save(s, snakemake.output["sacc"], derived_from=[part]) - verify_outputs(snakemake) diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index 50a65fbcf..1d4fead8f 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -1,13 +1,12 @@ -"""Rule cv_pure_eb: pure E/B-mode decomposition for one version. +"""Derive pure E/B from the concealed integration-grid ξ± part. -A consumer of the integration-grid ξ± part plus a ξ± covariance on that grid — -nothing here touches a catalogue. The estimator is one fixed linear operator -on the fine ξ± (b_modes.pure_eb_operator), averaged with the part's TreeCorr -pair weights into reporting bins snapped onto the fine edges, so its covariance -is the supplied ξ± covariance pushed exactly through that operator. The -pure-E/B part is a derivation of the ξ± part and carries its blind stamp. +All pairs use the same reporting bins; tomographic scale cuts are not applied. +Their pair-weighted operators propagate the full ξ± covariance without zeroing +any of its blocks. """ +from pathlib import Path + import numpy as np from cv_runner import _unbuffer_streams, verify_outputs @@ -15,44 +14,44 @@ from sp_validation.b_modes import ( calculate_eb_statistics, calculate_pure_eb_correlation, + pure_eb_operator, save_pure_eb_results, ) +from sp_validation.cosmo_val.derived_covariance import ( + propagate, + xi_covariance, + xi_pairs, +) from sp_validation.cosmo_val.sacc_writers import pure_eb_to_sacc _unbuffer_streams() p = snakemake.params -version = p["version"] -fiducial_scale_cut = tuple(p["fiducial_scale_cut"]) - part = sacc_io.load(snakemake.input["xi_integration"], blind=p["blind"]) -gg_int = sacc_io.xi_correlation(part, grid="integration") -# A part stores bin centres; the edges come from the grid it was measured on. +pairs = xi_pairs(part) +views = [sacc_io.xi_correlation(part, pair, grid="integration") for pair in pairs] +covariance, npatch = xi_covariance(part, snakemake.input["cov_integration"]) grid = p["integration"] edges_int = np.geomspace(grid["min_sep"], grid["max_sep"], grid["nbins"] + 1) - -results = calculate_pure_eb_correlation( - gg_int.meanr, - gg_int.xip, - gg_int.xim, - gg_int.weight, - edges_int, - np.loadtxt(snakemake.input["cov_integration"]), - np.geomspace(p["min_sep"], p["max_sep"], p["nbins"] + 1), -) -results = calculate_eb_statistics(results) - -save_pure_eb_results(results, snakemake.output["npz"], derived_from=[part]) - -sacc_io.save( - pure_eb_to_sacc( - {0: sacc_io.get_nz(part, 0)}, - part.metadata, - results["theta"], - {key: results[key] for key in sacc_io.PURE_KEYS}, - covariance=results["cov"], - ), - snakemake.output["sacc"], - derived_from=[part], -) - +reporting = np.geomspace(p["min_sep"], p["max_sep"], p["nbins"] + 1) +outputs, operators = [], [] +for pair, gg in zip(pairs, views): + cov = covariance[gg._rows[:, None], gg._rows] + results = calculate_pure_eb_correlation( + gg.meanr, gg.xip, gg.xim, gg.weight, edges_int, cov, reporting, npatch=npatch + ) + results = calculate_eb_statistics(results) + path = Path(snakemake.output["npz"]) + if len(pairs) > 1: + path = path.with_name(f"{path.stem}_{pair[0]}_{pair[1]}{path.suffix}") + save_pure_eb_results(results, path, derived_from=[part]) + if pair == pairs[0] and len(pairs) > 1: + save_pure_eb_results(results, snakemake.output["npz"], derived_from=[part]) + outputs.append((pair, results)) + if len(pairs) > 1: + operators.append(pure_eb_operator(gg.weight, edges_int, reporting)[0]) + +cov = outputs[0][1]["cov"] if len(pairs) == 1 else propagate(operators, covariance) +nz = {i: sacc_io.get_nz(part, i) for pair in pairs for i in pair} +s = pure_eb_to_sacc(nz, part.metadata, outputs, covariance=cov) +sacc_io.save(s, snakemake.output["sacc"], derived_from=[part]) verify_outputs(snakemake) diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index 1decf0913..9330325bd 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -37,6 +37,7 @@ def generate_pseudo_cl( binning: str = "linear", nbins: int = None, power: float = 0.5, + compute_tomography: bool = False, ): """Generate a pseudo-Cl data vector, born as a SACC part at ``out_path``. @@ -62,6 +63,8 @@ def generate_pseudo_cl( Number of ell bins (required) power : float Power for powspace binning (0.5 = sqrt spacing) + compute_tomography : bool + Measure all tomographic pairs in one part when true. Returns ------- @@ -125,9 +128,9 @@ def generate_pseudo_cl( cv = CosmologyValidation(**cv_kwargs) # Pseudo-Cls only (no covariance), born directly at the final out_path. - cv.calculate_pseudo_cl(compute_tomography=False, out_path=out_path) + cv.calculate_pseudo_cl(compute_tomography=compute_tomography, out_path=out_path) - ell = cv.pseudo_cls[version]["tomo_bin_all_tomo_bin_all"]["pseudo_cl"]["ELL"] + ell = next(iter(cv._pseudo_cls[version].values()))["pseudo_cl"]["ELL"] print(f"Generated pseudo-Cl with {len(ell)} ell bins") print(f"ell range: [{ell.min():.1f}, {ell.max():.1f}]") return out_path @@ -145,6 +148,7 @@ def _from_snakemake(smk): binning=p["binning"], nbins=int(p["nbins"]), power=float(p.get("power", 0.5)), + compute_tomography=p.get("compute_tomography", False), ) diff --git a/workflow/scripts/generate_pseudo_cl_cov.py b/workflow/scripts/generate_pseudo_cl_cov.py index 0364f1bfa..93602e429 100644 --- a/workflow/scripts/generate_pseudo_cl_cov.py +++ b/workflow/scripts/generate_pseudo_cl_cov.py @@ -46,6 +46,7 @@ def generate_pseudo_cl_cov( binning: str = "powspace", nbins: int = 32, power: float = 0.5, + compute_tomography: bool = False, ): """Generate a pseudo-Cl covariance into ``output_dir``. @@ -73,11 +74,13 @@ def generate_pseudo_cl_cov( Number of ell bins power : float Power for powspace binning (0.5 = sqrt spacing) + compute_tomography : bool + Compute the covariance over all tomographic pairs when true. Returns ------- str - Path to the native non-tomographic iNKA covariance FITS. + Path to the native iNKA covariance FITS. """ os.makedirs(output_dir, exist_ok=True) @@ -133,12 +136,13 @@ def generate_pseudo_cl_cov( cv = CosmologyValidation(**cv_kwargs) - # Covariance of the ("all", "all") pair only print("Calculating covariance...") - cv.calculate_pseudo_cl_inka_cov(compute_tomography=False) + cv.calculate_pseudo_cl_inka_cov(compute_tomography=compute_tomography) # Report on the native product (moved by the Snakemake caller, if any) - src_cov = cv._output_path_pseudo_cl_cov(version, "iNKA", tomography=False) + src_cov = cv._output_path_pseudo_cl_cov( + version, "iNKA", tomography=compute_tomography + ) if os.path.exists(src_cov): with fits.open(src_cov) as hdul: # CV outputs covariance blocks as COVAR_XX_YY extensions @@ -170,6 +174,7 @@ def _from_snakemake(smk): binning=p["binning"], nbins=int(p["nbins"]), power=float(p.get("power", 0.5)), + compute_tomography=p.get("compute_tomography", False), ) shutil.move(src_cov, output_cov) gc.collect() diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index 1a495b7e9..51800cd61 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -21,5 +21,5 @@ output_dir=params["output_dir"], cov_estimate_method=params.get("cov_estimate_method", "th"), ) -cv.calculate_rho_tau_stats(tomography=False) +cv.calculate_rho_tau_stats(tomography=params.get("compute_tomography", False)) verify_outputs(snakemake) diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 38de61f90..4c114336a 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -52,6 +52,49 @@ def test_assemble_resolves(toy): }, job.input +def test_tomographic_assembly_resolves(toy, grids): + version = VERSIONS[0] + target = toy.cosmo_val / f"{version}_tomo.sacc" + # The toy supplies precomputed OneCovariance inputs, as for the non-tomo DAG. + mask = "_masked" if toy.config["covariance"].get("default_masked") else "" + for gaussian, grid in (("ng", grids["reporting"]), ("g", grids["integration"])): + path = Path( + toy.common.covariance_path( + version, + gaussian, + grid["min_sep"], + grid["max_sep"], + grid["nbins"], + mask, + "_tomo", + ) + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + result = toy.snakemake("-n", str(target)) + assert result.returncode == 0, result.stdout + jobs = parse_jobs(result.stdout) + (assembly,) = [j for j in jobs if j.rule == "assemble_sacc"] + reporting = toy.common.grid_binning(grids["reporting"]) + integration = toy.common.grid_binning(grids["integration"]) + harmonic = toy.common.pseudo_cl_tag(toy.config) + names = {Path(f).name for f in assembly.input} + assert { + f"{version}_xi_{reporting}_tomo.sacc", + f"pseudo_cl_{version}_{harmonic}_tomo.sacc", + f"pseudo_cl_cov_{version}_{harmonic}_tomo.fits", + f"{version}_cosebis_tomo.sacc", + f"{version}_pure_eb_tomo.sacc", + f"rho_tau_{version}_tomo_bin_all_{reporting}_tomo.sacc", + } <= names + assert any(name.endswith("_tomo.dat") for name in names) + for rule in ("cv_cosebis", "cv_pure_eb"): + (job,) = [j for j in jobs if j.rule == rule] + inputs = {Path(f).name for f in job.input} + assert f"{version}_xi_{integration}_tomo.sacc" in inputs + assert any(name.endswith("_tomo.dat") for name in inputs) + + def test_xi_leaves_a_measurement_only_as_a_part(toy, forced, grids): """No job reads or writes a ξ± text dump; the ξ± figures draw the parts.""" jobs = forced[1] diff --git a/workflow/tests/test_onecov_inputs.py b/workflow/tests/test_onecov_inputs.py new file mode 100644 index 000000000..cd1e9fbba --- /dev/null +++ b/workflow/tests/test_onecov_inputs.py @@ -0,0 +1,108 @@ +"""OneCovariance inputs for tomographic and whole-sample covariances.""" + +from contextlib import contextmanager +from pathlib import Path + +import yaml +from conftest import VERSIONS + +# Bin integrals 0.5 and 0.625: not unit-normalised. +NZ_ROWS = "0.0 0.0 0.0\n0.5 1.0 0.5\n1.0 0.0 1.5\n" +UNMASKED = ['covariance={"default_masked":false}'] + + +def _ini(toy, tomo_suffix=""): + # Unmasked target: no site-specific footprint needed by this test. + grid = toy.common.xi_grids(toy.config, toy.config["fiducial"])["integration"] + return toy.common.covariance_path( + VERSIONS[0], + "g", + grid["min_sep"], + grid["max_sep"], + grid["nbins"], + "", + tomo_suffix, + suffix=".ini", + ) + + +@contextmanager +def _two_bin_catalogue(toy, tomo_cov_th=True, nz_rows=NZ_ROWS): + """The toy version as a two-bin catalogue: a 2-column n(z), cov_th.tomo.""" + nz = toy.root / "data" / "nz_SP_v0.1_A.txt" + cat_config = toy.root / "cosmo_val" / "cat_config.yaml" + originals = {p: p.read_text() if p.exists() else None for p in (nz, cat_config)} + nz.write_text(nz_rows) + entries = yaml.safe_load(originals[cat_config]) + if tomo_cov_th: + entries[VERSIONS[0]]["cov_th"]["tomo"] = { + "n_e": [2.0, 3.0], + "sigma_e": [0.25, 0.35], + } + entries[VERSIONS[0]]["shear"]["tomo_bin_col"] = "TOM_BIN_ID" + cat_config.write_text(yaml.safe_dump(entries)) + try: + yield entries[VERSIONS[0]]["cov_th"] + finally: + for path, text in originals.items(): + if text is None: + path.unlink(missing_ok=True) + else: + path.write_text(text) + + +def test_tomographic_catalogue_covariance_inputs(toy): + """One multi-bin entry serves both covariances: the tomographic ini reads its + per-bin n(z) and cov_th.tomo, the non-tomographic one the bin-summed n(z) + and the whole-sample cov_th.""" + with _two_bin_catalogue(toy) as cov_th: + texts = {} + for suffix in ("", "_tomo"): + target = _ini(toy, suffix) + result = toy.snakemake( + "-F", + target, + "--allowed-rules", + "covariance_ini", + "nz_bin_sum", + config=UNMASKED, + ) + assert result.returncode == 0, result.stdout + texts[suffix] = Path(target).read_text() + assert "zlens_file = nz_SP_v0.1_A.txt" in texts["_tomo"] + assert "ellipticity_dispersion = 0.25, 0.35" in texts["_tomo"] + assert "n_eff_lensing = 2.0, 3.0" in texts["_tomo"] + assert f"ellipticity_dispersion = {float(cov_th['sigma_e'])!r}\n" in texts[""] + assert f"n_eff_lensing = {float(cov_th['n_e'])!r}\n" in texts[""] + summed = toy.cosmo_val / "nz" / f"{VERSIONS[0]}_nz_bin_sum.txt" + assert f"zlens_file = {summed.name}" in texts[""] + table = [ + list(map(float, line.split())) + for line in summed.read_text().splitlines() + if not line.startswith("#") + ] + assert table == [[0.0, 0.0], [0.5, 1.5], [1.0, 1.5]] + + +def test_tomographic_covariance_refuses_whole_sample_cov_th(toy): + """Without cov_th.tomo, a binned entry's tomographic ini is refused rather + than given the whole-sample n_e and sigma_e for every bin.""" + with _two_bin_catalogue(toy, tomo_cov_th=False): + result = toy.snakemake("-n", _ini(toy, "_tomo"), config=UNMASKED) + assert result.returncode != 0 + assert "per-bin n_e and sigma_e in cov_th.tomo" in result.stdout + + +def test_unit_normalised_bins_refused(toy): + """Bins that each integrate to one carry no weights; summing them is refused.""" + with _two_bin_catalogue(toy, nz_rows="0.0 0.0 0.0\n0.5 2.0 2.0\n1.0 0.0 0.0\n"): + result = toy.snakemake( + "-F", + _ini(toy), + "--allowed-rules", + "covariance_ini", + "nz_bin_sum", + config=UNMASKED, + ) + assert result.returncode != 0 + assert "weight the bins equally" in result.stdout