From 367b408b187aad0de3dc5925d79d3de96f937bab Mon Sep 17 00:00:00 2001 From: sallen12 Date: Tue, 24 Dec 2024 10:43:31 +0100 Subject: [PATCH 01/79] add theory documentation --- README.md | 55 ++-- docs/api/brier.md | 9 - docs/api/categorical.md | 28 ++ docs/api/crps.md | 23 +- docs/api/logarithmic.md | 11 + docs/index.md | 56 ++-- docs/theory.md | 563 ++++++++++++++++++++++++++++++++++++++++ mkdocs.yaml | 23 +- 8 files changed, 707 insertions(+), 61 deletions(-) delete mode 100644 docs/api/brier.md create mode 100644 docs/api/categorical.md create mode 100644 docs/theory.md diff --git a/README.md b/README.md index dafaa23..e2c658a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,38 @@ +

- - + dark banner + light banner

-
+# Probabilistic forecast evaluation [![CI](https://github.com/frazane/scoringrules/actions/workflows/ci.yaml/badge.svg?branch=main)](https://github.com/frazane/scoringrules/actions/workflows/ci.yaml) [![codecov](https://codecov.io/github/frazane/scoringrules/graph/badge.svg?token=J194X4HEBH)](https://codecov.io/github/frazane/scoringrules) [![pypi](https://img.shields.io/pypi/v/scoringrules.svg?colorB=)](https://pypi.python.org/pypi/scoringrules/) -Scoringrules is a python library for evaluating probabilistic forecasts by computing -scoring rules and other diagnostic quantities. It aims to assist forecasting practitioners by -providing a set of tools based the scientific literature and via its didactic approach. +`scoringrules` is a python library that provides scoring rules to evaluate probabilistic forecasts. +It's original goal was to reproduce the functionality of the R package +[`scoringRules`](https://cran.r-project.org/web/packages/scoringRules/index.html) in python, +thereby allowing forecasting practitioners working in python to enjoy the same tools as those +working in R. The methods implemented in `scoringrules` are therefore based around those +available in `scoringRules`, which are rooted in the scientific literature on probabilistic forecasting. + +The scoring rules available in `scoringrules` include, but are not limited to, the +- Brier Score +- Logarithmic Score +- (Discrete) Ranked Probability Score +- Continuous Ranked Probability Score (CRPS) +- Dawid-Sebastiani Score +- Energy Score +- Variogram Score +- Gaussian Kernel Score +- Threshold-Weighted CRPS +- Threshold-Weighted Energy Score + ## Features -- **Fast** computations of several probabilistic univariate and multivariate verification metrics +- **Fast** computation of several probabilistic univariate and multivariate verification metrics - **Multiple backends**: support for numpy (accelerated with numba), jax, pytorch and tensorflow - **Didactic approach** to probabilistic forecast evaluation through clear code and documentation @@ -28,7 +45,7 @@ pip install scoringrules ## Documentation -Learn more about scoringrules in its official documentation at https://frazane.github.io/scoringrules/. +Learn more about `scoringrules` in its official documentation at https://frazane.github.io/scoringrules/. ## Quick example @@ -41,26 +58,24 @@ fct = obs[:,None] + np.random.randn(100, 21) * 0.1 sr.crps_ensemble(obs, fct) ``` -## Metrics -- Brier Score -- Continuous Ranked Probability Score (CRPS) -- Logarithmic score -- Error Spread Score -- Energy Score -- Variogram Score - - ## Citation -If you found this library useful for your own research, consider citing: +If you found this library useful, consider citing: ``` @software{zanetta_scoringrules_2024, author = {Francesco Zanetta and Sam Allen}, - title = {Scoringrules: a python library for probabilistic forecast evaluation}, + title = {scoringrules: a python library for probabilistic forecast evaluation}, year = {2024}, url = {https://github.com/frazane/scoringrules} } ``` ## Acknowledgements -[scoringRules](https://cran.r-project.org/web/packages/scoringRules/index.html) served as a reference for this library. The authors did an outstanding work which greatly facilitated ours. The implementation of the ensemble-based metrics as jit-compiled numpy generalized `ufuncs` was first proposed in [properscoring](https://github.com/properscoring/properscoring), released under Apache License, Version 2.0. +- The widely-used R package [`scoringRules`](https://cran.r-project.org/web/packages/scoringRules/index.html) +served as a reference for this library, which greatly facilitated our work. We are additionally +grateful for fruitful discussions with the authors. +- The quality of this library has also benefited a lot from discussions with (and contributions from) +Nick Loveday and Tennessee Leeuwenburg, whose python library [`scores`](https://github.com/nci/scores) +similarly provides a comprehensive collection of forecast evaluation methods. +- The implementation of the ensemble-based metrics as jit-compiled numpy generalized `ufuncs` +was first proposed in [`properscoring`](https://github.com/properscoring/properscoring), released under Apache License, Version 2.0. diff --git a/docs/api/brier.md b/docs/api/brier.md deleted file mode 100644 index faad796..0000000 --- a/docs/api/brier.md +++ /dev/null @@ -1,9 +0,0 @@ -# Brier Score - -::: scoringrules.brier_score - -::: scoringrules.rps_score - -::: scoringrules.log_score - -::: scoringrules.rls_score diff --git a/docs/api/categorical.md b/docs/api/categorical.md new file mode 100644 index 0000000..f0dfe0a --- /dev/null +++ b/docs/api/categorical.md @@ -0,0 +1,28 @@ +# Scoring rules for categorical outcomes + +Suppose that the outcome $Y \in \{1, 2, \dots, K\}$ is one of $K$ possible categories. +Then, a probabilistic forecast $F$ for $Y$ is a vector $F = (F_{1}, \dots, F_{K})$ +with $\sum_{i=1}^{K} F_{i} = 1$, containing the forecast probabilities that $Y = 1, \dots, Y = K$. + +When $K = 2$, it is common to instead consider a binary outcome $Y \in \{0, 1\}$, which +represents an event that either occurs $(Y = 1)$ or does not $(Y = 0)$. The forecast in this +case is typically represented by a single probability $F \in [0, 1]$ that $Y = 1$, rather than +the vector $(F, 1 - F)$. However, evaluating these probability forecasts is a particular case of +evaluation methods for the more general categorical case described above. + + +## Brier Score + +::: scoringrules.brier_score + +## Log Score + +::: scoringrules.log_score + +## Ranked Probability Score + +::: scoringrules.rps_score + +## Ranked Log Score + +::: scoringrules.rls_score diff --git a/docs/api/crps.md b/docs/api/crps.md index 9e8895c..75dac96 100644 --- a/docs/api/crps.md +++ b/docs/api/crps.md @@ -1,12 +1,21 @@ # Continuous Ranked Probability Score -Formally, the CRPS is expressed as - -$$\text{CRPS}(F, y) = \int_{\mathbb{R}}[F(x)-\mathbb{1}\{y \le x\}]^2 dx$$ - -where $F(x) = P(X diff --git a/docs/api/logarithmic.md b/docs/api/logarithmic.md index 73e4326..607b626 100644 --- a/docs/api/logarithmic.md +++ b/docs/api/logarithmic.md @@ -1,5 +1,16 @@ # Logarithmic Score +The Log score is defined as + +$$\text{LS}(F, y) = - \log f(y)$$, + +where $f$ is the density function associated with $F$. The Log score is only defined for +forecasts that have a density function. This means the Log score cannot be calculated +when evaluating discrete forecast distributions, such as ensemble forecasts. + + +## Analytical formulations + ::: scoringrules.logs_beta ::: scoringrules.logs_binomial diff --git a/docs/index.md b/docs/index.md index 8203ba5..85247d7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,13 +10,33 @@

-Scoringrules is a python library for evaluating probabilistic forecasts by computing -scoring rules and other diagnostic quantities. It aims to assist forecasting practitioners by -providing a set of tools based the scientific literature and via its didactic approach. + +`scoringrules` is a python library that provides scoring rules to evaluate probabilistic forecasts. +It's original goal was to reproduce the functionality of the R package +[`scoringRules`](https://cran.r-project.org/web/packages/scoringRules/index.html) in python, +thereby allowing forecasting practitioners working in python to enjoy the same tools as those +working in R. The methods implemented in `scoringrules` are therefore based around those +available in `scoringRules`, which are rooted in the scientific literature on probabilistic forecasting. + +The scoring rules available in `scoringrules` include, but are not limited to, the + +- Brier Score +- Logarithmic Score +- (Discrete) Ranked Probability Score +- Continuous Ranked Probability Score (CRPS) +- Dawid-Sebastiani Score +- Energy Score +- Variogram Score +- Gaussian Kernel Score +- Threshold-Weighted CRPS +- Threshold-Weighted Energy Score +- Quantile Score +- Interval Score + ## Features -- **Fast** computations of several probabilistic univariate and multivariate verification metrics +- **Fast** computation of several probabilistic univariate and multivariate verification metrics - **Multiple backends**: support for numpy (accelerated with numba), jax, pytorch and tensorflow - **Didactic approach** to probabilistic forecast evaluation through clear code and documentation @@ -26,6 +46,12 @@ Requires python `>=3.10`! ``` pip install scoringrules ``` + +## Documentation + +Learn more about `scoringrules` in its official documentation at https://frazane.github.io/scoringrules/. + + ## Quick example ```python import scoringrules as sr @@ -36,26 +62,24 @@ fct = obs[:,None] + np.random.randn(100, 21) * 0.1 sr.crps_ensemble(obs, fct) ``` -## Metrics -- Brier Score -- Continuous Ranked Probability Score (CRPS) -- Logarithmic score -- Error Spread Score -- Energy Score -- Variogram Score - ## Citation -If you found this library useful for your own research, consider citing: +If you found this library useful, consider citing: ``` @software{zanetta_scoringrules_2024, author = {Francesco Zanetta and Sam Allen}, - title = {Scoringrules: a python library for probabilistic forecast evaluation}, + title = {scoringrules: a python library for probabilistic forecast evaluation}, year = {2024}, url = {https://github.com/frazane/scoringrules} } ``` - ## Acknowledgements -[scoringRules](https://cran.r-project.org/web/packages/scoringRules/index.html) served as a reference for this library. The authors did an outstanding work which greatly facilitated ours. The implementation of the ensemble-based metrics as jit-compiled numpy generalized `ufuncs` was first proposed in [properscoring](https://github.com/properscoring/properscoring), released under Apache License, Version 2.0. +- The widely-used R package [`scoringRules`](https://cran.r-project.org/web/packages/scoringRules/index.html) +served as a reference for this library, which greatly facilitated our work. We are additionally +grateful for fruitful discussions with the authors. +- The quality of this library has also benefited a lot from discussions with (and contributions from) +Nick Loveday and Tennessee Leeuwenburg, whose python library [`scores`](https://github.com/nci/scores) +similarly provides a comprehensive collection of forecast evaluation methods. +- The implementation of the ensemble-based metrics as jit-compiled numpy generalized `ufuncs` +was first proposed in [`properscoring`](https://github.com/properscoring/properscoring), released under Apache License, Version 2.0. diff --git a/docs/theory.md b/docs/theory.md new file mode 100644 index 0000000..907b7fb --- /dev/null +++ b/docs/theory.md @@ -0,0 +1,563 @@ +#################### +Proper scoring rules +#################### + +# The theory of proper scoring rules + +## Definitions + +Suppose we issue a probabilistic forecast $F$ for an outcome variable $Y$ that takes values +in some set $\Omega$. A probabilistic forecast for $Y$ is a probability distribution over +$\Omega$. + +A *scoring rule* is a function + +$$ S : \mathcal{F} \times \Omega \to \overline{\mathbb{R}}, $$ + +where $\mathcal{F}$ denotes a set of probability distributions over $\Omega$, and +$\overline{\mathbb{R}}$ denotes the extended real line. A scoring rule therefore takes a +probabilistic forecast $F \in \mathcal{F}$ and a corresponding outcome $y \in \Omega$ as +inputs, and outputs a real (possibly infinite) value, or score, $S(F, y)$. This score quantifies the +accuracy of $F$ given that $y$ has occurred. + +We assume that scoring rules are negatively oriented, so that a lower score indicates +a more accurate forecast, though the definitions and examples discussed below can easily +be modified to treat scoring rules that are positively oriented. + +It is widely accepted that scoring rules should be *proper*. A scoring rule $S$ is proper +(with respect to $\mathcal{F}$) if, when $Y \sim G$, + +\[ + \mathbb{E} S(G, Y) \leq \mathbb{E} S(F, Y) \quad \text{for all $F, G \in \mathcal{F}$}. +\] + +That is, the score is minimised in expectation by the true distribution underlying the outcome $Y$. +Put differently, if we believe that the outcome arises according to distribution $G$, then +our expected score is minimised by issuing $G$ as our forecast; proper scoring rules therefore +encourage honest predictions, and discourage hedging. A scoring rule is *strictly proper* +if the above inequality holds with equality if and only if $F = G$. + + +## Examples + +### Binary outcomes: + +Suppose $\Omega = \{0, 1\}$, so that $Y$ is a binary outcome that either occurs $(Y = 1)$ +or does not $(Y = 0)$. A probabilistic forecast for $Y$ is a single value $F \in [0, 1]$ +that quantifies the probability that $Y = 1$. To evaluate such a forecast, popular scoring +rules include the *Brier score* and *Logarithmic (Log) score*. + +The Brier score is defined as + +\[ + \mathrm{BS}(F, y) = (F - y)^{2}, +\] + +while the Log score is defined as + +\[ + \mathrm{LS}(F, y) = -\log |F + y - 1| = + \begin{cases} + -\log F & \text{if} \quad y = 1, \\ + -\log (1 - F) & \text{if} \quad y = 0. + \end{cases} +\] + +Other popular binary scoring rules include the spherical score, power score, and pseudo-spherical +score. + + +### Categorical outcomes: + +Suppose now that $Y$ is a categorical variable, taking values in $\Omega = \{1, \dots, K\}$, +for $K > 1$. Binary outcomes constitute a particular case when $K = 2$. +A probabilistic forecast for $Y$ is a vector + +\[ + F = (F_{1}, \dots, F_{K}) \in [0, 1]^{K} \quad \text{such that} \quad \sum_{i=1}^{K} F_{i} = 1. +\] + +The $i$-th element of $F$ represents the probability that $Y = i$, for $i = 1, \dots, K$. + +Proper scoring rules for binary outcomes can readily be used to evaluate probabilistic +forecasts for categorical outcomes, by applying the score separately for each category, +and then summing these $K$ scores. + +For example, the Brier score becomes + +\[ + \mathrm{BS}_{K}(F, y) = \sum_{i=1}^{K} (F_{i} - \mathbf{1}\{y = i\})^{2}. +\] + +where $\mathbf{1}\{ \cdotp \}$ denotes the indicator function. +When $K = 2$, we recover the Brier score for binary outcomes (multiplied by a factor of 2). + +The Log score can similarly be extended, though it is more common to define the Log score for +categorical outcomes as + +\[ + \mathrm{LS}(F, y) = -\log F_{y}. +\] + +As in the binary case, this Log score evaluates the forecast $F$ only via the probability +assigned to the outcome that occurs. + +The Brier score assesses the forecast probability separately at each category. However, if the +categories are ordered, i.e. the outcome is ordinal, then one could argue that the distance +between categories should also be considered. For example, if $K = 3$ and $y = 3$, then +the forecast $F^{(1)} = (1, 0, 0)$ would receive the same score as $F^{(2)} = (0, 1, 0)$, +according to both the Brier score and the Log score. But one could argue that the second forecast +$F^{(2)}$ should be preferred: $F^{(1)}$ assigns all probability to the first category, +$F^{(2)}$ assigns all probability to the second category, and the second category is closer to the third. + +To account for the ordering of the categories, it is common to apply the Brier score and +Log score to cumulative probabilities. Let + +\begin{align} + \tilde{F} &= (\tilde{F}_{1}, \dots, \tilde{F}_{K}) \in [0, 1]^{K} \quad \text{with + $\tilde{F}_{j} = \sum_{i=1}^{j} F_{i}$} \quad &\text{for $j = 1, \dots, K$,} \\ + \tilde{y} &= (\tilde{y}_{1}, \dots, \tilde{y}_{K}) \in \{0, 1\}^{K} \quad \text{with + $\tilde{y}_{j} = \sum_{i=1}^{j} \mathbf{1}\{y = i\}$} \quad &\text{for $j = 1, \dots, K$.} +\end{align} + +Then, the *Ranked Probability Score (RPS)* is defined as + +\[ + \mathrm{RPS}(F, y) = \sum_{i=1}^{K} (\tilde{F}_{j} - \tilde{y}_{j})^{2}, +\] + +and the *Ranked Logarithmic Score (RLS)* is + +\[ + \mathrm{RLS}(F, y) = - \sum_{i=1}^{K} \log | \tilde{F}_{j} + \tilde{y}_{j} - 1|. +\] + +These categorical scoring rules can also be implemented when $K = \infty$, +as is the case for unbounded count data, for example. Other scoring rules for binary +outcomes can similarly be extended in this way to construct scoring rules for categorical +outcomes. + + +### Continuous outcomes: + +Let $Y \in \mathbb{R}$ and suppose $F$ is a cumulative distribution function over the real line. + +We can similarly define proper scoring rules for continuous outcomes in terms of scoring rules +for binary outcomes. + +The *Continuous Ranked Probability Score (CRPS)* is defined as + +\begin{align*} + \mathrm{CRPS}(F, y) &= \int_{-\infty}^{\infty} (F(x) - \mathbf{1}\{y \le x\})^{2} dx \\ + &= \mathbb{E} | X - y | - \frac{1}{2} \mathbb{E} | X - X^{\prime} |, +\end{align*} + +where $X, X^{\prime} \sim F$ are independent. +The CRPS is defined as the Brier score for threshold exceedance forecasts integrated over all +thresholds. This corresponds to a generalisation of the RPS for when there are (uncountably) +infinite possible categories. + +Similarly, the *Continuous Ranked Logarithmic Score (CRLS)* is defined as + +\[ + \mathrm{CRLS}(F, y) = -\int_{-\infty}^{\infty} \log |F(x) + \mathbf{1}\{y \le x\} - 1| dx. +\] + +The Log score can additionally be generalised to continuous outcomes whenever +the forecast $F$ has a density function, denoted here by $f$. In this case, the +Log score is defined as + +\[ + \mathrm{LS}(F, y) = - \log f(y). +\] + +The Log score again depends only on the forecast distribution $F$ at the observation $y$, +ignoring the probability density assigned to other outcomes. This scoring rule is therefore +*local*. + +When $F$ is a normal distribution, the Log score simplifies to the *Dawid-Sebastiani* score, + +\[ + \mathrm{DS}(F, y) = \frac{(y - \mu_{F})^{2}}{\sigma_{F}^{2}} + 2 \log \sigma_{F}, +\] + +where $\mu_{F}$ and $\sigma_{F}$ represent the mean and standard deviation of the forecast +distribution $F$. While the Dawid-Sebastiani score corresponds to the Log score for a normal distribution, +it also constitutes a proper scoring rule for any other forecast distribution with finite +mean and variance. The Dawid-Sebastiani score evaluates forecasts only via their mean and standard +deviation, making it easy to implement in practice, but insensitive to higher moments of the +predictive distribution. + + +### Multivariate outcomes: + +Let $\boldsymbol{Y} \in \mathbb{R}^{d}$, with $d > 1$, and suppose $F$ is a multivariate probability distribution. + +If $F$ admits a multivariate density function $f$, then the Log score can be defined analogously to +in the univariate case, + +\[ + \mathrm{LS}(F, \boldsymbol{y}) = - \log f(\boldsymbol{y}). +\] + +The Dawid-Sebastiani score can similarly be extended to higher dimensions +by replacing the predictive mean $\mu_{F}$ and variance $\sigma_{F}^{2}$ with the mean vector +$\boldsymbol{\mu}_{F} \in \mathbb{R}^{d}$ and covariance matrix $\Sigma_{F} \in \mathbb{R}^{d \times d}$. +This becomes + +\[ + \mathrm{DS}(F, \boldsymbol{y}) = (\boldsymbol{y} - \boldsymbol{\mu}_{F})^{\top} \Sigma_{F}^{-1} (\boldsymbol{y} - \boldsymbol{\mu}_{F}) + \log \det(\Sigma_{F}), +\] + +where $\top$ denotes the vector transpose, and $\det$ the matrix determinant. The Dawid-Sebastiani +score is equivalent to the Log score for a multivariate normal distribution. However, the +Dawid-Sebastiani score is more readily applicable than the Log score, since it depends only on +the predictive mean vector and covariance matrix, and does not require a predictive density. +However, particularly in high dimensions, the predictive covariance matrix is often not available, +or must be estimated from a finite sample. + +Instead, it is common to evaluate multivariate forecasts using the *Energy score*, + +\[ + \mathrm{ES}(F, \boldsymbol{y}) = \mathbb{E} \| \boldsymbol{X} - \boldsymbol{y} \| - \frac{1}{2} \mathbb{E} \| \boldsymbol{X} - \boldsymbol{X}^{\prime} \|, +\] + +where $\boldsymbol{X}, \boldsymbol{X}^{\prime} \sim F$ are independent, and $\| \cdot \|$ is the Euclidean distance on $\mathbb{R}^{d}$. +The Energy score can be interpreted as a multivariate generalisation of the CRPS, which is recovered when $d = 1$. + +The Energy score is sensitive to both the univariate performance of the multivariate forecast distribution +along each margin, as well as the predicted dependence structure between the different dimensions. +The *Variogram score* was introduced as an alternative scoring rule that is less sensitive to the +univariate forecast performance, thereby focusing on the multivariate dependence structure. +The Variogram score is defined as + +\[ + \mathrm{VS}(F, \boldsymbol{y}) = \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left( \mathbb{E} | X_{i} - X_{j} |^{p} - | y_{i} - y_{j} |^{p} \right)^{2}, +\] + +where $\boldsymbol{X} \sim F$, and $h_{i,j} \ge 0$ are weights assigned to different pairs of dimensions. +The Variogram score therefore measures the distance between between the expected variogram of +the forecast distribution, and the variogram of the single observation $\boldsymbol{y}$. + + +## Characterisations + +The proper scoring rules introduced above are particular examples that are +commonly used to evaluate probabilistic forecasts in practice. More generally, it is possible +to derive characterisations of all proper scoring rules, allowing users to easily construct +novel scoring rules that incorporate their personal preferences. + + +### Schervish representation + +Schervish (1969) demonstrated that all proper scoring rules for binary outcomes can be +written in the following form (up to unimportant technical details): + +\[ + S(F, y) = \int_{(0, 1)} c 1\{F > c, y = 0\} + (1 - c) 1\{F < c, y = 1\} + c(1 - c) 1\{F = c\} d \nu(c), +\] + +for some non-negative measure $\nu$ on $(0, 1)$. The measure $\nu$ emphasises particular trade-offs +between over-prediction ($F$ high, $y = 0$) and under-prediction ($F$ low, $y = 1$). + +The Brier score corresponds to the measure $d \nu(c) = 2 dc$, and the Log score $d \nu(c) = \frac{1}{c(1 - c)}dc$. + +This essentially means that any proper scoring rule for binary outcomes can be obtained by choosing a weight function +over the trade-off parameter $c$, and plugging this into the characterisation above. If $\nu$ is a +Dirac measure at a particular value of $c \in (0, 1)$, then we recover the (asymmetric) *zero-one score*. +Every proper scoring rule can therefore be expressed as a weighted integral of the zero-one score over all +trade-off parameters. + + +### Convex functions + +More generally, for categorical forecasts, a scoring rule is proper if and only if there exists a convex function +$g : [0, 1]^{K} \to \mathbb{R}$ such that + +\[ + S(F, y) = \langle g^{\prime}(F), F \rangle - g(F) - g^{\prime}(F)_{y} \quad \text{for all $F \in [0, 1]^{K}$ and $y \in \{1, \dots, K\}$,} +\] + +where $g^{\prime}(F) \in \mathbb{R}^{K}$ is a sub-gradient of $g$ at $F \in \mathcal{F}$, +and $g^{\prime}(F)_{y}$ is the $y$-th element of this vector, for $y \in \{1, \dots, K\}$. + +If $K = 2$ and $g$ is smooth, then the Schervish representation is recovered by setting +$d\nu(c) = g^{\prime \prime}(c) dc$. That is, the Lesbesgue density of $\nu$ is $g^{\prime \prime}$. +The Brier score corresponds to the convex function $g(F) = \sum_{j=1}^{K} F_{j}^{2} - 1$, +and the Log score to the Shannon entropy function $g(F) = \sum_{j=1}^{K} F_{j} \log F_{j}$. + +Gneiting and Raftery (2007) generalised this to arbitrary outcome domains by showing that +a scoring rule $S$ is proper relative to the set $\mathcal{F}$ if and only if there exists a +convex function $g : \Omega \to \mathbb{R}$ such that + +\[ + S(F, y) = \int g^{\prime}(F, z) dF(z) - g(F) - g^{\prime}(F, y) \quad \text{for all $F \in \mathcal{F}$ and $y \in \Omega$,} +\] + +where $g^{\prime}(F, \cdot)$ is a subtangent of $g$ at $F$ (Gneiting and Raftery, 2007, Theorem 1). +The class of strictly proper scoring rules with respect to $\mathcal{F}$ is characterised by +replacing convex with strictly convex. + +For example, the CRPS corresponds to the convex function $g(F) = -\mathbb{E} |X - X^{\prime}|$ +for $X, X^{\prime} \sim F$ independent. The energy score corresponds to the same function, with the +absolute distance replaced with the Euclidean distance on $\mathbb{R}^{d}$. + + +## Classes of scoring rules + +### Local scoring rules + +Local scoring rules assume that forecasts should be evaluated based only on what has occurred, +and not what could have occurred but did not. Hence, local scoring rules depend on the forecast +distribution only via the predictive density evaluated at the outcome $y$. More generally, +assuming the forecast distribution $F$ admits a density function $f$ whose derivatives +$f^{\prime}, f^{\prime \prime}, \dots, f^{(j)}$ exist, a scoring rule $S$ is called *local of order $j$* +if there exists a function $s : \mathbb{R}^{2+j} \to \overline{\mathbb{R}}$ such that + +\[ + S(F, y) = s(y, f(y), f^{\prime}(y), \dots, f^{(j)}(y)). +\] + +That is, the scoring rule $S$ can be written as a function of $y$ and the first $j$ derivatives +of the forecast distribution evaluated at $y$. + +The Log score is the only proper scoring rule that is local of order $j = 0$ (up to equivalence), +see Bernardo (1979). However, local proper scoring rules of higher orders also exist, the most +well-known example being the *Hyvärinen score*, which is local of order $j = 2$. + + +### Matheson and Winkler representation + +Scoring rules that are not local are often called *distance-sensitive*, since they take into +account the distance between the forecast distribution and the observation. +One general approach to construct distance-sensitive proper scoring rules for univariate real-valued outcomes +is to use a proper scoring rule for binary outcomes to evaluate threshold exceedance forecasts, +and to integrate this over all thresholds. + +That is, if $S_{0}$ is a proper scoring rule for binary outcomes, then + +\[ + S(F, y) = \int_{-\infty}^{\infty} S_{0}(F(x), \mathbf{1}\{y \leq x\}) dH(x), +\] + +is a proper scoring rule for univariate real-valued outcomes, for some non-negative measure $H$ on the real line. + +The CRPS is equivalent to the case when $S_{0}$ is the Brier score and $H$ is the Lebsegue measure, +i.e. $dH(x) = dx$. Replacing the Brier score with the Log score recovers the Continuous Ranked +Logarithmic Score (CRLS). By changing the measure $H$ in the above construction, different regions of +the outcome space can be assigned different weight, thereby allowing users to emphasise particular outcomes +during evaluation. This provides a means to construct weighted scoring rules (see below). + +This construction can also be used to evaluate multivariate probabilistic forecasts, by replacing $F(x)$ +with the multivariate distribution function, and $\mathbf{1}\{ y \leq x\}$ with +$\mathbf{1}\{ y_{1} \leq x_{1}, \dots, y_{d} \leq x_{d}\}$, before integrating over all +$\boldsymbol{x} \in \mathbb{R}^{d}$. Gneiting and Raftery (2007) use this to introduce a multivariate +CRPS. + + + +### Kernel scores + +Many popular scoring rules belong to the very general class of *kernel scores*, which are +scoring rules defined in terms of positive definite kernels. A positive definite kernel on +$\Omega$ is a symmetric function $k : \Omega \times \Omega \to \mathbb{R}$, such that + +\[ + \sum_{i=1}^{n} \sum_{j=1}^{n} a_{i} a_{j} k(x_{i}, x_{j}) \geq 0, +\] + +for all $n \in \mathbb{N}$, $a_{i}, a_{j} \in \mathbb{R}$, and $x_{i}, x_{j} \in \Omega$. +A positive definite kernel constitutes an inner product in a feature space, and can therefore +be loosely interpreted as a measure of similarity between its two inputs. + +The *kernel score* corresponding to the positive definite kernel $k$ is defined as + +\[ + S_{k}(F, y) = \frac{1}{2} \mathbb{E} k(y, y) + \frac{1}{2} \mathbb{E} k(X, X^{\prime}) - \mathbb{E} k(X, y), +\] + +where $X, X^{\prime} \sim F$ are independent. The first term on the right-hand-side does not +depend on $F$, so could be removed, but is included here to ensure that the kernel score is always +non-negative, and to retain an interpretation in terms of Maximum Mean Discrepancies (see below). + +The energy score is the kernel score associated with any kernel + +\[ + k(x, x^{\prime}) = \| x - x_{0} \| + \| x^{\prime} - x_{0} \| - \| x - x^{\prime} \| +\] + +for arbitrary $x_{0} \in \Omega$, which encompasses the CRPS when $d = 1$. We +refer to these kernels as *energy kernels*. + +Another popular kernel is the Gaussian kernel, + +\[ + k_{\gamma}(x, x^{\prime}) = \exp \left( - \gamma \| x - x^{\prime} \|^{2} \right) +\] + +for some length-scale parameter $\gamma > 0$. Plugging this into the kernel score definition +above yields the *Gaussian kernel score*. + +Other examples include the Brier score, variogram score, and angular CRPS. +Allen et al. (2023) demonstrate how changing the kernel can be used to target particular outcomes, +and illustrate that the threshold-weighted CRPS constitutes a particular sub-class of kernel scores. +Any positive definite kernel can be used to define a scoring rule. This allows for very +flexible forecast evaluation on arbitrary outcome domains. + +Steinwart and Ziegel (2021) discuss the connection between kernel scores and maximum mean +discrepancies (MMDs), which are commonly used to measure the distance between probability +distributions. Aronszajn (195) demonstrated that every positive definite kernel $k$ induces +a Hilbert space of functions, referred to as a Reproducing Kernel Hilbert Space and denoted by $\mathcal{H}_{k}$. +A probability distribution $F$ on $\Omega$ can be converted to an element in an RKHS via its kernel mean embedding, + +\[ +\mu_{F} = \int_{\Omega} k(x, \cdotp) dF(x), +\] + +allowing kernel methods to be applied to probabilistic forecasts. + +For example, to calculate the distance between two distributions $F$ and $G$, we can calculate +the RKHS norm $\| \cdot \|_{\mathcal{H}_{k}}$ betewen their kernel mean embeddings, +$\| \mu_{F} - \mu_{G} \|_{\mathcal{H}_{k}}$. This is called the *Maximum Mean Discrepancy (MMD)* +between $F$ and $G$. In practice, it is common to calculate the squared MMD, which can be expressed as + +\[ + \| \mu_{F} - \mu_{G} \|_{\mathcal{H}_{k}}^{2} = \mathbb{E} k(Y, Y^{\prime}) + \mathbb{E} k(X, X^{\prime}) - 2\mathbb{E} k(X, Y), +\] + +where $X, X^{\prime} \sim F$ and $Y, Y^{\prime} \sim G$ are independent. + +The kernel score corresponding to $k$ is simply (half) the squared MMD between the probabilistic +prediction $F$ and a point measure at the observation $y$, denoted $\delta_{y}$. +The propriety of kernel scores follows from the non-negativeness of the MMD (Steinwart and Ziegel, 2021). +This relationship between kernel scores and MMDs is leveraged by Allen et al. (2024) to +introduce efficient methods when pooling probabilistic forecasts. + + + +### Weighted scoring rules + +Often, some outcomes lead to larger impacts than others, making accurate forecasts for these outcomes +more valuable. To account for this during forecast evaluation, scoring rules could be adapted so that +they assign more weight to outcomes that are higher impact. Weighted scoring rules extend conventional +scoring rules by incorporating non-negative weight functions $w : \Omega \to [0, \infty)$ that control +how much emphasis is to be placed on each outcome in $\Omega$. + +For example, for categorical outcomes, the Brier score and RPS can easily be extended by weighting each +term of the summation in their definition, thereby assigning more weight to particular categories. +For continuous outcomes, this can be extended to the CRPS by incorporating a weight function into +the integral, yielding the *threshold-weighted CRPS* + +\begin{align*} + \mathrm{twCRPS}(F, y; w) &= \int_{\mathbb{R}} (F(x) - \mathbb{1}\{y \leq x\})^{2} w(x) dx \\ + &= \mathbb{E} | v(X) - v(y) | - \frac{1}{2} \mathbb{E} | v(X) - v(X^{\prime}) |, +\end{align*} + +where $X, X^{\prime} \sim F$ are independent, and $v$ is such that $v(x) - v(x^{\prime}) = \int_{x^{\prime}}^{x} w(z) dz$ +for any $x, x^{\prime} \in \mathbb{R}$. Allen et al. (2022) refer to $v$ as the *chaining function*. +The threshold-weighted CRPS corresponds to choosing the measure $H$ in the construction in the previous section to +have Lebesgue density $w$. This can trivially be applied to the CRLS to yield a threshold-weighted CRLS, +as well as any scoring rule defined using the construction above. + +Alternatively, the second expression highlights that the threshold-weighted CRPS is a kernel score, +and that the weighting transforms the forecasts, according to the chaining function $v$, which is an +anti-derivative of the weight function $w$. + +For example, a popular weight function is $w(z) = \mathbb{1}\{z > t\}$, which restricts attention to +values above some threshold of interest $t \in \mathbb{R}$. In this case, a possible chaining function +$v$ is $v(z) = \max(z, t)$, in which case the threshold-weighted CRPS censors the observation and the +forecast distribution at the threshold $t$, and then calculates the standard CRPS for the censored forecast and +observation. + +Evaluating censored forecast distributions was also proposed by Diks et al. (2011) when introducing +weighted versions of the Log score. They introduce the *censored likelihood score* as + +\[ + \mathrm{ceLS}(F, y; w) = - w(y) \log f(y) - (1 - w(y)) \log \left( 1 - \int_{\mathbb{R}} w(z) f(z) dz \right). +\] + +Rather than censoring the distribution, Diks et al. (2011) additionally propose the *conditional likelihood score*, +which evaluates the conditional distribution given the weight function, + +\[ + \mathrm{coLS}(F, y; w) = - w(y) \log f(y) + w(y) \log \left( \int_{\mathbb{R}} w(z) f(z) dz \right). +\] + + + +### Consistent scoring functions + +When only a functional of the forecast distribution $F$ is of interest, +proper scoring rules can be defined using consistent scoring functions. + +Throughout, we have considered the case when the forecast $F$ is probabilistic. Often, +however, for reasons of communication or tradition, the forecast $F$ is not probabilistic. +For example, it is common to report the expected outcome, or a conditional quantile. + +These forecasts can be evaluated using *scoring functions*. Scoring functions are similar +in essence to scoring rules, but they take a point-valued rather than probabilistic forecast +as input, + +\[ + s : \Omega \times \Omega \to [0, \infty]. +\] + +We use a lower case $s$ to distinguish scoring functions from scoring rules. + +To impose theoretical guarantees upon the scoring function, it is necessary to assume that +the point-valued forecast comes from an underlying probability distribution. That is, +the forecast is a functional $\text{T}$ of a predictive distribution. For example, $\text{T}$ could +be the mean, median, or quantile. A scoring rule is called *consistent* for the functional $\text{T}$ +(relative to a class of distributions $\mathcal{F}$), if, when $Y \sim G$, + +\[ + \mathbb{E} s(x_{G}, Y) \leq \mathbb{E} s(x, Y) \quad \text{for all $x \in \Omega, G \in \mathcal{F}$,} +\] + +where $x_{G} \in \text{T}(G)$. + +For example, the *squared error* + +\[ + s(x, y) = (x - y)^{2} +\] + +is consistent for the mean functional, and the *quantile score* (also called *pinball loss* or *check loss*) + +\[ + s_{\alpha}(x, y) = (\mathbf{1}\{y \leq x\} - \alpha)(x - y) = + \begin{cases} + (1 - \alpha)|x - y| & \quad \text{if $y \leq x$}, \\ + \phantom{(1 - }\alpha \phantom{)}|x - y| & \quad \text{if $y \geq x$} + \end{cases} +\] + +is consistent for the $\alpha$-quantile (for $\alpha \in (0, 1)$). When $\alpha = 0.5$, +we recover the *absolute error* + +\[ + s(x, y) = |x - y|, +\] + +which is consistent for the median functional. Note that these are not the only scoring rules +that are consistent for the mean, median, and quantiles (see Gneiting 2010, Ehm et al., 2016). + +Proper scoring rules can be constructed using consistent scoring functions. In particular, +if $s$ is a consistent scoring function for a functional $\text{T}$ (relative to $\mathcal{F}$), then + +\[ + S(F, y) = s(\text{T}(F), y) +\] + +is a proper scoring rule (relative to $\mathcal{F}$). Hence, the squared error, quantile loss, +and absolute error above all induce proper scoring rules. +These scoring rules evaluate $F$ only via the functional of interest. +Hence, while they are proper, they are generally not strictly proper. + +This framework can additionally be used to evaluate interval forecasts, or prediction intervals. +Given a central interval forecast in the form of a lower and upper value $L, U \in \mathbb{R}$ with $L < U$, +the interval score is defined as + +\[ + \mathrm{IS}([L, U], y) = |U - L| + \frac{2}{\alpha} (y - u) \mathbf{1} \{ y > u \} + \frac{2}{\alpha} (l - y) \mathbf{1} \{ y < l \}. +\] diff --git a/mkdocs.yaml b/mkdocs.yaml index 060934e..97c8769 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -7,16 +7,21 @@ nav: - Home: index.md - User guide: user_guide.md - Contributing: contributing.md + - Theory: theory.md - API reference: - - Brier Score: api/brier.md - - Continuous Ranked Probability Score: api/crps.md - - Logarithmic Score: api/logarithmic.md - - Error Spread Score: api/error_spread.md - - Energy Score: api/energy.md - - Variogram Score: api/variogram.md - - Interval Score: api/interval.md - - Quantile Score: api/quantile.md - - Visualization: api/visualization.md + - Categorical Outcomes: api/categorical.md + - Continuous Univariate Outcomes: + - Continuous Ranked Probability Score: api/crps.md + - Logarithmic Score: api/logarithmic.md + - Error Spread Score: api/error_spread.md + - Continuous Multivariate Outcomes: + - Energy Score: api/energy.md + - Variogram Score: api/variogram.md + - Gaussian Kernel Score: api/kernels.md + - Functionals: + - Interval Score: api/interval.md + - Quantile Score: api/quantile.md + - Visualization: api/visualization.md theme: From f05bc952bb41ec7cf35a542eaa17b7bded1b2677 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Thu, 16 Jan 2025 16:45:34 +0100 Subject: [PATCH 02/79] add readthedocs config --- .readthedocs.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..f72c324 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,22 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version, and other tools you might need +build: + os: ubuntu-24.04 + tools: + python: "3.13" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Optionally, but recommended, +# declare the Python requirements required to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# python: +# install: +# - requirements: docs/requirements.txt From d39db2bfc31fe040ff669cf8ac3165b58b0e2be2 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Thu, 16 Jan 2025 16:49:34 +0100 Subject: [PATCH 03/79] add readthedocs files --- docs/Makefile | 20 ++++ docs/_static/banner_dark.svg | 20 ++++ docs/_static/banner_dark_long.svg | 19 ++++ docs/_static/banner_light.svg | 20 ++++ docs/_static/banner_light_long.svg | 19 ++++ docs/_static/favicon.ico | Bin 0 -> 8254 bytes docs/_static/logo.svg | 15 +++ docs/_templates/module.rst | 62 ++++++++++++ docs/assets/animation.gif | Bin 149395 -> 0 bytes docs/assets/images/banner_dark.svg | 4 - docs/assets/images/banner_light.svg | 4 - docs/assets/images/favicon.png | Bin 13267 -> 0 bytes docs/assets/images/logo.svg | 4 - docs/conf.py | 56 +++++++++++ docs/extras/estimators.md | 34 ------- docs/index.md | 31 ++++-- docs/make.bat | 35 +++++++ docs/reference.md | 137 +++++++++++++++++++++++++ docs/theory.md | 148 ++++++++++++++-------------- 19 files changed, 499 insertions(+), 129 deletions(-) create mode 100644 docs/Makefile create mode 100644 docs/_static/banner_dark.svg create mode 100644 docs/_static/banner_dark_long.svg create mode 100644 docs/_static/banner_light.svg create mode 100644 docs/_static/banner_light_long.svg create mode 100644 docs/_static/favicon.ico create mode 100644 docs/_static/logo.svg create mode 100644 docs/_templates/module.rst delete mode 100644 docs/assets/animation.gif delete mode 100644 docs/assets/images/banner_dark.svg delete mode 100644 docs/assets/images/banner_light.svg delete mode 100644 docs/assets/images/favicon.png delete mode 100644 docs/assets/images/logo.svg create mode 100644 docs/conf.py delete mode 100644 docs/extras/estimators.md create mode 100644 docs/make.bat create mode 100644 docs/reference.md diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/banner_dark.svg b/docs/_static/banner_dark.svg new file mode 100644 index 0000000..5e52304 --- /dev/null +++ b/docs/_static/banner_dark.svg @@ -0,0 +1,20 @@ + + + + + + + + scoring + + rules + + diff --git a/docs/_static/banner_dark_long.svg b/docs/_static/banner_dark_long.svg new file mode 100644 index 0000000..3b02b35 --- /dev/null +++ b/docs/_static/banner_dark_long.svg @@ -0,0 +1,19 @@ + + + + + + + + + scoringrules + + diff --git a/docs/_static/banner_light.svg b/docs/_static/banner_light.svg new file mode 100644 index 0000000..365861d --- /dev/null +++ b/docs/_static/banner_light.svg @@ -0,0 +1,20 @@ + + + + + + + + scoring + + rules + + diff --git a/docs/_static/banner_light_long.svg b/docs/_static/banner_light_long.svg new file mode 100644 index 0000000..d5c77e3 --- /dev/null +++ b/docs/_static/banner_light_long.svg @@ -0,0 +1,19 @@ + + + + + + + + + scoringrules + + diff --git a/docs/_static/favicon.ico b/docs/_static/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..2b10f09687ff3623e7c6af5b9d480fcd9c06bcfd GIT binary patch literal 8254 zcmeHK`)`v~82(BxgK<$~BE+Csa0nWmxD9X!WXV|D(K#b`_7U_J^EXMVhQ%y?@s#ji&5ur}dA;jP(CvOO;Px%q z>)28ma{1PU9o}srhc5uTHQeZ15pXnD?sc~;_E-B20G#rOAz}G~`v4ERUoQ!{yupye z`$gF0{Ws+Fvi!}*!0U3@={pp3`8E>=4OR}LDRVgkcKdpL$n9&*k99W=n(<3vw|k-5 zt-7z!5u?IN(jJ%Z$)MBo<6W??Vf;8+Psr)p3tc`5JH2Pozs=XXG#nV_w?^07O$y$< zdx6dFXW{2OG|$o5b28v;UKVO@Ui9&%Hy(l)PI!miZtC`W=XW=4c{=R!)L`#T&O?=_ zpu-!9w6ql88(EUDuMD^s5f8_PoeViWe}o(@kJDZjmGmPqL)B=^xNadcm007?;UI&Qd? zP8b-q@q{5I_ERY@Th8WJSF>3(hJ4f`mhZ_Qk0+z&Zn2(QCFMG-zo5dNK&>~?8g@!D z0_Uv2RNL{SAxURH^DtdS1sg`YQ(4K@(paFYYX*E?QTe?Qa(L{@p3F|n*Yh<>&VSg2 z9G~fzOk;>qjVb!$acGd#9}wu}<;()?8x`e@!7_00koRt<=Xv}}rLI;*k8Fq~)ib|#;J&}$ZH8}_Ni?j!II>TbIgsfI; zzi4K&xrOGa1}m4c-N0lb53nE69Pa7QMT~@p92m6YbbopeobYcqjNxqY{K|Fy&yeXE z%2h){!)%Mid>iJA)Ih7V~BzH5L@I%l+3;bu8m zEVlt8RaaL}$NCbqm6)?x%;x17GobG1+ac~&j!!wreW3kye%5gIVE52G{H&_8T5qVZ z@@W8mmK55TXuqPR(TMLbxV#E%`5wT1%;TSmO<1V)U`7KY!yR{QG9k?-{_ry-`_tB6 zRtwTbt&h5IVef2cWI>UChc7}{`5xRkYm^)q*htLu6fs2%4Zy#n;gH3a>oQMWi1jwB zmA$=TQ|7zs^78vJkNi@4$XJ1I&A8A2c1-Z3yu#n_>iRRQG1Yx=%bXSYoZ=lhWZO43 zGVwj`iMGy)40hW3V(r^{2Rm3_tm`%6gu8v8S_9j(tid|a= zGN)mTzf6AmVx1qT@LBIb$5HiPWxov$Dl zS4A$sWdiMBB--)-tvnNzO!zMD8&4PyLzBY?w=eu|O6-arKWF1sgUEwoNLcb8In`q9 zjkRwS_$qt4H`W>IKGs&0^@&bHL(N>^pUlSr=PXQXGyWA6^?u&ZPGvh%q`%FKxW`oZ zRFfh%e@6z|UBXXQRTcd`K((dhAAOt6Mt`I8hkKUYD=1<%zwJZ)^$xZl(r{2}1K(8T z!9DOW?xn934rZpzqIZw~=jG)i4>HDdeS>W;L + + + + + + + diff --git a/docs/_templates/module.rst b/docs/_templates/module.rst new file mode 100644 index 0000000..7ec9d9c --- /dev/null +++ b/docs/_templates/module.rst @@ -0,0 +1,62 @@ +{{ objname | escape | underline}} + +.. automodule:: {{ fullname }} + :members: + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Module Attributes') }} + + .. autosummary:: + {% for item in attributes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block functions %} + {% if functions %} + .. rubric:: {{ _('Functions') }} + + .. autosummary:: + {% for item in functions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block classes %} + {% if classes %} + .. rubric:: {{ _('Classes') }} + + .. autosummary:: + {% for item in classes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block exceptions %} + {% if exceptions %} + .. rubric:: {{ _('Exceptions') }} + + .. autosummary:: + {% for item in exceptions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + +{% block modules %} +{% if modules %} +.. rubric:: Modules + +.. autosummary:: + :toctree: + :recursive: + {% for item in modules %} + {{ item }} + {%- endfor %} + +{% endif %} +{% endblock %} diff --git a/docs/assets/animation.gif b/docs/assets/animation.gif deleted file mode 100644 index e248a0952f4d65a34e51547c83c0bc08d8155945..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 149395 zcmdSAXH-*RxTv|4P(lb8dWXR`u|>E$ji$sQBkSW(yGzZYcezIv$7g+a99co+K7nQNl4gB zOS|6uK}ktlTieLU$lBW4OHt8RRn=cp)6UM$#l^+j+dD8YuonQ%0N@`0WGgC$8yd!% zn#NmMr967nWMYzRW0UdhS3kw_3 z=$5dsj_Bx~`1tbj@|v2O*4EbU?(V+i4KfNpLH{m0V&<1qomc(hSictm7WbWChqd_rPUa!P7idPZhec1~_y zenDYTF{Y%nth}PKs=B7OuD;=QBetozrS(nQ+xCvmuI`?9y|}*qfx)5o!y}_(;}erp zAErOf%+7t9UszmPURhmR|Gcre^<{hK>+atE!Qs)jX0Vy{U94o?gOfb)>0mI2oxF$E?v@KAO&FSZg)fTrr+4 z`gE*7qor~(U)t+nb+o1GLoq5G#-iC;{jp3vUBG&*wPv3)mY=Im1)+!)h!IDN3SR&;V|W^HiW`U`2EZ7-*tAO}GDZe)_54MjXCf(Z#j1(0 zQ6(b)t-fB@UInB3!hHV_iL5qxxoBTZ(GC(wW`gL}54Z~g^vHl1M`m8hArT(FGGcCe zbd1{NC$Kp3o`+3Jz&m~LNXS&L1u>w(6~@9&0^kbWw9Hh8LP`SPMg#$@VWB{THv;V! z&IejYQkl@BLXesgsGwNplrELmo}uf% z%T9~bQ}>VZ=-dhj!b``5l{|v&)%2tIci16ZNPaR!M_Hpa>V(w9y>vW8kyakI?6(HgyoTgf zwn_okMHE8~^Q|y+++W76zM}LX1kShB{5iy}(|EqXvzJg!of?2^+PIsKuY6y`I^4-J z?W7eRA(t>n^=8)!OhM;BY;ff~5(#hU*iCwWnA-Rlp0P9q-=>vbis5~IwUr=seDx(o zE!cpPE`d>eQ%MKmd`RAIs`1xd z_XfbRIn6+U`T8ltiQDzv)~7$aTKniY%53e^td*ghM(%%qEZR5z{kiIM^7j(I5q%qf zwVmyb|FxIwz9HAc^rS0|BHffVd-C@EK`8$=<{$q0^5o`Q21f!fkT@{oDH^Jc0n?4( zm^T0-VQB@DyLzD{${QfKC<=7f3;=sz0U{eTghLbrb3jA5Z+=X0VRSeok=RHmc`y)x zgxW%tkYVJ>9t2!hSQ3qgQb|pS(Jtqp8y!*gOqF26p~2+HN@Z>U!kW{^{0o0Yshp~EpFt1!3U~mF{9!2x zvgMqnqhltWX$g|3FaDH~XgC^2B+qANr03X6p?;Q$gI9_&Fldo_L<)hiXy-JW;rFKo zFKU@9B^f%x9m75nA#s5Fd>V-H0+ajK@A+1#MP5W~4Ev#as& z_B;kc2u1;-dXT0&q(ye=_VUD<(+oyQLNH9|1AZbszTasWL_?Lr5Eoj!BfaED_*W&) zF@48*8^!7~^exr}JxgZX(Gd`&|vnL~|QKr6cZu@=q6?kl~ zHN+w<0LrF1QhC^S2Ls!>{bXMUc`_IdTmUdlEcyL(9D!NKGj$T1V`?Isu|qV^J;@kn z@ReQYoUMBqERQtT#}AsZ2BmUygs~Rac;3 z1n2r18MZFymqK))=Mv^5LnrtEIonP%x) zztCgh;x6NgGaKJCc=a<=;rDA^h>B4iVoN6LNzVW)AKV6b240UT<|$^UP*kcN#ZAE^ z?KnDyyEnfHbI4i6AQs*bX^)({HN4m|t)(BbUR7{WrCR*5MA!RQ$1CJtKJ>Nl$=fYP zTq8lKjevFuG3!Wrz+B!lp6d5R9cBF*RS4i65(f#I5~qo)tI9NBH;cA$WuLNKp!6vl zhx|Q%#PVtO3q$|AD!%$xW+sw6amUj4HR-PlqF1h-VXVJ|W3DF zKMrUn4Cf@}c}W2{zI$=jYNeUKjd889*?1bHSH16I19=mM1ct|OAYX*3#U1JC`?b%a z5OoARr|4&EI$@O52-lWU8-hD+}B|J!dA4x}Q1*@^s7hosgCl508 z`fNFf9z0|5(EBjHBg$5WZG?iwLQzZ4SzY8p3cq_E12weLuW%|N7ih|!y_svkhPmfL z;qF>VZ$zi0EBZp*wCA>|)b{L~kyUVaX+40?G>>9iwcg;`rxM_$KA9t5;x`6w{XRKE! z8FQ;=Pwm0)j*$L2qm)+8YBQLs;yLmVAp z7cUt0BK3;!m3-D48>DTUY5YRl=I@2ae*n_LzuUKKOSGpp{zzi_F8dw?pT_rIOGuO9 z;t78poXU$Oq5g8rQ2x>swFLrf`za6V&Dwc3AM}3V9R4Ky{u2I)*Y_XDZ&@uMqe?A- zi}Ua;(qP7#QX2B_T%3whaB@=;d`<_Zr=&9^oj4qciw3H}h z0l#U9#E9Ha3!0$r}7h$S4ch&mI+D87YV1i`;l_Xdq$`#Y&Mwb~yl+Wo)(|LEj9RbckpvIG}VMk)r2z4r%(eYLEsA z6TQ-hIQj=+9#8ATV$H)?JIYN6_u#}=o?ne15F`+#7Kd_(Q)*Y=q6mGxXO9o$HTofi9mkzFg=UUGP`a;4PovUA15dgjS#fl-OKI zmJ>>-rWwTn)tU{h9`@&yqOIV9(k|X{RWY^lk_a2IgQZxG?BW>~`vNt71UCIlUc@Hz z-G%t;hgfw4Eu2es_~b&;EwndtzR&rZ5eSkXbJLeHht>Qm{qp?m(v&xIib(VgW5mk% zNgxYa=9jUC7kQ-&`Jh})Q;k=!SirI-ze^*(&0Lj}vY_8QZv2Y3;Fp1ldDj(s~L>S*azYE3UU2k)2Eat2RBejs{g@p)=FOUt!y5gyT(8WtB91%^Gb0D9fX*GfuPlFU>2H#kFn&!j#u7mohcKgd zxHQn=SEij^_P41Fr)=yFEmyc#CVI(c6kT2}qf32YfK3z9U52)3l#2S8lOW3x0>x;> z^7=uhrP0i3C_sXGe8(o|rC6z+MQO`R#b9hGAzMYv%T= z=9P_~#$i7kcz?8bcxSO3Ay^j6s81Bol(kVYAp`c@GcbxQU}kDE+cluEY|h+h+8M>l zjTsAcHN#|!MJ&rLs9Mr-%`6rzuemsB<6Gh}Eka{0^#q>LuvTf!`U@ilwYb(zr$)$u z3fADYZhX_J#X}ozES+HMJu?*%<^exs-L5QYREOFs9>Wfn8mt`EMI9*mOv{J<>XbyE+rS+ur|y+A(zv*h#prbXRM288ww{oi)zo zm-mlr+Pvv;vTVZ{G1-#7i<0e3y6E`2(!MvY9^Un?yQ{5%z@ZG*`%v&*gPjCd*Sog( z+U!5MC-%KOg0IH3R1a-?j8}Rge|k-f!t+RbVMp{|{QxyV+=)Q%W7&9Gco*-uCU{zz zX`-j6OFf9HpIIAMN8sR(ck0iK!x{AD{vPk=t@nl3*X8pLT$1)A%(aQY-%`Na{|T}_ zh#z>6(EjgFkF?xCrcVcX!JwwL*ozT$>Z8Gj-5BoxMtZfO7b^pLFVIG%LlEr&!g`f| zww*jygUN?)&Oq;Ny9b{psBllbw_JT^SzaeN@!r9OCTrO2PyV2v6;1pliY7@bm$TlfyQ@IC z=SbX;zP>e~Tb6VT55-llkaAaU)qM*~a)l4+FApASjV;(lD-S&_TCH-I8#)bWkj{1W z^fY4?tplS|3!KNvTw-J8Xyf5es7;U&{!`h%RSvXA&f6IvWsI-j2^e^dEF7p!PINx@v+(x=FT-d95NYJWc&4Vujg&g%T__a}6C?L6<411?;a(p)Dhdgk>Dmvzq|oET7v{BS%Kh!aWBvbHD>AtV$9 zE2XSt>)d6TUg;%U9k!g%-sGg*Sa~0`8uy(}2)8=HyPY$`L?0(@iWE!&txJS`Y%8^FE@drO{fUoPSSPn8+2!u@WEcb$q;cZ>f1CgU+Rt)7Jd>z5MY8C?!WwiFe?q-BXI>N~Gh9Oe@NvnKMy{B^6jhf$2B|26FK2y!BivzS{)WL3vZ-k zE>frXFkkj463Z``tS?wWCrJc9XoP>T z*s|Z%`ythP%JYDVyjRoY-?UjTMds6zvh^QQuBTeIJN1%3Vb(txKJGke{7L`ef@2fU zYKQ;%s`vcyKT6pXO^=7up6(P&`Ra*MR}ofM)cW5V%db2quBdF;uh%tGl7~Y+P}=^x z3VL{v+e@iLuAP1BecCMww_+8C;H4asrAQW!@F6_N{_~K%K`j3&^ zA9&MASI8gzf4_K|eqnE4n?AheF1cOv;jHe#&e#J={|7qVCxh>j5hZRqpH7B9vQdgF zLguO0K7@Q-`$rM|?AvnkwcTSpYoFQV?TZ7p%cBnzBBk5=5AZ%ge+f6wcUu1{%l$*6 zxTV$29DHMPs@q2v_pYD}8Y&RffLCm8O~>d`&62n{myD)`lYd-ZZRgnX!Ee~1G$CG; z#yKE9KOaWuvH6RX87@$3=@OVwP~SaKFKxNPEG%#B%Wgwe(2yvj{p~LqtH}L#SK7fz zZp)C}2nIbfl#8Mkw5YE> zEq!mM1-6NSp3mSCxwP`Gv(Cyj4PZn^(A@osu2ezi#idiE9M8y7 z{(Vn}4PQ!_jDvn79S8rz4Oi9 zpABrb%shR#_o+^>CF}j7*uk9BVzI#lVgRP0BsHl0l zytMbDA!@zrS<*jEyX!1JljCcDp4MH1eQam(8)C(I|0SYCD>$PD-1ZhVghNHQ*r z5NLq*YB#~*YKSuO!CsJ$P59>N%`8^S8-b#Zw9^Zxq#^=TLZ@XGVML&Uw;OlILt~zZ zllY2`01|wZIC9p?kvOuW4CplJ(Z`XeV!)}WIuRY0xYn&`?NF}UY(LduR63j0t4Z3r zzhJjD;O7e9SYHoZnvpaCDu>8`Ujw{=`ZX16_u3D3o4_EIn?nLmfE`RVR2GX&b5Y%z zhC5X9?-Ye4oj*;XXI4}K+$m+gVq)t#SBiOv4jd{H>KC`-Bd3wL%mkboH#PYBi$+k| zIi|9rmKD57&SY_OCJfW-hfwQn$b6}dGQ)hTk3HqE6LhJ z_=8`+1wHy7jkC1YivOf|lWr03+Fpwudm2{mU#C;M^_pqaYXPjz(TS@bck3y3I^25^ zJeDtl{fQs1&Q4OKuW_%8QofCYaYmjLu}lt~yd`}Y*oX^w^3`CF(s1G|m8{XPFNV>5 z^~Gpd%F4!zZwY~?JS6$uB4bxeW~U>kXwsH`@!{bdL;T(J0*N8nx5IJvl%IM!Y70Z=KY|wZm^y~g-PvF?vA%iFit-)G%B&n9#z+Q|2AaU6bm9DX#6&*JCWP=(l~fH-%1H-t6;(4r~i z@)8AgN|sqQT<4kQ?FByabSDNxsSV;8H;V##t76H(16{q1#h)vk&tYVF}D1gc> zfw@C_ph;1b#L|hkNTlVEtbEZbem7Ggk6se(sa;OxuKl+RrVu|RDC zS0DfaHj;^nI`xVDsb!JEL{u)w(lmh$NUs)u#=Xv&uGTqP?L`IREJ8ha^=xHGzoTvo zHHXL}^cE(gT0!hUf1!Ey?hiB(eabTp3N#2Ymr=Xkd|*6~W~?r1TgM`r zo`diRC6~*nzZO3N`8Orw&;4M zFd_{Rc~2Soe%gkFyv7jsUW|~3tAzNjs1Ok)mQ9Rs10abJgtC&F=P$i%ChE&*F5rf; zll8a7&~u6$4@PkFl@QBeL1b`2E2h zp~qCi_^Tno=Pwc+kG7cVq=$G}o043GzR>JSkI1{dNHRx(DWHAb+HTG1#qb?IEx$3N z*Ug!Y2|Gg0e&bfh=UFFDxP=^J5s{2p035{gyvA?R#jPd(GkjNi+;7VJbxYw%!miw* z--p2Cmg0Z#Jrt?`bi}Kc$Ysjs3WKo|D8_lOzc}~5^r=~v(x^!yzsE? z0-GD?qRVzKbmfi_d(qt(hLp>eYz-%|S8=X@kR|idI!3ZHh;F@ePf>|d8GH|6=4Q!+ z7dxCHY_I$F&7qsA_1kNbTVN#nEu-o#HblhpY^3mq`-1C}IYVs&O@^Pto~|G8NJ~&Y zI>|$fn;D^KA9szeDUTd58!lcwd%kh{OdGVlcPj0m5lo(*3?X2D95vQL`aNen@T+8F z$E17G_ku&Y1POK24HktWj)uYDWJRLj_0TM>{yp_eN~-x)hdWY?wA!~&Gq*~B=Ova1 zZKye~yZ(Pd){~<=y@g+twuYpSzI^98azkjb1U!*DFn=z!9#)f+Ulbe zndVlo0zew-{h&0O_)mg;Kd@|F4H!{g2W7sQ^tPb`%YVO>%rKC){!Ao76vG-QPfM!~ zoH{><;kLmLJImur2ta5S>c{(u)k0Jvme5J@l9sR>!&@0>|6`?6V=78qpIA8o4>L=P zVw^WM`7!Lo2L6!m+b&v1}X=^4};)SxJ(l5hPe9HPg!$cp2pL!Q(!=ht?VjZ9vd@&JbkqsW9>z_EduedEJW8+3R znx zmgriK#4kBelC2jK(}|xKVX;KKPoGDKZ#fv@@brJ9LCwC&fQ?W5E2ve!>DNU_{WH=l zhZOvKNDG}f%XzX3UrPN(%5O91>=}=oO;S)rYL_pRcthyW#)*(2&E^HO3y(*TFO-HK z3eq4v;e&oMg9&y#2i_(c*`)70V~jz1SE19L-V;cUrf>T~?*o7racHn_x+{^#mRY*G z1{CHEy7{8flK}4LDMkpu@G=~Kl_2JqT9=aQ$pI8!df!Ko+Q;4+=E?%hvuZc8qNYKz z8qkLeNd=2ZTryA|^K3f`LM<#opK(rv2GlbKTE+*ZMMrz7hi5H{MEK=IYY>XW1_fyN z4mhM$isnQ)=DbvlO*KcVp2t|>-)0tRYAkfD8VtI zZ93LoS}-mRD*p@CxE~vhOmFuU8a6AyLJN4LIR-Q`Cu3g@@1?!}RS+qb_kk!f(lcCh zG^`#O@zfETpyEHd7*-P-zFY}S(C3wx7P5nqeqku$$pL6F@)GEx8X#J0F~XAg(PO^c z`sw2IY5rIoX+RBZiztq>CjOB_*ibu+&_DOc4@we8+Nc3_&areG!K8_qd6lwHdjRN) z;$2g;5i+24H{fqysS!X(e9Ggm0hQN-ME(l7Jq(+a3g{y)6W1hEVF)3u0m9LNTGJ2+ zrmW5bByv!uhx5n(j0Lnk0&e@3X{VI^qT_6wv1Z~8MxuCIIDoG?rC3}rHCT{IP4K5F zX+CvGSB8HRF8KF6dy_%1yr`Mlt~qa9@C&EPrG;Q@e*tYYHCqaSo}RYBD2t{jizJ4C zS45xW^d0g%FS#6Dn=9G`nn*~#Md*6Q=1j4w24BniAt-9#<__uzKNl( ziD|rv<*48Z~O3KN} zsZdkvFf*HQbDQz-mz%uOYivEZDRqEwc?c-DB>sucf_}}%ff`Wq9@bDY=t2;ja zU25uo*S~N{Ndp-fBUxF)IXUBbdDBHjHxk%lW#w93-E3#)c60Oo+qd7ky3Trg|5F8< zo142)!F~)3{HF@Gy}f;OboBoP73{|EGWmb{U17S+(3=M%ZX5;$Fd&#rgd7F~rPmEN zWGRNgSPL_vjKzyd?!_51gY&c_jaeJ%T1>QQ-;qeXZ~gCn*R*Udi=+fSz#Il4k&zKA zmJF|Ck&$Epzy>CU5HPEx7(M7>hHcBi48R=fWfFp(OsFGlGBE-X zDNRt#=MaicH*0ZP8+%i?P%THrj+Rje0k1S0Bq{782h3gYQu%Pr<=N@TI$el&9wjb3 zvp7&d+Nb#-LO!a?1{QTi_-E`%Xpy5!lN?$2}KTQ4O zIKK>wIN>dXD&MBQi4kfYsl?E4^324A2PY(1XrE6gvxiH4_P3&XD^BRXgvyH)exL*n zqL!cn*iefEkvbEiEWz=5eCv@Rn=wWHVK(^pO3_UI2BAnXdX+gq*u-%s(=1J?oV|{l z1g#a86ZV9oO5U8*+dw_^d7hBUoke8TnTdLKSSydIK`=!HAVW{zV(4p5B(R5x?J!R^L7zZfIt2~;@o!YA59Vh~w`jU|BPYEXnLs8E5$0}%kYg)0)RNV?(u4M#w? z8pA#7*q`t=VjGuBv&A?QK}>-MiQ)(kG^w~%Pb?_NjSMf&wbqAaQQwdy<7i%O78OL`v(-U@Wr$;o$JL%b>4=H8EvU=7qfb5!awGENHktf zs-lto^Hx6Jek|BT3;$fS%Xa;_n=!m@N@POSHeeD)J!Bt*OwfvcXw+HYa^xEim|C!|iUzfY^({(CWJ z`X7Fm>4O1cW*fawFdj1UwGbc?@&FODIg~?gX-l&co+0}lcaIO-yZ`%pwcGd)Kj;1- zl#oU?5}>vTB_Bi*%Q-04)!ywb_>>%iNWc*A=E!?{;1stYI5RoXaDuVqP_phN2)}wi zWu9orAHFeuN^~gw(MFK@ZyPA_-F~X&(_iFB7@Ajkv5dR!~%ZO+NG47 z?E?=-2EgLj%r|ysgHE43VsHGe3kMv?ar6sVf{*4+wb_u7wSK(A9EWs4JDT2Z!p2VM zOU+cy`+LhNMD%;MfY50{?vQ>`L}?idF@GQV1y=gr6=yBAc^YWMQ4j*8&k|^iXs+KT zqD`3u|Hj8is9Gq&=yRrkd)PvCf5OK^zW^Qrz0fcMICZMRM<3c#aZQW!thq0!TW9R5 zd46e(x(FQD7g0!nL<=NkyuYnu9>b50C+a8x^$HAzN{WSAtAkPG)5^+>tS>^i`zvU@ z^yq=s^K8S5xX6S)x*u>PQJBWp$9L}nKcJj};*`qdU@YV%U?a;SI!lQVA2ZjsMM6hH z;4kmbas{LNV`*_A}>h6f!M4h?SG~P(yiFk%P;GiY#lS{J9kz3!>N>fts`V3h__*Fb!;%Nh23lZ9WkiMx|`fC#MI zk&o3^LpX1`Dg_y2%4uYj0;oEQ)V;@wl@Ne#uVOmxPlAn!4atxkj|^qn%+F~cPj}i> zMHCINKtDz91W^@W4Fz4_BzJ{jQQ#c35oR6?B95y}pu>UoghH(;Dr)wvBQMsj8ixu@ltff_P%l^Z427U0b#pT+iF5Hq6}u_iVS6jX1+m72wMBanNcc^ z6Mpf`*5ac90gjGOiOS=sdmKnal#K}PT9k9Iv1xu5Ey#ougZW@@biDi@6utJwiwMWp z4^elYmdm^D8l#Ms+RWSE6wl-@{Roms`a{Q*&(wF{=@tHw)P5cH z-v~7uq-hL>q>F7c!t<(~L|h%s`y{23yFKy)m2rCB)rx1f<;eRSEQ77F>WV*i124w?B)OiZhuoYi_L_e=Q^o$laA;uoRvXAp?aB1M2w@v!coCGptO(`dpMp`fcA;!gFrKlffN zzL6gMDfqI7+HBe7kVCU(o@Q*RG4fPp7R~-Pt88cV^@A4E0wKA*_~H8z8KlOc#YL`q zGzi4XKoQR`OS@fa=^2YiZoED>YyOqpS(LZ2;@0lF4;dbH6^hcd1(+6 z7F6w;W$GpE<0V(+)!OdmCKmW8=GDzGawh_SoYQ&lf+Y9CZ*78Rn|$=dS#)0b{7500 zMEY5L(u_uUmfC^1OdWWo?C@a@A=bQ+_rri-K1Tu`LI$oEdwVR@3vN~=(OcV`5YePG zRXN=a_h0Ybe~#E)@g*NgQy%+jo%*_lEG9p<3wu-*3qhtpG*Z0mQg&Yj3aF=Wjigwd zCI1R{`Sp}`fY_NzBh}0{_0@&z=z<%|ekuiW<5Ve zP)jP8U#4kVCO0)XOMChwM{XMp1%RDHbt&T+1DU;}WBqof%Tm^d(=3VzK>SsvmqvE0 zP4@M7S1bP1Ku0Tb61=(FM;B+AoOF{M#y1Q0?#Ae_DGp%Bk>3|e({7Dieo0nsP0>5^;!z0c?~bDH9rV|u6l;9B za|{^#UXfNSgEC$IKZg7tu>i>_28hGNyeK;LE2I|4J2A<<^rE^Ko6pplyOCRTBvYD} zTeyh}c^6w6hi53fG3LJSmxePGQYe&ZJC^KJVt5Bds&b3(`IpF9D9RvwXyeMfYw~jm z@_HD`AB*L~Ys-ia%GG4bskutjH8YF^@=XL3qA1JFEwTz3%D(uOS}xNR=Gq%-7TGT+ zS&u!)xAg@vRzBL#rtGZn_RoCvtHLO+O7~aRpH|O%2UV)SvT_HL8J8<9Uu5a~S4J$S z+dFC}=at3f(b$ZNtnyc6I@P%QKTxf$PG3$lXCNoMSDpVmwJ=V8Ov59{zm^x2DbP|$ zeo)JsTRTrxRmE7#6PuBiSJ!Gm`CwFp!=$G3AoYO^K-gY0q?szUPg}8E+na~^Os_D; zUyWgGz|S}-GKW^%#5LS?to!Iib>&yve9(}BP7`RUS1Wr>K$6zZ*l@I*jI04>2^zm1 zq;DL^0cT}N{*7$5sZd<-&x1zedkw#+Zspe26If!YBI++08;|yqrha2d)&;zq|P&0 zK;>eyb%0ZMTcet6Tfuv0D8g59yxG8+u$c%!!t}=d5Z>I@W)R<&mgSU__x7b_wlyOj z>t6R3z3muu_%>uE+vLFdrB*w!RQXYV>685SIA`pKFW5w_jtq>$8L}h2t|JZOkdxnm zkq0C6A)Y~e!&dH{zgf&I|{E9SjRcb||; zbKn`^y!svqNAPLlX-$s=pFg@aa{dqcS4Wzpk6CMLkc5=jxSsIbnOu8+!~<>NPwp$DRxF=Z1KM%| zA6U!JD+-RZ= z=c4pwt?i?~5UGdK+24-VF_A@A#Y@*Wy$Af#bDy;iYkf8sscx9{VePjRWvDLJP@eC7 zxiT5P>Gkh*c+&cIPt8S-GL`PzI1A^Apv}3&dHYUUdOTW%MRzryah)$|Q}NG+>6Yk? z(8Yh;(2|%3DOb6Zv=Z03dhdAaN~gk^eNFiOmY+@qQI@LgeXLO&uO|66UZ;WyzpeUi zQCEP59sk8xWEpg@^s05+>ag4xvH6&M2jW&D3z}AWy|%zlO^Z^udDMKbR-9;R^Z7fh z)OxZm`L56L2JIN#4{MDV$F-vG*ut7Va~bV|Ccn1lQnPqyg&)_s7k)`u-=)4^e2v}< zr{0H+7v+3h3v=6NJy(|W!Tmx1!lc*wuY()jJ0^7|YPGCaMjkT> zlSr|l6m+yE}!3a)%4ksuRy6v$sRtR{75`L|ETn9)zqG=BaHX>@dSB8 zz9UQJqobLrFQvKm?~3%-M81~#6~|{CuLSWa_#&gcjSM_>kYmg9PV4&vI`4Y=Nmaq+Ye3{7PrrmKY@cO8x7_ zt!tEpimnYU_VM+`#Fc&vr03z^AA#5U>nLNyEttpg^6fu7L3gxqh}j4DPmO;JTd4k( z;-%JlO4uZH5Tab*e`gQo+S?c)9)GSLPoa_Gb}IiDzl&c+!`#tt)GYGG@7gCB%>X=7 z%bxz9eixHq9lng?$}c{@u4}xmZi$rnNM;CxMo8q}_+6`k7P5Ro@;cV8)>i=yuEJCZ z^d_^&{dIUxP*TzBq@14P^^WkH$*PiDNV0{$JAVD--$@M%E;J!W2uJ}K-bKPk&p4rC zrOJX}UP?Oy5DvISHoAW(eOxk>%JsomXg?{IFp&k-{CiLQtAr(=^D_L^d1V_5F>z}K ziR7L@>b-czo{aB&x{T+DYibycfPRHhoq(njO*YjnFuj@TxUk7!cTkXl>`lDxB6^e8dEmsaY4ZJ;4?BN4@%Y$cYQ;C7g z$F=eHHXijK!=n9%>Llcf8$iZqqjh~rvQWx$wAB}?fWDZb@uj2TH-38 zoPK}fbeqgzSx+g`xbrlORHz;s2ICL(QzYSbK~)kl=%GLa2)SthLQIcB6?i_G;F4b7cSHI1E&hxNvb7S8o~TZ{sY*eFKm{}u0S{crHjP2+zL z2IJ)9ylMRZlkK3$$(1N6wQmA{Ha25!ZYv?7r{dyH_wFexEB{y8e?xWr)YJmCv_kat z|F>)>!pP``>ZCq?jM38i&$K`5`SY8!{|4{ebp1DAr_SFW8yx(9V>>PXgY9(vudctP zrR6`n{u{7!)Aiqgor!{i4;ai1*jcEqz6tziIy$~IH}AEz{r>{&+ywkL*Z)4>57TD4 z$=G3HhGCE!yh9=taSwwbrqvGDXU+kWu-xDsLos3iDjjFY1kTfpG-PRq(6mTZRq3_< z_k2HHCYM=4oEBgTBZf*ziMC6GS29aUFazKTeF&IELX;LX@sV{?f*LrwC#j+U(2FAo z%((e9!64RC{TL1hB7pv8r#1Hef93lmi5dXGwZPD+a_FZ~1q}l7v=^x`IG_R5S=xic z0P=8Z_=XHLT3bM$z>I#-c9T86PyK)9`(`z`@z_1oxB>}KFRS@XLt21o@KISuIp35L zo`wLijKCOX04&u1KjKul0IheC*OdE zZw&KHjbfb`@Rouh;44jyTE&dqs~(KW2BP7hbAnAQ8WJ51?0bzDKkUV;7i&>%8noG# z@jrv4a6qY<1HmwOjB?3982qk!29nB3YUE*b!A_3O*#zb6tEj>4!XRW6-&RbDa?}68 z+ypnQ4o-h^xiw6_uhLo6zL#cnn)Arf{Fra z*2(&>z4zIBpMAzRzPocXFXrt`=9uI6ywCHRu|U`wEr>x3kknU1cP}hV1pOamKZ`Z4 zNPV>t0Hp=CKkQk*h~h*sN2{}At4aeRFWQZ5cwkl&)-nE~AcYYHkV=MWxuLpUs6RED zxrF2a9hK@Qd}?siaNAyv>xhK0sX1dY4Kos2$ZQe|b=@EL+VA!?+vr3+(583>IZ5;J+vxjt=Ks*{T-0VPl|DoCb%-MUti& z;)kM8n5X5+;|?|rR@^6z!}Ao`zz2UNW~#z{NlGdMrujtz!KqmsBX0enP52ic8_EGz@fp<>F;#6z2 zhY6X5{~hlb8o6998Qaxe{u|zriuai3BqR@kiIo0>cYdt9j`IE7@Obay^+9yQ#cNY! zwowtzYU8SahY+8|fbrSty>|%sXE6|RbypApOPstSoVykiLL|J>2qBaCjZj7?66-6I zt1`GLBWtC$hmnn=Outraa-7@Yr=RM7o%Q}a`}GAwc<=Z5@U0iWFW&49Dc~bf{D5;s zsdz9x1Ij~*RD&0Uf6D0g^w$`+gZ1C_{l0=B5Pm8*gb06Ww-8d-=JQv;*memH@BIEL z*GtS;DGw$NYlOL?2qX%h2Ap`kN&NKgUw9|p3ql4(h=Q{)GHDq34Ufj)AHG|7YU+bf zQy#^_u|mR^TTOQY5MggAEMa!)N*4o#;9s;s_%;@t+H*t^Y^jzck?O-AK42`W$>s5F zXBD;k5K)5^R^%1NgWlozxI-I&Q0|8^=M&8sL*kOZBntsOXDhv{qvBgU_^JZkP5%6a zL5>uNQ1+J>q7a(YhhN28Mf)!j-|hln#BwkbT04HZahQVu6a1MNrTIt#VDsDwHqy30 zJ;MvPs>HG};zvy-1`$8|0)+741{9NZhv7Jv9bjZ21QA{l-7X5&wAE>5p2-=Y$l?!$ zcvykyXHzITFPVW2l$IQ^CDfH;!WMj=013zg;uh-ZOAqMC*Tx?A3I)*2%jw&#_t-VB0UktMZBfTacT%yVZzwhFj7c$ zd5ScPBr{zFK$ay89HJ8L_JFw%1$m}0L=!5w-&92WIE>O}bg^9W;j0IF!&P@Ru~8R# zr6yGN<9yybscG64QsW0jQuySwG0s~`0@;Ih#a0<+=!1;V0}f^i1eb#?3t%{ysTZlK zY|nB+pniw?=MzMGMW1|iX)`w!DWK0i1_mS*&Qx zPXw^j;6P4i+5Igzb3XeP50C{y+R_^%PSYENly#&j9fi*LE~_vBj)usK*?Wi4CJYqs zfU)dQFqZ{fb@h{THYSeNH3N~-dnk{<1Ru`7qNj%iF%bqYuv{!w$a`+dyB({N{vL$1 z)fxG72?X1T*r0cfcBPcI0~{qP1jCc3gkSc8WSxrkiLC^EDYtB%;7?3(FStG? z+-~km=^Ej@wKmI&2L;%JMsm5j=e~9Em2OAX1%giJvu5JU#rXPzI1A^^NH00cw2%JW zDP>oJv{kzZ9X%rZA_J*RoyFmuwM9B_Lv~Mjb}Qj0*q>|4gyYerw7*tDG}@az zZ-x3EblcB%8~L3?RAA{1SWs`fVWRZ#7D)xHU3ACxVefv$a&0%&u@kP(V)h+ zZ=@)*XS{bl;<4j_#TO^d1vYb9vCryg?*3d*i;i|mWN6}V^lc~Yr}?3; zaGTmVBo5IC8aD$#c~QZ*tVb^Oha5OnBL~u>k*GxPXQsRWk&+zwig6*5UY)-~ef=I__YiSy zOQ~!Lr)M@g;3q9gW_*>d9F9P4+i8Vk)af%J3uVuPFWfpknF!@yeBto_+Tk4?WKkgo z73YA`CjpxeAFw~*phTE1+@g&D+!V0P44oD3hFeInq>w5_KXUOW9fTn@?B2Zb08IJT-AZ=)EnkgBB3XTgN;~$;b3STji2P@g zh*@{J4m{X)xORj`dSjRlN0`$AhsxE9sAn(ven#^*;;t{_?4Wm65KklS{wcb}2;64W z3%MSNDZso6!SDm;uNq$7A`pd&xeAnfK=GQ@sXxQ!3_x^0X)&MekKQwhGzYg3xXvoM z&T}&6h(Biv4xV1r0&fLk`G;k+5ihjPpKo)%I1^1+vX5sBjxX|n;JPhNxHgD5)e%&E z*UCGq-TmT<{o9r5Gd9n6S&0Xc(WQ?-Bf`(mp-C)FiI#X|Pgk4>qmxiYN%XeN1hC`` zmgImk7#U26-XWPGI9Ws(5*eK6a*+HIoFZJxvtgXV4&&g2Io}|4jEs}{suCTe%+66S z4_1;zOi1oYC@6xHrB_oI&QsabSd?gmG&Is?Ez%IBDb!tQ+K0rtt8#dUiPxOz1{y@; z)Xplyd?eZFW*RIHtK^p7K2?;+V11BrP{PBek?QDRCdQ??WswL}rMtmk&L*o5A6Ro4SXS^jG?Yu0%q?LpA}hKI8tWtN5S;#+Hv79k_FE;S^ioPvbarH6 z_OLt9FPvTgLx-c#g9e#q*BO=BRMjwPvF#ViKIjH6Xw$WntX+<#Pfka2P9G9+c97k# z;p8rnZ0VEr)`T8YB_+lWL{w$;UZbb8rSK$jMI`bnP4dKDk>e&glZQ^D63Tmpc_`}q z@TSBk60%#affLdBp>w&pwAuU7+0!tsOP~C6T9;iD#dH4D`L2TBTuIEd0OPH!>+Gzq zY>otn6c}Ar(;=BxWFgo$_>Y7F*H<(dIdmzDCeX3f21* zzsGVG{~6*VQ!l#fNR@h>i)<_wDk-s+$leOd^BFHO>&jYQ$}Z$8HR#InnJ>1EF4dOE z5)&w@JSaJkA zPR;QwJlCkAzD|9aRxW&0$$XI_{J9tnPrlbxS_-d*!K#LYm}=9js_Ck0xuqe7)i<;? z_cRMNIsH;&YHov40v&5Qj&P(sIlQ}k5S>T^%X355UT`Lx&D9R06Iza>{(4Y?)t0PF zD@<|#l{Qw_trR9{hE>kQB>cK6*@M?_o7VM|P$rnxFN-H4zSST3k{orKNQBp3I+7ek zl^4Prb|d0ds;hw-mlaO6)wQx;nj|##k6URg@rhh$q8gj%nnr)PEnk)+dK!t%>L|^c z7|haI&`tf0O?~Yz{+7)H*?ds7{aQkFY@Sd^Pn{~Z79}azoBigknAYkaPF2jUeQT}j@DkFR`UmumZK_F+{Mw9pU?uHs z=2GptJjE7f4Uc0T(71qvX|ugn`z<(WKDyeyCi9QO<7>$}mv#G&XV?y+UmZ8g1!t0- zJ~bV(v5*fFO<~6>ej@b=Ih~0p!}F~=ZLO{ppWOEo?dWxAA!Qx&uTDKYP37k1QX=KR zJu<+zy`Tpg=mEeiTB?YYT=vK!j=CFipe|{E(yf*@B1H=iq)>HF4?X%&j7y2WmxN7a zwwTLwQdMpp**tFy&*iUT+D~(Y$15hao7}5NlNwaC7G=Q#&z55cI%&bgdT5rD!E1Uj zKF~*~J!ESDaB2(55DQD;rftr<AgTfic7HI~QQr_+@0$~!^a+oLzj(mOO>^}D-H98woIA@RH0OCQv?JJI&Li}}uEi~o3v z^IMRsMwj;F$*=BRW=Ma@T*l z-_scBiD-ISjn1j9sTM8mS|0}K^^HNF)A@Fb~iU^CdNLU$vMWp z+1npl(Y*7VI`h&*6TBpI(^D-^Bj$j*IcA~-f%pXmty`i23tkK@l6?ybea-#n3-UT$ zrzd4fI*a}c&1yQbtkZoG0kNNimZF>%y*-!sbe4$L7so=Ez`e~p(@TN@)4$-$mLHZ( z`Wi(7WUWrSa0GN8`F^CPo;hCj6=NHbAC^0v_`S2t@-MWP`srd;ZJodX=vT{owVsB{ zw{mf(9hXP%k{DNBlGI=Ku4Dwf@28{vt9I3nac1FWCY^EZt90GwhFr2vZN53LM3Vk|_{YS{osg1!8 zE1jCR`u#Vg87l&16sP|*bl$9N$b4AnUaUM^Wn294y}CNkRPlhMCBbz4Gv8+IpN3jt zrPIEas2iq%rXVI6c(fpBybY})Ty;!TQO#7sSK7oz0(MP3q$NRYWgkME4u$b>I zi$UY2=<}V;)8b7H6|uYRR5!FBrd`?mP1MbfmGB;w=BBp8rezvf<80Ft&GLM2FUYhA zi-4Fs`E*56U^~F{RCbH2uRzgl>j%j`8EL`u8#U9-%JZ&?;oqum*t&!3o%fITSL4wR z6q;W3jg#4wpFjuBy4#agWPZ;-N7ip|o9q@-9&Bgtwp%`EWIT+Y*_qV1#rFKb_H4ev zf%0|yVZqr_p~eCJox@3u!|-IfJl&%Jrp)p4)tS=Q>UsiRYObIy1M+;cG&+UcIBh_Q_4lneB%&YFd*|@{T`s3v{Xh z@cGfhN9WO9C#(ZW?IY&}nrYMmM?d;cm7k=RW4X{B#MK6GB42V4&-w(-b(M%VH9Of;^{EysXA=ZLe=S5s>; zh9Le~=4>3^S#U5ml#`|!6QRO7Z54)2g`)b>q(sH;0+tnae{AuQb7 z>-uNX(%Ghsdy|ok^8xHv|#Q?+C}+PmPLXhaf+}Lwt85wV&)l-SKy?+E+W!HGGP$mSg13XyF59Hi6({{kQt)FMEG<{E!OY;Im$4kWMIHC z9K(lon{B4MH#+d2_c++aL>W4zEEEZg{pe$Dc*MWH#e@{?o%3&()WH~Pv65J6isrpJ zd@ir}6lDTJe1noZm9s^+4_1qQrS!YZw$5n zT!Zl_lmXmex3EPJ2yY|GIsuQ;0ux83j2ghFiwhcr-)4%oNzuGJZj-7j3wxIKK)0$i z+3?A=4GeX~uafyB5LTY4AEsfOX`KwpBa6GD`@%=Gm1 zKibXr|Ch8I9FZ~pAA}n{rhf@HM*ku*aPcrHoNz;@7ix%t;`+xZ9Fj2-Xn;w?7%_p- zy5UCkAP8-%4ov~%mQLG$4l2hLp)hgrP5{SbZi@@u2Ec~K53^80{B)qp z1?F8_3!w3M7AM?TjQvZv;eq4rg$5VJpn~~rhxGvY(f=*nXy}1aIN>IOgSZyVMza{U zg`x#4ETTb})k2=uVtvrL?oKQU-0HCYXHSdP&w1fN$8{+?yof z&Og7dzJ)SWE6nknGT}Xoc?GdqUL-6CDqdiIc(Sz>idg0t!`~ul&Sky@x0<+{2@8tz z4ij%0A|}GC{20wyUb(~i5YdqZCLKY@yOm?MwvZH0K;gs%jjYpN?ZN8#J|TMTL%L2R z*kZH-OulU=RZlhhsTEd4)r8niqddx?2B!2j(;^~z%)Ts-{gGSUKpW-LXIw#0D4@>y zDc55;I-OAF&Y-bBlVZd&3+{1Fyju5E8#8nJgIEN(g4;Sb5+R1RQn_qi;~7OkTR z3v1ui3e$^Ni-;6XPD9?Mn@5G!PDmK)hcKuRQHjHw4Ha}HqVbj9XNNp9XeZ$hDj%kH z;=S_$dLB=0oyfT7wa)VqDS+Yso5(mU0RJK~KmR4% ztckAZg%a_V>Va{<69|S7KLGWziUqR)HhW<0bpripX)Pq4Y^}Q3B%9ZK%y}cJ+``lPp!WI~y~$sA>H~07SMLjqJXkjwAQ_e?EYcvyW2I?^Jvk5K{QXq=ZW79R z7-Q)95(Rsq7P3)>6K=Gbhy~9J@Vrrk%(V+pJMVkQ{qrdG#m7*DAQ{OPDk_VPg*@@u zFq`Q`2+D#5tOyMGX}HC_t=^#;J$OT7QD|)z7CY7tRn>`3j&n6+BE2h+NLHC_sK;u-_%Vh3pWTsZjI zi1tuw<2?vhHh?*Zg{cpW58|~TG@MlcBU~Bfbd{2g8=o;(BkE|SL2w~?d5*hV!S^J# z&8U5MGHtT%>8HF*VqQUkZG$E`eZv(MiWGs|wjrx?w5Ulovx-r&dyQ*y&$zGk8~Vb?G<)?iPk9U%A zx~NyyxCLTaQdR>h;~nfJDbaM{I?qf<0Kw#FUy*MW^6@3a;>EKO)W?4Jpk|9t&J9C! zL^2WS!B&+hUb2HiWLnWNsOaSk0#wMvBtfR)>lY^HHAhEG~N_L@c6diTGyW(i}Vs{lOk{!MiGFg z1d;J(wxz)|?MW;Rl=oT@6Pk68100oBOy^8t4yR37(Nxs7?m2%Ah7iQ0Qoqk4usP!j z;}Xy$`b?|H*QOt9Rz^W*YA#<<2eMat@WT}Q?SU&ZFc!pZUA}48+DzM|Ef03=n&;RW zW)(WWL%H^hoCJqc_`*R^1yaz3D_4Z>3j5EO01t`e08Hp16&U_~(!w+>>!~9+_h!`~ zS!87Z*c%0keOusiCCQAj99jgkI-o*TLiiY798-wi2>^da zM^pA4?6zqE+WRJ@ck*s)8lk;+#1m#krw@O5-z;Eq1_=C!gmF(=yke7)R4)&*O<*xq z5_l;;sdfC-g$EQd(N-Vt|M>})^*+O>IJzeCLyqx+sqC%Efv!;%rk4qmyE_r* z06~8k5`FZdKfZFjl?xOOIsn7(M-6Br=xwCSt_v(oU$xZ9Tx`BY=?y$G9dR%2pDgS} z@E7P=?5{VzwM7A7jJ#L-o#a4kg%3tqnVPm+u6o};_mqb%dxyY+@XnJF6Y83L9|LJS zv-S_k@A;l5Y;sZD#mW%q+^=CVl!X#^M&G9M1}FGedXumfBKZ;Wr1|`G&AdT6v@}4B zS7U4$e`dj@XO!uqWXQAusAMgWlVwf%-JGb(IlFa-&$aGZF<(INp?fz=aervvSjYWa zbx9tH%AX>6{S{RNNa~whAC+L>(;>!I5^NBN-eBig?Z>MAeNCfYgX9C1%w(|Ri(i8~ zOaa~Z@$yNb0EtI&P-sx9&3&QVMstxzDkho%U|Fcg7d+`JVTKJa`9uYzO2qvz#QpUO z$mAW5US>aed`L0ByQW=W@aKS#E$IR=sJ_}Gxa;Ge3oOX#k8aP>ysdz`sX^Z-2LMfQFE2sFm9`)X2q0ALsF0N|3S|_(8}Ei+eW9;&4!U&p zkd*fU>4iA&fXd(F_3%>)$>Zl0J~!rvK={pS-RQ(C$RqUJ#Xeg~sXdeV?7LnO`hy~( zL^%o~AE>DmOgHMr_Boo#=q2lWH;xX!?!n-PHCv4t2tgxAn4jJWF!CYpX`Msxqm4LP&6ewJ;*vHsZ`UdLqisTha8papDL# zQ3DSldB@>&EYWN`QH_n2hChi?HED_6?bOy?ek_4lgO?y$35EjRMcKn8lG*i>xr7nN zmB|d*TueTSU(a84wl>BFQ@JPujGzllr&fA-s5^Y52gg{uy+KFUx~ zhK3KiJOeF`$6>FLRB(un!C{(RbN<8Zc8A-Yw9++IX>F@Hk>@#YAc(xK)FF-BaEshN z_xr|Nxl`BlGviVa%WNNsyng+>?jdAtdG4Bo2iiepzcE+TI)80EZ-P99wJLQxnrev) za^9A4Ad$B;E^wxi^Tmf|28LJ@O}`w^o$*NrnF`^r<&0D@q;;kcI_A9LDga4jkQ|}k zL>EL$6jem$WG3gjM;8@F=NLt2GsR>VM{}HT<^3|DN%hf1colJBtBV^CFerC+j24nl!B7+R4#%LPW=PFv<@l5y6Q zlk8CDZjwR1kYnDR@%Yf3oV!@_i0ZL}x@b(XooSLbT3Tu%*HSX$As5ffwQ_U)GL(heA1s zy35eM6inBYzf7u>Op+%+SzdJ2cl49rrxcfWrxIM3l}%LFO;o+vHFwphYS*N|dV30Y zX9ld*_(#|D?V6Wg<~K>EoUw}Q}a!;;btPeGp0dPvi7eD zXtS5@_)!DC6iG7&>3}bKUo!b50tx;Vx9XUQrr$J9-o@2+N6nWsPNmgV!p8{}k7C4elNkbV4i4WEJgL@c z+LDBcY6+)=Gy+7FWQ!&}Nm{vO-%acLO8d;uFW+swU1pU|&jDNGSK~y&PO6bp6~F^)P-%KbR~<>z z-cZ&wEZN!c!<3h(KF!Q=hK)?{xHY<_^QZB{V{}(8yC%@&pZcpSr78}##eS*TokFet zxTc{o);?Ic{V&s=V}d~OqV`uc-AU)Fw*dbJqTWQ5YTZD|8>ilQlq#60A=Pa}6o05hyR)lPoW519pYeh+i@Qr<~=>#Q}b zw$RP9_11!DM8z7;%s*maKEgMpC{Z|K+KX}2XLSAjmh*(D+hEk$f8clWsLj;itzN#> zZzHdHhi>scj@lb_A|3;#kb;d;zEfjIu@8taQpvTg56v4uN?|b!qg(6UeeUB*P9t)m zBSq$8CB&HXU8!RK!J}Ai6lS9GWMa$g)|-V1M&kbR3#I1Xu?XV+2Xko|(vw|Ny}rpH zOz7laJ$+D)jbo(47ln=ccZrSiT}HT9G|x>?{Kbv&+m6X7-j>Gi3S;@fx5nc z^-;XO=~jl05NBy3l7>=kq51GB%Ez;pYtwM zrFr{$E!OFBd*=|3ryj-vC!hrempNyxnF-`V6;b@$(P0-YI-o zVWZOSZ<%UT0a!Wf}c`**JcN zrG~oKU)JGqyR6gXVJh~$lUd`dI@v&%0hSt23dhRx_(gmou4AEY>lK*HvOC|q z8@iSFyp@oi#*|aJgwu+r)45rBtLfA4F4tD`>sB2fHP+A z9Nk@@t3MuMb70$DAStUJM_T>G0gfS2OFk=3r2ACjaTr4T`Nr~7&Xdpgj<)wJseSqn ziqEzLNoB0+KUc|a5Bt!5B0g-WuU&}duFyRW#x6gkDGa=&q;{mj|-gk5(BIDUd1NG|Qm zjJnO00|{9^BOUpEp!D_37NOjp-=TY@=!MYrx%nKO72gGY!Jf4Va^d#{$Nf)ctD4=j zPN+92rTub0Egp=Be(UARwSBa}n|M~!m+C#ZC-L#CYkgX#{#O;U@2fJYb5?_;^heap=4SHT%S zjAU-qAow{pH*@!Y_-QMD#17xiBKN@06aYRaV%KW1WxWct zg@3MY{}mMUe*~4i@8E=+=O#%%mOL!_H%XmyX!7?M?_(uKgjt7OEZao+$!|i?3+fpdo z7@&K*FVjnQ&r12BN09oK5eDBXjHf}kSG+~yZFU1zIJu~(p%LGOFQ;GIsKwDU_gHZp z#2RXU|CVg0Oni^)(Nb?i8@n(5L;@{JQ2qx!3KT)6>n^4Cp|-=r;f>5A@_bg+3w((` za4lB_rKt6HlP`v)W%}xlKGn-P6tj*J(=f~0bS4${`~)BryklDf-uompOV5uOy@|+G z*et`yiKs!AVanR-VF*RTE2Rio=M^O~F@O6|65+Um5F)--oG-&&Y7qjv+(6tz_ z(trTj|58Be|DnmO9%uyNVY!!3nSX0C+ECJFP|!#^9gu}+79?7u z1egi}D><~8 zovX)&3qw(Rt;lqMrT$#``=gqVZ*ZCnsRR&Q46+!XrlM@wN@oJT&g;D&o1o;*tg=|n zZAL@^VfjF#8x%yKCcYk5D>(Lp9J8M z#YS1>qe&+$i-;pUAvA8Bm`;Txzm2qtphyzXH=7qz)BSzFey#>S*Zb}u8I@mNPKLT; zuL<2NT=9TX$x_;H1*?#RD7*>#D%%@( zJw+m!C3diA9AI}}c(yQDe0ApNw4=rJYc#v)RB;VTN>BBWSvLwnEFX0nmGhem35sc0M0~a@3v=5{sRj4-}7>v+K`=k#}znN;99|9NdI8f(YP%cMPs!R;U1lU(6~Yb+rY2M0*U*8b4M1n03MD=ktbkT#6Xbuzw4%7_TMA^eeuth%e;^LAp~DSQi2FrwaFO$R_qGSe{7B3 z`|Ee%IxwhL2n`B`N+(Th-<)ZoVRzNufazxaewfLBqufepC$0(r3Df*0#X*?_U9-z* z6*ANT6RGOC8M3tys+L7;g)a|z7o1KgCtia{I1m3`qCgNO0+9b!59dwGBC;G)I{N_N zjWtmrA005W?DOkOofJ`oszHc@6d^kjS>&)?&RXKa7*4GniVm7?OX47cJb@L;8cWz7 zF$(WkDX6BtST!_mi;9LbN>x5M>yEx12)@8_>t|)x699ndo3GGN-3h|KvVhB|QZi?rPJ0fMM>kdJ_K(!8^~=B#DGTRrO}w;D-JP^dE{k|&B?wAgL$J|%aQ6SX2p7c9>% zsG#q9=;k5S$@z0W7z;8|b@meCG@Z7!c<^-2rVQL2ZCTDA1PZbUP1HYn0OBuLM1T#x zuK))1sEo3^yD&B8Fdz&L6h~BRDS~1p^W|4wzB;ISAhJc;9uB1G#-&AE9jJ3i&8M*M za9C0)r!!GKx+POe_?Jl#Rf3hqplmRcnDG>9(Hb4FZ^;X#{`p z$kT+T*)|o<68-R)i`0;O5clfbDgMeA5rYlKn zcnzB>^ufzXJN9TCXbEG!{Z-YVn>OKaYOxbVhWxdz#re_24m_yxj zw}tE^BNGS?xE6DMPBqH@T6MQI#rVmT+bMR8&PHx-$Scm;zj(Md*BR_l>eC2mx>}mb z(*06?-!AliqqvPZsVC8r^wHc>VeO+kt7Q*$=yevdmE$y5&&u_-pZvOck0JO`+`w$~ z6)UUQVc*`C<$#JYOmu!JN1N4J!5Cxd^n(&w2qISisJR)YDUZLk-&Z_`4Qs#kgwDzr zDGM>mKkQm6CTg^QEGnTrmp~vgmeGGv_r?EWbt@g{OO|IGDEtG0Z16`?KKs}mUVvf{yqU1?yTJNg#{AKvVnGizaW<>x!H>PX6 z%`-wBU+A}lwBeAKXT`84GAKb-#j=K@6A#2!`V4cZwO1cJ6J2gGx=!(9{1ycr6<4T!gTp8Z{D zoJ8}q6{=-KIh*X4O72-L;Q5H)g{Uh9t9r0h(e?i{^W~JgEiN}Tq~8y5nN)M>7H14pcVe3hHY+7x z6nU*p;XL9UN3|T=FHV7}R9=4&*wGo6U=$Yl%I#@J*t|W}G*k&Y8e+SW6>r!Se@q21 ze~#bkbX?@Ll`6C(QH%Qw<=L@UfHno6T_r?OBvRV2^HbP;v`<6@ClUzLBAz8+y%U=_ zA6DIY?YEqWze)~h$fcIXfff@;vTYA(BwMZ&*PDHR8z)D6O}Y@Gy%pj>e3-lfqN#NU$g0-b=YBL2{LqykX5gtDYpr#U^0#x=`6k^g4AGkv0+vICaFS zIHb)4r46_6efN&nrDfCOl4i+{*D*;?bxj`-0$3{3%@3KMI7qI4jZv1!pr-YY z`M$v^CK6L903Z=>*Gd6CiC zGmDvD?DJm3vO|4RH=;9>s(@^~{BZ~R=qj`vZN6(&rWA@(bv)m(Dsww11LKp=;FV0^ z0omM6`zn#~@=&{zE8|Ln1}CG0Ii&nGp8jlAl4yc|%%qTIJ3Uq3`cf6cnPp+b$9J>@QYR-Jppm1eD3?qo_urjhSZUG8=hE9NUnb(HOCngnR%m#q&Xf`?XZaVDf3p^c9!=TQkkbh&7B{CZ~v zPLr8}Ruu_Rem1F|Ux~B2sjikRs=bjYR;+2eN$gmY5WXl6?yl)wD=#mRZ$wuO-jFle zmtFbP_OAr%nHJ7yI(^BcE3nPRE>t^y`K-NK9X5WH)E6Ty%u-ZvQLj#UhPx$xnU)LqDO(aWI+?agP}-hnx60MDaKjNZ;|=%y z(zww8!%^$QUul!kNa~3uv&lAvNk|u6h32GU)U&z--)75S=%GV8=i?Ta_4MHDl0x71 z+useRUv{`_bqq_kCd6Pn$V=4h#71-c&YK6q{d?=+s^r3**f)_D64J3v zKb!@UL0dGVFM8pos{NKO#^^W3UlWhgnbLdm^%<= z0b=LJW7Qt?#p(m>@_2mf{c?T~-jhMj;{g_HeAq0&mpe%73}|ng`4CHj5Gq z7L$XOwTD!-2m5f1{HYOC9()~%6{>f9rb|<~&YZ8$RJ-e}7FM7waHb#egH1ik6|HSrub(=`x9h=Fq z62nZC@Qx0zo4T=0KH%-Lzmlkw?j7zaDqNV%(CSTRn5v5#>+BgHk)BND?J8uD9E=;N z)EdRk`cLKacGZ|qPn}Fe{TeMsPFF~E6h|t4be^u-XcxDHlt5;JV>_Q{OP=^w@Do?t z5Wm|$nRL$?`O!N)w$WamB>O9O@?nm7Ir5#YREHfo7(d|M5bl2O>~H^Rvz+NN>siyD zwoT$WuuO{~gEX~HtECielfUHcyfU@ln)mnTi20`PJJk&u%o|9)Gl5Ut>zm(}Zc(!w zXLDIN{oQ(FzCa!@!|4Pss-0Jeen>N40NmUsG$(e95-2T#&D*aZ1KKzKDb5ZmZ$gD@N>l z6kjt)bbi$RePnEt7p{U9ut*})IK{paditK74|C_MOc-AsP0maj$*P54Ki8>D`iELb z>}q!13h#7J2!&ihK+{L=wUX&&ul3cs_-1z(bS>k$IYUpQ%X)TiO>l)BT>{`$j5vt>@{+QovI6(K2D_hXK=? z--XLtN}KW3)pkuF%F@jYiFH>EuZ6sg(trgg#}`z19}`SAECng4hd)LhepFOk1Ytis zp6=S?QUsrsC7|z0%4|Xt-sci!ur6-7c9)L^Y$KnnJd63j5cpB%ae0CMM+)rS9V?l# zQxYW_(oDP=X1wVg#wUee15yd@?s5h$lh>%rEbbgjmOo6-{?E z)Aoc-OR|cW)MfXo0~(3?m30FvMY`XZrF~L}DXs*M>*#(O{!^?QykV@nKO0|=wXhZ+dE)NLE1VNH+bsKY{tFb zqGsEAxW6u-v;BG_7va2?n!H8Ke~`ly5?T27A@i zR+@ObZ!Y_#!tF2b-Un_t=!(0pK1_aNLT`ZqU=CZQ?{*4)-P z2}fH>0gSgFy2CExOg5l-lAg3Ta<1e$W@zu9?GpBnzdp9wH`>TpMXu8H?z0iocs~1~ zUgl}!vQz)w=l{AUPt?8Tx+U^8n`^_NgeFL#(ep<#wFw|_V&bM&oX>Uat4a|RBDyz! z9s5%p0BFA({Vqi`N?YJn%_M#lncT9({4kIhyLR65vQM}v*{;;@4_}TBY#P70HGU+l zLw`0MdTIQC7bR&vx^yFjIDS<$$p}e$VAHtR``-Ka$MGC-IAJ@8b|~QT>bN2sEAmKH zd(_i{AtH>ruI9en%kTkmwsEIV^ zi+99sygbonPyzf0c7K0;S@>rr!z>fqG@HSW07wZ4YTRJl43xh+R*LI+BD{FcnqaCK zdzaBLuihoKY%Ft7zGZIxqPD0T_eo)QoQ&C6wrU9JfsP@ciC>l(3ZQ@pSejD4GE9+)7T$kZ-hX3$9b& zcf0_N&h=v<*~DiXZdN&;D(j3W>@e`Fdc`j6&6f;-5Yi)&xYWRA_o2Y^`9rb7f|&v) zIS187AN?hVf=bDp`t-aL*&I^gA3IZd=IR)Pv?524 zJ4;c4Z$S3CG@Obz@tFO!OO(u_Ryf7g5(oti6AwC&K)`^&XQw|U{^-WEO-XqH93D(1 zB|72|L8A_HjHEZwbBtmXc6W?szpdzq;Db3{isebx*CHb$L)x{FLNcUXi#|KMIXAbR zh=_}v90{Iysi^#oO-OT=J6c+y|Br-BgoQb}1wUeU{$~XFhM~4hxZ}C_e`Aaq?ZA%Y;dJQC0%^a@yBk98;8~|Ub z7i0i$WVvj}A@l=^404aAA9SY(gNUH1SIyuCGK#_4|05flf48b#o?!|GHsejq=*1z{ zUecLV&x5M9<}``y;0boLW>pCS1hWZDc+&WXqTJ&2Mbo|2G~bLmr^rWvp_T)cJMogm zyNq8J*Ko6{lFP6*fBzPvCnDntBG;XUlb+S)-*D~ zI@+}|mKw#k9vAH>!wrt2fik<}Ov8FNmE}44T}`{dlE%iFX1*_ zceMsXdyr_d$?ic=m;p+WnX0({XX0HYm-g%T!xbwBxGVR*XqjqI@2z6BnLRih4|a&3 ztqsC+N*#HMjnSrlmz}LXWyZ>08gXl(lg>j`6k5S->Zv8)qa*-yVXD{DjnBHS-gieF zBaBdZQ3SR><#uH`%(5jY7;MiD>~FTY)rc2g%OQEjW>JL^ zD4;w&W-t>(CFRv*r#<7fcNu+V>^QIA%~OuND8=cULV05bDgscToQpd9o^I_nW)Vu^ zGWqoW5zZEpw_@ozc$-!LCR35glul zRl~_2}+j+aX&D-A!R+47>nJu*t}wk6e5Z3!yhql~R@;Pgf_kt^R7s zT>eGLZb~>6;xog6sECTZpiAF*QopK>OkZmxo@}?jS3X1(N#mhEQGTzyKNo-XJK6n7 znzl5KQ#7K(07^WXA}9PZ@nBer2-*unw!Nd2CP)BK2e60%E{1soounigj-uk;COI%q zNU0b-0xCpyq3_EBLI6)<36HN!I3Qs ztW}P2m#Ik1h7Cw>@W-fqj)XMk;L#Ih>J!?0=U11KIyfy=RYZV44;OHsEp6cn!)1~eChVjLE|cLcDhhgO{wu-m_i2;XGV)$=04x zOLsPJNjAXDbWj_AF;-TM%!bK&RP5fX^mGSb#83KkxqNM7@6~la^CKheku~BBGUz}6 zyFVr;mJ>%3_hs!mdoY8JbqpSQ1&LKJ&Di2B?wRyeRXe^BtK5qOQK2B_UQ}0Q`UK1G z>F~6Vq04$+T~QG4lbXzN;|}(J4b7AV8KzFq^HW;TJhrR{5=l@HC7C*UP>Cb+Hem>R zsdZG!0GUokIh0Rup6P`fufA47o#<*5Rg7RZKd zR)Y)#6g}K-nRK}k6yP+UI%&?EQ-Jy9KRtT+(1M>37bcXA2NA0{w*kfK^}sXR*q10k zS09EUPium5LIuh=%vOs-+4+%)EQjCz>rAGTnXhqe0`G%*hHlhx<41S!ruYbN$EfXH1IN7MqI!bmK_sR?fXIAuh8wf%01xczL`e5N{~XpM5K7S zrldiYaNnFrgfHYqs=m;t^Y^qt-%cysPY=mQ$*{;-~EA3sG``LNJ(naCIJOnBCd z?PS-~1`{Xsm|hBMjlD{6yqZMJcypJ&)NODiY1cn#^Fw^yg+x!Cq%~Ihk4I?flEjm; zieKeKlp*No+ta2^!>I9BU(qxt}7Br3|7b zA5|Z_>Hj4nm0`necwLVGBqKUg)67%HhyCe<(k?8fddkw!*GP%QrOlwz=I+pe7BwZ{ z=`T^~ubdc&^KJ@p>20|585xpbGd&ZU;mwor^bTFAySq+YMk+3&Z6LkwD8uv^X0D{7 z?U8viAoEgMW;29FE?3)bBQpWWYP`VUaQ~KzM^+prtKox+w}%w_Mpi2*yZ$@P;g!sL zqp8%R+6MA|UW?i1!im|n5SOK#ag(t85)BIB|Pweznp=9M4j(K?HTiQi#Gu0oj~P-)v6nFbzW%^gLb;%aj*``9Db ziz59OJX1MNQZly?vPmah*NrcB|@Cp^8J4?{|X)LR_P@9tdtGHKp zu6-si<_|6%Adafn>X-5@m0CMuE4cGN>X*sIm$o|zpf=N#yUHXy)!xA~H4y2<94g(~ z6wP3!AJ*kMOXX7|Wo>17hIvxb!WFNklNCI(u6CvFu$I}^R?O>FG|yI$;7PB0Wz&U9 z5W>QV3ep1lO;G}+htlrC@ zia2NcxO1%X0(-%daqZ)Hf~V_DO}S^oJi)VUDewJeLPb{XOSZ;3Yfl&! zch&NAp#@p znhu7Q6?Gctp>5^QItxTxKWuijEyHM>#Wu&%u&2erN@U0((?Bom(X zhBQoEx;`kPJ6hK*+15I8+8u*cC*Zo84W7qh)m_7DCJmLRoy0(!PnQf~)2%Hz@y|JL zDSPw>&R**gI@Oxpe7<)LW@`Z^&7R&T$k{?b?cmze1fBDyK=^Ubk2*QB>GtAly`NR^ z9qf43R>*k_h;{-WfdZYMhA?^ck-{$L_aGb-cm+ZqYN8fdFN1IZ&Jz>sHaVIY!ut5? zdzrT>1)BQ#C-COJJaU`^zyyAybAa2f=X(SM9XhbL#p-l*P@1Y25w2{oGVtk?jh!$k z-#xg*!IE?^cu@CT$qrOg_988~UzUO6hTRKOqvwsa$_`XjVu^K?c#x;p(6zH}i+1H( z^_4d{Wz=Rk{U!$PoIRud9Dm;npPxSv?L6$yIrLH_x?X0aHNX3KNjbS2-+M~Wff|id z?H(P!-yk!ZeEKY8Wi&T&7{dujp~fE8kF@2p#w3o`Obm5~NJ#7{7Zr50qKRX+V7dhVynXr4-Nqvtbd>SjS??AmCBX&_ub|*f8i>Qnxw#Rdf zF4;|F6tw%Osw{6mdvtC1Bh^%KL3?rE@E@ph+8$_?+rT+!z5L>d#Z+GYQxC9uTiHv| z{C1Cmmz!SWPKJz5y3>1{Q`U7rXZ`eHXlsv`3Siu$rt)Xk(wF#TBvgfIr(I9w<;?dP zWJ24ls2Z=>Q?=!nf{$lpmxeS|rZ4QgS_y51si_HFZ`2d%m`k5UEw>84nw9mQ5>&Z+ zP%uULe2UYM*+O(q-+P*Fi&8gfj?TN`+|JyU$B!qy)YNw#bKV-+oPDi3-rzj(`r`AK z+j+s^jPq{4aFV|Y7ANK}KcDe<0DOF+ev`V9Dq(P&G;N{eXzEx|e<+q4R4?^m<&C%Z z%=pq^c;N!-`eO#a(ImAu<6aHxs*5&BHO{M3vBryX&+9{^UW@H4YA4kdhh6+7vyfo3 z!2E&ociB?p@q9wiQsd47Us7EUVWP%+S@?SG)#uCQN%OcR#?qwMI@`-W2k55f4Y>Fh zW0T9uNh{UV=<(}S_3SKN*O&XwP2*0MmkJk+4OZwZ-`*Zw>d{*MK>b!DsiuTl>!Wv8 z%Y#g3skg|3cLVV?x4E?r&sC?f`<^_0XHQ)<$8_=Zd9Cvaz419MvgC}<=Y)oZFH#aq zJU3a>e!X9QykPgNPKI$+agpHdU&cX0aGHDq!2xLE;418-GbLz^r{Rq;yV0BxL98ct zUo36moVMUpv2px6(TA%kZ%ao_SLW0{M4l_(n$}jDDrF;1E1G<)dt64FeM!G3m1~!K zta)B<_v7Mnsd4ICsS<(F(DYWirzWATmGrf=xg-Zc4~#pFeBOWGo{Uk zHC7fP)t6gbZ+|M$9X!8u?_ssnmWoFM!SD9lpKPC=%C4VTf7&sNx8_odDVllE^X_#n zmHpHgf^m+v*=kbJ`lsiOY%w;;CMBbbj3Gpms-9%5Jx6liV!ilgyJt%i_QqqA!mb+H zx+(L{sieRuPwiAfrODRMw|NJ?Z#r&opS{fsY~CRDOl_0*B=>`>9%k? zFY>WyX3?i?E<)rnP3y$Yt0_VP+g9BC4!c3lW#6rxqFs9XDg+9c?%DpJzGmtn23Pxf zy1V8YB+-(*3%dN_wh|rxySuyMID36sq-zkhMj6#@f#)=Pg+18$Z=dNee?9jwQ<`o) zUw$8dxXH81DmAsoRlJ6dgO;MKR!zR8!qamY_cg4CC~kbDw!L|*{!OIu!`X(!mxtdK zi}x$xF279Gw{O;apmXC}VAu~Ei62J2pHgUOaKs-Z z*;pAr8OZF=RpNWS>$iD;jC}Qn>!ZWr$)w8!qJ-&2AK&Bj&j>q+A8;pph2V%^hGqhr_I^m%_EzVWpB>rc;0NC)a$|ND{dUaap+ zm?=Orjy?NIb*jV=xx)R+^1;t@+7485rwca@mL91apt#KsfBo)0i=09m_|ie}ho5gu z&e6(@0JLn_=Xze)&_h!&Nwlx~msWtUS zb@i9aUvmCyI9+3ky&2^yxMM(I9~QBX7SbHbioCHcP?Wh;DP zMv4LM1F?n>#+$rPJo#&r82zsE-t*$QdEzbzwp1+6~igi^uip%O?gLf;neCQgm9$EIw6AI+?qR@62ts3hdYk1 zNN+XzKM_g)zaf%y=gyHpq7(r9-DCL+k&x71q!`N;Uf%1Z5{rz?A3ma{_P6oN($exT zL=s|P@ZU=;78Xf%c2$4z5htg=_(-v@FDb*)@aWOs0hYhkBrW0Le_@e-Wg>sCNl30Q z5))adto(aTLQ;H@u*gPxJ84t0{rvfVkdbeFeaFMYzb7X@e*Cz-z5V<1=l%Wt-@kwV zlTQ4vIFk^G>3=~adjCKqqL=?dB+`*`e|ju>5yt;SB$q{VVC8X_NrXh_AA}@Ijv=?K zZG7VI9!rL7F1xrG^I$kdnY0K6AWq^V(&8jOVq|Je;v?cB%%HC?IS0i_z53W9L0D5M zSvXMF6P3;lSwOJ>{B6F?j!{n%SY9 zt#>s7yR()%06c7BtMF<`cVO(I(*1{aV{J9FRi?$wWpV3xs1_V3phSF%ZKtxWd#)-C z6MU`36iV|7>5yfXPmGoa)&pc}-#!cczL_LX!0^;X6pbt+Op(+82X18U$+~{}F!sWQ zF(kl?VU7VbplD_UzbQnL)&RSe4$$2FS!+Mp^s}u=C@Cdzpu}#IDzr~??1BL z&SRIyNHI|NJMeg#m-YVsJ)|iG0!IND4(@lzd$AhFn477n5oqx&{=9QiCw7zbkBQWP1E7S!ANZp4*msJw447z%Qee7P2 zes~Vie1S(#J0RA{NPF~JhM9@gBFZ@4D%Ld|J!ivhAEz^AJP-Hyoa=rYnw4^Gj2{QI zHnn}^0V}#0`H!8Gwz82qm&VMUK^B-J{#NvjQ(i28?YcBK_cTv0SxKSq9q^=R@+BuWx3 zV8B%x1~3~1cmXs@7p{{30`RQXynJY^CJfFq06M70zK%fB&xUJ=ta^}=F&A(qqIG9saAl-yA*uf`=JSf%;|@+ROf7>+=D5?%ivLz90)Br8S< z^6A?bDb62ke}9Rl5Igt}L^7lJMYM(Hd?GOLuRWF+vBL%D?47#^{zyz1lvb8>X(eYw z{ahgz#gjO+9R>vB#ya%Fg}OBCxdj>ydSa3a=Ls@wDQ~mqU zWaAGap^|RH|6wB7^X>>sEQmsE#596VPjzYLQ%*?z@#a5zEF)DWsbp$}JoIE7P3Qr- zp7&&-dz`ezXNRPriP{#i@jXX8n7z#u!th2VQ~i+b*6&}Zd&BQLsW?s1Gyup5_0E6e z%1iap-SlFBLQj?18gIL!msRtyt>?FmCSda1fekD}doeW}s~rKdAhvC!n(oKd z$D!3-bYQdacb*g!NB8S^J$YsC^_9-$91V&Gn~-y$D8P0OmwsgUsWBf70A0da0fgdtYNRjlo|=#}p;*ccpMg4d z5@341eIR9`R2Y@^DC{LW$+M2<_#Lzd$v-U;_hU9j>pWPqhCRhPI~J%SwWHw}P0xvxx&jBv zmT?vBLh5K=nXJIN6(BZfx&Sm z+wl&OVQ7CxaE=-G$ko{E@Nkixi!O@ABUr@8G%1#ZJO*LiS7~`|{wAi8l@o9>W`JO3 z7k=GPK_7OA)qXVial#gJH%HgSK}S@H4&YqNjJdg9qN-12g^B7lf}$N6AYXKBD2N&~ zp<2uw33-SjW?80c#xk)7HE#I3l!B6u0AvJ>DDh>j+Kd%Maz|Faa}2xU^5QXo?stUI z0b=lQHNB|mFdP=B>;mAimjt59m0y4L6X0Q=$PuEJdX;W_ST1B7sZ2SOU_ymcHPuT+ z6`>(#f&_SVKkv=dz2uv>J;*L2(T+%5*lYenNd-E<{6ah|p1!}-6xe#fx@^7chi5@e z)?TO4F)TduL`w5TUX@&|(&|71QG4@LOFXgb2KTGOVd>oUHVigGPYm~|#0_KWX4dFU z$%#35Ii<=4eNKPyrFH5H=Y%j!_*otC4qkdiBu3YU0LY-j>Dn$%5JenE9c84$O#&4l z6@ybSYHGkd-xqV@V4j|0jwTPZfOZmeI%vHbLp}`~_6FDYq&tL@?Y$T(C45V*YRTe{ zK8{4QM~S|X)3%Bhfd|cysCW+K|8X+aD#_zvcsj$E6ivmY46X zn;UM=?Pi8OiS3p)e`YJ!bi2NOcDbcm3N(PHc}kaz+P+Do2)$m#WR_(r8}{PgfXX!JG%8>!FokWTxi#VUXzxm+M0k7vWLxVBH<{uh3Umw|=SY zsyLfLZiWB)@bvUO@Kr(OXu@Vma+Pn(F6V+Mp;Ei4{B;JpA3$OW7n(3=QmhA2Y6)%P z?bIpfb&0p|*qFI=&Gk5X^w3$$7xUNv4aa~xPCv2m2U-G*)+QxiO16bgU)6#X@s6uI zcLaS=pCi!z*@J?fpqUJ&qP;q!kyqtex_964{VdQZ-)YHcbofLth7M4eh2^{-0P70U zfu*tcC{Z{D5$t1cb!U6ca=tz}09I?AKiTk+DA)ZcyAN>1M+WiVZ_N$+=(_0iMxw0gvDRoFU1XE_@AYhz!I1p5t0aaaJe{cN?aU^(E z)Zz}3**oo;L`t1o$jk@1y3lXd4X~#5YPG@3&+*Wm=6l6O6B&%UR>EVi{EI z31eH3y`sCIW}-kB_Q3(ylVA7bu6^ilZH8r|qmc_e3j+B2>tp_s+Wl_i^GpphvQ=;L}ebN@OIZ6b(H3}7B~{u%xnHjJ9>BG#9uyO}Canxt zt}xq#>DApk@2L&iyoWemyRDoVO=s!zP6pZi=59QeVSzdRO|)0LE6X;E)?2Jy-!ShB z9WVUn_&)N4foR8a>v%K2c!oCG+$Elu-(_-m>=SeBa#_+%`4mJD9aWs_ z8@XgLXeylzHA9dp#i%Q5Zt84P>Wd33pGvjQ!_#I^X>}H!4g|I_pIJyf2_!*xtQi!MxhI#8?kI>#V$H56{+4HG)G@K~0{n5>t0j ze#dB*C7(ukdrrkdemOkDp(JBesGvS9{Wq&mAH1+5D}U%hkk5wa*kY~}oX&hB;jK-s zR2)^4P0oDh!-dYYlfL8+8#%)8ys!FF-;c9-;(Sj7i#BXhwsMufb*B7?%jSqH2D*xa zn^S+=;K<`y7>?;xf)YXO*$lZjL>I1Cui)JWb6P|Rb1(~)k}w@xiKh*gCNGsoIg3oc z1QJwg=aGUWI%e-Kmaqt;8Jl9ch0EF(ic2AMIHuH-xUxClWUe1&EAaA`xDv+BAc@gZ zOSYu9O@*IE%8hapU&vZ3c$QCBmlZQe8pc=H#FxL2DO4A#xD=PLO(;OA{ zp6OK`wEKY-_9t{zvA{)-bf1&7?RHk9&MLvthlsrZb^Y?Y!RfZPrH_KE@6}c?BEv7i zt5L^Zxr=2^W7S;yUTJRCobfeh@JiRAYj#34Y`M3#G1%*yH5{N@%i#sVwY7C!MgFx} zrVbM1DST4bR7;omBO@1H7G+z{w(4;?J{#YalE_~i)`>=F7zF`AVi<_n1F)04V zo_6Org`TP57(wlPuQD^J@spnG`=wmya+0l8L*Z!6?_ikiJVSYI)4ZPDUVIZRWs`jn zEzPr1Hp+B+p<33{=2(M985GN0U<2=Ix>a0NvwrhS!7CCa@zL(IGe`z!&c79JBuZu?BN zT+LLZY-m|)skLE@vb*mTB!J1?SuE+vZ?=?q{yd>`!IoTgmq6wlvD`8*}g zpo6_jz$L z^;Y+T3kC*-w|fz2zpJ7z=C^w7r3SBi4XPCMdP%ukQw>gYTw@|Ad4vv5*G1jlegRqO zDc@J}u^XDM6XL)O2ZZ+Emnm`S!xmKCPv(Xm8VyC&KYOk`^2n}xJ)x@P;>a(`ZXW5; zl={x;e4WyZqs+B^Me*Fs)1$fF9bF;};^?t>yV0h)oU#w27_au2{kyd*gZFM}TwYXa zq8hCj=VC)EKdT?DxaMASHdb-=wBmHIf?=Z4s~tT99uJ+Ew&^E|%CuJwOpJK7hlWnR zI_tV8GP2S=R_xVYS}-2xHLkg3{(^IIjdNTx1VCVv7gQ%*Zz0+^r`Ee$CLO1eSEjmG z+Kj)Ae_NTnoBt$g=;g11mu#od^QFq)w~-=^J*T>Xm$45 zjJocAf$A0Nw%@^{o{wL96*e+T1}?YG-wf@o>yt$5@#k$9}9;?f})Wb!2 zZw8E)$f)ZcifMGJ)xH?xiaM7u`gph~cnB=}cCfa#R2BVhGNbZb_MqsyLF(m_mFe}J z72%{BUVZe3U(XVQM@Amc?TXDM2EY4O`0mMM^>*0Eufq36530}92yWgukh0!Bdt8?Dvi3b zU)Hz!TBl(x<828A*F|%aD#}eJ@WW4VlRR0X_VorVpE_1)_mbOgb8!e@w{pqbzT_1C z$?M(6gTkUK4F-2iQZ6R*XEZOkCV!zRD$1i@D{lShy0d`I2Pkms2jp{+>Z|i{zSwsw zPF&eabc>7V{_2Di0jttxF+H_>-J9JhPsXWu)W2H*K#!Dftp zuJA6)4bSBUmg=<~&f>3JCYknqi&&Ny!tXy^8ikGvZf41Ud#Y3kLjfDaZ;BcoeCL@i z z?(O<&{P6clFm@0#Dn5$$DcAoht%IV4&>nYf{|wko$PEJ2QjSyKAFhnjAS93Mo_^5% z`muco`Zg6*N&B^?_)CNQ9OH?32AiI=k!V9sp_C4&G+NM z8)t43uHh^I^WF(_*hSvUkXs}IjfodX$SRDRg_+jEfgDIeVz?YWO&4^Z$a?vFbCCKb9K7ngtl z12FbLlKtxzoICieV}g4mM%T_+5!geM(LfctLXWFIjO5Ci#npT4#!}HRg3AkJ&y*YHGlvJqB2o3%ARC~3iWDvt&#jKa3lub*3bzN3&V6{1z$j>1AZZq^9gAd@ zw2R%tYeLA$sTffhH5Cwql2tYk34*}r1$;d2DU0GNCr8q6-BwfkdqhHVZv|*+ zg%}$CSL+sOL=tOdm3ZyiUyvll!6C!R>8~tG`Q5vuR{j6YNk|Klzrl)FFI4ic3zGf+t(=6!MlAme8!_VeBjFkQ!2f*AqI`%gkNg`$HrKMO0NJSI{g(rY?LM@Wq_0kfopq)vs0aC}lGnNu$Kd6U9UupLt+&&X7|HvTiiM(J?px0dbBF+fR z+i^9HArnOV)`K&n{5_5pus00?q2MJ#EjgMX{GSd=tZz<$0C2Gi8_Vbg1T|OBlq1SoV31LDrb#%wtji(UBzh;i3=z2 zEqfw_`Thjyiymsy`QCyg3&t71m%W>?QC$(*A&QKc1yqj`)Fv?X;!t=`q%z@bgt_g< z)4ESpw5-_dk-YasBmZXai_$c{(*$|*|)ANhm`;p0Ak#kj(%tdH@ zEkUy95{n1-{wF%7inFTNo1#xCUZgbqIx2IFr7m4?iSi=K!<<>^{h6GUh$DjllceK~ zL~GX?7ug}jyRl)EFr+&VfpsZGWWH+WcawKNI5IuDx&|YIJO3kPr7!v@ufJ$ zIRm(@WABPf3daWN(I3iL7OH%^Bz&XLa-y7_Nr4(n7Oi~D5CyQnw5pB4=6%{UiUiE2 zh|&T1dR^0yr!lbLK3g1S$t!t1ImXx2j!HTJy{t76fTPv;Y}QPzi-5|SYKD(sFsc%~ z*jE-QL$y0kt!>xh`1RI;Ar9T_E!q@_X?Z-^p#1l6f%nK3W1nXSEkLt@_1+HHg}#)x z9iR~1!$VpAza?C%lokB9R7nl`w(W1@lEb3^Dd8FytV-Oc<<2o3n9&n>eDKOh;ejl* ziLTg>xqp;VM9WELD@N@-07AitI$R)^Bmhu)Q(Pa6w;b1x5Sw-wpiw>88=%!hlC~jw z2RnoBFGIyU2%41MQY2YH?a^AI!tT)rtgiU+$Be7Dl|D%YHffS|As7HKrNeX|rSwNE zDT=~Kle8IqIq~SzU-OY`TvMMj9;D&P&JAJ*=uTA^oZjHYVYsfiVU5FfmIPD)LHoa1ZD^-oePZD+xJ7VVuc)IYiyhlR>omV79Q%c>I&jGN~KS9;S>f4C6+Qp)rc8yg8cuh+8> zM!J@~Cwiw~qUmsEV1HbAPI!QpUuoN*pA^8#L=l_;#VAVgn-M6$x2H!nk31hr*}Za z^>Q7s2AfoCBq4 z$a*9a46GR$8AKs)0kyPj6bwUoMxtaEm1F5)mZI3faI!g^ZD_Aff>ZsEU>FUKCPJPFwZ&{*{oEZx!@&0=6+iz@WTW+NI`? zHl^sg+Xs3!Uhs`8{;Xon$TczqUNbf0uI&Yo1NYU`HL)8!u;6dsv|n`>xhmwVKg=0& zbaBz)5Wz;i8A&zG3DlHLzYw7&-3ReBn+JR`CZA19n&N6EQ5k3vK1op8m_CMng$)qW z0f8VLpz{@cl;|#g_;ClLI;EoBnyU}$O|Z4U%gug%^=5|r?h7<|6>y^B_2xKcLOhk7 zA*==;hcKu{^V8$G1otlVXw+mf9go*&w+RC$08wS^G;oh0#f-@dj;LB-K8_hR)awM7qBa+uM(QFsC!4*m2FSfIO`psO#b6Z4*z_s$#Z z#c`_wszk)C1f~XV2{e~s$934XMhlST0^T2rVc?%L1(?^9n${fmF4Nx3fMLU$HC7oC zZeWpcWNO=O$63wwarJ;U4|Pi^hH!!9sbiDZ)JPOY(r5d<5Z#eClho}`sBSJ+Yw%V2 zCQykJ?yZRCHa*RN$7?%>XGYVS;sMguHUTHXbg``0@XueLoAO*VqoM7%-m9B6@JS47;XKZDlwW5 zp0%MLr$IX<86G>{&2I7e)MqG#yRCq_WI;^3Z77M{CHMJe1!o$$K^^FJpZvzJdw10M zyyvsL7w&m4KJ`}BeT1eXBZ~%BxnVt8e#R^YueiO=eG^xR6w21J4bXxqU^LJCVe9+) z%zziJN!(YOLH?(|6bq_gSW1PQO#Y+iy>%tcSa$=uP>OPz58BeTcZ{={Ju>8Dx30tn zRmbS>Yn31B(f2>ZRL80=xX8o+bb^xEf^mg?F~?<&e%7IRIW}q?HbdtAeCcr|tSkjN z>QYF++(ofEC*G$yekB69qpjL3nBc9Mu+&N|9cCp-wj{O5+<{|~0Ho<>MdxS;DSpP>zh>*NX_<8DIBGYmIi#&HlxQ=ln- zro||X;UIM*BF&prMyY}6z|#XrS(G@Kj)ykAhnjnCRx}}X_sqavpll-;{aGy zE;i^Kt%s;}bWRDJoffW~*OBWB#}0i$C_VXb??a75uE zyl6EDJ32rQ$u0N{XIZWW7(V6iC?%{ZsU+4UE-7Ws<_H}r72oble8iS?+?g@q4hmi@ zCU!D*=U{(W7r~Vi*Y#E8vB$GH>03It%-6%~jE7}Vzl^u9w@%a-)YQX}Zugc23U(n8&}F9j#7+cNfL zGgId!>35~}wTlv2D;U{A`D`l;Ps-?Qm4t&+7`rOcO@rW=Oa1td4hybUL{?>b*39m?!h@>Z#%f;exmp^eJru6>4623zanx)zKiyrQkES&sQd}#6 zjLw?s6Dm9Ya#f|ejS&~UrMk{dEZWniU%7q?>%2J~{W!jU)SxWIAfpG7j2<%e3Qi%; z<|U!CvMq8S&$v5LEj^xEqLO!|@`fi@DK|uJl>Fv-yxB!1>RPsT(lCm@nWx-%U;uZM z6TfVb{;oFB&85NVxUoObYNsm$T9?qr+juRvsfE>OB`=F6KmO)HjjvL3iJ%Txr1{u3 zy{%LHUR-mYpf;6C)`oI}RzM@?abq1qB9)4ORWte=MoU*0!rUA!t$g66Cu`Mm zEAu(r(29e!)o2+rkI2-ijv#g{uv$(qn@Bxlpe*!jdKIlS!IE|7-1)2+$gga*7i@bH zP80XD*;_a5x2dW-Y1`B6swa{aZp)YLQ5c0Nfo%U}r^RNV9nl`_4;zIrW3*f1tYD+f zn0gmH4N}o;D5dcUp!^A9&Tlvb#bs(xpbZ zb)@&y#KE2E8J(^$dG z7fn6`y72G&soS+51c4m(#mRBNR{CpoyS8Z{Eb;ko9L-wqQtF zqGu@33ULKJti;w4*o}@e8Af{yNu3VsUi6shw%^hlK(V#MsjMt&aO$kMc*uwzTU*bO zv?E!jU{8tPQ;>`GNFQrEmSezy>}4@)TN^^gmF=aOKmGB^sHOFgx zV)M(V)-4YSQWNV|L;nFR8(xE~<@U@#D%mLeN5Qttp)|G;^zvx(5I(}AIYCIK%p=|1 zqcEjrJW;4qNN}{2Z9>q!In;Wh(0Y_zcRZzeyeX%i+FAw|Fu|Vva@b?y=5k}p#l-kZ zgQWjxn#ttlgkF7L&Sba8$bmpfy6V*QN}~;#+*)k?m+0ZGi%FBE@ee;AHHyD#)~zQC z27QWs_2r^&@fzeLXDZ;L?iQ!q=fk=$?*=dRUIVptXQFa{0&2F$2VnZsh)*?rDsu2o z^#?kZHgnUzR`B@oDW<8hqJzv<@tG#sI(nKn6#B^rLfOB3N2JIAMaa%ak8tJtZ4 z{Me~0^um&)yX6F)t=OmX=%+=V*PNeuw`{^jl7iY;2me^`d8ON zCgzRbfy;2&cfp2BC;$D;Htrp7Addd@+TsbG&OJa{UlceXDedxdTY?9Q)?vTez zT`H@wUFLooV|*dBMP~47W1l=VZuNjXkBkW7kh1k`Z?j&-vOaE$Ypv|Vz91FFdlB2$ zui;w>CEKa4d1(^czlPrXTja{lY{^`C|7ErFyS|L!mAM<&v-?zD>c;P!>gSMCZw+$q zJTluVbjffj{m@vMz485nW8QkXc7#&jO|QI^E@lSxy4}Z$8-;6^Hb*|j=(qM|)&R?GK{FZ;dH;Op&)QqoPAE&`QL*hH z(UJ!9!k0UG2m8|3w&uQEwKy_b!m^2cVEQsdF7ozA2b4_n>ov;pp}Gix7stZ+$8`B| z+D9@6YhRJ|i5o8DVj?Cq559`5I|(HoiHn^6%!?bvebK!0l2>V=i-`zy_scl#Y4p@* z{p+#182Gc*)3A%v;My2IduCh83A?NBvY3d{Z-dP8AILpWOfjI=K>?3$e}5J5y{Yyx z^C!~($%`MV4-{qXNa@#pbY1+YZ6@W`Jd>_H)^u_kDLk8v{bpj|Btv)?dxE`rLSJ+5 z^CQxuyr0=Qr;23|kQh)-@hj~0*?cm@cvG!HXq8B?j-#wAG{+h^!B`;^6kXcT`pQ8X%yY-7r$m+&-z|nVW%aM zvGcWl4;t(Lef{dWdKy&J3!cBBvFUJmd;RxoVJ967$r zlJbytd%(L9Lsifb+2GZ9%n8x4xlg!_2Z`;#4nnY($7?Mq8xR)$X&&313g?tOfsW8HBZ0{VBWhb6!bRQ?YK z1%lN>1`hw{nn2)DsnXC8B9xX8$;sj2Dd=@G^jf1MIkRRjMw zkLte&Ji<*)qphs4wzhwt5|SMq|KU*;KYB#qQT_K*!oQKI{j1El|P?pyL^~brc<-#-TF0Nf=vHLdjw3@uMQjB@}#yPil)s{x!BR z7uE)#=g@23Gf}}+010v8;I%y>41O9RMZD||-Z z#C`o$?|D&<8m*r|>gI6U@+dt zfeXwnh43C>zsq3h#!N7HfHtJKqC?rFce1@kGdDGL$oaEk-5EOXacE3FhPofkIOhkt zeNeM{u25V4{-!eAFbMhhGZbAyoWw&=t!qSO0Lza~))PaK*a}%t*Nr(+lm`;y{KjL0 z5)D@0jPr&=r-KP23Y8(y_l%P~r0@m=Lp0#zcGw6vqtWBh-(ps`+&k2U{Y95L0O;kp z^+^X}Rq-`dhe=UwnuZNE;8G0`&uu(`N7a2aD)^=6bp9{Z`iJT-eHY(9e(C=|9u#6P zVKW0H|D)9-UGI26Apjjr_TLT)JMVDASe?C84TyECKd;Z+XaDM)Sl-1D7kUwk5L6yc$$Mcs^iu+$YDrehxqSohk zZ;{?$div|rY{}ZM&pkhEhe(-+^WY?shSBH8oi8@pzD!EJeky|xEJBk$;dl^uqf3r( zP#(F^=9Sy0#z6Jyq z6uS!4gbHBV?&HzcUT1bRhX*XszR{R+vlPF7({MQd%}%g(*miEFw!khk0z(v*rM8LW+0atv-r&oJVbDKbs_)zo5B%uhK)E>*obou9 z=v{0Hva2!$gyKcKm)He}v7odIwE{5veDmV9NTq=jBclp>t>hr+FcWQ<%+DRgb7mqU z;FL%k&(E_uG8s}8PmEfOCTe4=u`_(nSQL(o} z4jh(P3=2%#Y=e!ay2$IMYV5tlpZq+|9`xvDn9tA2nZR9E|GdIqXI~Hg>uhr7}9D1B7v?j-pFWTgU}-5GCUX8 zDJ15})WW?J$=msF>;(0%-Juoos-~3q0C{C$rl9b?c^4krT)p!~Ndp`b%B)0u>o)fJNk4I@X-L@fCx(NX? za0b^V?cIbH2OmDv^p(}S8ymnh?6q9us5`mkp2IW(8TX8F_D}>BpZP%!sN$X;DzUDc zKY&EjCqx{^GCHIY12dwD>GE){^GD5M_nK1JH{UxwQyqo44^p>H?p*i(Ibr+R+lPs|*bTx{{VhD_ZLlGjENS2vHUdnVy}XoDB77 zJh*R=?h)mOW}%DXdk+W@_dR_;BG@V2`ODGiXz3e!^mnDkL}4 zKKmxz^TZYNcr2R7Jt-i4*4UslWAIpPzlB3fICA4MT3yyg%-NUMF^9b4E+jOj#%GU7 zgULKx-nfLEfcX;mfZITl7&altwP0LM4Oc50$U;A*x_6nH2U@A)YJcy1d`o0S&5U-X zL-1A;n67Xw^Si`=XHdUZ1+f;BZYJeD&?3(!aYsj$gWEIq2JYdl*FI-_sgt+VxV@>W zgR~7oZR(%QCJ@6gBtr+}ha;T9izOWfLo-{YfRF>wk08r(p5lG*Vb>7m`?^hzx|(6m zi)wjzTqMCYGc}3LmgnYGF)-H=39>Ho4zLD%*c`=yHlx#f)j<#?voqj&@*2cC|bb>Ym0V&l^jH zfKF5=Pb*}DCq=5LXXpf^z!HbD+wP#akWloizAmXi= zV0kl6AX&tTm>4C?wZX}YI&02!)lD#eJD;Y72~so(jN!)kDAjoeQ{U~q0vzVrPAteT zl=@K8x@J7Vz|38l(gGS^`qpkjSLii%IAAO|QdYEw6EV>()B89YGPNKlY%u(xx6K#K z^$6UR$BuORqE|=^K}$wAlR}^IUA9?}rlKD3I7{=9{X*TNX^aXSRkW*RjTY8)wRfDcIH63~?nv&?{j@N%EgPpz23V>TSW%*$Qp(7T%v z9z7jatEzTV7H*FZuihrV0gHIT7m>A1BXKVxKpN&7Dbwu(JXnk{Ig3bBg{)x%LNmg{ z&1J4P-i^_Uj9Zj~V{a$=M$u_#*MY;+VG+p{QdT?2Tzo`+h7?^xNO4DGsjrk6hg&5+ z(o`BvVrWqB8_{U4WNI9fT{O&UJ7 z1Rpb}?(4xBH?XHpg1djYC>=KlLw7F5Zb{ok+J2{kr#M}>WWxUWq7qT$if)ga2B^bt8Y|>J(EtT9ij!`iQ?a* z#5g<|m=nqN<>BZU*0IFX&uZMMgMt#*=M|kFdu^xSttsqU=n=?eF>JBGkfRRADQbb_VnMQ2M-8 z?UdAz+aJcVn)W?ijj}(EWq;C^>SWFz<R&1CT%Sk00J`ZH73}^jc+N%pfWPqNi9Ic@m!8e9qbno$tJY$7mG)O15}G7brjPd0~?*i%sTO6MtSIGvfhtV)GZYizNOA* zmXyt!)HUTAu*fLf$TI0F@|$J8?pMn4C??X^&pKLuQUd5_D!a)Nvs2*j^sMYAe}Owo ziEJhV>{l7}th%gDdD5)%xaHqhx~(TvSPVnfttEsAn9L{A#OF90(Oz(HB(*kzc(tT1yGgS zWtVRemBfz=ma1w!n=4nx#fV$WH@i^xw`u$=>cXq!XDaL9vyw00*M0c~Vlw3UDo{s; zEBt{#37Y^%o%Mg-)ThE5K(k`R=mwJHzqWziwjhLpUg0--2Ee zcRUsBFf1y4NZS$F&FLg46}#L42e&+;?M(W}?q$^(;opfcZjT`lr#`Y9T1lm`c2yoL zJTd7ib7!&A?JDT*s!YtPJnw4oXGy5;bg=4PR_<)h?(WJa;7b7Yv6hze?q`iHPm8-o z<>-|*!M?>k6Rh<3M5$@kUV}th_HR8S?!87q>2n0t`Z=9qmDF3-J|-g$f@=LkH8pv= z)J}KbLIVr^M&AKxKLbbi{>Sd`=XCAWy}!EqS&w;@b^HI2;+Pvr=_Px`Fe9}Y0F!R;E`7_9r*SIFq4dmcN0IpqhFF99Mv>mX6h;0%`G zRrdft*$@^5YPXYN^B9UMXiIM%k`I7soDbb#8;-^e{!$$l@fc1*4rM@w^>VJt)C?Pu zk@x71nCe{v+YMf3k4$agvLGWGvHdr#hi%!2V20-Q9-|gD6l?{ucPEElkiPU_8(Hj@ z^(Gstp=*28L(p}w>;=Dk>@lp=*zSJ#@`WCZ<FyJv}F5Ko34oZ=9WNtbB5uY0}2JW~gBjx3MW#LDsLjBG>8B zcPDkY$9e)-(GOsojJF;sx)^^sIqdUJzIU?M^-%sbTkqxG$H_sme%305S&yE?vZ)1+ zz9u&G_ys;yRA{+*s>^C@t!EMyFtK|vHP6~JA~bc>(-V4*`!e~8>(Jvw@Aa?Pfgkb{ z#TTz>Rw#cTzD|f8J`H(QC`e16qevFmsREkZqn$}wwnQPP)AXj#bvx*6X1<}eQ}Pxz%_^dAQ|J~%A>A6~QIoZIUB zUIjwDNHf7;cYod^u+tSZNbsy%lB>NtnID#0it|rN4D{&`9LN>LF??7&KcJ4SmC4EN zCyT`v+AL()G+#d+l?s3N8NpB;celf)&cF+ROur*lSiY7`=@+;>Jk^-6_cWKh|3%ZX z(){wj!=A8 zEqIsKV6(8>;$Gk;@mhzW9Xnh6X52?;tI7t)<=Uo$jb%jO7U}hlC5*Dr^)26JN;2y0 zeE+o{I(N=Iw!g=1e|Ce`+HJ|lZ&@r;!U|SZFC`^_&D+sX;*>YqpSu+EYF9%aYCe(r za(~U%ACis+S=h!Yj=Y!4f@ae{GP>+F$*VTR>E!r)sLO43?-QV-TJ~JqLbC4G1igQJ zeMcZ0`YLVLpJL#LLa8JQxlz6wwAM`%1KoQJ^e67c=`~b4qI(;*u$Lb`KZCH^p#iks z9oyZzbvc?x;7!wD)}#HPD+ks4P|0cV*$EtG2olB|VCxRT;6!D^AWddNm>q}>{W0Bh z7gP0O$ICbDJH?+CP~??E)~Qm%Y3PQQ-SZzu{`}=3%UYl$ja^LfdoS_Z0_2A)C-ZHW zC>_JkAD(~SQNJo&qqeAI$UOD_{Os z_=~TC{a*z)zVcr^;iEheyK^G>fe?c66%T^KcP@MLTgdTkq|Ja z|EGYNjg9SZnvWbQDK|H_prD|Hgao0kj;5d>IHNV0nDkg!2uvRfUS2}Xd{125?O(R& zzhh<-6B9zr?4zvgr>Xf&N9To+(LZFLfPjF005}E!0z`)pHb>pQ9cyFrPu`q*@7~|N z^*k@Hl7|ofuC6!!2g=Xi{)2xa=YQk}EiElQJv{`_4?%uFh@1(OpMQGm|0K=B!^8jB zqW@{E@9gaSP4@YJBx(L18|%S(?0*H!5+($gPd*4vutpOSW= z;wU3_aHdX#UVeXwA|iv)u;t$y>*ER_&{fGBOaOZ@Nrzk;AUO#jtg zBNJ#oGYe1wUzzaCF@5+RNoh#=O<_v)4g)MYd5=8S1J2ZV1!l~Mxg1>)ch!rgHR7cvUr-H|@Y(StG=vvZJsLyu@L3G<~R z*)xm>WOwZO-ec8NAQ_xyHkTx5t%&1cEffOzbur~uQ=1u(A?%v%Twqj)0qdbaBf1I= z#OgFv)&PJ}CkPL+qffQ$IYOw7L9Y5dZIQrqV$+I`8G^v|ab`}MPtCFZ%9|Deqq>%Z z)nMV*ykcL*O#fJ5jQef`_okUK7fVzAi;^a7b^UI5@T>se=iyYZ!e2O5%H}%pkht)k zlJb(C@6R?>z`h0KQuEhb8Ett#2}F`8y|o)6S(xl3Q)J&8dW}?|NdeGc`1A^rEbTFf zT(ZJ7w|=k8-}YL%qw$bXX0R>=eNNe#GI)sGOBp*gxdFc(>ZLk;y_XMHkK9BQg@T8! zO&9%N(|j&`s5xQ_PF~6X_-{0y*OKKz!!Z603X;B;t$HAvAg!U1uL-C0_mfOL%bpDcwIlXs^sXHIA}&=z5QM z)3)cuF@F8>@L>Vp(Ex>RhvLO)3tj%jw?${AA);ssOc41cpRtId#Jux^3yKfxeM1yT zb5u^GWPse|#c;xg@2|7nJAUMhzG4`Xj-q^^*huj=Gs<7?2mqsIK(6FRL6dwNAl}Gs zAM$drJoXAyiVqGqJ_^aM4hpTJRiw!BV!qPD_6zNTdvvG@h{JA8Qs7E@r@aFB9SbS0 z-mY_>M2B3fzS$@5m18hxC<2k*LY4u+kREVImDq?KSSxHtzRE#nWs~G7j=_U&=APaj zmaw^tF^UqX@TNi@-Ki=&0{Lwa%IlZMnIg~7^4e!F|2Tod0@>_Hv?W4Gz_=_6 z=>%OFA7lIllh(y2v_6R!a$+y(ss}Ui6>$s~&@==ArSQXr4=~_qiI8W8140uXxxf`k zeF3i$Vm&Pt@S+0<%4cs6GUTL(dey1$Ax%S&nH|8sF|JbQO%BBFiHBB-N=vLxX0mp9A{g{C#!ITjwowL zz81=_H?{~WCRQHehhhXtj&N2(YDA?JoH^l!C4nt>%uaYIKuokx=+WHT z4B8o_9(chTK{}2nb{Wc(0BD3w4Na~*QwMBR-m=pbG`+zdN`BJ7HFR)SrX7+zj&;)W zxKI|62Vr+B_k8)nD(u^J2#kr8W*5GwDdz#E8$L z^+_&YiRBWJRD(ANq_)Sx1e05c8_dPhU$S-cs&@KR$m8P2CY@A$Ww3H+4B*Jt^x)mk zSH7=KvWk^&iBLbG-px7A7AC=;GOMO@oF^C2-(OMl+NSGGffB3G6YZyN#iV)?)5&{9 z4ZPPf0$)DRPZ`RKTk4pWy=CH{CmCWkTky6a{&CG6a&C3I4M{_D?GlceU3F;*7JH~L zZ5nFY;Bxp#PhtEfYV?drxY~VMsO9ar$B(3ts5I1~!e^q3R9c?*3t(ta8ldRqZd%x! zL>YgzvJztm@Zzrjr2Z0Oq#LH%RT@dnFicpXy(7>5Iho+EEn}qdiJiS@p-|2fKs7*{ z*a{%znsvx4Mb>ShZpF9u?oehRtNXxcy&PD}?L*P8&J-9DLq49!-NB@OHlvz>f@ZBh zfljQrqI*Q`lEHVw;h-=6E~q9lZRm8Smrit{1SS@RFXA{>Xc7_SJ`ehK_NK-v{L)#P6;lWHC-ov$O7x zF3qTTU7ct0-4?SvdaBGzWUz0!eS;V3-g#F6chp&{=6LMt^&(@PgeNRV0IedaOauKC zuO!DgM-uz{A+#6-}nyDF`1gzw#GgC%QAQrmFrwtsNfrkmYpo`-}6 zZE_w?jXq#IzIO3Bss9DpJybROZu^&CCFDk*_q>Sp6_3zjSKfqNGw(U6WXFxl+0wG2 zgJ?f#ECL@Ob--M~rX-bfLn`S@9Ez}~{%&paRP;`{(o>En++6PBM{dpYnAS8{SNp?I zq#ETss2LDmoDO`dsr~IO=Oq|Gdp{LgV8~9TkEb!t{-DHwb2HtY;`Z<&585bg1i~*_w;L{~=m>ApWbCLlMm6^hv^mbR5a8gbSuHwhyt{ z5R*nG>~FjZ9Bl|2b4r87kpGO?_m!`vNnG1YID;fG_$AT}hx;}qo=7K-&-x&llbBo; z@AElbley>Hp{UTFM8TibIz+wzOXi$Un!T4S>X)>@g_6z0{tD44{hnOQmz)zqX|9zL z?39xI5Xh`Z5gJR$EEY5zyXQ~=kXfX@=Sxj*ra7-jl^+ee?~-6AgVx(iDm72D8cXh& z<+@RsM)@f17dYK$EUhhsGEyU5;E7vW2ynA9-F_@J8Df$QyUfsaO>cR~8$RYp#CLn8 zBg3CRBXK(otL;c6ou0ZMpY&5&)jK59FFCstV*WETD=H&vJGF#4ql`H=)q%<}Giy-X zCFgNugP%%BIOUjmHdw;td1ZD)lzixZw%~WCO}?DO&MX%v^HF#fUR$Q4GiTCIz89X$ z;dD>X8$0KkX1t%Z9r%VTqNSb^tLIXVQZ z=YP`9lSk2MN9EVg-YwqG|7jtYc`E`epvbL}Nz#?g=_*93qr|3>N+9^q?km!c7vTAC zZ~7HBX2!oBE0j)&KY$mVgqY?Zr*glOFjuD($Se}L{1un6UoCvME(RxIuo z>yw%$+a>+wv``5l*TS9EE?t7Iis6keVRuVMjF)J;rRRW3)uN*XgUoNSh(DLew{R1; z2-Rs>Dz*F-k;`8uGM)@_E4z`b8edsvE8s7Oq5QgCCX%emrBg1EtfCuTE}pD%bG#gt ztU`gP2&(eEty3Ykt!y1#ah*$9Wvs$MHcn(S6MG;b@l-3-vO=t)Fw?U94^K&Mml#nA zpvh8|rV~iyRwZnr`0*&MO7`JgI`HCW6_0~Lg?6=ogMz}Fq7GTzo8bWKo9b&O@~E)V zLD|5B=8Bg(Vk6rM6XVj+H!5F06KhH?o|Ub+{0_-mu&n+*l)oHZD}*gu7ZAJI3A9?S0*^)KneL4`mX#yNG)t5I_3dLR3whx}tyiX9OF0w%u+>$Nd)nB=<-Cni z3}BjTGuLfrG)9OcWgb?y?nyHj9NCF2%I-3-Z!3)HsynB2MEto`F&9jE@i-_OaMKrKJJ z`{|581+6kiq_~)d)*KTYaSm+C5dydFCq0DipW}Y94kT^#(RmEKMmM(<<5*T`Io}LW zT?|AVF-sQoJ(nBgzMM2NlasqrGvIC|C*m>SEjT35GZ+?TFKIm>9Sak2mlN;7MW}`C z_z!V=bfP_muTCdH+@mTXmdg~a4yu+@Z0z9+z*hA{U+yMOS z$J7tvCAs!?TIZCn<8!>usWkv%P!@`l!mmMHGc zwcOU~(;2_RH zd;J#-sZ{tV`Uf2AtK@q)mmRt$Qq7zzO856FNBzo7|+m|_kDivOtjjLY4zFml0 zoeZGZjB8IhXV$b?dnH^(Dmu0!y!NOXTIjW+75I)e+p1h`UGn}qyc zS-AdOVPoOIE?a+#Wvy8opbR?P8m-=%_(cP2Se1CyDMd;|;{)RJd?z+xvD>vB8rP_Y zrvGfeM_ZqLZ;n`3cA7G<*2YT^dtPAQU222Wn(9Ek- zJW{Eu{qT0B&ijLwkAkxD^>N3`0^K9fP!NFh+NHDIX@C=jgu)r77rZFm40b`qvGl-( zJjP%<*fWE_+$;kN3?}UnE{2j_Q0qAb`lH>f_{E&(g(d~igCok46E0cB17F*NQ&-!q z4G!tlEQkdS7B~Wbqkz!eIc9hPp-VVsd~i%( zf6V;)82RNGsq~fm&R3rPW8VC)*Al>-R8W(nuPDlsR31X8OeB8iMB?g+RKkhegA?gH zC#mx%DwL;6f4-{TIo0@bB75^x_s*$i{)txpsmxW_zsJa)gcv#Oe~6I@J!NTPVnUMq zPgR+aBonI21TH)Qd_u^OZ}ai}WzUt6AW%>KsVe`)g;!M#_&-ojFa`!8rlth&$=?Fq ze=98iZ$NypkI(-Je9{^kN?^njz$gEt$A7DH3F$F`eKMAwPO#_xyE?a_fuPR)n|<;R zCjPj;|G$LEXQQM45flIadzeg!kVPdVCvg@8Rlh?){oNgaHWX3nncYZNmQpK8ceBfmy{w83C4H7(!aQg#bP&W|bCW zeQRuH0+|;RVFXpaVqX@c2ihY@K_a3w)8e-{7Eyw7xS$wNL2ygN-{2EYPCM8lxerZ9 zvZ}HP7|GNEMaL_X-7)v0wu2W6G-hZM8Wf9wnv5PIiE~4R!|ewJ!(ijXat`r?L@Ek9Dgzwkpu3jZIOTAZ+9lE3>$$q^b*m*{4W4Kn zEsTY7u>oRy$$qCp!?cXgc9MzOQpEsxBoT_PqHnh&>ZLLqshw6f0nU1Ky9hmHd=&xv z>H-jr=(pT&h<1pkV~4F7Py@(;PW~tz7FOGOL!#c`TMwnxlCczk>=R<-?oX+8{;1ROXybevkYG zJ~2;pOJn{Ee9}0aWy;=@p2@*>tYIMjvKNZB50{X_Ka?{!GLA~`HPv!F{}=c~Ao;et zu>wiRi#R}afWuUaBkaD?uf1Xi)BD^@9I(SnX_%9lPy>e6Sb89dV#M{st-ru0r49mD z1{aW|UOZab!J=Zt96RK@+NpdXuR`;Pl*eY4W}#9!LBZUZ^|XNRl}HWTeTk*?V0kGE zy9iXFnKpSa-FopU`<6|NeU*>BL2#JFCWXvifhvW5FD4hM7qGWmS>dCp58NNI3yMso zPr}`CyiycTEQ-esT!ZoqffRoo73+In0^HMKp|>>g^&Vvhhs_U-#BkS0UAATnBRJf& zXxaF1*#N_7)g3+rM&jZ1f;)4h2p>Vl-JqgiRpda$-vasI1ri4;f7j) zhg4nUa^TCwl?r|$Hd*u1W_c*GXzkgho zZsU}}r7i`8p0dc6!r7r_{19=j5E629qx1SYaRX?S+)nI?D>6ZL8i9b7F_jWj1>=rkn?h(i z+LD4bFLW1^VSyAMX9g{_0eHQSz8OTbX%ngy$00W$Xt)0e zKjSM68a4vyx~R}YkJQa9K};yGK^i(#7J}Oju6INNTUQ@ZVQZoSEmA%rD^uT|F?`WY z1CVZ)wmE48A)@)-irE9e6K+>31}_-dce>2iMg=IfPAZZVwK#njO!T!%702+zRM=-I z@R=J*M730skg`PsGI#2a-p3!UHh@D@7Vy?oQ|gG|EF3?Vjwff>Qx`Pq^}y~mb6QQ6 zlxTLA3@TIZgR)&JPT=^|Kz^WwO@$AaC*~EbvtKUq{Cl&HUJgZHPNk&F78ppQgYwiC zOowo4lE4BuD8SQDUOIZ6bpAI;nxkNW_sy)VhLyK!H!jEe4vnce#fC+c1oQ&Ilcg_D zmvwjAm_dM-JslHTPsKQyz_kRETRJzFbogkQ-91X-`ga0%TWcDxdAm? zB1!cl!Ai9uFyt z_L;9=zLV-WTfJ)9bFdDc*y;m?XOlsL%`YKxXE?wBwdRGS>QX!#gv2(4o_wSvbvPSJ zm^}*LeWD5uIU5c~H+Xs51r-&Xjc}x`MSdJj%^5r!4K+O09U%b*x6WP$PkSp_h-C$f zZsN5JzB+J#Z{M9Zp|%okz_#;2Za2)`DP=N1=`_CAeGUdmhC?^dgKvy#zKzG5kF*p8 z7a6471sL#!Ph*YV2S!Wxi+p1shI|iJJYRaPFSAa>M^&0a2c8P)z=bl1Mr&UW;e}J( zr)?XHVpMa5-(XO4d;)N*&AN}%&1q7QKWS%tv`-a%1xf@i*_(=0LrnD|S*r(vc%-5l z*fJQv#yY!JR)r9drGWq4Xw1^3B}c6ryh!c~;nyLjzVzwnm--KK4aWKJs2mv7Hri+a zqTrFoI?!6D6zwP=z%ehl0SBl}@K+V-BC^p>%??DrbP1RsWTbC$2*6WqzWV3JrW{9J zYeBh_{^FW5e`WA7`l^iOK-0u;JUA^m)hHVV#Af^EEzZ_@9&Hl^?0iP`(!IW+p;1J- z!dP(<0ReI2W-rA$I6rv9XFTpsd-OO@IZ!3BnW49u$D%k`Y7c(*AVI5pk#5|UaVm4; zy0>kLpH{TIsn_TZw4ZvQY;54xooP8uXUNat(MwOH^|-W(IfjVmbYd4Dw)cH0p;>n@ zetH8}VIs_Fa9;Z~6}Xgd1m8STsIWn0@^UUar0b-9gdDrG`%A7h`f7`&oKGB<_L_p5A&Of9f>HM2nrc;~%HIXJwY_y00XfO!5T1gamGc=KP%-EEWzLC~ zC8GGw8TCshJ|kkTRr``S7ObbaWm=}*bX&XIn$6*}mGvketvx!1prh&PY8Q2)cNcK!U6h!$vvUy19ew+|#Kk;TqxaBe zjBlkBM5pglB|FmKElKg1+wJO_?_KCSg3P8dqqB4jFN4_jg4$^{w|Ij&wSwEWZCE0M zNh^ZSA4>hWwDY)r7R-qE!O(f!hzv%V%hrnp%AMU+aFLlhgW{UBj7xpLzV{y(sTf$Adk)G-=AkAMw9SBZY4}pSr|vR!H^J zg#R>Gip~I<&l28sB=`v6{iIeiu!zo>C)*K^X*J*PVZWt@3VpC(fG3b@O9|h$ zX0=FQmmz!>!@-PJ&{Fk$oMbGSMgbfV`%C-wpx;NnStY$$crWn5Ex$#lERX2RajE_tSMc6Rbrq6N2Hg zZy5z6#uNl!(!_OS#5MbLLo#DKV^w$JQMFL&jKT zPJnsFR8&Sl22jqJ>k(o{wi!F`Dhm59fNIYr3$c3$&l~W|Zqd#Yet5?r4%SWKF&|j#FTUl@&(TidF+e-zo zp__C3g|sGmORj}q{T`nj<+32G#eZg9MTlCe=_kk(l{}U52`)mg+*i09C=%|9*@YJq zb;WcMq`A+e+(V0HETziQ(&Z7pPnz?TyHK{7&(+;ZSQzNEt5DEKSM?7{s3mV9X-Y|& z6?0rmZ?o9{tSq$>@Y>kOx7E=D(n=k&ioBbFLcg-l^wyJOWyojfx6pDIgpE7FQ>!BP zAiCV5#9ZG#BjA}xaR`KjrNTQ4;?12JE>IB%&5e0i{>G#t0Z|z#V3n*>8UL*E0xUyc zRSCuyqfAdR$L{n-I;L1+E(yL!rwkRBEB}z``5F@AT4G<-p!316YR`Z zO$`1`<%!%r4oyNaETt+<0;El%F)ZClk5T^3rJGG6-A(ekjMtQ#Z?LvV`(Jt3+^nI? zDDWsur@KYRh(-i0b(6GJD+vAAq}9xw?xT7uy1SKL6k+nQRg$z#tB~|bajO$6wZT1* z>v^j?DRubQwtKAYbl#wk8tqT~>F)bWxplX{#$M8jZnS%mcD%9!y-@3T?A|d$MZ<`Z zdNI*4ZkUC&>PX5aqoZ<7?e0hyr2UmBmAKrQh^-R?y3z!@ut3%`+OCRhE`_$P^5w2r zY*X1pXA>*j@+?)|zk34J>_61i;}5%E(%sG4gP(3~aPJD??Vh&M z0&TR-R`#LOw|-NAgl6a+LpAgO3ZWXRp@X8*#DJ(Es3^*!34sI% zReA?Q?+|*&P^5zhsB}Rr2&ky2fQVmQetYfzF6WG~ug=a*lG}`2&38WYr~bI3bC5&h zg-~byC+lUiu2Hl0moeP~mF=6AT??KJ+&{WMX?1|GoodZmuc8+L}_?du*)Z!$KrjyRTmBp-R=m(c2G;zx4XXbZU+DeNbyw?CcE` z>MnuwU)dCS^P>+Z+|tPM2qWKq;HYKZ*`(ppZ#&jUNB8!uP4eIY8`frF^MSMSZ5A;D zU$eQ$eFL8Yk3Qm}kHZeM7z}D>H~2UWm~Gn1H4b`owkQV;26;6m`Dy!&)fZwLNy0<7 zM*XLghs1;gtXPL8D+kF=L&}?jGZMpmm1((E!}68kFV=?Heh9~UX_v%OZ##_?jMWv_ zgH{?we$Jod4(gAWA6TI2Hpd^^)cuVB|GQI%TNUrFO$shem1uHtTaVW)PQG{G z6&ahtDAYU_8|@9B3Tija5}Hw|EumaBuGh};^iqWc zXUS7%L20e(nN!!xN}4B(m!@eI4zmkSFU6JCq_+yad1?{Oi_XR0zEJdd^@;D&Gp~_D zt`}6JmZ0Yt2Zm+NXSc?mUy9~c!ahBEubve&qx!8*z607xn{||&MKOD%-J2sX6_LhG z6LTALFBF}%MqJUklu@1kWfBzIHD{+iuRYJ5K)(cud3OKWwE&9+7SDx<4PMXcg{C+v zYuzMGWbx;mUK-V@He2B< zWm_(1%e#>XAlR0l=2o4h@thc6j^3;?!^2-FyeJr7Fxi)T)NIFX{`4(djyV$XFz!Ww zTxA;o;2Cuv|D4}vE0ovcVyjsIDQ0OT2yx?QKNK$VSZ10-0cPuE#^4nh5lH7D$4d$J zo+FkNa{?Bi0(Wy?&Iy&BK|tn<^*0sP78ai?Q;r~=shYa|&h1dY*YKLcRlVRo8wUo% z4;P9lz;k@KN&Kj`CW9FQB8Ws-1%qOAUVD4xpQS-H2(On~HLPY{JKq>y*I=L&LCVi~ zW@ZzZYXDv<2#sF#=x#f;3k_O+8+-^gnS5)4Sih{g9ud4A5f3$?&#O(W8*i^CC_rpn7jcfdb9BcU%w>8qeHKq%Z-Ueytq8ZT2@6lrK=_L7i{QKz} z^yT zZtj2PJn7_C9R}0>PwwoOu zz`($N?BDRSXaCvtOm=n6^YWrgzmyv{itpb26PX|qiLtSOSU%Ko2 zo7U#!{2BH9SL63zQ7yeNF*-UrGc&WexcI-;`=v|Re_sDxVZuZh4uDwE)?ukY8p{b1 zCN~$(Z$Y#aNrkY8W+g@|(XHTUOJQ*OsYsLDjtF7;U$g(wz~CQp8>tQgizuV{0pTzR z^NABmsmjCxkrT=y0JxMMmJn4|nU@3qJIkdPQ5ZNzF-LjtkLmT^2-au{$fUX6HEaj8v{*koss^N zWI=(_w1mh^8@s~giLMT(bWN^Xf{#Lr%T~ZxqmPu1Gs6a!-a`+?9_q5c2$Myjc0Ny@ zO(3{X3DUW2Q!MxZ|I(E(RuxG2HGc)(ZX)nflF{6^(JFUA@#X6kC*h2t*)XtmYfeeCVVU0>*>?;5%hSq z{oJsJNlNf%+#3ohPtC?C!6c1>)cWQ(#Bp2yDj7L#bUzb{zu7buiHc{0onT><~+T=jAPzbHH;lI8zaJ`$B_SN&hyTc3I$6K zRpS5q5n#9Hywceoph5>n%$rX z%kN~IA)45)kI@4`$RN%n1O!iSbQ-GYTV-%D05b%fQT;5t|8AF{Eda2RFZRmQWa6Pn zHHdc)19-9@wTmFth3YdgA^>=u6rjFxk#$jp%np@dHpVQ&FRqe8St*RX^3b}pmL!xd z_Kb}K$63WgHRE-j$H@a7Gu-%IvvX!{)DF9qBsshe?(+); zyT9a&vBhVm4X23-7~$BxZ^>FyN0h2w>Yr}+Mx@L4gSlO!fZa1B#Aea)t9-<@yZET4 zYoX$ZB-)KDZ*`>ia9kgHK{U+ZgF`5-tn)U)!w3W^jPCt`4iL!RD$cXeC#ih($MxCC zz+sm&uuQU=>9W#o!D4sN5jA})k@oj|@pJ;jLakydItoy53T#D`ri30tu^?Xqzw ztPiDS*kDVuP-~dA0z~h@Xp-VdcJfE{ z_;G})*p;xeO-411E8?8Xxbp>Xu7XY>uUuhUB|)`sN1vd&GEirs630CebkKQ8$g>r1 zhb&>s?(eY1kTDZBKvJDvlVY7Xe@!pgDGKDw1WM(Bzl8B@mx7n4NrBobNtOp#DX2Aw zmQxp@-e3N>*K~2R=St}Ag#qtPA&h4b(6+}$75_GR@gW1z5Nzo!33I|93PB;s%<kIhaOkxb@1o|h+X_fW_(kICNNsQmzi4Te_61S`q!#SO!FHzED&!J;PaL*Rmk@d>w zKE)2(5%2BD4T>+*nXRUl=TM6a<_3(3+jaKTar-{~H%J|Xw(@=HUcn-7P%Inhv>wJK zssOi|o}3uHS>w)|Mo9lFkRxY%?0KNbHz8K=WHov3)d(+g3g-m7#5&`D48Y~9kjemi zu^u7pxGQ?-hp;4ElOkM$=01Ai^--3B(Y?^b2k)ewymcU0ijpIk&_CT!PE7Q1RZzB8 zesWMS9z^1=eYl$rR2jQe%g*&P2ll46DXL8|98xYQ_+B+U-1SG??=uC^Xd@Jg0Kxf< zV=Qh@n@nn)tZO5 z9=UH~7{qBC`36$$(0djLAOhN6mlwN!Td7knn>$an;-mjp>7Z9(!t7!BWbePYp#-*c z{Ba=7*)FA2qhCSE{Do7q;js}S&%>#;H4E|L3u~Iim*@M{w_{Db5DHnb+GP)*bBf}` z1s_6B5udtW`7m7V_V#;;>OFQ1#)_0SvzsIcOU5$+f0m+-kFdS#?W3bb=yCBA%7xKjt+=zS1Ikb@9v3-rUTr45* zYcW5h$LEMO69GUGc6~U%yAc@4gt}zM=>Y;K3MPNW|K6ogA{VQedZ?zczwmJ!av+px zJ@m+N#K^HQjW+~JApwO6mEF}~!5b+Y4iS{dm}Y}Jsq+ zVJn3fZWxnx&l4lQ=$Vv|WClniRh%1BxS2c2fLBx6JsgX%wjBz;fUchFw>eo=)utd3uF=sGq$ z->!-r9R|~3p&qS6MK_?dkJ*H@%tn=yF+7vTUHnmPWdAL-SdIbGGw-mWR_uwP5R=(h z@V78^_vopNi?L55!U};ys&8CBOjn{OW_dmKAr3=Y4_TYllAYx}n-M=t4xgV@@7X4Q zK%JE3M}Nh|KjPQa(evAbVcCZO@6Y#{NrbdI0@UOFYZzuUBVkvH@PK}s(v}c>hW`+i zxLz2dqn5y5p70|Br(c2l~!F7ukCE^uA$A*b5! z#cU0wp4yAqLEpq@s_QocOLl2zGo#a7Jg<4AmCq*JERQcAO7bb!Q1#{f>#uYP)k|-O z()Bf>g_$x!_YRXn&hS6OWk~nAI+kZ-8@L`kkB|3Pku*LcgvmT=;0lh)^c#-6E|!!t zmmaj6oIj_^8j)7)k@eSXTDeK)U+(NXvsuCo7f(=tV5#f^kBqccu~r$CA3UsaVyuo*&tHXKkGh?1b5w~TWQd;u9n&hT1Deal$wYlg0ELYCu=3Nq?K&DMT zdQb#ZjRVRlEX-cJ?u*@EXJg${2p8*Na zm6$)rZXrw0kErY1%Jdy60MBFuUPA|RWrhTx1Bn?#=CY+P>5-bHC^V4TTGsqoRbpE> zL9>(-9h}Onc=8NzCbPV$zI^swxmta>tZYT;CshQ#VnrV!js;ijD@Y*0n%fo2VSqRd z(&!*(X$`bwRW@=$I-@JQO;Nmfpi8sL?XxYHch!)rw6h*~Y`*G5!F3d(dTJhdXmuC# zRJQ85ri4~~^^&QClpAD4u!c3vj1OD$>{`vD7_%a!W^trusYkT*bj{}xkybZ@?SR@P zWuA}iwFm7YLsRPC1gRy7!27eco7bp5p@^ecD$6MK)+f8YGgMA3KG>!@y)a?o!Xrwl z;~1?oa&cv7tP}U-Hr!K}4y@;R!g7?Q{?kakQC*CzP`%0m!X1v03v6J0!XiRxP#bL+ z$2No%g7xGYM`=JExKTN#ktIUG;r*yRDnZW`)F$tb+)}HOD zfvpsw_QDRBV<4t#=alzWNb>>sdF@~ z7STUGQ7^pzvq5eYJufT_vn;npxF2Bu&ItWc&tBXm#Tcwc( z9c$nf*J2&3Z$)eI$=A}>YHa+-@hdyWRx=g9R)9Bwrob7>OSZeQfr7Lu=9 zFw;^UqYB9P9Sf=xsp^y4Y6NffWjV(5`Scy^clKWD|GwX4)Y-?FUE6mCVmel1&#ZIC zi=Hp(Ci^^Es-T_Q>Oa_^Sse7KusnLw_h`!D(N)%g&!+uT#DSq`gi^qq5pfhhvLlkDSUU&5b7(OP+d-H!MDS zam-@DsbpzuTt9pKWzcw};JqpHu@{R~=Qx_*9ZU=f^1k1iP>(@;)_$xm#}#*C!a|n+ zSJlMr0A4Q^%=esb@k&BJ!=&Ck2ifTy`%+=Q$`Gf;sn>Yk&AZ6I zr&uGou;bGQ0XJ_kQa@|*7>QgxB|_PtvRSb`eWQ6w=_jFDekSCafaLWV^{xW09>DDS z@MVR9iGl9pYU8W>dSA1jB3PbXpXa*tbLQ;5Jhj!s!8;e8KM#PWZ$Cfb`P@;{M)Ly3 zBR4sGscM4p0%>WsFKZ@~XD%tXGG%Eft@>#NGs9}}Tz1#XP`cef|cEfxYtLPl@Rm;R(Z4${s*kI|MEZ@z=>pnG9MGR;FpWpFZkgMCf*P< z9{0XZh<6est!oio{ahtD&U#YH7!Ua{3E~4Z3Te!IlOUgJOghu+VAXSccCSxQu$`;H z*ebq0C|(xgQpbzFxdlfxz_!L-eoYDd~@T&8$I+}y={oD_gmdbNI*9v zs2iezg%EThDoBVN6)ZylAH#x$s32Qa5Qp~(7#2vc~rojgT!33@9 z|Jf8wz*GyNmHu{X;c9A%sc5=etA-W=z$4a8!e|9PP}!95BAN$iB{PE*oT>GvhV%iT zw+Mwr)EdZLo)Q1EDR{-qOOLzpRd@?CF%O1A!{86~lz_$rIy~jnw=U5EQC$+~n}RoT zQp7*&=i7)NvLxK<`yTz?6uhv3BaRwEu=7y-jZ$u@#Ue~~#_3_CkQJLf0*f;s*qtsw zlf<|a0hUAazX9z$0F5r-yXf1% zyhZFtVl|o{z?%T-;_pqtV-q{@rWVG=P8~RE>J#|bYr``j=f0;`g>Q$Uj)j{jnXKBA zYG~>!Q7Cr5^Umdc@~-2g%coVnAFk0X4-XzW|&upyZr#< zRmRI?YlpYCF?9!j_JV2_4L-Q?vl6Nk$tFI9H*fv`UM5N6D*M(e4 zL;wdK++P^y9>9@A@GkVy7-9swZ-;4u8+q7t1=kH`k(L}gRxD`NJFt+jtwKG_YOCx< z{g|#Q{8+lVv{A)HkhNuHB&w(9B*2RADSg~XgJVcu1_5kO2Oz z)@~z236xL&Rju{<^bkv+kEH(R1n{7+0kb$B(F0-S#y37;L~moETqHD@S*8j>WYyYU zfgto?dL)ih=qDsDe5A;?k;CSkUN*W~`)%H<<^8um3E*1`+h8UhnoJ&;*L%B{4JV(7 zn&Zv3_`XU$RhP)c{yU=j^AqCXZ)tFH9x=1}t<84`KP3zlK zfQn_glBW3MH8Q7q|9xxi^*%PWsww(v5UcKw?eLtuAD_zm(7XWmknzEnSpNG5AF|Ip zZewAws|S%L|GH1!IUI@%*=1qAvk88^Tp9w~ecAHDSCjjWT=o7A-^Gp}WXJpMY&+1E zufDWo1~aoCYFY5NEZv-ml(9rma}lrZ2G?P4M6_SM!M1WcJ& z*O*4Bl-&i5#ug$}=U@!6oRWHFxA;NiC`OE163URCP!(7)cM^DDFN>AUg%L2RC#X&v zw{M|Q7X#KA#`GKAGB&l}Br#G#g8}25mr%fIRv)G(vj+Ypb5!?80hEU54H<$0>I`*+ zm$T3uv|X6tJ^N<8Jk`4|%Z%=BoJhW_*`Nf=DBzYhlt!mnALFzrSMef9-#oAae&T41 zr)6zq*4>mKMN;Csc4VMaIsvBo*67_Eu;WmaGdDkh=iYvzo*lv3&PLzO0e3i{e+LUn zz9SoloK6#6Qoc_j8yP5lIVG7ijn@~sw&>s;CG~7GTsg;uP3&~c=@K^$G$GVh*;Pf|8c^y-k=cAl!OrE^#70_dWF$kuFU_$5uk4tGxY0!6vVH*N zZHDlz1h+t6*=M@jEWN&;pqEec*c@E?diJlzNETCu=7put?D~x>MnaD8%gf)UCVb!J zF0h!sTHaR#cYQPvWBu?_^2h%7&KGeXf2`b@`E|(mkguudq=Wm)A%qFjbF(3oxo}~H zS;8(|?~{ADP>{>9q{J>EUvv;dm6}jh#lzGNg00^M%s#@7S;TThm;a#l`W^9@aeCE| zg6y=Rgbz;o^)jY7hHf-|Xts@Jm1#9t;z-_Q)U!v?Y=bgV-2FO$>;0(3jaVtx6<#yz zUggrKVSUIRnRij<>|Tb>L62JR(Fc0Tkdn25Gz3Lqozsd{oWP8Lo+}=yPY@mnbwDBu z%+kPCwG|aIjHA37sro!F^>7x(M{=qrMYLu4ZY~LPNO?Ie7vc>%YAOa2+;&lhCcb@` zWhASZ7RG?selAP`7h6q|V(MIvIL8dt#)w#n@s_XZ9nLEAFe}J^XCLr*RbAM^z;%b6 ze!l_>eq?3?quq}0I2Q2?DmO{r`EXTOoKLf6z9VeNGNA+0v&ldVijsh91#Ust+Aof$ z330C(8%rPc5@G_;7>?VF>8)I%B*EEIWdYBg#YV(a;lFM@2)C|`R3LM<}!S`ose5wk8fz!4)C-C_l4nBWg zeN=`B3$ipp!tRxYU$3Lk_|BW$*pn>=a}G+D-9O0>G}cbu0$8lOcNh|H&RjQmH~kj< zRwpQD%k1HPb|F@)vo>h^YK_!s>4)T=1=QDLFAkzvqNkTGUfg|qWPc~**SDp&DS>Yv zc76M92wu7QtMuMjI`++Jg4nk!6X(^bMFb&U!tG=N!rkRZr1qIXF&=lk{!>DB#i8=<7da2t)fTq>mF=8Wht!n(h~j>Pp*8N=LE=&> z(bGUd-!4QVLsL)h(y5^fh8b#b+6{|!XDdI5f^x{24BfM>fM9y~o3A7p4i9I$lU8=1 zCiMs)gOp`pEFK<#DT~NFL%fj@p<9Nz@txqa8*!Jcxw{hLkJHvD15WQoy1^p%k~t}c z`M{XOUP*wq4KY-6a26dewa-JDHTRe$El~*MOn^9aYU${hegMbwM{NZmu6^r zl|`owk)zkR)z+iyFNK`bkJ*?GwIWHjmW3DWs_|g$x`v{tei$iSR39$3m3Sm8gPs%! zwpou|f&`OMag&hXg>CXPe&xdzbpYBLSA6E)H0b&YPLnl5&B-o)WY-tSkMB#qxoM66 zfK%H`0~}A>f3q9oaVBn;KYpLAwnMo6(@uFj0=T7~@ObxrF*zY|_da7!!k%61O;rM7 zE`f7SZOtwb?w{zD94%;qQC5pO3QsC|lva&ov&)M|p4wuv&|m1I4cl-$;2BMoKP0IzFCifksv@LkHHUldLz6+ygc?m^!a zOtdLaWiO0zF9e>vO+7+%wl_(;2#>6%j{*88x(=sZ_s4i8lD%bcoNZ}$_fmYmr1_Vp zA4m(_dzbD#ofLH+>Z70 z$j!`*#eozAva*{)#67a0$!2C{S(R-eJ*!!@!$C8BryC@*dCo>E=*71VD_L;IcKK)B z#aZ z_H;i}*&NLsoL-sUUU_O&ynCda7Ql^9QEPXo;;wTV+OHhZ{REx<(zN=OAlv6UwdJguQH0>vJ2f9P*&0S_)--EJty#UM zYj*cxUDs;Y=W7$u%nH=n@3Pc*K<;{9Eo@Z$#{iWr5cZ^<$|O`5)?bj((!rrvk3ko%ap*<%e>p8No^8xCb3slWd zV2#mwsmfYivqt2P+M8b*z?J{8j;_1x+04k)>`~dg{zlJdy6HtU&y-+`ogCHcb?Ed)b7{ocI}$BVWe@8^0$X-9 zMFKint}WD&A+00PJi^pg(q@f6OPlKg)yA)tEYuL6jfrKcN;Ye|5(9{`v}XzNM^Cpk z@3$2#)E9JM%0?^hY_=y>R`Un8H#k;T%h5H4T6fQmj?EfsSR3Pyj(77{v%a+Fnoio9Zoh!&Nu<-DUXXVRs&F`Zne7CijoJv$b)qjqm zdhtAD+j zN%=6yfyhe`VTi za?t0*%iE)FHf79=>~ zEQKpg?hYLBs?4l1iyzC&4YDjW&y)VtS`suY>@ZT<`3RRaVsLk4A=CDn(daaszaq${ z^&sbkw|4Ks=%Y+d#g$>^u@T03VQqhdM0Iy|LLf+78dRvf70& zE_h|xBmjyz&M$82CL6T_`$!&;%sz*_pBKBM3Q|5VzdV}wS9>4JC`oubT z5R@xp(_-~z;$D>9sM92(k7FSHalHn7HT6Bc@}8vmTJ=1@h00%Ar5dK!V7eS>+ri(_!i56h&m-SR})Q$_63N zDY4By^9zKy&y_07J&%G?>-9UF79t@FjpT)Yg78P;=3sFV< zZqv#pfKlFHi3E@U62t+3*xYEpvjVc+FPSC*1Ra8IAD2h39d}++3tm$dg-ZH{1AkG2kEj z^iOQ|pH{#hrnyjA`6m?chiSg=>iTaJ0e^b4|EqxJzrxLbKGP|u*?***afb9gGZ0+C zlnyy_(o{%#IXO&xhD4kYy+O+?oS6_K2LZtzM49|%ocfW5AV81Q+RofX{U;zmKAtM1 zsPG$cGQm|8<(U9Q;t4=#S*Q|cX3`+6AjbzPnG~K=3T|LZpBIYgt++i5VpsHIXVy}a zn`$}qUK=?pxk4x?AwkJJwl`$i0O7Q%$+xJe)jl$ZOBW1Tm=CdJD1I$V4hN*1*+eN= z_s@%uDF$Y?V61z=ctj-LXZZX@`edNmS5t8S2=n3PNacEzJ*{cLv&AkhGHVT7CmudS z^xt-JAP&NR7K^pFf888P0BD)+TsPTd4LZ=P`)9{0t?5&aHJ00u)u zD9PmbK}1mq<0JRisRlAqtY?^Iz3n~EDA3yh9bcuRv83PLDT#UlW^d;A-0|EE=6e1# zzc+ITqc(V0qVW`i_ZsIt5l%yUeYyjO-0XQ<7Bt4N{#u4N;A^tQ;#0bYnJxRMmyq^M zvLLT0t$pmcg1y9%+h>@=XwKn+^`U@Z;|iE5)pjmcM{va;BNu>?67+d}zrD~qM0nCE zr>3|G281HODI~uWmQ~@bJ$m3KhT=+9x%Ja5VzV$q^TrpaHOv*))&3_#XX>LwCgh>5 z*QZDPSF8S&a*CWXuVqK30Db>aLscD@%|Y}+5!&=(02VS%gK*#z=tTz#p_fH@mOy0J z+@S+bEVoNP2fahvH2kF?sFxKVsqk&qE&ckpxxXo=TMI3ep*C6tU=X~sqR(cqGyJvB zx#4dOwcD3P$Zt3b;JJe&A`(7~-+dAGJ0RdTO4VmrmZJ5rKGgWoCYEqUaev*j0l^D2 zj!?e8t>WU;d`JDEU@)&TgobYjdd=;Bpb)q3ZqZ^sgfYdbV={6KRepp{d3`qA0Ibk zlpvpA>y{g!FOYGN@EMrY9ReqqLbKwYg!hM+t23iB_CGUe5r6^Pw(_DHOSQ`77K2T*3W>dH&&Icth z!raY){l-uchWw+DS(;oDHUTt5Aez8|182QZWC7U_&~)y>^Y;x6BaiZWi{;g;u<+uc zv8otxP7+m&O%e$+BqYeh`kfL-=>b9xU}TgeN*oFT4V7bmktUCTD{$fA?nYwnJjUUP zb+LqGJ#g&dj67iHrs_Ed!!X+vO~?R-nF3HI-{W!S?sVJ9T0b$~CKafM0L?*BeCY{5 zDL~;&MXa&hi!_;v+9rSe?3OdtK;rF(!omCm#^BF<_;dOwV@bn;XSYDp*hg6afA7W5 zuA`DGoE*%vmz6-T!ld_X!usWBtv8B#!=vfihRjT;drSQ_92=yN!bZs_Q{m@$UhkUZ`bx)H6x6K`vT zKX_ZKt}}5k-NMe;6XY1A`lYYWK3#tzu7g9u{HYjWcBFMElLwwb6g3vQboRD>s!IgI z+xAE1w_v}YLg!8f%s*3k+2F^M50t!}pKoU`D>uq!bvANb*ynBjJa)?To|)5%>uH&I z$(q}7LI$u~XWJ$(*Uan7X;g_5YA3H(-}ySYuoduE$6N!z-<5q~Tl0uMG9ZI7Cu!nT zV|r)V`1{!!zYFB*7U@%LMh}t*7fvC2>Ot-!QJq|PmRW19PX7<+Psd&yQaB&y?QeWU zDpFBPC{;m21OUE5fA+Pd`oIKT#AtkaS`aYl-BnTAW=viVzN_lwtJljehfJ`C6Cg|6 zhJ0tsV<3&HJ(71!nD$j+(4s+xitSHgvfIV0?&1Wi6OZKIM=vZoCk`DZ4KiqUlB#`o z!C#wLBYk?DpV{GonAzB3moiWV5%@^?8q%msVP65sZx=@G;|D>2XxON^sYZ^LZlQI`Bn+I-(w5t4ijQk zZID0X6w1YQv7F%HtIf5SQfT#Y=`En5FX&lQfXMf@5d$zL>4)4N6&CuRx-1nmZ@}aS zeN72k$=5BH5_gb_0>`LI=DjZgP3}!}DeEF!pMLA77k~V1T;nD8$LIVDB}5e;qH>C) zlYJ0=@0yDQI-ox%j*+Wpv2IZ6hSU*?e4Ch>3sLIb>F=9KGnex?c(!VPFzds!%R|M~ zsqx-%8v2Rd%vn!aIp4Pu9-2{TmUX6il03IzFVKb&>gG_^r-Slx*f8>Ury^gL*tsML z7*4K+BI!gKT;U+a-Ew*pbHrvitenkqc{JWM<#J1-7cam4bm+enZ@FIh>%*&1zWtfX?wvUa zoX3Tm4$Xz*WUXSIccZw@xlPe~&GdyXpI?Csj1RqLSAsrS1s?1%UfvtM@N<1j*>Ann zYJdDgCxOM&8vSLIP*V~2&I%I*XdPAly^kq;9Vdg+L6DY2^0 zR~AN6@FQOR5@y&+q^*URpomln2oqAwf}db)r(vxUKE$O9L#a6qx!GlepT%4RVS_HN zYmA)>yUb5=;a5|k07o(+9{61INHp`_#dyNh?DB)|!jwy&$p*Hn6R&Xkevf2Vbw7?H zi}*#RkjPO(L=RQ|g8_0-YlJf@iohRnS2`fgj_iWr|J)qKddc}SQ?&0o`Kr&QB6pRQ zCawn=(E_P|Ibzf@)}lEe&X;g8$?nl^4O~wnV?YfTxbeZ=t>~$<{(I`NCyQfT*N8*2 z=&9Ecu)LrqTncBSNzY z7l`*@w7)ptDgBkoi}p)AxP9I!UhJOve2qAK-C5iM=ggwk(OiEr5I5S8*;e zIXy~s&pTQDzIvNd`knh)!^z2X_PU#Vtj|xp?uemG5+|AGS9;2rzA?|c6uIFLXPeN| z0!nDNBME<{egqp#4yArm(VKy#eO1wWi%T=o2&Pw~FYo#BI@* z`{0SXBs@>)(xX%F_{U{+fC zp@#_S3=pb8S%rfcCSzdNxEbatpWxslK>l+M2oeqEz9vEkjOq#bPwCMx3Pe;^kO5I3 z!7O;fx|83k^f9`H)4& zO^Xa{*yRXCrViZ4k3%f?>0c91YZlpVAdKoY&SVuI`4o0OtH^l-5e-$lpjq++Uvwq9 z*nNZbH4W@3TXKD#wSiOZs%GgpRls+o#4n50vAy*6eCg067_Q>pS0m%swf#bvb0^E)?U$< zz~)w1L2Z|g*r>>MsQhsY*gjR+F35d(t&%FpEQhG7b*OB0RqJi9fKl!aWK~itM1&|+ zgY6ag{i?(Eb~hp8>d9*r_oJ)lG{wC~s#gQfyt<_}I#SL~s(Ct69yJAWPp(Pax2@wZ zc`XYI%vAdvT{f>*u{B?v9#H)&pf3r*^(CRfTeDzH$-hdg_&q4pkg6;~iN z%(TkWpw36Ixa4Z>Tt%tgYwDb4g;!sldvwLIm^#0TuwY`nvuw4}X#MqyMAaWuKL@H@ zOvCkSmV(x`tqzsXKC7e6ihDp9qsk(9N8RzwVpZiDb(TsFA&kk7LfB<(+(M)6KA*Kv zQ%MB~g2bGyEac5@(#|dxG;4CGENTEZz7nk1tZ2UaqsT+9B|od&wWIl@AiW__9%56K zsCw9ZEQ=*sqE$jrEWEM>J<=4V)p9ZbbZ(|KF`y~sQY*LMsSDXH@s5SO$hxGMvJ}U* zXsv>?GMZ@%lw2*%QY}hGL3?3#zS4R#Uq{=qim-aEjws zND&p%x$S?aj@Y^0)>vTH!R^pK_@f#$s8y0LRM`+>;a%4wD^wg6y4Q&ay@ z=YmHkjoAHMfOo8~`|(_R59HzKUf0ou7H3U>K7F3Z+A4dVg_u6m z*eqO5zraC`d8~<^e1?l&?_-w2vtEy^Iy3E^%+H&rJ3sBee0;!LV8Hd@k-itZQ22oR zu$#kH`r5moJx z2V<$KJw4yEPgYkYh0Iix3YYGgjP|gWZ^>}_1&wr#C7wV4JizG3z)?fE#bJ~6SaEdT z;>)2&onvfKJOoGc`q6P{CU2$Jc-cYDG+`*^YWvhy(m3nbo1pQ>0?c4BWd16vnFNsq;i-^4Z%` zsE;i~kqX`{W-dSiboltu`uVQ8pON((Jsps7;e?U(3ESzpJT~Y_#C-mHNRh?7miK%a z+X;G^6hpsjM?mDhfMqDhkKw`Mbc{&&jS<;t0CanSb{$~H{ylD$03OPvw`UjXs~0DJ zE>4OpP3bJnTv&S6wJ5Vq?^%P?6F3i2IhWa%R~44mES6tfSblkL*_3v1!(w@zZTbDr z<#mx4o4Lzh-!FfAzr0hu{9fc=PTAAw3{m^PXNbD2tba5~RaI5GCaHJi2t7h-dGx5Y zsHnA=n4PpV9V5CZDP2-lzKX$Irzc7c49v~V>2p?el6YHRKiI^78UO`O%*H_xt|Nj}B&J&?l_u zvgDt$cDgJ{2a9VB4RllTzf#4$!9jX>^nWK+{I93%L{s74Lss-l_7or$PWL3~3{k+8 zXeGi#-?0*M)Ra<` ziKhS&wcpq5LsoxZvkQQRo*bQ{U$Z};i;{|H1QacW(_`Xa2T=TPt6c!-lRu`Ukrar1 zhO6jRcnM3X3On;-5n~0zZoz73xbM->YVFaVF^-M|vff|Nm z?**+GaP$2T))6poMHkeOQUF5F%sUD{(42QW$>3?S0hqUUX{@cm5CveR1@SK4oL(Zv zVDhJac)We5FBpUWpi*ExaJyE@&E*4fQk0!xviWfJxn(C1* ztw!(_zDMv#`2z_3YyGx#t3<@23ApR^|8e)8UrjB1xAsZ_LJz%TsM4hqiW-U#ih>kr z76hb=L3)#r1PGx82oQQ`(xrD)no3hpK>_noziYLW7OF8y zQB9D;3bd@>00ZH6_By@u%xAT6_f<^k)nO(9>`^SkktLvV zk@fB3?sBRQM<=@&ieUf{nrjwWCMtg1TPZ616%Q_w+rf2E^Hhi{#nrDO*@PORu&kkl zfqENt&tHYJ33fk>&ug7H)AwPZP;8hXFd81cefux*pP$___y(9TlE-3s$5*dHzQo`) ze7`=mymb8g;q&UiVTSgdz#WiM*xk|HxoYt@Cq2atP^~JlwDBdG_($fXh!uX1fi3^n zD+c7uVdVf|T|c2FfJ~4cSpxD}04A4ULbRvP5siY!izWkFpoPE~U;y=#{59Tr%@1D3 zf{KT7;ejOpl=5Z@p|?bp!k%$#iU99(xZ(GOhU}fA5)5NJi{!d%+(2Iuuib%RsLKW; zcp#o9T~&fZZV4s^!pa+joRUFZ1~R#^NY&3^8Bqz)NDB&_j)lfeTf@PlHa#!~vtig%P_40!#B*hNgzgHaCvdYL1s@KW z`~0%Zh}Lu^Fk*TvbuR;uC6xKzkolR!X|fX+w3M6~Z+fbX4y|0d_l$^u6*rKh1e4`i zqRz^_p-R0JkTcfR7KEXsHgRAMJyqb+n|IW?ek<1euV6VCO&YcaXlDp96wSR)J4^I9 zsah*n$p^YnUsacA;v8Eclqr!4jm6C$Vd664QQFr#Q&1seP%=_|(EkT5bN`HJiE z1f}{p2LXt_sKBU(#&Ms3VU*sFVDnH!QeHV0T9~91DM_?F!+6q z=X!@qOW#p2x*hDsl~bd^GjQB}ao~u;I~(TN`1rxo1FDD6DL;$ zOm5FejDNILZqJNZ%A!=$!Fp*v0D}cN*~em(p)R{V_|8m~TM@@c&gY;{7_5A2y8fPK z@;Eo$4g9?+`sxuk)yTQw8`h1J(zW2Md8cmKu*lF0W><79j}LN$1+4E#dw=P59Ig8( zeO*-9?bwe&h$yzTu2|gPzpM1*Get;w&pRpJf$2E~_s&{KYQ&y{!wp`nVc+>j&Rc* z?1XN>ONWaBbHv(LXI~m|jYHayC^lXxQpwu0QUZ*Tl|gdp(3p<$R3jN2nSwGJQR*4< zzXDi=6q_(ta3wnEA*bunva0c#Or_m7dDQ6wn|%V}Sh^9XAIV zDtbPJM->iFQbiO<6tuyYJKp6#i3aXtW%D-Vpt?UTU|;!`3F?C&?L5~e)!oLVb9V&P z4cjQJbFKIK2^v7+UdEIqlMFOh}>ZWGZSE2i73+!{*OGW(Xmx+Pb%?U#$kCv2#i zp8$3yH9%Bkc~G?_Exn@Ql*|vozE2cyRy+SD(ZdixNM-M)G-8dQ3OmSwTTh<^XB&}F+_P|0>OFaDP^E6i>5liolmwB-`#QrK)dcqc%X zvSq#>{=j#c+g@GD<*3cmp7HsScQ-|t&s|2&l^G;oj@@*!Rea2-C82q5)_bkL5n3#A zE#Sf9+)w2Cuyd%8@3Ut*)hA-wHSZ2mx!_+49dkQ8R7BhYw!V;z z`(v4DwvXdpeSY6aShF+>c#7X$s=21UdI_J}*qH9v5Um`(v(meJv*a0oaoWn2>D!pW zg;hk8U9N^l|G0SNMYq({CJf!a_?Gl7$MZ09+v8-E3p+!^le*%dH~w!%SWEkd^=}DY zyYh2O^fj{EvSgcbNz|<}DstZI&Nafe!Gn``yZ8eye7z%ed*T6OfXO-`C4tG<&^nJ2qvc!%LfX zqqc@b>EbThC4B;0B%@Q#&)t9l7rSA{qNI~=BacNo7C4%@}No}$}xR?=){kAsT(y~o=k!} zyXm<@C!V1*cJ`tv_>6tDw(e*~>RiY&IvwzpevMCO=t;K|0B+}y^K6eVuabvtkAFdA zj@TaW^U36yzfLwx6poZ0|Dy2MZYE;>dJQ@&pxodnA#24aNHHi{$(PK(i&yKBZMSpO za>|xr4jQb^);9OsUd{G1(%+S%=$&PZ3S&U@N63m>3d!H7*E*c?~ z6TL4Q5u6n_FBDO6n1eT`iqexY>q4$lxhXwDt`*4{V^qgB=ErGje`>RgDzz_SoZEVy%?pWcmT&|Dbs7?1>@ z1IfHl)uMa8yl5L>G{4B?6L`|R=#eiMe>3nB@R(Msqy4w*-0s z^PAE7l}}4om1onVanNsSJ76vnji(MK`d*8@pe&h|EUHi@?gI1TE*I1*Pr&kA4=$6d zgH0WJ(M?I=%M2B2^lIBTpi587wESqAE*1S}S#=f4m7^5gDk_wPSyXH)jr7XvxV245 zy0v*y2_)Za*wr2%Yw2aRaK69<*Aa|c6wzo+_Z}u6}TlZ2p;0GL1R_8=~_Ue zm04g8s(9+k1B9!g3!vAU+E*=Trwz5^3QBbpYN85C4P2^E9N1pbsQG4Indnyy?UhW6 zs@bVk@QI*l3D@K()Dl<#a?AOq@QKV4UFl+81(an>RS$l8a(ldNBXGDJJxA!t$1CzbVyqqv%H3WYs&hqV<~NFt{jENZQ-@p zlM`)mwQc%+EqU^7#wqRD^6i$arKZvCTeX5>)4Hg>T)F5rt3E29e}~DBTnDbYqTmh> za}>9KqpLoZ@K@V(2oUrl)W$Kv!DKr zKFS%zZeRU8WmbbEMat#PZoGa@%t5DD#kq|7>@X37eAjHTv7YQc%2)56uP#NIti36& zoiADpiYKyQLk5kOS^G?VWt-Vwla9C4XI4-2cDQDHrS#pN$a)n5`jFkn9m%cO8+Ea; z*DexZLG{Wi79uk%kA$?eM)rX{`x(qR+*$i4{V8tX+Z-bO!nJ~5T>9VnXWo>&^Y+JW zy?y9pr`bYEhPH_ImVbTK{wa+*lP^Egj|f=&a!r5Le)nhL9fsPQW3B@qQ?m38>G!i1 z4H;`tOd7p_kkcSvln=tKH8`~f{nR(eXq5r297OaF@Yf24>pO`a2BdxryPM>78=uVM zRW~H@Gwss5*3yol?g&oi^F#RLA(j4q{JxuEgN}mh@Y7L&#lm59L+S_Y@WQvcU|r7R zlOwXJ8RE7bXJ1x$*FvFP_s$#Sel#6&Q@ZET!>w`tzMesHj8Xp;(ffA$&|3~8S<&}i zz!)4H?z#2fPn^3LqU2@Um!nvA?^^#Y*D(S4pF{I6Mv6v{PPvYHA5hG9tm3Tt!bDTi zJ64`-V~GJ2Gcs49(uvTmG5XnXEqbhp3aJVhf3=yd_=dSisZNcV;m75{7PevOd?>hO zq)T*6n%M?6GqLw0(*kuOLus-#Ad~4cN5iC5Oh3*HFy@DkAC9I{Fjg66G=~`1hZt5e z5@RerJ$WuQ!y16t1OZ17j~2d7eh-KI2(n$DeE3*D?F?2w(3oXLY3fm3s)ZZmlFWq{ z4O1U}kZkDhA8!MTRFSiY>EM*{*SlymmL7?|rd1{e!-9^Gr`VKRInq=ucOE0eo=7Td zSt7JdF;C>uo}6laqQtIcf`S^spQ^D#H9!2V_t`+?r69_5w+{=Jp@*AkAYM3#b3g-1 zo;f)TaKb@+%^nMt6+&~$g!4Z z0qhLpRp$fPpZhC6KW6y+>ec7;2GYRW&x2xlo<}`=9{lTh=!fUH<8u+G=dS)cm7~$A zoWg&ja(@aY>G|d3|5}*;L*;&(ef0dYm>8YP*-J@XR8gTLId3hk-$?HGam4d zR4(k)saPv3I+eS6_3D54eRR3+pZqe#!{ZN+s|^qTgXHLn-@g}3wkIUeksLk0{11}5 zef#!33gu6`c5$NL=!rGqo0)0Yh*^z*ETj(u37x{5 zi~pXN7pG>V9;2&$S|kfmFo@rnP)C6sX|^gWFTB(8uZ8&p9g4WBst`yN$JD8Hj0q^k zmHqCRY%sU7Xs1_9&R0LN5(h--(?MN-Q@L|oLX|WKTSyKgGsx{$;&ABKzp0#ig=7bU zMG`Fal)=7(F`-or#s~;lT?F7pM5lIXHf9c#Cj>B;HHUhlDzmPr74qX*#4OsKlX8+7 zV?-$kESe^=t@Z{$)QrXB{193M`XUeq=sZVVr17+oh)v#=Cd|nYp4de@j(S2NE6Gz0UNWOZL3om6d)pIG{{X- z$bF#^w3jG*Nf(4rt(MMg@pz!g-3x?Ia|7ZWzETu{B%`yi7*wzO49?Gru$@ zi_L?)5uH0!;0973wnX?$<`!LhT5mluVzg~Iz-b;El|9BSs(nIQE}96Vw;eM&&0MP# zQb&Q2FL#UpL2S7|)Dym;Oxd?4!>*Q}gw$UmB0o72&v9{QrD3IWk&j(T$omq&vFUIw zfF6;UOV!-?o@0lHw>bIo!@1`@l^NhKCBDyiEoY)t4vig%!I@t1Ky&IlBCxGXe4+M* z_xdctMcxd?sKJh#leim-g)t&!=~Rxs^NI~LX&)t=R~(nGk}&Sp9PwGHT|LUNCDS6o z!$L@Y>glpbvBQ7wseFROd>&h=q_DU{6valc4--Q;a42dxf7TgBrbrZX_M`c81t=0^B(fM8tnkIr=n^kp9b$9FkPk z;os9hzp32T=LyvZqHQcv0f@ohEAhb~MonN4DwT>gVo{zX<2mSoC6d+ z4+X!yN-vnai;0)*9{}DhaR{XEy`w~@ae!lkF_4#J4W-EUH0u&h8UC4p$W`_@@q>-f z2UsuQh(WsEySlc(zUNx~ALEzWo~jJZb}_yF#?%)w5WRf|a_r}4EhnI2Yqa9xkI(hS z0;pE#5rp^GaVzWN?+<%6Sz-qu0k5AhW$(pigNtAo!&s{&{)(Yjad8psTX{)4z#MD& zNVOIM%;k>3F3_9kuw1y>P6=4hw9n;lZLb$$g1}zw7^EEq_XJ}glYrn`1&-vl9Jm^LzDZ+PhZHZHv~F`(KZ3c;8>s`RGI-MaRwxa_DtRD z1lgm3(w&UA1TwI79-(Wzle7<&02qahK*qu*0wO+!2cLG5h{1-YgsRNDYH!T&f@O%B zcxH}>APKAKOzV)DlhlArgqG*Ht*K!$i_uQ1&q$##AKb`-FN<5h*06xuTpj9(J$pK` z(zE8!kO%}k&iDnZL~yH)3c|}^Z>5}|JnJ0{@=GCTZiGT|bGo2a7W0BDP9k5m(5~b& zZ=?hPI7Hqz#tG$P>Y!OG#3Jh@Lq8gOZ1It4OS@{nLt=Hg8sm+M zx^pQ7^r`LB=f1r3``X}lcgMl5!AfRxx4iCT`x%GEpmTL({RsP#sQTjRU z4o$uv>NNbP*<+ON6{XW8-0}wY8fZQSKF2SOmjVSPWTappNVMb zqt%w}VVVMm$#F7p!tG?09?3LEyg?kO!|%b*=HPCNV7B=kc3%3+ldgcNfhg!t)iT-k z5rTWDCZ`ztNP;#R$4+yCJ^mRAU-)(;g}Ssjli-L`*^<2@7;1GWraY@Mf(i(syQG2G z2Mk|CMoLsj6Urn^bg#r$<5 zl_!)Rl;ah!%IS|%zQHGO41iA<(f^#NY=$t96O$Eq@2%bUU1=f=li^xP?PqBiAqMl^ z6%-Y{XAq$axLp_Ym3v_IvkU%QhLG8AMu)SR1Ftok$sdv&W>z+vjh6?2KpW`pkTj$D z0Mz|8I*(=kNf0Ep)_*g39NVoMZ=Y!wpT9mCKnT5P)@jW|*5?QSxgA}i(I7`4@V0R;?S2(v3Z zuI>8q)Pj3e=iIbOK*095fk?Vne@6p@z~2HT3b?{vbavN zzcNT@CE~W)8QpAq^gDq|CBtV=*k^cLeW_$tGVwuenVK^Bo_VzE;gv@_6*roD^)n)- zKIEqPFx`H;B7SV~JaVDJ^4eNRcGuIpr>_so(=6<)OFl^NuQa?C-Eh|*zkfSj^#;CZ z%etv|Q9SMYBi}-W#lXJ7_B4QD;@75Mb>Y~y%GdskLuC=pRsD!xD%yO^C4LXJnUgp# zg*8XB@1#D6USnKZ9c2;XXv zV!~#uf_9ZhBO|Hw*ZZSPk&D44*I{zxZKh*-ciBqYo4Y^`sW)P{hwm_YUpw5iF4bxT zE*#w9i)Bye4&QoR-_A1KvwOGUS6pi8UgFceeWqtlyA7qk(vMyje7*hiX&j{LYhK~o z$CSUYpS}TgvvI%N9W#TjgS(G~-HTcL!@i15@ z_n_(PV%1P(s5`J-tua}G&9Y2$Mahbc8_K)GJxYT8rkSv9%z7jstHKIw1gtQKq1 zX`gg6*3wHQbe|o~*e0UaeKJZUbPH=Teh|@2(;lk@} zhs-&~AIu}PqLpc^rjtj^Jf)_iKbq;Mrc(;b657}2H_tM;tW`tIGQ6ziKbobyuaAOd ztC@q3pUUPoROcN~M*4C|KLr}+vUv^F8Gs}sA$F6V2(xuIllfT7b(qOMg~yN%!E(3_ zA(9yIIUz23!Juw)&et74vKitg&(5Hldxe?(6eia*Klc$F60n~8hztsn&zn>Q+~;${ zgjn^;K|VcsEpVZuJ9*gkL)cp!nBYuJrSTGL^GLNYz0aVueQK)P^=$Ke5*E}PnomuJ zee{47&F6EM3()eZ<<8vni*;FK0gjem=UdP)hHz>tsO~wcqFUH2U%>kLOuKx6BSq`> ze12`Z;LTKvqJ z`c%H;g?T=msjZ9^Sh;zh|56;ye0^KDBwAirRJC-qmKq6y?2Q#HKi1lj&kJRueU;Bs zeGGyYlw!w}n4=1CUeTa}T>V*Eajlg2k`@auc~<89si~mIMklu9i+Njw4m8u9x7)tC8}RVUQY8`ReH2ys)U6 z=rNJ-;Tj(Ko5$@duRGt&Qpo$uxt3s&#k6#jMyiqXE$a@h2<$0~@Tg^#uV@sm6VxsC zQz*a5lBG3R+7eYSD({o@whp3O)mM;<^Ft2QW%wS{w;qryNXQ2Y8JX)16H(*|`NsYO z@}ZGSy%kF?DX8q(LV2yO?o?g+n`%0n%TWy~UDj(n5UhRem($#P>-k0owXWeyUBm-m%Lp~NL8`Uar;JCxT1+u*R=8F1MJkhOrAldS=?pUL50aqRCJJQ+D92X zP8Fv5L0j!bI;sTQtX=Drgjk&V+I=RHg{Ez0-ge;SKLvtlBeG_>sI}w!x`4tgF> zhqhTr3L$>zhPomrMWd{HEWmeWOMVu)gqiq0K75@ejM{?t>Mwu%SRu7K3Xk$w8=L0b?eb*rKykS!4fox<@8kc>=f{8 z3dBAQ{V>I#Jk4-?8g?7Xx;V{zd>VRtnk#mi^XfFug~yD?o&P<^+C;})JpT=M{d-;C z33m2>JT&?ewh0f9rI3&{0&(H*u0B6~efkErrKKf(M#a_DHQ31Lp9pJ6NXWl$sQebs z=rby=u74t|lq*;Mh-Wv$!|9-l9%Q8>E_#afk9gLG$NxrL|FNM$q0oIZI^}v?T1xlL z{-jv{$YxvrH&d>E53-v7JMKy{5~QzSvnrzLw5x!=hE32cpwlisBfP25@3KDOy!14M zEM|3*nIIU%jUos$_rOG&%nf<7z^oe0|5evFp;0fSti%fvq86(uG6TwZnvj~ZP`#tiT7_c!C!O1eJJz4m$5raH>(KOK6Pzt8(lGYL_9b{H#N>h3J;@dntGV6os-T zlliE!MGMIE*9d;VUfLP7#8f6VTq*Qe+h-6kQMZ!KcLs4;<|T1U5Bx2m;ndr$G*Rzp zaWX%+hV!5hph6DMHqP7*X8;jYp##iV#&Tx2Ogir3e+EDgNd?^8WbHA|VskHGy4luU zO3DljIRWnWkZ2(g0Jk7myq%#k;B)*6A+b`5Ko;b}wR*ppqMbYf+UahA4jC!{bFwgd zRw+|2&*dv702V6fIidtrpjhF{%z<(Y`_Dn#lrmyaZbNP3uKB4KbfkfBTTCq9ee z82$C5#M9Lns@+L5=+b;Bz8SiV`wL;Qdb8}T6jp1FN;OHtOJnRx^*C^6POQF@tk&Fh zo5n$bB1b-8P#GZ8SEXf#0paFmFayvn2V>x-Ka9Y@(&&2If}td{%%dy?G#mH6FB%R$ zBMC4xJ*yZ3K+A2lX3)wS(9V+y)sW zs8}4U>I~MH!+?&!;3m9pzCFLx)xSO=JL>jdLDoNYeT9u+`lPVV09dd=)fi48-Bz6s z`g>1h0+W8LTv{JbzM;<+@rG=6q8nQS(oW^(ji2jlCJ*{I zA8m^puc}S^T^NKQ^>gvb<#5%W`k{Wzjg$=*fEkgyKyYEsG=U^xx{Bm9guTh zI%!NBR&8=ZWh3nP4J*fQj8xFx3Ce*^pEd`Ur$!kdY%Oa3!95BswDHmOVfRjPSF6ze| zg$TgNVT1`POO07GXz8|_Q~#+ujdWow5A11^#}d#CCxJ-rhk2ApPp`9zRz{wak`^2C zPbh)L1AwVn^$RyO*aCoqdtpcTnKOxrs<>Kr6bf=tGTzk5O-`)F)E4~o>R&#KH+iRr z41c1)*=Qiz2s!kz!yEDd^SWF+sBAy?3<2J_nix+* z<|vy_vuRY;Z&&2M$@L@}7 zU?4E>!mJRPm4Z`IFf*|%fZ?M^uPKTD|Y%@%|g*>ySc>~og zHRxkE$p}_g+CBSDn^yyGxg1>1Ya(MU*5~pp^}EW((it`--_fs;oocjSZ++CfT{!+HO&ax-~ zR_E`x?kVxDP_%W!xRDO|zvf?c-dh^*=l}AFDC(_k9T^OZxKey6YUNRj=S1}Pl~2>$ zYug=>SMMrnmS$A*!WCV{V_%&IH~d^%S?s~cy$Pt2xY{4kO#daNg}gh#u#tK(>N-TF zqw3U!?ql`66Rjyb^^PApcR9|;%anXe57+U%z(bwIrC42xy)bKc_~5iskL>qr@nYT3 zHRpMWbTNxbrVm=eKd!oGUh9wwTT{HMzfe8#wQF!uH!>#bWn5p_i~h7g=DO&sa-fht zp=TGZr(P^ha(G#8Kl~70t^DdIM`#a(_5rIn+0GKtIViDmE`UAy&DZa{^AuZ8qRwO^ zE6{c?@1ZX8TG1QsJkYtapF#1?Iny$$d#+uD*9pHUG8F+m6EY?2`TA#;VRKy%IlZ3x zZW}D`o4#O#*(TUO`$aRrXOSgrOG z3DwbeLJK3XV!SJ9e?2#B6WhO2>H?zNin+tF5%f^x@Cov2!)h6WO<%^vOF-!N^^=@i zCvJ&<3okj?{B-Yr)8vgW;m>}2@I84YX}jdB-Tk_XxFc0ZFBb2GL@htd+*X%1OI(=D zdjj3g+*(WW!ufXI_?$Tz@xgli2C#eQYr^@kPXd=lxu?7LFNK`sy!F>l@ATCE*ROMT za!L<5SdJ`znmPpcmHxHQbpA6F7BuI%YlfXadwf0#%Vv8Vw&TGT5ldnS-fS(JGvoGQ~@xtU;6 zrb}H-xU^>p#=5yjaAiOuy?wYcJor{S6VrE)>r6=jHR!+0l6XMKH%Up~03_3B(lmgS z1QTNgjTg;`OU4G$L}IcJ*y&V6DvZ@LI3{zB&67Tf?U)>dHZUzuezc=5K=vycWjpo- zh}I-c?5H!*Fg0^9`tGmwb5iOKsr+aPX-6Hpli0bZ&ghm>y9Rx^|%HkADO+{4b+chzam1jaN3IUso`Z{ll`pDIkc}G<<6Ys=~_zb8v1N<_K=($ zI-hcGJtfW4Un4(_2gT~Jo@TF`J4?**drN7oIeBqE=P^>$zaqD$hU_b!Q?i#4 zYSw)I9l4YFV^jh$zk)eGhiD-kS%4LC4XntgGkw}T>3k)%$-E$DH1`^$5G$CA87_=* z%I!)nAVyH`CKrZ`QXZHW5vVtUAVnBC3g1rAXip-o89Xyq6yswU4=MIv%U*OYc9)~5 z_Y}28NWG~o4ynn;y)AY}`hJivaTyg8f|qXeBo^ftH}2&h=;raNB0u^jM9_z^Yl@(H zq$OtB=XFA+ZOOVWNjJWjC93oy5d^V8f;I>u=13tw!g**G&qi`gPk}^T@}uN3RzHGq zayhR;!t+CBU9k;38?UZf9e#(oLcR`vbF4h4rVzPMZb1#ova3)=@^Flo=?dd@FK25O zR4NNnjW;Ug<%&*@6W4^QP#bu;FPR__*M5aeC1fSZ2SPh#J9fwKe$h zxC7(z;HVlgVn}OS&E9TVpsG1- zoO|m0(I5f6dYZxwgUbf(8?iYo4YxO99l{I-{bGf14fp(F`I@r{U+NOq6Q1lFjE%<} zKJ{vRI380_*VteYhXa?795ikxWj%?iJ0~drm;8z62hnMswQm&?(9jSE z*ry|*?%C9o$I~@_>p(9$zM$z{U2M3-EpT-6vt7rl@0u#A1cKHNST24d=WG=J_{fO2ES;0jjbuSmDw6~@u0q(vQs;&`o5@nj3EonMUtTbz_ ziE#drN`?Bigd{7TX7t7Vw%}?20&nGY#Uc5cjiXyDyxV!d+p2#>_GI`68L9$@#ywuXH=;Cw9hTlg z5e(XmkJYC4re(Xb0JKiEg|^vs_d(vGMqk^0w8nrEcXJ1Evlq?;Z7)C5tzR^<3#kJs z-RmnH(7%R8zyXZOgnvnO5_FPa#LAQ-NPSu|JsCr4F;fFzttt@Y%RU!0oEdjE(W6G z22lkd$CkkI7_ck~OwV!)&^4wdfXN1cst!Rh00#i#2f(rxWUx}eApgrDCDCDZgJI3{ z!`d;!$f9AS5?IU)g5DWMvGEwDLXFr)On(la3K%gr7_o{OF=ykkj2SUex@V_!&)VwV z>4p&((SOSn_Rw(-=)d8df9$1`|G{#2dC!Q7+DS?}%E`IN$$9*16_<{4=sCjQILFX1 z@?VW)|ABLGwzkQRj{itYnU^o;dwc)!mgp?!Pl)hOBiSF8Ltk9@j}YN-R@R^R;6FnP zzq5mXRu=wW#5p7*!QbxE-!v!xx4U#QKcDGun3D-+7RgO(RG=39MmgCif)H~Dte$`t zp|WT;{?|NiqDGdGk|G}_4hz6-x;g`gYtXdNG#^Fa-9?p6 z==M^O9P?n21R4TRV9RmmVa&T$Ho zlb7HJnse2Sq@HjYa>L;prr#S^r#LeKTC~rtCB%~&=8gOAJE;L(USJef;{FwrMb!GE zuT9g0ZoJ7CEnZXy;&yR%3`>FrGnkoB83;Lj`#>#6i|6u_3!wMcZNML8)Nq&k&?Xe$ znoD<2Y|$nY`o&!4x%{k!)wQM3Cv-lY*#sDCh(47Ma-&>Zk5wVjAID5CaVzJ5C9G`3BnEKD`D~wr=zV&} zW8;7B;G727VA#k{h!2Oo5FsSV!3gn%13(wjlb`YHoCZ);gSJLGj}z@<*IR&grN_Om2Q2KH;kJmWiN6!bEsP?vz%8nVaT+aVU-Yu zP5Wn!{Esq)zwM>|Q4Z0F|IJ=fyU%efpU!W(-9G=jy;KSqLD`jT1|Bjf(U%sE5Zu5l z$}<>a4!u9rrtIqz%cO>Pqq3ujZ!`9eSHI2vZ7(_1+ZdiKvuOsgo?KfCxnmRj_xgg^ zFe8r}_W)FWnvQsW-CX?sDyHk9k>iZJsxd7B(h&Zdc=SvM3pZd>l${_D)4fdTx(&92 zRAR#4L98$CuR1+|Lz*A~6S_ks_*=a*YLzb#Z>lXn8qU-|S21Q|K0R<{i@7>5dRfg% z5WC*Ld*a77?HYUlWW=pMkaADwo&HXprN-dZZE+wLBjPDF06nykp^TkZ|ETT$=4qU` z@$xq|f$cAeqMzhh4Pk>rpgeQzpc|?|kEpFP^lJV|l0nC8g^`~y!02?Dz^rtP5wi0* z{n-!;L-#Re@+UYjmvHATU-PvmbFZ>)kbzmjbtcqk5{$vE$Ou{qSK>7(K4Uh(>Hbj8 zMDp2DGL334?-i|(=_VmfjyIA<$){(V0G{vKg%_IrOncm_q`8Esa3cWqb4o(V$3*%T z3g$)-8!p;2soih7;3l1L5yk~but+8*Atc!x+6-T1eo9ufneMO5Qm7@`i1I>588qaK zWy8o}S0-K9I560UwdOE&)yrTTDyL4tTRXLy*S}N_E3m(W-Y|VuJtO83{($o{%k|-c zQS|+{15XC0;OJ?T_9zqFX8Lnn;)tt!=FtoX(~LxMtIAm?1XlPLH#;XXD)%jIK~lL&IRl%y-iHLNLn{DkGxvTubY0}bJqS=~GuE-?e4iODf9WqW zi#&pQR)|myH$z(c666XpqsvkiBfu7K zf!wcR$mxRHma3+jR~WF4u(Is)vV%7rD-+$l$fhMX$b}5|1nN$l<;WR9q`4^(Eb1(X zFmB0O!N4xZ+J%EB_LqVgt{-EU^xhJJl#ND@EOiB*H4xOVU1K%x%tm-g|d z&IqFvcl(N;-Gr4BzPu&Wuu|U(7rMSPJa1ODyTv^!6A7=gaka&kWHH>bznV0~54P{s z)?~47Vtu^+?2+0_nJbu+dAWXx;l-~UilJNAFK$U%E5={F7S8I!J2~!nS*t>x`_@RIQlft8FC`U~^SKh+mo~&TwdR5Wcu-&CZW{Uq*?M{rpBlogz#AShJ z!nl*EuNEBgMr1BtaIr~te|t(WNGh-;L13#hWK?jdKAS1!8lCxFQVujI`INHH56Cm_ zyPddIkeivg~tjS6251XV#PGM8O zJ7)1r&!Sg_9y?ez!jx|xJ5qS1ugFwHAJngfNIH0Ob))oG&(P`%${k_N z4$nUIV%Xrcqvv054v`IMO`k&FupL(V&K>kNgXIze&Q|LcY6o_<{b2O>wJyB&r1^Wt zZZ6>A*LU%!m9ocOF0&M)bGQ)>?>qy__8FV?QCG$Cm_cbl=Cs7>f@YS zCeLR2taeY;$L^qzqwc1b9SoT#4NDBkT4?-GxC814A6s)@AyvE+bO8JIcm8@nh~myWoCmF5Fz~ZTyj+#Aii&oS#JBXHLPVE|=_gdmmjbdmR&Yv-n_wD(cCf_RHNW zy?rs`hWfnN;VX~q=kKR(sNO5S@kQ#h><8=nZ%U5Dxr?3wZ8zNgg^!H=)n#<;#cJGr zn$FTu><=^jciz}En$B7^_ODe#@+y`!!eHWPd@@=4t9N`7S=+WcULeBY*J^wspaqh{ z9q|FUmWSOq7T%~Py064!Rf6R)<069>Qm++Rx$CdC8tZozAh0{ct^DP!Q-DS|i( z7)m5CG)F5S={gNT-3PGs!E@Op5SGAnUgCuOo0m){-~bSvm$*}*3^s}0)I_hjgp&h_ z?loY{Tmr9Zk`oeOrzJYoaP!lW9AG?`<`NyKDzZySwy^67aztA}fR9Xcq6-^KM28xm zEV+btl8_u2;azsJRt>ihHaXD=AJqb;)eyD1L12VdksyBJCaFvi57>Zi`H+<5_~cbn zY-@bFVT8Qy6n!f1LOV%zE!j>gRT4?Mw?~pClI%TF#pg(4=oHye;=`I$aXCH_o3tlu z@rS*FY5H?1ublAaXSE*5#Vf>v-ua~Q6H^zQ@ayhc@7J)?bLq-XvO6{D9DB?Pn2Z;4 z@!}=zS*#s9<_g%H<_20 z5ScsoJnu#=b%u#YX+S%AEIMqS%0bOZuEmlb=VbX(2ae|C`(m|wbh726_2%=_lVkK+ z$ThxLX6`Jz{DP6AHuRM1AsZN!16h*eEGe62aAw5Q;sQ75! z{hq>B$Na0qMfu*D6Olz_NBN2(i>LFW)c1=HbTVE#$BpG9XXYawB_mh!BluJSQd>zv zctN04)|&@e+sVZ_-U<{*X-pDzCAl>10d<3!mMp-X8CklRAK9KvTTd!%?Jk9~M5x&6 z{@f3HqDx~Ij(q(^m(MS3aXJf5PZpZ7Ht!d{IatgM!IQN z&GW*%I_e-q+Tz03{HQ<|RKP;e^#j8sQV=sHJJF?D{W-gr!c8)LsOgJIN-svQER*C? zD{BU-!q=881lg7qX;W&!j-WSZO>63|&1Iq+U9JhK8Z_(Gae70Y$Ll(KG2_9s-h*nD zqq76B^=({ZMpcd5H!Ls;AuJjWr-ZMeu}TI7)%o5E!FrAJ z1%aebv=x)ahbwlA2Q>qFLC`B_-Z(`>w&pG`++tegabalsG#pK#>)i8dWecN7_H^;l6vwbR(O z6MFxsJX|N82X3^PbLHXuNN&n+G@`WwJ~sWkx}BczLSTM(F8#J$$h}Y-fNcrj!hmG3^m?s6)KhSY zWnb%NPghD`|BJpmKl%nN`}*|zlo5Z!5iHb<+&@v+Kfc-jaH9W_;_YeI+cVL(=R|HV z=-+-^c>CFp{@0tgpZE3uyXrDbXCvD{5-1Tw_bOOf>C2RV*oYbc9A{&bmX=mjRMgbe zqC%gRp{r`iL(6tvl z9hJDa{8QfkC*MU246FzW`jhW!`5T!0BftEqaQ`RY^>!Zc}>mYpPHqA zp%VHU<>ch#)2B~gzI^#)e;^tL5SdaR4@RFxV_H%mL-%1p2urrL1pPqHgfH&aYQUEp`F zt4*f`D8iWlVmcaPAh5-0OM2ashN>WFV>;moAgaL+4ANWMRXq_fR9&(rtGcH>uv13C zf>gaQfLmF~y%a$WVLfgDbllL)-KSm-ZT`Tp1jCjE0!*|2KkDA{Ey}-d*FGnJA*CB6 zrMnSy=d=%E{FX%J~qK#)*TB&3xPBm_Y~K@p$va$VQ`yYKaE+p}Ii z>&^KeoG*^g_c->Q5x`QqWWK5xr$Ez^!7&izi)*@2C&0$oI1XsUV>BdT5M&{9rEbZ+ zTD4c|5N#4&N+(?ReS?%$FUgF0j$D0$mF_#+C_wd+700eBwEccG!gykLh16gGq(Rg! z>zOe1sf-m+k~Ud1qN3r|?Y)PAKOE(^8_SR?UXy zR)@Gv!A^(@$O{FY<7|8QO8fhY+mM?T-xyHlv~uDqjB+2ltxM}!m4O|;)WEu5p{Hg8fBVvYO(P$A`Cv@_m*g;rPsD0v_Vcy;PouQ3IKr;>fG&9~r$u^NwC zSvYf}6}rO{Gl#mR&4f3uqYs%%7^uoHxGZ@~QFowQK_%1tq8D2nLbD#Rp;6S`Sg=*G~@B zC{;OP)v3*`ZiT#*d)f%5un+q$-qJ(|87movI6&f~({O+_QDesfe%Tea@UDyqP9Q-E z3R@<46ZL%+lSe*u6J+nNmWosp##db!qR~V!z#rL}s&^6ov3Pnv6@1hEpc`fv=KEt_ zE+q+`TVd39zSH^v=L=lDSWz5m8S?#6OB@+~{dvzG#W^{t$$7-{LDHLN{TZwDE&yEocyn`(Q+$A2lQkeJmBine zhl*XFZ%j_ph4qK_gB#CQ2E@GYv41BFpxA0v1tl@5BW$5aR&77ZosC6ulKBo64md!( zYzolx!$A5Cl$655cP<%vLAhyYcWwuuZIlnd;gon@CHg|k0nS)1Y7l~RRT8GwYyf4Q zA%UMDUDVXgoe~VR0iCA-7{Nu~+v=D#BFk<(?|c%2)#l{vM&y&qawZdG$Hq4^mS3Z< zh?S}vrYTof;Y^}P-Mx-%{2}w~tmi@WT!Z>$k~!vbrv~M�^U3mwm8($B}4kO~^2o z;_yhE2HXli|9qLRphRMnYOL=%qQo@Ro0^?-9EuJI7UA^Oulecy;$SwJ) zRp)(I!d~-$UHjuEKf|~#E9+|HGuAD^QO#?l9*;L^>lh$ z#wNRUh|a~NrIC`@Vir09NB~T1T&i&iKpF)Z){k?zi;QYq6=@=M)?T4Y3ZpUn(U4?8 zwxfN;PycEH((@ER`t%HWt`nLG-9ZPu?ydq8e5}+`oD=J zsk*N*HK=|s+hWJ3!`qU0&FImzYGB0>*g@5U|6KtS@bM(gOQyu*DN|Q}T2o#2lT9u< zW`);*`wj)OpCxZ=J)RtqY|AJb6Ez^6cw6e*);OE*HdvFLKhl|3D`HO&8ffQF($e(g zwCUjBXns}GyVKz~JwPJfo;)|ShbYA7V|b3~HoIJxrB5QVIikhu{h&$M(?Yc0LUyv_k2X4vU#BfQ2P~@z6PBP0;nvtw zA=%3#!SKP;rXtg}^Hq$^H)c8qG4xOyu*gfvTD^@TfkIgtL7MA0u z$$O#t`r$lR(x$N9!RRe^Mo~|uSK>cO^ZKY83mW3|L|d4gx|IfqKfDsNiLZGT?0y`7 zH*Q_&yH4)4uR;B;0wgL|3I{c0Po8RzduWIi4(^Avmv(&a0KKxAiu~$YA>kuvRGL5i z;Hztu$}f*!Q`s}sHPRHS8|LQygz5J~Pje5gW1%k>-42ycn;sFqH~6BciqJo9xohAl zcZnxWukIf{^vQnCs1vx*G+tqvM7saP)r8%lNmjEhAGm*~NK9R6HN1bHnX7HM^CMxH z@9mNU#HY@8`_Uo3B5Yp*E8oqX%IXn*Dc_3p;z3{k*44qir$Rz(FSrx*ZSXYn28&p+ z;7+n$=hcIx6M=yx+J5aH%Na*IO>`gF=FV(c*uKWJ$gJ|dft^x19cHVc?*XCbona(2 z9xwWs#vdgCewSZMo)vwf*J00(bx7~=3=`4gQItZ-`BZ;j?w}6)A%&WAt-H=0&f2B; zdSm=flV4Q(vu{5n0}}!tS}B#9O8=bSU<&H!S`QId=~|*WQY?A%YyTbD$qK_!Z~2$Y zTU15KCR4v1uoDW!huHaL^=qQvCVsIVC8pODa~g%YgeHH@h;g2l@WYOZlpdFS@SasM z!j6)2WS23pj+ymqS6Ag7qlRR6Nc+c<-;{o*<(%6zz5KUYU>>>%2zl;!8}K zdGzTTGT@7Y*DblHTvn9dHj|wQ+!&s|qb!TuYbZOPy7?5Qj~qMf7lu@k^mARMG(S#u z0-=s5*E(L19mXW7gTU61DsH3z2lzS%l9~go%z-3aMS`sX z>e^^6c&$4)U^6}%ikJq`|H18cvINDhoHDn{>V~F z$gPd|{5;NL0&I2|==(`R5bSR3nhXXT&NT~4$;V>m5;DRQ@<-W9XR#O$tYc+jOrA~& ze`16dJ3D1kF6^%IAV*Gk#8nQ!X(KUg)TM<#DXa_qC_D+nM}O5n`4OK>pY}a_M%mWw z&=5qDA@<5M?c_Ur6wznN#eCRtY;uGLcCjO&OzXy5*u5%#*~=mRP-RZYa&3ysT+%Ee zHQx$EEH1mU9n#5|HnknvTbs6C8}f8Jt(YO@>si_ztCR)pyXo4pd;B5z=b`*A4L2H` zVL}>_Lb~AtG-E67$9Bj~SVs0}JT38E2Fe76#PE3-j!Y+bsXHTPPQ)FJgEL$WzMZKw zXC`o-q4XUi!ju8_RFg;y&$rIPcz|}WSuuI?A(fe;4CzXD!-P%JeszcWBjn%(fn{e| zqC0_wCP{_@S?XaPy64#f-(g{ovN`9n&3eLIICC8DhMmgi2IXbr@k_Vvg3v=bv<$gE zb-7Gd*#SB^a(Nk;jod3%qIU)IuJC1hdgh6GWP~VOhJ`QWp$YP!KEJ+wp_kEwKFrYOXDa7Vs~1&gNApJg3jecvNs&qgih)-^k>bGp_Kk zz_*~Mu$IYBNwuhF!S~3yP)#7D^sPe2xsM5P@q>c=D|`jrb;YD|MZ*h4A3}@zc8b9X zjH`?#?{@t7Jr$qZ_&C^y3b2|DxdCFMzMQc&~Yu z>N4E_)>HCyhrL(oewQWc^TK`IPsE;xGWCQ~k{_Ap1*MngK8oV1FSyH^LrG_Emsi!G z37E^i-}NfbD0x;{UX}}7w<*86dncJIm%Ff{_R{kHT1XC6ueah@?yg!z<_I-cQiaq# zZC{(ocF|ABD~;O(>N{0y zoKyvO(i7WOncqQ#Kht&Tb$6|=@!0k7+^tbgy5l4GfQ{insBS<=;RDj|)$kYEUP*4h z9#uyQ`j5F(KCgH>-~0J^$t}^lInwYJ@f&=>xFemy1t)`OWW(~7U`MqiuNP^LHEi3w>!6 zNr!urKG1U$`&Y;HdW(a|hzZncbm|G*=}wcf8ehwM>xv6L5mYcEY1F0X265Cc+BWvr zKuODirhCo3BzIbo6kEH^+evqHCzE<6n@@f;C%Hpu$~hnPwv1ps)txW^2ki^4R(65r zv*%ECN{DYjE8QXJb#KeNLVaC0^biTkK@+nPLg*S=8J5IM&DyR-w5f?hwQ+3%LQqj5 z{ITMZ${|F0?@wbxX#gUNzed0zB4uC!V=xO2L^TQE4RYFg0O2G+RR$7R1sm%@jIVc? z=yh1)>n%kcwvRjP_rQFVU?n(>zA=-FUFU7DPFJr^L{X>oV5f&&r!Q$2qM`H7W2kRL zSBOyOKW`5c_}jzEf4e>Wo!Q`T57+PN#Sn~ zs;a7Z%Jbi`O^Aj@xTa>*zY7}w11W>QI{fG4knZG^^KYum|4w;+ghKu0$y8QW{x`&P zRu&%a{AX$NuY1F4b2Gl3fv3uR{lj|A@6i0ZfOm_+%opi{|W+zz&t^;6%cS$;j2>^OMTL7l2P27;EJwR?M4p5N; zgE(0CB>zSxx-lEp3V(?A#U+eAR1Y4&`{IDUHCE~2#Y}RO3<^mF*U5?MIAIt^JLiC& zsy#t6ns#Wk5p61C`YDVpyaUEA)DTYL-Xr zV!{4jskP^z85_z^VoZ1|c?03#tqu;P87h1|_O4oO7nDlWeu`YwEJ%zOfk|aXMZll%RKTt=d%8jY?;de8M z+LD)OI_vmJnJS<$qUd;r1@~8I@!@d3b96$HHc74AMxu zG5jhDw9}2rj)w*0QiAAB+3z70vFbrmvpg_r_0~W$L$$jE$Pmn-=Pj8V4?TCdz;#jv zGUp0C%Ro-#^g0Kt{}xIO1eEES9D5t5JEF{o?7%-yj?+0JN${14zSjdRmJHoaAk{qj zr6g|LAK-Ja=$e~cIK8Ag5iv#%NziaPsw&KF+M6tr>B_>C%Pu$w<{=vM{QQ5$dL|RQ zauu_B_}^L2!VjNa?og`YYe@ffdnlG9<3msmf(eGb2Ou<9e^nCkWjsbk`2xUDX+cHW zCt1ILk|2@hX*!wGm)k#?HrCodA!CKpC4$#r&USB)*+u@ap2=lqAK=$Rq~w7mEpski z``@Oe=D`4gsCoUjC*J){VM|i>l%)hW-#7Wak6>VzAzkw+t6oJXAluh)*Ok4&$x`pn zHZnX8z&yh8i1WoXkJ57`BJuOP{g~utIR*OgXvOXPhD+Jt7SiJ0&(70NYlv`Ua20Wq z$SITLUIF8u99hXRNr_QC&TGb z(E|mJ*^FRkYf}5FfcjvAF6UcVX5P=S1ooIIseEQI#cHpf30wxF4WztdKoSxB-3p6=L-#26#Z7}YQo^oPFGDUNO!XbYs1CuYy<>hD}iRc5Ghh!`b6aAW)aSBv#% zWiax`x2|XO>&>>Ix2FLIKzYZa;@(^X8F-FpaO2!K#M+aqD4Zzu12I>4K4WQw`B*=n zyDuS=E$3h#V-HE6S_%Po;++&)C3XF2Yf*?rsxKQ5nX=zk^6bT%9C+pEAjz5NxT4Vx5l#Ef>_bJIKdN9=uqfS~HU zJJ86faKDW@WF z^~3T;H5&1lV=izGVsdc8=2@OL3~c$4)-de`ZQ%P}D*@2}kyB?PK!$DDWnID0K+_j! zIr-SHSTCJLXzAJs{b!V8%Ly*qD| zsIH3UO!yv^`>(Nn>aRz?_VIM~J&Mex56pj~?OC1yx|L1O1&m^9v`Og-5tSHY-UJ_M zDb;y9oi3ZEpB*RF)7PC1wA4`o-`drA7+p<9)ldiGrw=+5JuKwZP+_5`Rdu#bHg9Y@ zMLRF?%^*J_biJ$jV9%fb^f`v$9>BnRz}TSJy*VLc=(H8GP@4H_Gzs{QeVMk6hceqwMnf z-VZW-ZG5G8HO%~Ykm_}NlFOTKM5cx|+pfQ&Avn&9!44h@_oz+o@N>muN|cDaAS(zp2g(kGghvoW=V zfU1bp4Z|xdqh6wc_aj7JX?w_5K8kJ^@c8xVxLa(D?fYWORfBQI-#xFV4<6F2PJd(q zkdq=5Al8SVE-HYI){;4)9g@ipi~%-%6;K;Y}| zS1wcO!(+}Mayr21%|k7o)L*|f2fn;ck36*7__aM3W?_x|{p0F|&98*ufd|ru9)8cY zkmr1UssSL_cFgZiJOCAWgO5^-BbLc3O3w^=kQ>F6k35i7qVM+q@R5~MJ38fUECs?( z@U1N~!uKEn#l#l_T+$-Hb%FG!QSV3iwusQA52EF^W7of-l*0Wb&(H$3zJLo_v)e!N zS&Uw||BW-uei&egM!&PfnAZB~SjWK;J_ciPTKv8Ygz-dt7#oBi$%(XOt+$Z;&;1*FB*rB!0IuA(AnE>Ro&+ zEb+A_7R&D|#3*yO){6liSICcj`Y!6ecH#&<4bK_&8-0}Nqr|o?5cp80ar+KYcTyR@ z*CP*5t95d7CnGy9xoX=>U@Q5ndx8^9a?KXD{VcijT4Lq4@1|8sFXB!Cf6DGi>|||9 z8zE|j-}iEYL3X6pQ*I6Xo-ws_E6L?JHCHxy4&nPMKV{23@qpi#F+A;Kej4$l>~=ol z5fOF+n_5AbepKsYGM~O~32H;8pUxxH)@09N2#dGLu$}ZmcWOGujQ61+BF~J)n$#2i zj7-8${tt|O4`u|+I0kO{5xW;;wW@XQX6k5fs^@pn&(ugtd4%v9dV zc8Je*)XBBgyorP4>VG3Q49^+a0$pj(MG(7%B~@7xwd<%bBkq68EZ6>cYt=Oc;T)?fvx1-H`^3v%wd>W>$6j@+)NDG=IXX;UlY zdY5LFmDzl)u(F^)XoS5RUC8v!-~v`ye=WD$^G=&WQR|MgML<%^LV?r>2n^4U@x0T8 zEe_xDyvCXKLam6@(kODMu&{>Xh`R(dqW`7^zty*POw@*_U0sQjr|W7ZKo@=hn1{9$}%({UGNILJB~{h@&E^D zC_a!Vbk^licHaPb*H`-LTH@VKa-OnI^$$|E9dyN^Vn`-=3y^Vrt-+&87ip-hIK>Zu zgu$u0T&5f?D5fD^Uu9cgO#+1*Lu-@jZ%)?VIIO?X3TdnVgUe9hfJg!mJT8OZ3CxTH zQ6K&Z?ZdhNDF9-00t>c+UoAF@mo-c&Bf;J!sd^=&Hqg8bNJN$^1o6$EiLW;T2lakJR#wA zQqsS0jK8G;eP(8SWXBIGZ20)_k=;X9*8BSP|Av57SO3e$_*)c+jEw9DfFl6-4FIu5 zMi>i=^jo+7v#G!r1^yy3$^!%c@-b3UQvUKWI{syO?@dbji-7%G5%^Kf$|5R6_wiLC%Zw;3Fp!{43K9|_2C1Q>0b1Mkd;mGb!AvXPy-MI@_c(!) zo1K|jkCvN0hd+e`iCpT#(2;q;6bS1csUN70Bv4TkCgG|Lq@*Tb z<0WD16-JM%!=!tm;tQrP8h{Ksg!c9!YN!7ByFon5Ji@q4 zIdWQj-f{N6{Yc~GjAy4Is-shQQERa6=0O4z|MGVyP8BxT!`Vt zlw^fgo5>vpFHB=lyNnNKkd6zoA9R* zI()4PC$L@dUoD|rCx46LrQqapun|A6TH<^{_0`B9jSXVsqTzS?5dX6EIg|Zwr>gg6 zNsOBKy$GW-##5*svFHS%4YhVTm)I(G;1Y4+PEcxdnX#0no_Z;0$jJ@(8cbZu+WExc zc(IlV=Hh^e(_89H!p7_>y>xp+TD?FtBj>su^=s=n{Wxg3DP<8;AT-?n9S6IfHObd0 zK~1uUN%|+ORLCGn1ua-U>VnoX4@>_w9hlVSwxD*U&IG{Ow4PYPC&cY40LWWm<~q|b zWFSs+*8R$jABWtqB5=1jnymGm<{(t4DhmL(uGM={rz)Ag;$foWf;fhdO zE?jqpB$vnEnR=>;9PQbS4h)o zo3J=>=wp|>ec3692)6H$K%e3=e7cuO&rMb)RCu$Ie-u$kc7b-2$Nm*KZ zy;T+Os}o{=u|UBPE9!%ky%1V#2`EucrH_*Xbez1}t9T|G0O(-8a+NRT$344Ohx1;a zyhZxq^w~pBC$QA;^6#4OX7Xhe@ zrcfXN{1Q(gyCSbKn4@lgySZxaBQpS+E-nKm(zdz2A za&WKG^A1Ov0@w-1I#a#$9%K-dk>PoxomZmTrjvm3VY23h%PQE3(m&oTaNW90L2GQ{ zp#WTMFS|mX!J{Vpo=LzuHbr^iREOY|?|pBtPqY36lM#dm=_SJ3JzP4mgaX<3!A*^&ZIH7XSSQy-6E+V=!RX+R@=_pAs#cbprTVp z`Oen+2JKo6+ID)L1U7ost${BS-?7gc13ErJ2@Z_Z>Mugn#Ap+=zUWHSgzgnZmNYmL zal9B<)QU?f$aUVUd=^$4^ze>}vl^$+d;{}DP4;skv%LbFQnnDW+0Viz+VZoHl5&}N z%mE6cJ)4)*`&Et4^@;S>hvsW?YBVL9d@|muWE^~xWaIhl$K5dHd=%WRTEL;2k(^6x zi0+xN*s(3Q%OR#%4V%erof>AR#K_d4Z+Z4ep{O&VWlUS1zBf{J`GCoseLfrJp-uft^!u8n3$8nmKK~ zd;Y;xA?rW-cJGsOG#AKTbF>$co$UQFyFS^J_&l`8@v?vMML#`D*_BkMbFTB4_^s35 zFNA&Wmn$s%)*CQLHISKiVIe>_hhqDG+27t!P}>@Lgqv8A;(qYhg$r5ME(VN+#QjQM(9{zIMBIMjN_hBl@k4*ci~TjeGbA~CbnKPw z)iv_NvmsKBufg20AFmpHr-T^?2E73`$s9qirf-2D?aTATP6HFSQZazK)2ECy5<+A% zq97l+9p(@V7$Z-ltvEpXMm-Px7(;@*G$9spvPG7-Q^%LX1h>LY38CWXR;@xX34g@& z6SC`%KAi8~$-7r|{UNHG<@(dm-yMX6hf-Y+E)~?G9DimP(1rJWD9HFQ_H#MF;>_`O z`%!Z!bn@b0v%}EgQ0;PhlU&R_*W2m1`9PBtesw1kn)=l&S80c?X>t1Y<%n_@a!(<~ zZ74GO)blc-!C!2P4hl!qUTq_AmH4Usf%2>+?4i58*~N9O>VpZRW5W!ipS)k95Q85c zZq~IRemmVfzxM6od&BRIn3U3=p5K(;w%y!#&~o*H%kag|@hcH`UfrCq>+jE}X^0VX z?Eh`m8MTq;{+&E!abvWZU_o zEB4)J82dK_skJ!>Rf=vNk(_f(Fhi2U+ZMn z9jEp!+*RJy78`$aE6$20{#Yj7tNV7z064%pZi|ju5|`k{=tPeMtho~oLSmh26ISBF z(OB#PFU_YgYz)@%wwx>m;h+yoJROaU4!<3yotOffsXNhBUShu018G`Hh-oOfP^y6(8$FSt~Gdm_Ys=wC7-&&SwG32J=GY+{Ep8CLoI%+=c zgS>q;e>(dJYI8fSgf8`5KK-#p>aT5Q;zQE#%CrX>e6hE26&h%er!y@)lAtafHyc!+ zkE8CfEQG+N^9uT)M${X7Y2!^sy{U zPwS5X*Gs=FY5EKi{Q+C zdR|B%BTTnbgzK_fnasl*njZ2(A5Iq6-?f$T=jjnB8SXKR$N>BVN*)v2XmMN_wUL># z$$3Brt#K-Nzf+vd3!Qe-*;7E`WT4ZOx<{T!pE#(L@qH%a((mJ?jv>&mRKQCY(B=7k zV9`ul94gfcCO;*np9B*nl@SZdsQZ^Q7M3&pfUqu>OK(7KOhT@6R9r9nlW*lIA>xN% zK^*v+J|&pZSds`1NY_hJBSCoKuy8$CX|WRTcZW0MtxQ$wFDiLfK??p*frnLk^;P=2 z3`Tf3MtPO#4?GUG+W1bjg>ALfy=wa(RsYPiYxqpt^Iw@3--7?=OY}cm@Oavz4g&)o z67a`w!oy=HDERjo8qa+Ek8C(ZP3^9Z&R^N^|0uuXGi|)Nd5XRLe=_Y~NWh=+dq~LN zOq-mXjL)cUVo8ilZY~kMI$E1K4UY%NN~(_+ zA_8EAiP}tH5RGbuFbTLu3Zo-L0|rDJMn=cRuKZ|o^ zM3G6FGOy3nbyP>k@wc*>FO-!7(}ce&oom~{YVLfbj?cetnanYnwjJ;w*Qm&cgj1C; zRq>~L^A=YM#UqS8)J&Cct~_Pze!sRIVP`1K5sN40zkf~VS@MuGIp;ND&aZ}5&SlJn z8N?)`VAS{?%yepn(|3muAYErtn}=dqo!6+!Zk;h&qu5-nIkDsw`BCAC7;S&z8f`i& zvKM^%g28Ou1E|uTcg)@xr|KM&Tf}f}CxN)&xx`P^`3~x^lgoCjGr1ycEpj8IYSHLx zB5EZ^2Ehl@6D0;u>wcSnn|acaan%Hk zq5bw5ORxQodH06>&gJGQHKMB+p@V;9+V#-uHWXL0fuX*WcV1tf;DL{RPsOl*Ou^7) zB$PA1OqERj^D2Z=gK_O}=*nXurar#QS58O&whpIAR57!X41l&_<#oePO+O#u{U+WI z$znmc0TP}H`^FcdpBf*LF`H+qw8*j~KYFbm(WlBwJHvYNP>o?#g%8q2?ch4V3;Gmn`ELX+Ow1$q;#br-z)mn$7smq9(TH{ z=nsynei1Za!Mp>eYQNuiZ_mT5I(WOG0rhjXrE*8e?}^|0uut?anHzVXKYFe3RjRM~ zwB?mt%kUQ=zUBaF7TcKP^Bitse2)vGts-Fd8_u;Grk+Dy$8L?gdVa zgXQ`dAz9BenIY^O#CYKzzD>1ounoQ4RRFE9uvTQaFE}Ny~_SjYYZI zuwGRd&~DZTZdJGvh+Hm6T)AUvEowi?r!uaS#*0*~gys@(o|c){w`DU2RSB69zDzL) z%q`g1MY97sVbT z9@O2)3b(+;A!~;LgmJ>~mvnNv+Uqhkz%z{*!L5iAAJM`$i7vuCS3e?*Us+`fCe5&I zJ>Jx@aJd16`BhWIxd}PHm@?hXsF69XeXg4{o#2!X4%`vY)gv}2r?7Y+Ra;|{z9{(| z6I@rh;aYY;d^xLrK@nwPG4jQ*VAl1bIqgB$5V^lQE79*p+7g}KboP;`_z^9Qzn;}zKH?$4) zamQ_LSE9_fvjru*>!CyR?v2BjRH?)Vu6*CErc3FIG>G{e`9`nKJ?*?*!M4vr&}uTx zp4Ql@Z0~g|qPuneTPI6{kADn5I)*3{BddA2)4e?+PaeKm1%V%n4?#4{7EXdGCCDf&@T*}K`FXF=yX39s$9J=et z1_Q++X{CBYWkA1sRuEp3wW z>?KIWrN;1Y_Sq|;z~l9KAKmIUvyuzj-L%?z&YYzLW=?|@qiU+HiRG?WNKt%?>WBwJ;w6w18htej@Mu1SHJ#l4s> zAspc6x#D`FIB#f>Ds*DRPuq}k|I;@zg4r7f$#rbr%#Y)iT78MTJw-ADrmgiw#lx@) z97v{GyJz%pEh~FFG+`~_n*=r}4 zjesC<6nV2tlDSqsHRc&OH9@|EJ@1gPkR}jn1%J;wD#jr8&VNZ4y-Ah#x!>70U@0ka zRp6#*r1-nQX<5`K3BS)np6u(1kSAL;8el-yKbSZILqNQL?!LAV4E1vYn6iGjIToLm z(a;Or()!^p!5#|z?MmP_DkeUCxt!spzn5AK2fsfEdD8$1qsTJ&v)1%2JA?&;YZhmzZwpM>VP9mBc=pMK4=xHqD3KkV5( z=zi5r(M=Vxg_pOV9=okX?$Q_?&E=K0RD7}CTKRE;QM_zx@Q89@XpWc<3jvcH&L7M& zBNvz6cL+P*JMt5oi&?!muW{@85_iSAr^Wl+`Ipi+xuS?omaFwRxtpi6y~khf$JEiw z11|^eTSUD1{Y$k;pYWjnvh~NxuU~T(gL2WCr$HCrZPbX6_X&WDP-LD=5Cy-@y;;eV zE^}Y4*s}z)wg<>Ow_rv_q%mETs9CJ4TNHPGtat|)gp=Z#H{%;c@pqfnU$#X}^9AV< zp*axdG~p-(e)Hi`bY>1Po`6oRj!~ROi9aW1)50WHqtwwDAq^1!CCYMS~`wN0}{t-5q(?R`dWCLYq#m=Eonr!$tF+2Erc-{BEgT}^!KND zBptQ%SuBz+#z-oGd^6S~JmKtw#x5Tl5pE*Lh((24(eWlaSzGFM#l+QOKTB&`-Ns(H zf~>zM7Ka;RYZC&WV;vw#UuhE?tdo|_6I#x&pqxaHp`=g2L0z!qw>%dgJeUP5B`^cStcCeqz8OpI8z!y|OL>pgtH@7X zw6@T3ii?J%J{yK-52f|-B=5h`lcUXzZl5Q`fQ*MCWyLA_vhQ{vvF87^BI1S!j47>=Uaq?#g9CeSh? z@dtQ%GIYYxf(1dWIZzl!jFe6m10ADYeip~LmXt}xjU5~GX+&*|%kLtq?IY(%*$;Tad`Ea(ebprsXzBF14nm31hgB4v2CyCgmt zCNJPp6NNQe3-D%>Vqz%XHw-_7++2m+7=Yki0e_Mq5+Z@;%Hdx=#lehtqFgHg#TET| z_C$gO4~qm3!2=4#{Q||qcZ>09_62bA_aHu`^5l>aCJ@bS>r_OB8E-%`$Wa>6s@O8x!I z{roEY{qYGAzm3L=0q`O5FHEitiTvAD{*U_-9})*MGX5zBSZQkdn-Kr=>Nztr^MBDc z`akcT|86OxG?@QHL$N;<TQ6Bs!NjbYiS^h&*2{uK9cFO48b+3Z z2q1}9qfMh#Z@|;z;^jbK771~B5KC6RjM!rdv?{SzNiz${J& z$R?72M8)yR5Oqomeu9aI${$wPBAxfB1B6hy(f$89?jcuV-@*81Jip(Bp&yNV(xo_*YJ0%2=4t7!Gm;4s{-; z67E_K9D1&d*IE91A^qsJ49t!gEDDpj_UhA%zT>yrd_F^Xdjc4et^UBPeac*jTum9K z&e-wT%xXX6wGAe35elk2BMzE2c-FSns|)gS22$a^fmDJvf0y|w52TAmE=QRrGJURvhe zs!12>{cxjTuJ>c@19ph@qx+@zCI%Wy9?xVg`M&uQS?K`ToKRHn>ust0HU72z%X`C| zNr>b14PqYSsHH)v*HjLdOO;D~b21OvqsuqUZm9vmC4S1E%6fm_*nIrL&-aIpM{UQi z?_n`rVA$&Yvx|hzQx!wx`m+yjKYvw#hU&#$Zy2a#)fR>AdIMn=ICE zE40U2=!UDu@lc@+sw^m^Dr)$BVR15JNnaWfxvbZm^z4gY&7^&j0kMjw@SUu^mT$W$ zon(@#csx$O(oI8a%7F|{VPXv;e6I8~0IQ>>AA6Cn3Fi_~ljZrSs@|=xLI2g8I6RBH zq^{TOZq{U*XALo8*M*jh*Ws zRlN=g&@b6D4kDa=9Oz%~-&A06+AH7mgATXLBMkDQd^fc2U3G7zVWL)rZuGryMF#J( zorsrIk9#gM)Z+8$*B=XerxxvUk3Bs7I!$L5U$r(jF^O^OSry&Q?#!%SFn40KV{xrY zE$#q0MJ(AquV3xZCu!CUD6(TAE0b#9XZEaQ>6>oObq_tK@=zIHZq9l1c<@-*_c6Wg z?xIicRyC7LefY{}Nvk$*wl5!LiyT<6R!>9K;3|7wW*3WPjbB#wbUt`VzoPE;Qz37C zd&#tV`6RSsP}55|+yt;8m=kFt(o`|xwC{r2F%r-XY$SNRbf#VI97cJH8)yNXbT@?| zuLC7uJOHxc>#;M%7)=60>stszS>X*|!4T2`heP_ZShd_*7t*qIJluL5ufxppRE!lk z2B2K|U&-LDD1k~m;5R>z5I%_(&_cd4a|a5G?2$4J-4kw>64a~9uAGxL&t$YN>u76?+q!_b!N%r%} zLipgrf}bzkHaa04p|UiHNt*dMU;fRU`YQxt}r_~>JzDZKpt*m7X# z`>!b%1Jrh2OsmcV;^+Vlkh-cJ*A(&f#=y zq;D^%Nnt*&kTy0ws)pc=&E#5ve@RnRiQ3JddNRAAbsJJUhFSvCd%hu&zL%;C3TCk@ zYXupNLTuHA?-!Py7kE8=bGmouYM;V$_VcGhk2aF>`sAk%&I%L{5Xp1y%R*u^Vkn2> zMY2vpL%+L06@{mWFLa+BNqnl}ZWMBMD(^NCpT9q6aD2A^;v;fv~s8jAo3;RZw zK<%|l>pqU&o@;W+J4!4W()^hckTha~Qb14xT2e zw5?S$8fUnzHCi5Et!;!lk-ep@)sqnKtgXcnF6W8WdMLkLKR0wN-zbPPxe!q5#vNC`Tmq@;in4k0BXAqa|qgouQ+ z2|nY&bspz&#eHx0%jbHx{{{QD)^DxvXUPJzdCHAdnz!6iyF0X&npEDq7+lnhZ|F!x z$O^aH#P{UTGojPIh3ltLmO}-5)7}fED&)|(Y@`kiYdBhfRut2pT%yA#rY{a_SW2ZF z=%*ucpud+>LlZ8(qK51wrfJTJoeXDecBmNuEJW_u!$hoe6P7$F<5wA!iUo^Nyr_+$ z=O4}}*VCT@0X2P@n;UutSagSDW{FVdi!$gF2W|%H44(N+<5;L7K*eYwX3d4THpCxFVehQEO~*Uc|PuWw+r&ZN%I1#q5pJX#{>s< z_umeT&^Y}&ol}Lw|I-#5Jw4%0lu$ao#LG*F=3HcDy$QDLuW0T+=GZhfBaDpxNADGD zVez-|%5-u%&*ds^-6B}7*x1;=i>Ci0i;8d+I&uF#p<;RtqFQTfJFl3YgQ)(C+4^5p zG5tH2``dx(oTqc@f04U|wF&+!8$>D@tCIa^hfO_3lO=_a&0!O&s$+ywV0qPA%wR&s zgk&HAXcJWFQenb1ef_agATW!l2t9~7GgnghiD-;8z#=KiGNh@aMc{Ue3Ne7*4-BvY z%%XHa0|84VBmx6Hh(4eNb;e1j30t{MU>+e#0g`dj1De#~C{i4?6$EfLg9Ac|b_<$r z!@@CD(RF{Y-6G8NL_`E^cMbvyhf~m?K;_2T%mmK89I%3ef{1U{2vrRY7by@V$At3) zS#g383=zwXAIg&=75Ba^=iW;&UdMd04mAi*oumN(wa26Cc%~UUz@Vj|Z+Gc3>30CP zGNG*{n$H-0C9xpY4kR6LT<8JA6joE}W8ana$&~PWunXrq$k|m(xrwO+#GkI&^QUU zsR(&~Zs;lTy1#(X!_P}q>_2q^5mBkk!q7TFCvWYe%EU|hrSbQNq1lf>%XZw&6wSHh zZ4|_J5d~c{54atY9xvKbOl}k-!{V$HudxSJnQ@0wG+SSf|0SpI@a;p;Jp?VxivBuP zW!AJw#7n^_;30WjI#AEx&+buJq%dq{B5sp^W&O(8T?F@M#9VQVR5CLF=K6tJu^|Re zP}?SKqZz1xF$Gw_)LoYMPCC#`=8HgO0gr)BwQx~#97fnqiA(p&nAAv>8D*%z3NbD{ z-YX^w4wkZb5yM(HJ+2aCQ^{&}G*|YPQV554u9W?#(<|+@Szy2eL6&0R4;@0CTo@}a z=u@%sApk&DoRt?|@l-3Z+<8=sEE`3sKR{f;X;xD8e@^FuwszW=dT_pdp&ubzEGl=Nl*ggV?zi%fP7+fRkk5o;PaJ`)J^B}Lw-+aF{P zCd{$1KE&V!XsZx6o{!j*AB>8H{fbs0>B0z%3Ws6*UMMyc^@_1fve%C)k#z5q!DB<4 zUyXqY7o)qVq<|I!$+VW6eCnqyFD*94?loFzum#R?)ylZOv3I7B9~RovK54!FpxZA@ zrlH>J9pV{A##eUq+Q~~FD)9%4Q&V?>OoYL^b<0vZ~5+?z&)cMg|0h<`TW?-=*NtGuKSt>22A>T#8xy-G37fYWfI3*T}u+T zXItKIsy275*8SY?Q}Ovag_p_rwP(mAskZJT8F_ltLn01@p1s>D_%%A4AgO-*VSB3|p!j3a;j=4ER z(pEJ=-Sndw!oSv|8ew$`DrHb|=@l#G zjRGwfGMP)^D_;AiCNLv>^<#7|FMlC=<6ed&4gvN?q|yw#7MgZ#V86sGp2ql}8!uPb`E8(M4CMEYVF1w87LGS_e1Yk*eFGsz@%WJ;%~oiE><8k2>&DfY zUngeuM5qRFQp9qYI1bMl0z``=md{v104Ufeio^)9OdEnH0-s?>JP^w;FB?4NH{@*! zgcHG44UDKsCG@gE8Onl5B#0%+or)IdCu|Ou^$9`bO3w=C`b*|++tYPLD6mhr zyOsJ{XotYf>QFTg_cMXQx!OTkbANIcgV@z-VHkce`sD~Yxx?FT;)+KPHa^GkIDqFQ z|}Gwb$M=4RABF0PgDCn$mTPH|%jJGMp13IMFAyQ3GKury!E_hj1JD}b&j zKaIGf<09$hI8lZd99!+&#~k|@(Wz*L>xx1o5ooJqJ_t*saIN;4xsA%NvLJ!)TE}$< zG4fN+v>Uv}4nBVB%EW8pf*+r{Kib!mzqi4ab~5B7F=Pa}QzEJHLzARFz-cl3lU!bz zxjb-upUq!fr0R!O+@s@|w@uRYc#Yo>Z05YDQWXC5eG;ia5LHiZl;6pREAGCz$HmD`8=ddE^<9Qc?{nKR64O6|DjVfmz|=_vHqlcPLS zo*{+b`g4c(v(Y3yTP?$Z>n&O93mRo#Ti4AQZ@O zI+C{aj#fX0g8ud|rU(w`9lHDo*`r`4n{eXxsN`7M)~^_PAx%~Ua0Lc5H)k5xZr?T#So11KrB=zCO9YDwkq}`B>qlI+!!QQwJQEy z3~jwtoPUU#*Iaz?u&QnpsD3rRi`gT>i@*YHaFP#uX9VPw;I!9(1Eam{_2f#GT zf3EPLjQ|Z2EQrOzpI`|$pkfnPvhP^RYAkIvmU{vuvrNNUovDS-WK)82Y-X~tWN~U{ z2?%E)l(M)Avi|9TeiJ-U%)dR*KO2gKt2+f4Oa%@e{2vBkBxwU4WXnc zE$yeF@z2A%Kpma%f2C0WrKA{bW|m@W``;Ui6#)T%o1uR$8FeNl{ipCLE9*sJ;rWKq zdGfS&4(9!fl0)bw{;&2!godL29}g6x#YPH%$i?&s7AOluMrbEyWsx!wmZ+0JU^4cM z_{xWIdSc0NnJRr+H#EqCKx`7?1Ue4(u5>LR5u-6IEx}fDp%DUR zmk?zFy?OQ_h67-gU;uI-k`qqt;Gp7YdJsMMY2}5!o|QN)5GkINb_#4kWD5Gm-MeOB zMh1vt8-cx;iA;-Adj?JU#9oZ03?M5@yWEFB!}nO4QBhHSsrvF}I&#a)I3j>pik-gP zZmbCDs7@}eKR9+547=PfBveuDrYY1lrikdREsTYR)3;huQvmGPz1xhtiJ2{K)6yK} z+yFQC831QBKt*B603A}{Do=e)={(I$c96pdk+>GkbbdPemvNaqvh*_|JOC=1LVDJI zbU-N^rNvN3$;v)!22jNIObM;Enxz1`Sbd7FNj{MaMRE=U16MH0d>7<(vgjvsGFgYS zMc;^piP%`O2PWUBx4_bX{6QpyN$QW)ATn$V%mL)wN+}lsb>u9+z}Fl_5!d!@LH4ue zbx$+Na&ArG9nH&ucZ07t0WSkS;4i;-b7lD8K$D>qVsO4(vS>9S2Sbu;a8LW!~1u{HC7%F(U8Tp=D0iK~9W@%Ysxsw|3@H1Vk zX7%$6&f0N6FFo~pmWK;><&f=}nViJ~j{3Z$X3U=9R}}imWpUMbKn9&ZX$_`!PBqnc zsK|@d*MU@6Jbb(iCIJCvhT)ZHvvH-(Mk0I~S?i&R>O@^*v{jbX4HzbhbaIJ@ywxnZ zSXmvy%dLrSMBRkgAcace*_*(k&$#`@(wfHl@HP6_=_@3a4x^v{pBjomTlGyUAwb_h z8j7nCY&MjLe>W6YJpT}N%`i~jazL9>ja+k}7sk8oW43o6{qdZYLwnYVin_@*fnVtB zlQ-HQLTHOYNLq%UH#Ku6)HM%?FJJ0)7395lFe>{3H{&5CqayQCq~(_LxKPY8@|6fz zi{Qk_uF=sz7jsg;gtnww6WLd^^ z`<`cvc)&XB(l;r=3*Y8(y1QT3 zEDlQ_tPiqsHj|Stk9=PbwG``r&6vKZ^ie15R_W*d+m!xb=RQuAEjbnUpF0)PM&qwK ze~WYuQ&}%N@AXn@G=bZ=uPg3~u-$AvtP|mqQbJq)mN?>OYdAeT>#nUs?JNzsp7u}W zU03_c<=miNvhiSk_2*uTMiY7Kv~SNT?ZXC*{$HK;>1QO_-(F^5h{zEGg9&(O{FjI` zPn_cThB~QYnv|;O(G@2bwerJ)mZ9T!^!1KZr$Iz zkg?1+mMKgfyJ>c@32>bcRMu5^M#@R*(d-X2WF@L>5`NU9(;w)FN>sT_GLu1FWJ~Fa zU2uEzAZfD|hD>A^e9SGWKYNAgyKBozs(FwtJ1-;GH>BjF%yd_SO!f`t=J{>&ps{zn zEZ^TyYTlMek8EA~Lhi1iCQX|WVZ_Hd^G)}syEZmvpiBmN=H8KjHBOJ0S>e?X$Ywd4 zK$EcZKWA>l8B9rGi*7t%xIF*Nc<>;KYMx!32=+wy%s90()`qCf|Hb9Dsk`*=paS0& zr9d50`Bz(RO7Y9n7#dq*@DTnZ_9tVF6t@b-Qkg}23r3RoQQmK2^JPY=Wm%Z{ zrsLeJ>=(_Q7PbX;i>h8NisOt~xQJXQX`+kebD+RWwhNYp3K4_!POIg@13A^x z2^5%E@e2=j)qfy$XpfEPdS{GZ(NEqVM?BEH%O#bedp5G5&2J^!P~#kk8@=lKwG5_Vk7lYojXkc*%wHPOx?9PZ|JkAaE0aTx^t*0$f) zHlzSmn8pAC0_Nw)$*CI=`2YaFXF=Lg@fZ<;ag!%&AZE8Lg(0pxlPzCfU_{l8$GIAi zq23MBBJNGg@$-_I*^2qMckXGSf=Rw(_WKOMABi-H2g4xLi4I+ofF^t}1x$^1=;8r- zDF#zO)N2l1w0mDdI1tObtT=pyI1sH+yL|RLcNRvm>qE_B_mrx!q)T>9F5YNWxZhW?89Pv;*G_=E+fjuq_O-~l<0(* zzSTJ9Wth>-y_nPy|6yRsKNpXFUox_{^vp}Qh(@e@U@~{_-$|rfnC0&x<*3e6k7WLXAc)Cx)@^*(6Mr|)0b1q_sS_mj1P7xTSF>dd>x3dcI%YO|*{ z?spG4{Z@HrxpvTNS)3l1Jv~E7A}GP9`-Q35f9V%vc{{1PZj>eavdX~05KWJcWyZn~ zMTVll`W^qIrK{`Fb^t`cYM<7pXw|Y@`laTXR%SbUw!rTv2=k6xS%&u}F9a(+_vq>M z7Q9p7WUKtju?ry_MSZY&5+SbNR~; zhTq#K&Cjq~{ts3<-}LBxz?_L~!&(F1OBHPPev#S@z0UKVLQ*g5>~x3rhSKVg(6^E6 znHQLuZjWo6hIGcD_j4mxRJ4ghPBV>eRy@4EuKjGFFW5evQ&4S<6n1--%I~Q8YZsAe z#*g{mzQ=99OP+-_{3Pa4|E8c5{E_aNb}3zCYFFeLLsZ5uCd%tSOjRR}4Vlx-og9WF!v{Hk-@ z9^TNd^Fslnw-)(3A@Vg{ ztkfS6@0N(7l8W={h;F2#xByEaq)@tM49-;;+Djl1+5^p;aMVx6v6rCg4EFAj&!32k z%lG?y3%X`T`anA>?VGFy4#tQ}%y&x&c7_h&00ZWvk8LV?dk|3!UH@{@Kz1CiN=zG> z(rS~^@(I$Bm~u|~{L`L8|7p(&A;8QC5aFhZgc`(xCG_UNBGhS9LTM9fT0t} ziT{BHId>>}3=BqWY=6$EczEo^#0YIUCnVBUQSrPj_czlsKuhZn)6>}4+1c64%j>^b zmGd!?bUVAh$)344Zk#hh{(E$Deoyt}UvA}`5%Qd{s8vuf`QX8y^yaUg90BXO($ccg z(XsQ_G1Y%x68T>d;1GNYp)D7E;V+iwUqv}Ah*UOK0gEO1(~~32i-1Wv(qbzrA67^Z zs&YA1h6ZflWbIfvrbKv!HaiRiVZ2u*4|-rVIf+wddDId*DF87)pp z7U3?08*^ZkV8lRWOk*V*-kKL6`*bqMUz!RrBXr~DP?tmr-8h*1g#JOz+lp7%U7DJ` zKrAi5VM62}58Fwzh`z2AXb3AeF2*UKkHRkks(kDc+BG2}BWbI{*;hGqN{(>YN*q}< ztg}%#hBUVyjA-+WW|OyreexOU_v3k+M<2SM7-Vs&oT66laUs@o?usF|5)yOEdgEGB zTrL@^B!|*?wjh2WT3=^&IXaLC%>;zrZKmUn00mu+sN_dr;wzZ%T4G|iwGHHmD|rFT z&8U+CH`-IVL`&WuyfcVP;$O3b4c07I14-WBwYE6{p_XZ17o73XaciTk?flIav7Gwp zq*=hlXoNXbSfJzvi_^EerLijV0XnI`5TEvIS|-|PJSYtp0w<~M2DC}?W)3)&x4b3@ zxr|T@R`qC3+n5Z@fpIh0x`cUI*1#N{$df8|Y1B)<@G_i+BLoH@Z|^3VkR`TiB}#wg z<0cc24Ow0Zhj)bJRJG1}Y}I#stl4_hb@G3^q-s6c(}bO0Qk`ctC~swzSadY`f6zfh z`{92@GOUd_2$d##$)BtyY@p*)POTONl*x-g2jMl+Rw8-5%!TiP7wrwupV6^Xcx-Cc z)ebNO+clcQVr%vr*u(7hb)Ke)d5s8|{&E=<@n!GBi<^4A_LL%d;s27TpBnG2l&vcB z3KsSaJwZcdEZZyF0jHX}INc>n0p^3rHK@F?(*%(A-JY}JWt(!14q@Rme_okhg^pw!d`kqe8TXLEK^az zozD_(Q-H%cyY{*Tx2=;XFHP zF;z7$*a*H9y0+7)5z))Ez^w6QQ?Lj~_+l zD2|PN=>RR1_*jktn5=bJ8tv`pOwRz@371mQ0J5y+^73lh@|VrAUq!V`CtxRs|Y;Ghw2Rd|_wplrV5}u&&e^H`T0`0pm-w z$=G#vg#r!G0(JJ9XNpjZJWXqFv%|!_iY@4fLF|PKIb~wZv{40eEVR>s*w;6Iqy~)5 z@0b;kbS9E=2tPHEKXfSRs_ZqGH;mb#%I~cM(e*xQxuG@f075e~7RON&c5C05}fq{o!wXn>pSUKL7H#4K|$ z$0d8M$}*qCm?-@yjdoo~QmN21eM+`WwsF&RKL1S9?I7oYda;L@%#X1LCO4u+$JWh> z+FqyT`qjyVdFaxGUgE7mfYQo0E^ijhJn)$*5uSK$d=UVyDqn&~dw4!U0)Q!}B>~#B zD#+!a(JGHfbGruoUUA2C47LE5X}w@@hpu~!I4w7Ez+94S)|nAkQ7KpF{*FN5ZJbSf zbjSM%`QBT0d0g9iG~lC+iF%)oYg2@Q;2vx-)OENvQCJkYhha4J8m^58)=TN30@|EC z0nOeKhcv&ig`nH(iN5Wc_p4l+6^ZIz+%1qDJQ8NIKD?a8uN$t$|ndpoQ= zeOJ16-1z?6mY=oMG>LW)ky@sYJDb`3_{s9wTW@a)aLgiJ)wrN{M;99cES1*jKKg#` z`8=?I{iNQ(HebIdQcC0a*=5-fvAcU#9lLVuf@qVH7}ZlKN>~w4qa$5THh|EnJ_{E{CJb)#j*b*=2sos9e0Y$sLxijb05o_O_@R*dp} z9MkX6!JqAZ)t!^8reAUV0^Y|WSzRAFRi8{fjCe+27rO0Z#P>XXv8Py3ZL481_)X|1 zXWKHh4bL8xSMQonDxA-Jc^I6-zA4uSv)kg=+q6{WxE!r2# zm0n}x>M8f_Np#%IU#_qY99uP8E+xP=d(?v&7}bu8hU{C6sRFel7eA)PN+cwIQjWha zZp6gI{=T>|{);epnU|M97_=4=vKJRWzq=uD2G0|Z^Sc{cTiaWjn*S%%AmQjH$=W*Y z>echRo4lJh&pkxBzkhXb@OkKQj%6q+Dr)@;%h2(kZ3F=|_&huNWl_<;*8!TE{`L_6 z8$%C*fq)S- zsO>~Zk%FEXDFS5E04H`IE+q+u7?kQFsk!l3VH8+jdt5Uav`j?D48+IUMw1mi=L2m^ zUGaBBRkOjb97pG%Ktg^OsQFPqG7LjiOH9qjvV28f;KHEPrrC6tST$hoY4CJmJHiO@`8 z$c?g8Ggfi1RfCRfOpoA!;}qd0$Sp^M~E7accA`8esw zU_dgT-4#Aec$rQ7JYS(W9}`~y%es>VNbETy-|b@oxYb$p*-WlcDa8mhwVc}ATuanN zi=7frL}HiB>^?9F;P}iD=5|W)c?Kg=`Y5&OlhTN@wmR1Qzn~a`w#0cfVv+w0I(Wm$E>iL0Q2&8q zIBWwzAX@AMm`V%5y@&r}An?09;4g3m**|zeJuiqUvELe3(eToKss6j})`UW$rS;^< zXqU{Sk?5A}3j;lIxe0v|m*W||S9Zs)`>1)PmvtA7;nMcWT%+K1{l0R|S2dvg z-oxZ46rf+MKTPvuE^lq-$#6DV)RBoip_qOy(YZxUh0EMge@Th2?TY06Ug%ByEO~8! zc}ZoLQgj)2&?mEmmY;}mR@9cVGEM%lvi-1I=q%n9f&V&W_(k=51#>!n_fvvX2%y{c zTjEd$B>8(O{RNf!;p}>d#vvzwTJQO)K#^)1x$C@0R274YlQIQeo?2Q8c!U2QIPO@U z!UaCIdInb}>@zwKPq9|D?}lPYZ<8$pq=_o~#nf>fSC`3-iF;XA(xU}mv#ofAXtD($ z6CTlVQ(YwneqWpj6k&H7u&dHZ9BE9VqFW$yglVyZX<}7ko%)AmpI2+%h-D}$Bz+tb z!DaFTP9#RtXhq&n;{h=pm<|!b9GFGnmLG_?VLGiKq zVB&`tpcs>sTSjaB+pyKihs>lG^x*#3>FiMvU(|?ztwpXv$UAXs;>?-Q1Gg2V6Z0$iQOEl`IaR8KBG%PP zuACIP?F0PmWt)x1H*ej{TyyEaJ)$jtW|7<1>e8S5-MB8wqWC5h|2_w1>}O(`#q3tW z*EDZZ>}^@pw!LxmTENU+ApPD(z)j_u5tFDM%VL*Hg7MH%ohU0y99hVH<=-R5zAlz& zuYbA?4u_7!+WVKxdbw$2e>WF-TVFwT&rL)0?W>d%nKJPT9=$Rav&@fTMW?aXO_&%N z^V1PTWd4<;dor(Sunu~<>nNIg9%mqs{9Q0GEKJUiMwO{!balM}$emFjt!)}$oe=4Q z;d_E;QF-e8c+K}78kcAh8RL8?!5!PKuG^9g<#qy~R(OAJO&~Ph+XDn6lO#q4L8(H- zh&Vf=hWwYC1Q1>nx;Sv(xC2xiAx5T)g!EN9KvfW4q`DK3K7Y*P*m}V0u}IYX$5v3A zw=l&2r#WO#{vPz(j(E&F+j?5-T0=!2lQi5yE^WyZu9u*Ime82ZMGN9thBdP0dxKQy z;AD<(yJRwG;6vp;xKrH6kPdV8I3j>BzxIV5f&_iPk0u4M^~V<{v;X+!&9U$;3?RY`af*} z;9T;X2buY!oDtk&O$$>|=CC@o3Ws00@|FvF?GshsWXLBC|V-+T365qL+Oz>=*! z;H7aU`s!tu?saU6xZ7!x^88xAOp9-lhg+oY%`A;-*)ztsmuo+}>|=@T7Xi zDb(H+lkm}Zl^MtjQpk|2-*svCoj*|j(9%Mo=DzXsH3XCYq&C4jLe%%&%Gv!xXphfg zX6K^2jza&A`1VC@)oH2x5Z0}S`@;5$P7v$+B;dt;CLYxfGlqB~)Z*UU`yVS`>q%Y+ zKkAmtTZqY;J9;TPykD%aw<_~sML?GEV7ZumebiEE@@mA>y#B6eTB;wU;@ts-+NQKD zdCJA}RmbpMXXgv;*Loi;dJtiXH}6^Ap1J-hpm|I3^EExh%$wcbkQ-{d)X?oiUWoPg~fLy(LK&RQBh2) zKe~28sHE@Th+MKWi8u{CJ?=W9U{`u!;rrxlY;iB-YTu@L#?Ou?v2$IKs#|v6$k?`)KL?h%dd2p<6Fr?XCp(tfS06 z7u@0U)yq@;&bfQGf7M&OuHo4a1`=8p=+3Fn|qwi=(u_)e&jR9QVL_vpT*@dEEFd4*L6l;e(zgsk|n_TS_OyovK zw3Q|FP84XKDXO;xx~%{(R7LaVi|HX_b!=j_g`oOEP)))GK4C4)3?hOD(VzjS8Au#S z&_;2#`EfQKarRYlf(XJM7>%=Lyo)}K0XqI>72$)?BR~EIEWuMh{-3=jPJ-Nb{>x^(`$e|J>MX zYW_(L{#9`LAJ6O)t*i(KAAi@Ja-5t9m-Yk@+rz-Xf3A?9U)tZhcdr41IZqIJQd6H{ zvFF=h=OMz--Mb@sc?9);-g-JGvaK~YZ*_GI3=I6QQ2zwE4}f4Q=JDq0Kr%qgVwhmc zNyrFbTxJOch4+dQWQYM0)#7+lg3f2WSg4i_K@^atw3w?r0OS&qI8{_Q0kDa=DFnnN zC(Q!tD}Px;5chISKwL2?NLn^KBVG+)lzYbcw7XjtV8Vh(67+$x_)jwCGJt-8S^oI1 zcoSfaoS%6jiKNQ{EX}SwF-{Ew3>0^&a!EnJH4+NpB*u#V=Mf0Gk`y;Lo*rXn=#7Ia z8X!&k_YL$@YKI5hASgw53e)<#1R-gmlQ6B;s!0$;{;%w2ABjsvW>xslL<0LhvTmmuf%g&0MT>LE7a34Lvf z8`nqTQ1b7fh&Kw16MJNS38*vPJOR4(gB1aTa16%QdOn0#SQ_IG+%OO4&lj%ZbGS*b zYI8Iy0Ko*U7kagAe!Pbs6y7Kbn04PM4*6KUaX+(rHFc#Pn=r9G#8zGFtg^At z5Qo7=Mbqr(t-84}Y{?_vgCAQB7A*SPMT4AIx2&tgJ4NfpXFc$VQ__askDK2fI^r(d@ZQkeJ7tVW_!o#Om8X7G4^Ba0rIDZ=Y|89QbTTfzV zjqd2?OrgvpYY5$|V?KLLFNLT0os9BgGGuRTfUM0l4YB-u`+As9MRcZJ=&Otrm1fDD zw7#S^-Od=daaWAA;?lPRqHibLspD*9#fL+!l=w%JFT2<=0Y+45t#zV$FJBF4j(9#6 zpnKO9IBj`|KYUYew9PhuGvS)-+ae<`Aj`{DdVku_*B?C-_(JWm+Lv{X=e-luc3*qN zzTX+{b!GDz?)4On*n5{OsU9ew`8(}MhkB!1a_R1upV;M?$S&;)iQLTK4nkOcgzT2QMYagAAF#?%dpcrjW$eWDbnm0al~B{DmF9%mU-6F?H5(o6RS;B*3f)ca$Ou*_>(94C zF;F3+7?S1uV@g0_OV~U16D{UhT|zgpk7 z>uMQo97!M9?sC#u3;uJWjR3=sj7SUF`yfEE6GS{^L(Jy{o*c%LsZ*z2QCWl8I%vxF z>_^{PC?-idvl${wHp>!2aSGMLMs=YFL6Xq*0+evS z=U(x55Xo8qfN)oJ3Sikqy$-L(rv%#5mj)XjeSk0yJST>}3@4xCWnx&?J7mCws`H`9 zxb=a}xNOXaUrvg_ycfYY#>1g=+)Si6Lw^%&AQT~>n#nTGyK`bTwNcJrCZu)+g|O}q#XXAH0_+dt zK^3}kSSB<4W!^kY+1ce}5;Wd{K>;Y`42lKAK0D8o01r%NgMzE^PN$U&MfPp}M*G{s z3R=-GKkjhIo_ekIfGlT3#u4>u6_;#2+xE}Lv^~s;0?=&3ROFa;i){=DqtZvtj_KI$ z1i;vg4}sZ6SE!KlLAHL)>60f^)T&SX=??aCMk*IY^~SsH zt?ZK9e?4ZIQy7b%{!$k%CQHTa@KXM)(C>rTjaK2jao$TD_vgRnFlUE)TF>9w&rNlC zcCBL~aC(#kO7V6XsqD`k(Yo?9^;1XM_gCgu3(6_1uiG}UxXLaD*e{8BfWnl=tvYX2 zZ?#EjI2XE8+6OMs`hQlA6lZ-bv3RZA&Rv9Lh>-hSNx0?Rsekr@)tg@+XkFseqom1Z zKGRaBm4WHAbN0@&Y5Ag>`cpZgHzIRFbO3Qrnn&4A;X+$_VaTMniovaS9u|Iq4TqiA zgRFhVPXxB34(k>De!>?xR07(z+I%8@8h0P~oQ$oUe)0cm^p^M)TMw_jf|`QrdtI`l z=b{N)X`+voAA3agyXL}Z!WPhD5$%%>zeF*TgavN(Crg-xgUt5r;UWLfUS)~Hr|ume znTHCR!hAM8))qeUZXHd=sLOakuKB$aP=4oh8qiOp`~m)D?p5u5g+pb-jYII^ly}(2 zri`by4!%qo&(e<)Phf!zQYdqwk&*%(A1Rklbz6+Eb*5kbSd-?H)`4jBZTkkzg@F-=~-vpOd z(z7Lfx>@l&_U*f~6TRU(yGaNr#~hA`0s#=&12HRXzrP_HK?jxyFyhnd_g9hNqcV*< zcl00pn0eN7++um>6A2jL{to@~;0}lh5d^W#zXkC>kDUocf3<&gXwGHvg}-RT#%yea zk)3l*Y$qZ@c>gBU{R!MD!tL`vxl@0z#Cm%FH%)A5Nq7SPTM!cla?fAE|GqL`qDJ3BiGZ{dSEIRrsG_OG*O0)2|ms#$yd_#8m>zeW&8!#EAi z6HK{CL0~w-EFm*1JLj&s3=AMu$xY-WJa}KsR!xC$XURv`0w!F_@?8I+fe~`Dtf0rm z_`BQymmD*oTucVymP;|nR0Ejg5CfStGRp6i&59D_02u-Vpe)l2+q*^ZfeB=gToqu+ z6vUcIN}Aa}VXi!8{DbiVDL`ee3OEd5Ejl6r5TZIq6T?RZLa@})jJ)p!NS(ZP}c z)Oo<1!>6^R7kQH6jybJRK+gb%EC&PP%LDolX?6<+6g9E7{C&j_QmAD*P@%orCSTg! znd{j|1QtpJh!}vVb*1NH>_jiZ>9I#RXFZti*{&6-{_HN~5&%-SxII=X6awN z2z*-j*&d#OpEvKzenaVm3IoS-Y5_{g8aic3EE-4LUTGd?L_-Dqvmp-sQUMoAKmmJe z>!|>BfsS!hCISnhkQ3BYgipd_q9xYfwSthEF<~NwQMt8;q{zLR#*6Fc(AzjT)Uvn^*o1o zVV5jzEDlWRR$1*rMnjLslU|VJ{+A#gy(mB`zP(Hk!~z%$#iuKzHwx~6;ch5dW)Hs7 zWUz9KPC6s0dwIl(MRi5E)8`Fvjkhvixwmjl4DdOLb*&;Zz9qk=HYmATEUotkZ*4NA zi)USuZZb-}b5W$WT7*c{s{ym<>Xp%Pc4e|zU4F|IysD{!YgL9$R6Tw}`E#RO!|xro z6`u4?y!k=R&E1a0A4#8|ZkMu|Bz}%X6w6H|l$oAxdq3gV zi>|Gd3*Po7-d;}?eWo537WGtJEi5%I7yDQ$5iwIiv(7s>wuJUJ+?`mM)j#lN9 ziXOzqT?Z}mR}@l*9HYYKVErWIUahTf}y;lB-~Q3y;)s$ z|EXA9bGP=^S?H>o!p_8XOxWU6?N_o#_&4eu>ATvt;Rg>*?teY_>T4%EUupP2^>}|Z zyV*#vea%l2BMv-xFN-Q(%ajcLc57bT)_pVdrrRSF+GsDj!ZKHd@nR=mhPYUkQ@in3dNYoBXTHVH=#FJ4B)ZN;F1NDkP`JMZR`}Z zh`f(n={tnB5tFU7Oeg4vie^Gv5c6R`f{v(YHuMw`A08xl6$R!(`x6m{=LAhrU~cpj zl#dxKsDd>y@xc|_wF0XAHc_EZeZePn*PsW&iX?k2?q`<-nOOE-m2=>5N-}_!rxhkH zx2^{PFyr0ce+A9^%U|b2;k`T`MnSi&YI?f8yOB7|lKhU_ps_7j0&Cai zf_oy@PZD4lgx&P76Xj~$eZD{g-Z2?N$n5R*^Jzo&C5%|*kPW?`93jW>)LXkd)*ivs zOW}`md-|?fd+mq4jabipw)+AvU@zKH0_q6rR%u-JFI_&XB&z7W*iRCBtMb8*KC(ux zeDp|1ts^$7y`P;|GEp*qk;XK&TLpoh>YHRFYw+w5h?u+M+|_tGHRr%NH;_go;Gpqj z&W_&8FXDNiEEiS5Rq{o@XPk9w1VSQ2ch|T3XnKw80pBR%7@m9YX*RF(qfILQmpVWQymY&&OF3+m?@6?wE1EHY;NKy@y=4QljsuOSk5-xd}Ijj)cZr zg~P==VvtYW(;vwU0c6p78+8KQjR7zFWFW?QM$$KOkDd>1yJL&5yr#QnT^-2^puV%% zqD@+xK5lzxvemX%WCb=RUiH*YA)BbB+#$Wu*Dwted-X#1Tbr)? zUk4cLzruIwgHN5h%eYU~~G}#wu?d- zTo;HHEMML6iR@hXp#C*r;LdJvg4W`u$yXHcZz(EGkfJ94Rf?WhG5&s@CG;@PpJxg1 z({uC=q0w!MKoHa@p^))UHTriU_8IUX1V|LwRtQolafKm?mmzwbnlFkH2@Yx?zUu7NF8si*~Ju##Fm*9 zGZ1$&9CBA3P){-ku_wb|$-IDJ5~>W~ojtg#xRtC1Os27cQ@BBixd6Td1_#X3pR{|uVN$8*`5fKy0rbuTC2?)|b>ZS%l z5@~`UpeSNU*dR!Wbio2h@4bn1RCU(>1!U=Y8M% zL37XxEuUF=J;z)HrI@-;JFrUvfQpIiJJxjAuZ z2&}@SERdU!jYv@XYsdbsprDK7%myV&e}86iInQvV9s)mh?(N6dq`{H%qWW#=BfL62 z5Jf4a*UJ46NRBWDveV=>@eX?^G*Oa0zcu=!%PdAB+c}qoeS9`cVwHdOqS&<+BrljE zHq`kpj_2QFp^S2*F*!Q^;8kSPpnI95n2-_~+I$EA^;I*(K~=)capl6deA6riK5_J} z&3Dl|jI%{x;e_hO0u5<^UW^p5^IZH}gj}>3B^I)=7=7v#j>;~X=D8GuE+|@xl~wYL z1W^(7ERNTW63K6|QnZ}#=RYXg;tA!_7I2U}k1}Tdka%Eli0t4tGH4!|+j0>}88ebS{aeea=koe1KlEWU*QT|sJ?!+pPO)vv2E`F4A&u(>?x zfpk&Hb?+jY&_nFi+Dyk%9=J%l%o)7IflSqNZ4KFK)1~gwYHMQi+1gm5s8n4H(ZM9M zIc-)7n6Pt_fwx`G%745xRdS_qXeyRcIZD>(xV5U(jBc^EGuqBRxj*qa<>3dDyh5g* z$ySwh?s0|EjRF&58M{TOPdDueRPh0nDbOUJ3sDGB3sC-5*U z!j>srHP*vp@#{(#N2SSLlB$Jl_3)10uP;M-93Lx3PuO!QbM2Y?mW^9Vm+!tTIpEyv zd5`<0JIQG25bSZZzqQK&Y}~Ho2*R1FrZWmGkN0@r@B4Hg4J>@GmVZ+9{)PDaI{_b_ zHu&hmLvUQ`4Xj7p*gggU=;#Q_y4Jab)iJ-`B6N*2h%E*ecd#OFML&4KAIaIWA#5@# z-NQS>8npE&YX=x$Xrg3#Y8pRuJYgv9DD|{7wgDB45(9qHvX7+F|A1Q3BZQURU~)|_xNlX_ zP<0>R5q1==;~kNrL@ewC9S5&dpz%`NVdv;L4q_ElZE%4tD5LWjag;^$7Zzqg#|aP> z*aRf0>^Ag^{ACoXoqwaxGkQp{*LX(7fE(o#F<%Cuc(^v?hWw6n}zqp!K zD+8O2y+yI<&4(oU9DTrTxFn;SD5QjG8thX^CXbl@r97rGk!?U=j`A{epcXl*f zbqmfim%_v6UHtd)3Cjy^o_T?&F@wwBaw58L`rP@K-HBuxb@Ba3~KZ7 z#q4p7>Th}1IdCTPlGYh*?5yW)`QaRH)+$%A8OcArEMDb7T+J(r^gdg`d>;*hI ztpV$E=}d})>b^aqYHI~K+F!c11-LwZ4|EyVLW%+dq3SY2Z>+zI&5LAK5GkcGQbBjHy)y=j_A1R z6+#0(r8T|$T^{92cm80|4}+@vbr~RCR>_8}Ec8EGs!Hel*mIvx999G?+HmaX-e*B5 z$k(dUKkT~Rd$DyP4aSGW7fNThwFh#AU)N9gJ=`zEG)u)@L@rnjy@atzZD9FYF-(%; zj8*S`C+*DNF6_@6bvHhDMxIc?b81anB3@IX`K6M0pAAS#6{O+y=mbyp9+({2?utKh z+2hU`xXLNJYxnCGv6rV!FRUe_ZiX$gS2r3eqH7|*U0mVW>r*t-R)pu45u7YV9a-9x zYRP)bV(YOY>Z4z)Q_ilrD^45YLNYNx*83=WCuXOAPjMytHXxhG?jw#j~ElAaHPg3c#IK z&H|pkuwKkC5GcTX_BL_icvBJ5Q2gSvk5ksWsShEje^c8{01ObdKx%_Qq9M`X0|J3S zp-?t9HW&;Bhr>BII5;^uxwyEvxw(-@Bsgpb<3(X%VQ}OQ8WlmUy|}nI3Wbu8kN{O8 zXfzsw!AMI>%gD&c$;rvf%PS}-C@LzdsHmu`t7~d%Ubt{UM@I*X#o}-{U0q!xBO`G9 zZe?W!nig$sZSC#t9UL6+cszkX0QnA7E|N$jGMVh@>FMR=1>)Y-t5^N}{QUj>gM))_ z-n1m==2Uc6{;Z{N+# zJj%=K>goc~udlBU%p!+|hQ`OoK?afC-}?gyCcw!qW>}Stqtapx(e%Z>&WKPjcYV!J$(bgr-eKq+5DH=eBB>@VgMANjz0P2b^RE4FQRU{`b3J_(v{^irb zUj7@9T8$qYPZA$Nlv(+pyVy}21}H$(J#*{r;5@D=G!8B%V5m&MVbJ$6j^tlk0V8$E`s-LkCCAPK?KZ%M$=i0+wxB=eK;+L7YR&IfJzV&me zmVVD)tmXQjTq`h)poJWb0_BNp*c_9!HWt?R$R)`wS$>bD4 zQbpJi^|$#XozT9y_!7K8{J7l?!2iS7 z|4%=;LKow3!JtvVKW4i}@XDj*-s3k~Pro`9?R)CAM7o$*pHyM1Xuou0obUjsDH9x2 zn*Jp)q`E%HH>}AB;~UY2%khos3Y#Iv3{J!cjqAz})s2|zlwY5)dS@Cusf3TOA8_<9 z51e+*;tH9ONHlABMSfZy@YdUYFy!4_8#kjNaOCmTxe)yQ8_W-rUxpZU;YTO0E>Lwq zRsYbbgda7r@=tu16T~?~S9;CO5317er+n8k6Z=EgbHa}uKFv+^tL`bd;e2aLt77=@ zWl_6%^~ajKVz)mB%_sbvYuT>2vE9bt4%?xCyuI*J-2dj+{&T}&dvQ8Ezn8}?=W*M&+s4lz4qbmcIc0#X!G!cyr+rGO_55AJ3qhRdG7vtJ7RH{@qL~roOzfya`$NM zr#SHDTnD6PgwmCJ66#;j#+FbPaZEB1dS)hx-Rjy|p;k9zrJ~NKxGLJ?MHIqFqKo&~ zXPVTYDZ8~NU9e`y6CJkb;*9Rr9kq;73v)l_(9|vPeKeA(Bj%38srMWcx=g#&>c)p@ z>OrWEMOZky3Gk}xv)Bbt>{;D}em3g=4&DvN=ix<`)eIP^0e7xk#2+7MG}x`*z3so^ zDpsp@>EmR;ts9Z9r?MI^t?uvMyesL7qNy3q3kKeZ?s1XuX*8Tw`5K%^bdj`HGkRki z7?jTHBBj-6G!*3VY(thsW~t|gD${!;doY160ep=wzH zQO-tVJO>00>uD0ogzc~mu9MPZ4-}M1!h$FCWe(|aX@k#5@6eZ=(Bluf6mq3fISln0 ztm;ygK|`jOu~UiW!_1TdpW02sfK0_FEoOyc`9O>2^s2(;cBN$1~_C1-XStt5xogR=K zHmI!0h{t#;esmzEe!r9Rx3@Pw(2-bg7hXzYmXn7L96cr*!mH=DJmqdX5%z`dHkU8> zwjaD>JtmyhJBPp4*W%1|sxhjM%`WKo^!wT`AvJFVZ9`c8n&Z|Mj-H=%zZt6TDxM~M zZ(WdZE2hcyOoMRDwk;v-p*sHDlyEE~l@MOsgqQg)eE(>YKxtNYlNS<+g9~|3`hCxod@pB#KXY+&fSw=%k6z&j}G@*_#P^4I&TZ?TB%r z8XlLXM3U6fhzXcx50mdA$vRU+x`qbPLP#{_k`O7$sF`T3Dw=9xM@n(fAlciArrD>F z(!82UP9dV{_$g9Gs0JCICYo_Yh^&zr(@geg5Pj%xM}BmDDoXR+!Eo>wU7oj9(b_)` iMkvSXvf*YMx+e}t@0r)-3J+} - - -
Scoringrules
ScoringrulesText is not SVG - cannot display diff --git a/docs/assets/images/banner_light.svg b/docs/assets/images/banner_light.svg deleted file mode 100644 index 92e7c6c..0000000 --- a/docs/assets/images/banner_light.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Scoringrules
ScoringrulesText is not SVG - cannot display diff --git a/docs/assets/images/favicon.png b/docs/assets/images/favicon.png deleted file mode 100644 index a2f74ec8bae91882d0e25021cb0ab7f363f46705..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13267 zcmZ{rWk3|~_x6{T?gr^n>F#cj?i3J^RJu#L5tI%=T0oF)6$t_9hNY!hKsx_3>-YEK zdGUM>FtYCK?9SYC-{-o{Ax87LA~psE1_T1ZR#uYJ2DWv;cP$<&@RKW1Gy>Ql+excS z1CJ$P-o8Kqey6oo(pHB+{Fot-kZ=g(8h9vV8v^m0#PcY6G-;+Xf23mx zs_%4gbLw1AOgV{4^re`1tg%50y(k0=;Uy6d9#0gOO&OM+@&PPqxUhOGL&fcRZeYbZ zEdMWZTjja1=WdnfwE3PHhIKqQYK)u}?fe1=dA6wc;bMz>Xw+lB?T=B95p9PuxUGA! zG2oAIG(ODX>wV@*w_Nu>kv_PO$mFJLKgBnJcAX^sQ>$SPcY#r<4`eEWZPskU9{af_e}l)~D=1pt!VNm1MiNl9LI$ZS z?pmF0j}<8qS+~IH{)tJ-Y1$6i*aSgfDh5N!5zbZ(Atfmg zieaBGjAG*EWz>|U1q8H{k&rn)#pepU{~5~U?Y{cEHIgq0!-v+L0O1xJ?td?#2gfhJ zs@a~YCwy^{7ue|zE7n6JSHyF7l5Hisrt9iDyC!tP?s>j&V9DO2rY0rn`xo^jRx%iM zwJBjvnyhQ$Ui-7P)_ozeQ8?5h9{s6SuiKVfniekhTHQ8(y?<(O+9|TO@|2F*rXd2{ z0}lfegj_V>@@Vx(RA^1-^=@TNXJNDBK2M7BH%2@Quz=r!c>_Unk`%`I=ixx>NKe5GGa^_H#y2wEcT9(#|RI|#3)76<|( zGsteBCnGR8=KA>HaDB)H!QP+iu)D}xlG1Sp;w!XLCrN66DUQB}`lD$bk&QJSijg%d zO072x+TnkJ82oUv{BVEK z$3rAHKJfOqk0$t*`{-&bKluJ?>w4-THnK~uUxCm{W}1PLkgb0yyy-D-?Vyq z73hr15h94M2zkXo7KJtq6o{>46#9Wi2ryO!LYNtbFoxYX!CL7N?T$}XY1uB46rg+GAWAfCX+3r_0LH+isnjlXMQp?_! zh3&&+)cS|Tt?|Ev4Y8PUor5C>7(QzlQCi&D!?s^rF7AIh5CUfd{-oDo0pR4Y^_#oV zWvSJ3Z~7{@FK0eH5(M!rDmDe5JzjDi2+k?|e0Szq2Y`rK@?@$pr`~5Phcfqs1D6t? zE37^E;r{OCg16CgaGKqu3F7j{)+Ll)P;132vpX7}K_Q+D0gFZg9Mq2yp2Ime?c`-+}UuuVP z^n?%5QDVy1Q(3su(FJYgdP9A;s6*5O5s721qp&FjoMyQ!$3H2jGOHu@CecU+-TK5v zCp>MZPLa9&F*yUGBi{TF0KJb~mg{eSnOUozE#N#~k6ti95vq_OKJ?fYt**pYeK-rl z#ufyz7S8>Re45PdIxrbcmT46!Bmmd!-EXdTS_{&(glqwq`A1auyGI65NM)f6^dN}+ zzd-&s@6s}Y^aP`1B+KkmYbPg!zCg(gzZ@2X2CNSYhFJF{QA>K6!t`F@CV7O;eYG3O z6Q|4&db8FOLl{~BLi<2!DTDFm0vzt$ufx=Y*M?nPgZ%$4+;jGPgOk_qJ^k6}tL^Yf z*J4eR65Sy=c$ih>%zch7vCGYGoVdFk6lWjRo6C)KzLwY`e1+u%ZtaV1+Qt*DLFp>l z<3nE{XYNKTC*LJV90#u5^dRB>4t#z1);*L?HWG`Rr{@xJak`hRd$w3rUEM|S^|6(8 z(E>IR(NR}rW$yFd%VW`CgX5xX1WPG*_g4eOGM38{Y(3)Az^x+XBP{{Un&mpBV%XI8 zVanWW%qSUeZsHg07I79mXUx2pJV_g<;y8O4>r0&=srHI(LH19{un{5)vEFIre7tgxd;ezmjXFH8*z#BJ@6ag6J!!?@bpM0>8W}v~(cSFg;}v`W)IO z_;SY87_@Zlxi^#Ytr1$eD?W)X=n8H3{rd+&0K5}p7<$C?x@SMI|NQx!<;%g+JA!mf zA-{bGXMf5SFW&dz4NZ*9CB+CKL>qZhDZp&Wjjxcrqvh};5**u*Ur)g}HP;)VYx?RB z>$rgW!%#}#LzwRkvJlJG?$o(guX%9Cm(d;MMi({?ua=OKKj$#T_V1r5G}9ykwRI4J zcR9|O(p2-rd`~t8QqRm2J~4cMbn-hF%oxyf{ypkfQ?fNH(ug&CyTFVcPt9715gJ@bYzE zl<29s`pCy8F^UFat!rb%r%K!YbS1-$>?X0XInP&qlDcy?HbO`_1w?M?&}vwLiTW*( zP3@R^GMhpBoAurVND2@^P{d@7dbWdJ_t?nEgK(qE`DUT9%NePn66CBCYsv0t<=a~{T-3*br(=pVfs+?d z{GUIEhg}4%AI>bcemB%ZvIYNAed%Ig*nrY8*wh}k&`$K=6nV|3;MEw`TK$YAo}B;O zf%jMaSuZpo)%CmhG0g6HQ@xLo*B=8ph2B0SDW&Ko90)r*P88IU2Zt|W9$2+aIeHNr zP%L!SI{k`DK$w%BW{(AAymIRW6M<^e9Z|aKGJEHj@wNVkSG!N6kPKT-_iD1Uk+h<_ z;tClVKYiNnz*zrgGAjQ489rt{`pJKn9-+a&v1ZzQ>ZZPfg`FP=V#8LCo$+GTK)c<) zt5KVq9HiDo3J1Sk%FynQlVW2V1v+XXm5xpXl}D1xF}cS~li@iBkaYNf=LJ;$taXyO0_xE`dhL%f=x;J>T*-hnId|Tp`+r&QqX0w`AuGMQX+`$Gx%(Rz76Ib zoq?eUNT^t3Tm^p_u*WthhF(r7XlR)wm!q`8ud4!VVlOMZcbV_Lo8sf7$zqmXf?~u6Q@;P(SqOxlwPsdTn%;!%hQjQzHmZ zqo}|4`Jq#XMyaFioyW9>-5rsVDu~VlQb9#b+u>RQ6)6XwfC}BYB@Y_}>6#^KUX9uE z2(N9J2>Da#!VfiOS_!Cw9ucj;2O8%_nxGKpPFup(pu2+5ot^HT2~rL!(UOZf<i19a+|${3x*oA(xtB>=FOwyhymBHEIlT*K8Xg3 zkS?M<-p;pbK=fW5D@~fi>gh48>dcr^5NwqzMC(v*kdp`DZhSQXPTFTO$w`mJ{Z6;F z3O<5J!v@ENGvUk+S1+`Dt*fg_sNKr%I&yX#YzUqdCoGm&UA6s+XgTn%Ue}T9i_aV5 z)+r3&1UfON2DLPuf_QY)B+8BR=J~9}u;lrKMN&OgNkZ{mh6egy98BGKh(xhmiBIHA z{W*E*5-jZM^8C3@suJ(kBpuH?NyX2`{FYL$xxNHkg&^oZ#iMqvjpnN|#Hj0u{%qdi zceXp#J)t3acRHbQLI3ibSs_`Qu21>VyKY(Le?!>m%p`xn-RgKPl_fhnWS63Jg62A# z2nB-l*oRhl&I-lc=fy_=4eYdf69?>PNzgjBC;rWl=CL)`b=DX!ad z8d_(sWt=sIK{b*o!2XOXLrj>tx1Um5AxEMS# zjElrP>Cf;M7LxhZ_IFmf zv`tR#a8y*#?Hk_~{H59J)78LeSeN9Buq7O|EFx;>4W(w@PqeCX1*I}-r>)%Kq(V=B z{d@1Sdf4%BAzQ8kK|g~}QxaA8C74&FI=*4uG&Y?QwM@j(+Cq>pZl0l%=+=sgiru~;t1={wgZ}2K1jXKw+dsFy49?J>KaHv| zLNp|D`396^)FzU4qOShkr=>TDE4@2;;#jEzHawZY1PP!-!3gQ;>+AdbYLX!4ybMs% z^z-&i6nAOqAY1tM6^FEh_*sY@4pS9+mHOr0()ZUhK%y2`i$)_B)LyO(s~qG1NGHu4 zq1e+jd(a(@ZuKYH7bC6~SK@NzEw03oPr>NcpnAord9mEd`lx*qFyMgHJQ7h>l4RpI zEeuP-4K+Fj1OXNIv6#=1{JQGg?ti|$#R)ar zi;lq5(W%m3X_!$+!oi->h`wTGw|1NXS3UNl@0gtJzz-=SkW8q7GW`xUbwox15xKXk z4(w*@D^=Zgdt+lk_s*n0u^F#DE$lO(+B;Swlkx7Rzv*Ddvm4df47_9N7F-VaJ1n@o z4HyFip4)2C=sbE1qUsH!PP_G5W#y}>;u3du5u={HHr_4wXWZ5@&QkVeu9PCIO%LO$ zoE@j*s_`%3DARqoBsD|lmi9NQXRDgNK??|PcQ$!dK8RA^7%M(weSezr&p1AS< zTz`;wt@)ZOIb)mhf=(V?_-O^leRd3vcgqCRy`$Fm^wuvJVpkf1i<26EexIa?o`|$)N>ui?fR8 zl!Cb-9b#{2W=2^ttvex2uX(`nPg08jE!fu&N|Vx@fWsBOD4R%KRq05onarXd8^doB zQpV~mcnpa3FE4(eW}OyZ7^yOL$hrdOj$k8XXVUT2YPK&ahk!tH`Jth=D<5vt5($m# z`L|N(Lq~#`6r%>Wsb6|eL;p)yU{jFi=@Xl}-~8w-kz2leDIYjMU36hi!y_ao=jN7{ zp6|HNqBm5&{h+H)ui~NU9M*z666ye%=d!g!8rZmZGu#lms{aPDm;^;5@$O<9j z@^Z5I^dao_{sz|daDM|hfxwz7@_hS?ah1LWpPDCWh3X+20x4abf?w8>EylQnw;G`c zBLTzlKQrq={HTXeL+rSRFb@{hh2hpM*T2{%1Jeu0M?i8GNR7-#6qgw*V~pu}q$#(k z6^))%#Y({=bN+7LI6V%Xwg1&Cfwm8VB*V;Uyn8(>pfLitBjPO@6o)E(iy)>WuV#rP`8+|~#IHf)~lm7Lfr?4H) z-h-J3NPS^EXt7)#L#R87d8J{yKA`wW=QuMrY%gw;QH!RayAXLPtrOPf_o?{B%+lS< z-NT5Y3v?V1eb*tcq9XPFGn02hEjvuk>Cr3asq=4KfNLW+vNQP( zwgy8`a02;Uc*O8}LS1mlV?uqmE2#^kZooqJArToZJN$O_m9uACLwoJ_A0THEs_1l* zxSmHQ(j@#c`u;lK9Qe_jQ6RbomE3j(?(g87{`VxnS+}0-T+!0fuZ-R@(x0BM*i7K} z*dF~T8?_twfxrs9QzEe^{E2(}4D@I_6%3^hov)M>-@OZaE3Y=3_Q7F6S}B`{C;c-g zJeB+?Wek!9t>l&{eBQj{ES#r2=_dg_sf6yG9Z(X`ui=igxMQEWV6$W-?@huDv#d`S zbEq8uoYxTGj5?Yb5k883(trE+;Ob$yr|XMfe(!~rNirc}!$I^OTRNv;Q}fb{aU1_o zta`pg;PvT_U%vASeN9)ZlJq&Z1E!mUgMC@_8+2<3(aK)U)sbFR2K+X3Kb2VM-mGpr zY~8DFJ2j7ZSZN9y(jjp;`%xY>A13BuZ)m8AXW0-<#)5Qnu4mxr`!)ft4O3!y{;9g% zz{XIdsa5FfAJ~G6uit)H{*sfEL)8n|Xo=L;@e9f^hEWALRqQ|@@{6Sey~{g4^{kFd z{prMQg6!55R00^mXzBrFE?;Amb2sq=f7nP{2X6=j@N$1k>0E% z%z_Ccq?;j0?Bo%|MtvX)5dvT6;}L?dC-U!m$X+JFrk`z*p(f94Q_-lx8O9NOTnXJ1<4ln1ZA>Avvzfd#7)F50o z+sPjxTra+dqeHkrSq!--M!Nj<)7Og`Mm=VVb>Ic$uV|MSDhZE_MosJH&9RCU9Yvp% zmp0EYy0CVm(WJe9btNnP{b*80L;P$~$1lI(;k&wfJ&~0as81oVhzxu0rZxnZS#$_! znIc0#iySqY;ziu>@2`6j z)p99F?Qjq~FX-Taph~Bf;mPcnqLzflJ41zaI#= z+`k{h3B~{*UjUmuH~!5+4%8xvSuwRpxghPyNB!^s@t4xMa)qn*fP~eJx4BaP$ZDix zwLXMcVqb>kaxfo;Xd#xRo&&dlV`YjW7{%BBt5_i1_pHX{?1`Ez&sAI)J&>in)mMyu z#j8;-&LLA3(kP2#q$Rhv=;-~|lr$-AYS8Fzqw&(m+0OGcu?hO*iJNd}r9p!;8C_n$ z$pA}!073Isp8sLX*5i&5sL<_yp9V+l{&GD@^1hi9THD##na6FpWhW{*akC#B4YCR0&tJi_KbPKV;EF>B;8GIXPp?5;ecBV;d=Qi zjcvE1qD!&2=K=*jdTErjwUwd^SygOpojw~Cu!JSH*E36w>>#VC7#r7nKL2wxVxc8* zy+hMRSkR2cMr2hq+-45sl zN+vYKv+awP4y*U4^W=X2LuHjZL$L(%!jtm_dHLV^mLqwf6!ex;5luSI9T6gHhncBqjBcOQrbtz3uCDj~Fn6!oyIyRvs|b(7 zoyrz%t#jCNSMWoSCA^{>KX$WQhBk#y1E?6lLG0!o5&JvDJHm=V){%<8d;I2hV# z*)G7mK`@Sc*miqX!^Ffv`sLNc=HUGE&6_u@GKQT2S0|f;#1a05kUeaE)JL4ykX~`P$y5GeMDu+8hr!Q2f-rYW65oxo{UQ)VcNd8 zUOOD%^wat&$hY%wg{25q*idlrl?pNcP1w*w)sNtUzzDVg5+px%V;< zHAgb|9?;^q(jA|N{xZt3FV)YHJO(*D#w)m?6$e>TDtZSjf(x$XNFNrTq zh*A$5_XCil`}2z?BnQup6v|b&I%`_q4NP2)GMIss!`WT7FvDOSlG>+7Rk11pgouK({Uf%Z3v2hUyDd%Rj2)Xi+Nk?-j!RR{f5d&91=MI={2% z6#5Sjp7NjxoQrQ>M z@>Xr8Biyu_(-BAMk_yzZ$d3ek)~c0?dFkd9{7)uPye0w@-b`ZP7A-g0sBSTINHP2`>&Dogn9j zXU?jP57I9kf#W^gg}m8~ZnN5~cwY%=7!KRMF*4FJ(qbFkNJ^BGR{)~I0y<5;1g&0U zP&sW8g06ryopctKVgX#qoGgb1#EI$zV;<tMFJBg5 z^64Fqei`vU?A&sJT!Lp0p;gjGSCcI1(Z|<1Obw}Nr-7eXjI`(i|2AamCSkT(!9CW% zhM_~*Dbh*H0s(Y%v54bzxRn|*@Tps}GleRus3RA_XLIT{pp=kI!fB@Z;pr$TC#tb9 zwRl|~;D=&a<~$dv8A+lXeA*Rw-ZX;fP|Et~S|Zk1U3$CNRh-1;7gP4zA!&NaRvlp~ z!Da&ssWOap$$RV36-&u}c5j8d26Se|Q3gd@JHbgLUKdfH(XswnN>?Hhpk+hPKs5KB zTA9HMhwS$=4vgmu6nOo)Kl%Euu21n*Vdx z6c3N=Q)(^lPqtG^m{)$_4n8#vB9``znj6X|VtyW<$ZZnxYGQ3IEG(p4XMJtRc4QK^ zZ67B;Wm&r4B2gceB30)Q!;S0HaIvy`A(2^X z-ADMVtIUKoR@|8(L3e;bj1nEU4y6)U@E$y}`@KKW=Y)EFfr~xW#FRrJ4mPH!!7Aqn zBY`=cYNgW~vP?bxM4ry&DWCLXwjYsiE<~1ZEZqIr`;j%qdG*af4)-RuQD>a?P;n@P z>{n>J;Vl+toB?{o!5;bVT7rYE-}dzxjst5}hR~P`TG08|aAfYGw0E9**eQP z?hdjLAkkjOhJfy+)`N(4ESDsRMSBbl^{TnhMnchYU~!fz()%AVc{2VzX^Z^aak%~UCCj+IaEDJwE($Yp96*AvJS3F`H-#6PI7yk6C`9^t}N=L&40UEA0 zl(lq07(4OM0BIx5{3e!2H}w(5@%l~AMGUOy;+1E@#82-5c!^21!9V}hBYW6JJ8h|r zw-m5rLp<8lG2BPqv@C`(g)jp?sgYK<3z_c^rH+{s*`xlXBrJ%}2i>+u$)K0Qcb*}oPwENR zdO;MB_NhHXD%j*=;?aj6=UH3%B*waC?F43-$+lDvMxku@?B-xPr$y(ppy~`- z@iklNwMX0^eFaiW@Zua^FXKO1Xw!XNzM>uKl^ukfA5+SjEs#Lz`V~P#Qrt%A^%1|M zEz;sjr^0O7PtWS9Gad9HRC%?MI+AA}X+i+S|nvlNpn6d>}(j0pe7QfuQEL%iJ4AS4(*PW`9J> zZ|(|qM#e=KhV+Ma~urw=SCwjkcR3 z8Sf;n%7dmw^5f(7ysIC$zOp@ooXbAe_N;j1uWF-jL~E$OZzb2rR$Yl#b<$d2mgWat zzk>K&ZKhkG*7Mx$JVJl7TKru#BE3KEdsMmx({<^o#!i0W-8NIhk+1x-e6d2-fC&xQ z1WY2pfV@Cauu6vk*~t@a zX4e?+rj9o3m7t!*lPOXt4Ydv-b^;iF=3RNxn@#~Fd6zP3+tY4SGA?>%ab-1-Nq_z&#ikmLsfXfEM^Ha4yE9Le z7K6>rKR`HGaLkaWxl8yt?M{Aq4U|@x)xOMyXcf`Z-(g%aOjXiThI{uG4WeOvF%FfiZdpZTUmQS0ao8>pu_{Wd z@kEvH3B0~!?yDyLF1F~F&O{6E6f}iRo#r&21K~w0$qB^Bgm%S@-=#uiq1cg>Tiwq> zRtxHZUl^D=X*Hd{fGrXU#|AH=ES6Q^hJ$+>5}iRcv4^DgR5;M$SRzen7LERb-X_e; zdk0^|L)OtX=1~$5q~0?ZzldE(TLb;U%E;}3YjH6L%a`C_PdyW`n<#p8jyE_+C3Xtq zvxTH>e0*|SNzsk58AvYJn#A|oYLvFq$YGL5UNM}|Cd{q21?b25V@|Yu2{o~w2z4_H z$S7c4H4us7jtXm|Y@vXp=^!+MVSLWfM1|zco5c;NHQ-2qv z=q(56SS&+v_LofbrlwJI#EcAPpg1`aOPVk*q^;lld_YfFRn_k7--o@f^F6y-pjRh} zhVulNAU?MG%)c!Ms)H33Jc)gB2BRgxaQ~B{Sl|x!m()BNJ?JfWNkee0IkL&ys&D!l zDzc~jb0+ii&9Arq2ne(;Nd*Pzq%L5awv5m3dLc0}v9re`ynlZS`=w%D^u7AdvIVHS z5EazcE)6eA+1jeb&mrD!1>;atP&wW1O_#w*dbUidwMMNAdoCN{SOLZhs*7ruHO?tQsFjd3b3Bh& zZPO7^WZz}J-$}x|`Q%d^75O6s3@KkDAjns2UH**lQmk8n5!!ioeZIHA`TEfP2L{hK z#a=FbczavXg+u*TtC5P8jFk*M9yN%%aASANn>%6x%hvoI5MI?-)Z;rvpQ!-WF#NO!qomH3FyINhuW1_D<5;{C-0u6y8zrG{@re7A-$&lr{NM?Bvc$a?Kt z>$}o%%M+AWfAIURV_1G&*cWKqay=ni@tj!05JYEh%0Klf@gRbNuHFD0EcH-E!xHFa zeam)gp1zHoLrZuY*JlW~cRoIN3GJZBE09$Fbg1>CckFC=lmWB?&h1~Vd5LM?c^Zt~ zsp_Gp(HC@yJ~lBi2BIHrHp%!HB{V3x{HKu%vv*Pmu}#>oL?3bYmG{HSj1E%SJ-Y04 z7qor~-I#8Q)mvF{sJa^ZJ>Dwk6mN5azn)tSE+#pJ*@eVsm^I3rY!9>A4wa1XRmXQz?XO823#+ z(N-eUr-(Y#?BB5=gS^!T6lB&^F^FrQi>RLNp!pv1ok2- zp&!Z+Jf+?py^|CmwPourWI!>pvv*IveKWwpr^zg*%x<-8M8nbCgBbC3w{CWICe+TbG#za%m&GtsumW3v@|o-f7e_FE)k92vtP+I9)=vP*a}$HU z+0M)uJBCTIk1+fQnAa^rXJc&lNK2{yBnbQ^h6g5vwFaU7wo-2N%me(z5Dc;_ho_i- z4+;O8%VM{Un_J7Y8K2U6-kE%f2)v~FM_uJUd{r~*wpPw^jE@Y|in3$N%Xi*vp9#3H z9uCm7__Dl^lz=~QVe*hS^sss1@lwRv?Io~*@Nn_)v2$^=bMxqO3yN^_i}3P2<>C?n lzP_uD{y#f7yV|_6_5J_vpql^J3D^OmEdN}tTE;T${{Y!r{;~i7 diff --git a/docs/assets/images/logo.svg b/docs/assets/images/logo.svg deleted file mode 100644 index fa9e6f4..0000000 --- a/docs/assets/images/logo.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..2732bb6 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,56 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "scoringrules" +author = "scoringrules contributors" +copyright = "2024" +release = "0.7.0" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "myst_parser", + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "numpydoc", +] + +myst_enable_extensions = ["dollarmath"] + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] +suppress_warnings = [ + "image.not_readable", +] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "sphinx_book_theme" +html_static_path = ["_static"] +html_theme_options = { + "repository_url": "https://github.com/frazane/scoringrules", + "use_repository_button": True, + "show_toc_level": 3, + "logo": { + "image_light": "_static/banner_light.svg", + "image_dark": "_static/banner_dark.svg", + }, + "pygments_light_style": "lovelace", + "pygments_dark_style": "monokai", + # toc + "collapse_navigation": True, + "sticky_navigation": True, + "navigation_depth": 4, + "includehidden": False, + "titles_only": False, +} +html_title = "scoringrules" +html_favicon = "_static/favicon.ico" diff --git a/docs/extras/estimators.md b/docs/extras/estimators.md deleted file mode 100644 index 4a82417..0000000 --- a/docs/extras/estimators.md +++ /dev/null @@ -1,34 +0,0 @@ -### Integral form (INT) - -The numerical approximation of the cumulative integral over the finite ensemble. - -$$ \text{CRPS}_{\text{INT}}(M, y) = \int_{\mathbb{R}} \left[ \frac{1}{M} -\sum_{i=1}^M \mathbb{1}\{x_i \le x \} - \mathbb{1}\{y \le x\} \right] ^2 dx $$ - -Runs with $O(m\cdot\mathrm{log}m)$ complexity, including the sorting of the ensemble. - -### Energy form (NRG) - -Introduced by Gneiting and Raftery (2007)[@gneiting_strictly_2007]: - -$$ \text{CRPS}_{\text{NRG}}(M, y) = \frac{1}{M} \sum_{i=1}^{M}|x_i - y| - \frac{1}{2 M^2}\sum_{i,j=1}^{M}|x_i - x_j|$$ - - It is called the "energy form" because it is the one-dimensional case of the Energy Score. - -Runs with $O(m^2)$ complexity. - -### Quantile decomposition form (QD) - -Introduced by Jordan (2016)[@jordan_facets_2016]: - -$$\mathrm{CRPS}_{\mathrm{QD}}(M, y) = \frac{2}{M^2} \sum_{i=1}^{M}(x_i - y)\left[M\mathbb{1}\{y \le x_i\} - i + \frac{1}{2} \right]$$ - -Runs with $O(m\cdot\mathrm{log}m)$ complexity, including the sorting of the ensemble. - -### Probability weighted moment form (PWM) - -Introduced by Taillardat et al. (2016)[@taillardat_calibrated_2016]: - -$$\mathrm{CRPS}_{\mathrm{NRG}}(M, y) = \frac{1}{M} \sum_{i=1}^{M}|x_i - y| + \hat{\beta_0} - 2\hat{\beta_1},$$ - -where $\hat{\beta_0} = \frac{1}{M} \sum_{i=1}^{M}x_i$ and $\hat{\beta_1} = \frac{1}{M(M-1)} \sum_{i=1}^{M}(i - 1)x_i$. Runs with $O(m\cdot\mathrm{log}m)$ complexity, including the sorting of the ensemble. diff --git a/docs/index.md b/docs/index.md index 85247d7..9c58622 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,17 +1,10 @@ -# -

- -

-

- -

- +# scoringrules documentation

+`scoringrules` is a lightweight python library that provides scoring rules to evaluate probabilistic forecasts. -`scoringrules` is a python library that provides scoring rules to evaluate probabilistic forecasts. It's original goal was to reproduce the functionality of the R package [`scoringRules`](https://cran.r-project.org/web/packages/scoringRules/index.html) in python, thereby allowing forecasting practitioners working in python to enjoy the same tools as those @@ -34,6 +27,26 @@ The scoring rules available in `scoringrules` include, but are not limited to, t - Interval Score + + +```{toctree} +:hidden: +:caption: Background + +theory.md +``` + +```{toctree} +:hidden: +:caption: Library + +user_guide.md +contributing.md +reference.md +``` + + + ## Features - **Fast** computation of several probabilistic univariate and multivariate verification metrics diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..954237b --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/reference.md b/docs/reference.md new file mode 100644 index 0000000..d70c0c2 --- /dev/null +++ b/docs/reference.md @@ -0,0 +1,137 @@ +# API reference + +This page provides a summary of scoringrules' API. All functions are +available in the top-level namespace of the package and are here +organized by category. + +```{eval-rst} +.. currentmodule:: scoringrules + +Ensemble forecasts +================== + +Univariate +------------------ + +.. autosummary:: + :toctree: generated + + crps_ensemble + twcrps_ensemble + owcrps_ensemble + vrcrps_ensemble + gksuv_ensemble + twgksuv_ensemble + owgksuv_ensemble + vrgksuv_ensemble + + error_spread_score + +.. Multivariate +.. -------------------- + +.. .. autosummary:: +.. :toctree: generated + +.. energy_score +.. twenergy_score +.. owenergy_score +.. vrenergy_score +.. variogram_score +.. owvariogram_score +.. twvariogram_score +.. vrvariogram_score +.. gksmv_ensemble +.. twgksmv_ensemble +.. owgksmv_ensemble +.. vrgksmv_ensemble + +.. Parametric distributions forecasts +.. ==================================== +.. .. autosummary:: +.. :toctree: generated + +.. crps_beta +.. crps_binomial +.. crps_exponential +.. crps_exponentialM +.. crps_2pexponential +.. crps_gamma +.. crps_gev +.. crps_gpd +.. crps_gtclogistic +.. crps_tlogistic +.. crps_clogistic +.. crps_gtcnormal +.. crps_tnormal +.. crps_cnormal +.. crps_gtct +.. crps_tt +.. crps_ct +.. crps_hypergeometric +.. crps_laplace +.. crps_logistic +.. crps_loglaplace +.. crps_loglogistic +.. crps_lognormal +.. crps_mixnorm +.. crps_negbinom +.. crps_normal +.. crps_2pnormal +.. crps_poisson +.. crps_quantile +.. crps_t +.. crps_uniform +.. logs_beta +.. logs_binomial +.. logs_ensemble +.. logs_exponential +.. logs_exponential2 +.. logs_2pexponential +.. logs_gamma +.. logs_gev +.. logs_gpd +.. logs_hypergeometric +.. logs_laplace +.. logs_loglaplace +.. logs_logistic +.. logs_loglogistic +.. logs_lognormal +.. logs_mixnorm +.. logs_negbinom +.. logs_normal +.. logs_2pnormal +.. logs_poisson +.. logs_t +.. logs_tlogistic +.. logs_tnormal +.. logs_tt +.. logs_uniform + +.. Functions of distributions +.. ========================== +.. .. autosummary:: +.. :toctree: generated + +.. interval_score +.. weighted_interval_score + + +.. Categorical forecasts +.. ================================= +.. .. autosummary:: +.. :toctree: generated + +.. brier_score +.. rps_score +.. log_score +.. rls_score + + +.. Backends +.. ======== +.. .. autosummary:: +.. :toctree: generated + +.. register_backend +``` diff --git a/docs/theory.md b/docs/theory.md index 907b7fb..1ae12df 100644 --- a/docs/theory.md +++ b/docs/theory.md @@ -1,7 +1,3 @@ -#################### -Proper scoring rules -#################### - # The theory of proper scoring rules ## Definitions @@ -27,9 +23,9 @@ be modified to treat scoring rules that are positively oriented. It is widely accepted that scoring rules should be *proper*. A scoring rule $S$ is proper (with respect to $\mathcal{F}$) if, when $Y \sim G$, -\[ +$$ \mathbb{E} S(G, Y) \leq \mathbb{E} S(F, Y) \quad \text{for all $F, G \in \mathcal{F}$}. -\] +$$ That is, the score is minimised in expectation by the true distribution underlying the outcome $Y$. Put differently, if we believe that the outcome arises according to distribution $G$, then @@ -49,19 +45,19 @@ rules include the *Brier score* and *Logarithmic (Log) score*. The Brier score is defined as -\[ +$$ \mathrm{BS}(F, y) = (F - y)^{2}, -\] +$$ while the Log score is defined as -\[ +$$ \mathrm{LS}(F, y) = -\log |F + y - 1| = \begin{cases} -\log F & \text{if} \quad y = 1, \\ -\log (1 - F) & \text{if} \quad y = 0. \end{cases} -\] +$$ Other popular binary scoring rules include the spherical score, power score, and pseudo-spherical score. @@ -73,9 +69,9 @@ Suppose now that $Y$ is a categorical variable, taking values in $\Omega = \{1, for $K > 1$. Binary outcomes constitute a particular case when $K = 2$. A probabilistic forecast for $Y$ is a vector -\[ +$$ F = (F_{1}, \dots, F_{K}) \in [0, 1]^{K} \quad \text{such that} \quad \sum_{i=1}^{K} F_{i} = 1. -\] +$$ The $i$-th element of $F$ represents the probability that $Y = i$, for $i = 1, \dots, K$. @@ -85,9 +81,9 @@ and then summing these $K$ scores. For example, the Brier score becomes -\[ +$$ \mathrm{BS}_{K}(F, y) = \sum_{i=1}^{K} (F_{i} - \mathbf{1}\{y = i\})^{2}. -\] +$$ where $\mathbf{1}\{ \cdotp \}$ denotes the indicator function. When $K = 2$, we recover the Brier score for binary outcomes (multiplied by a factor of 2). @@ -95,9 +91,9 @@ When $K = 2$, we recover the Brier score for binary outcomes (multiplied by a fa The Log score can similarly be extended, though it is more common to define the Log score for categorical outcomes as -\[ +$$ \mathrm{LS}(F, y) = -\log F_{y}. -\] +$$ As in the binary case, this Log score evaluates the forecast $F$ only via the probability assigned to the outcome that occurs. @@ -113,24 +109,26 @@ $F^{(2)}$ assigns all probability to the second category, and the second categor To account for the ordering of the categories, it is common to apply the Brier score and Log score to cumulative probabilities. Let +$$ \begin{align} \tilde{F} &= (\tilde{F}_{1}, \dots, \tilde{F}_{K}) \in [0, 1]^{K} \quad \text{with $\tilde{F}_{j} = \sum_{i=1}^{j} F_{i}$} \quad &\text{for $j = 1, \dots, K$,} \\ \tilde{y} &= (\tilde{y}_{1}, \dots, \tilde{y}_{K}) \in \{0, 1\}^{K} \quad \text{with $\tilde{y}_{j} = \sum_{i=1}^{j} \mathbf{1}\{y = i\}$} \quad &\text{for $j = 1, \dots, K$.} \end{align} +$$ Then, the *Ranked Probability Score (RPS)* is defined as -\[ +$$ \mathrm{RPS}(F, y) = \sum_{i=1}^{K} (\tilde{F}_{j} - \tilde{y}_{j})^{2}, -\] +$$ and the *Ranked Logarithmic Score (RLS)* is -\[ +$$ \mathrm{RLS}(F, y) = - \sum_{i=1}^{K} \log | \tilde{F}_{j} + \tilde{y}_{j} - 1|. -\] +$$ These categorical scoring rules can also be implemented when $K = \infty$, as is the case for unbounded count data, for example. Other scoring rules for binary @@ -147,10 +145,12 @@ for binary outcomes. The *Continuous Ranked Probability Score (CRPS)* is defined as +$$ \begin{align*} \mathrm{CRPS}(F, y) &= \int_{-\infty}^{\infty} (F(x) - \mathbf{1}\{y \le x\})^{2} dx \\ &= \mathbb{E} | X - y | - \frac{1}{2} \mathbb{E} | X - X^{\prime} |, \end{align*} +$$ where $X, X^{\prime} \sim F$ are independent. The CRPS is defined as the Brier score for threshold exceedance forecasts integrated over all @@ -159,17 +159,17 @@ infinite possible categories. Similarly, the *Continuous Ranked Logarithmic Score (CRLS)* is defined as -\[ +$$ \mathrm{CRLS}(F, y) = -\int_{-\infty}^{\infty} \log |F(x) + \mathbf{1}\{y \le x\} - 1| dx. -\] +$$ The Log score can additionally be generalised to continuous outcomes whenever the forecast $F$ has a density function, denoted here by $f$. In this case, the Log score is defined as -\[ +$$ \mathrm{LS}(F, y) = - \log f(y). -\] +$$ The Log score again depends only on the forecast distribution $F$ at the observation $y$, ignoring the probability density assigned to other outcomes. This scoring rule is therefore @@ -177,9 +177,9 @@ ignoring the probability density assigned to other outcomes. This scoring rule i When $F$ is a normal distribution, the Log score simplifies to the *Dawid-Sebastiani* score, -\[ +$$ \mathrm{DS}(F, y) = \frac{(y - \mu_{F})^{2}}{\sigma_{F}^{2}} + 2 \log \sigma_{F}, -\] +$$ where $\mu_{F}$ and $\sigma_{F}$ represent the mean and standard deviation of the forecast distribution $F$. While the Dawid-Sebastiani score corresponds to the Log score for a normal distribution, @@ -196,18 +196,18 @@ Let $\boldsymbol{Y} \in \mathbb{R}^{d}$, with $d > 1$, and suppose $F$ is a mult If $F$ admits a multivariate density function $f$, then the Log score can be defined analogously to in the univariate case, -\[ +$$ \mathrm{LS}(F, \boldsymbol{y}) = - \log f(\boldsymbol{y}). -\] +$$ The Dawid-Sebastiani score can similarly be extended to higher dimensions by replacing the predictive mean $\mu_{F}$ and variance $\sigma_{F}^{2}$ with the mean vector $\boldsymbol{\mu}_{F} \in \mathbb{R}^{d}$ and covariance matrix $\Sigma_{F} \in \mathbb{R}^{d \times d}$. This becomes -\[ +$$ \mathrm{DS}(F, \boldsymbol{y}) = (\boldsymbol{y} - \boldsymbol{\mu}_{F})^{\top} \Sigma_{F}^{-1} (\boldsymbol{y} - \boldsymbol{\mu}_{F}) + \log \det(\Sigma_{F}), -\] +$$ where $\top$ denotes the vector transpose, and $\det$ the matrix determinant. The Dawid-Sebastiani score is equivalent to the Log score for a multivariate normal distribution. However, the @@ -218,9 +218,9 @@ or must be estimated from a finite sample. Instead, it is common to evaluate multivariate forecasts using the *Energy score*, -\[ +$$ \mathrm{ES}(F, \boldsymbol{y}) = \mathbb{E} \| \boldsymbol{X} - \boldsymbol{y} \| - \frac{1}{2} \mathbb{E} \| \boldsymbol{X} - \boldsymbol{X}^{\prime} \|, -\] +$$ where $\boldsymbol{X}, \boldsymbol{X}^{\prime} \sim F$ are independent, and $\| \cdot \|$ is the Euclidean distance on $\mathbb{R}^{d}$. The Energy score can be interpreted as a multivariate generalisation of the CRPS, which is recovered when $d = 1$. @@ -231,9 +231,9 @@ The *Variogram score* was introduced as an alternative scoring rule that is less univariate forecast performance, thereby focusing on the multivariate dependence structure. The Variogram score is defined as -\[ +$$ \mathrm{VS}(F, \boldsymbol{y}) = \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left( \mathbb{E} | X_{i} - X_{j} |^{p} - | y_{i} - y_{j} |^{p} \right)^{2}, -\] +$$ where $\boldsymbol{X} \sim F$, and $h_{i,j} \ge 0$ are weights assigned to different pairs of dimensions. The Variogram score therefore measures the distance between between the expected variogram of @@ -253,9 +253,9 @@ novel scoring rules that incorporate their personal preferences. Schervish (1969) demonstrated that all proper scoring rules for binary outcomes can be written in the following form (up to unimportant technical details): -\[ +$$ S(F, y) = \int_{(0, 1)} c 1\{F > c, y = 0\} + (1 - c) 1\{F < c, y = 1\} + c(1 - c) 1\{F = c\} d \nu(c), -\] +$$ for some non-negative measure $\nu$ on $(0, 1)$. The measure $\nu$ emphasises particular trade-offs between over-prediction ($F$ high, $y = 0$) and under-prediction ($F$ low, $y = 1$). @@ -274,9 +274,9 @@ trade-off parameters. More generally, for categorical forecasts, a scoring rule is proper if and only if there exists a convex function $g : [0, 1]^{K} \to \mathbb{R}$ such that -\[ +$$ S(F, y) = \langle g^{\prime}(F), F \rangle - g(F) - g^{\prime}(F)_{y} \quad \text{for all $F \in [0, 1]^{K}$ and $y \in \{1, \dots, K\}$,} -\] +$$ where $g^{\prime}(F) \in \mathbb{R}^{K}$ is a sub-gradient of $g$ at $F \in \mathcal{F}$, and $g^{\prime}(F)_{y}$ is the $y$-th element of this vector, for $y \in \{1, \dots, K\}$. @@ -290,9 +290,9 @@ Gneiting and Raftery (2007) generalised this to arbitrary outcome domains by sho a scoring rule $S$ is proper relative to the set $\mathcal{F}$ if and only if there exists a convex function $g : \Omega \to \mathbb{R}$ such that -\[ +$$ S(F, y) = \int g^{\prime}(F, z) dF(z) - g(F) - g^{\prime}(F, y) \quad \text{for all $F \in \mathcal{F}$ and $y \in \Omega$,} -\] +$$ where $g^{\prime}(F, \cdot)$ is a subtangent of $g$ at $F$ (Gneiting and Raftery, 2007, Theorem 1). The class of strictly proper scoring rules with respect to $\mathcal{F}$ is characterised by @@ -314,9 +314,9 @@ assuming the forecast distribution $F$ admits a density function $f$ whose deriv $f^{\prime}, f^{\prime \prime}, \dots, f^{(j)}$ exist, a scoring rule $S$ is called *local of order $j$* if there exists a function $s : \mathbb{R}^{2+j} \to \overline{\mathbb{R}}$ such that -\[ +$$ S(F, y) = s(y, f(y), f^{\prime}(y), \dots, f^{(j)}(y)). -\] +$$ That is, the scoring rule $S$ can be written as a function of $y$ and the first $j$ derivatives of the forecast distribution evaluated at $y$. @@ -336,9 +336,9 @@ and to integrate this over all thresholds. That is, if $S_{0}$ is a proper scoring rule for binary outcomes, then -\[ +$$ S(F, y) = \int_{-\infty}^{\infty} S_{0}(F(x), \mathbf{1}\{y \leq x\}) dH(x), -\] +$$ is a proper scoring rule for univariate real-valued outcomes, for some non-negative measure $H$ on the real line. @@ -362,9 +362,9 @@ Many popular scoring rules belong to the very general class of *kernel scores*, scoring rules defined in terms of positive definite kernels. A positive definite kernel on $\Omega$ is a symmetric function $k : \Omega \times \Omega \to \mathbb{R}$, such that -\[ +$$ \sum_{i=1}^{n} \sum_{j=1}^{n} a_{i} a_{j} k(x_{i}, x_{j}) \geq 0, -\] +$$ for all $n \in \mathbb{N}$, $a_{i}, a_{j} \in \mathbb{R}$, and $x_{i}, x_{j} \in \Omega$. A positive definite kernel constitutes an inner product in a feature space, and can therefore @@ -372,9 +372,9 @@ be loosely interpreted as a measure of similarity between its two inputs. The *kernel score* corresponding to the positive definite kernel $k$ is defined as -\[ +$$ S_{k}(F, y) = \frac{1}{2} \mathbb{E} k(y, y) + \frac{1}{2} \mathbb{E} k(X, X^{\prime}) - \mathbb{E} k(X, y), -\] +$$ where $X, X^{\prime} \sim F$ are independent. The first term on the right-hand-side does not depend on $F$, so could be removed, but is included here to ensure that the kernel score is always @@ -382,18 +382,18 @@ non-negative, and to retain an interpretation in terms of Maximum Mean Discrepan The energy score is the kernel score associated with any kernel -\[ +$$ k(x, x^{\prime}) = \| x - x_{0} \| + \| x^{\prime} - x_{0} \| - \| x - x^{\prime} \| -\] +$$ for arbitrary $x_{0} \in \Omega$, which encompasses the CRPS when $d = 1$. We refer to these kernels as *energy kernels*. Another popular kernel is the Gaussian kernel, -\[ +$$ k_{\gamma}(x, x^{\prime}) = \exp \left( - \gamma \| x - x^{\prime} \|^{2} \right) -\] +$$ for some length-scale parameter $\gamma > 0$. Plugging this into the kernel score definition above yields the *Gaussian kernel score*. @@ -410,9 +410,9 @@ distributions. Aronszajn (195) demonstrated that every positive definite kernel a Hilbert space of functions, referred to as a Reproducing Kernel Hilbert Space and denoted by $\mathcal{H}_{k}$. A probability distribution $F$ on $\Omega$ can be converted to an element in an RKHS via its kernel mean embedding, -\[ +$$ \mu_{F} = \int_{\Omega} k(x, \cdotp) dF(x), -\] +$$ allowing kernel methods to be applied to probabilistic forecasts. @@ -421,9 +421,9 @@ the RKHS norm $\| \cdot \|_{\mathcal{H}_{k}}$ betewen their kernel mean embeddin $\| \mu_{F} - \mu_{G} \|_{\mathcal{H}_{k}}$. This is called the *Maximum Mean Discrepancy (MMD)* between $F$ and $G$. In practice, it is common to calculate the squared MMD, which can be expressed as -\[ +$$ \| \mu_{F} - \mu_{G} \|_{\mathcal{H}_{k}}^{2} = \mathbb{E} k(Y, Y^{\prime}) + \mathbb{E} k(X, X^{\prime}) - 2\mathbb{E} k(X, Y), -\] +$$ where $X, X^{\prime} \sim F$ and $Y, Y^{\prime} \sim G$ are independent. @@ -472,16 +472,16 @@ observation. Evaluating censored forecast distributions was also proposed by Diks et al. (2011) when introducing weighted versions of the Log score. They introduce the *censored likelihood score* as -\[ +$$ \mathrm{ceLS}(F, y; w) = - w(y) \log f(y) - (1 - w(y)) \log \left( 1 - \int_{\mathbb{R}} w(z) f(z) dz \right). -\] +$$ Rather than censoring the distribution, Diks et al. (2011) additionally propose the *conditional likelihood score*, which evaluates the conditional distribution given the weight function, -\[ +$$ \mathrm{coLS}(F, y; w) = - w(y) \log f(y) + w(y) \log \left( \int_{\mathbb{R}} w(z) f(z) dz \right). -\] +$$ @@ -498,9 +498,9 @@ These forecasts can be evaluated using *scoring functions*. Scoring functions ar in essence to scoring rules, but they take a point-valued rather than probabilistic forecast as input, -\[ +$$ s : \Omega \times \Omega \to [0, \infty]. -\] +$$ We use a lower case $s$ to distinguish scoring functions from scoring rules. @@ -510,34 +510,34 @@ the forecast is a functional $\text{T}$ of a predictive distribution. For exampl be the mean, median, or quantile. A scoring rule is called *consistent* for the functional $\text{T}$ (relative to a class of distributions $\mathcal{F}$), if, when $Y \sim G$, -\[ +$$ \mathbb{E} s(x_{G}, Y) \leq \mathbb{E} s(x, Y) \quad \text{for all $x \in \Omega, G \in \mathcal{F}$,} -\] +$$ where $x_{G} \in \text{T}(G)$. For example, the *squared error* -\[ +$$ s(x, y) = (x - y)^{2} -\] +$$ is consistent for the mean functional, and the *quantile score* (also called *pinball loss* or *check loss*) -\[ +$$ s_{\alpha}(x, y) = (\mathbf{1}\{y \leq x\} - \alpha)(x - y) = \begin{cases} (1 - \alpha)|x - y| & \quad \text{if $y \leq x$}, \\ \phantom{(1 - }\alpha \phantom{)}|x - y| & \quad \text{if $y \geq x$} \end{cases} -\] +$$ is consistent for the $\alpha$-quantile (for $\alpha \in (0, 1)$). When $\alpha = 0.5$, we recover the *absolute error* -\[ +$$ s(x, y) = |x - y|, -\] +$$ which is consistent for the median functional. Note that these are not the only scoring rules that are consistent for the mean, median, and quantiles (see Gneiting 2010, Ehm et al., 2016). @@ -545,9 +545,9 @@ that are consistent for the mean, median, and quantiles (see Gneiting 2010, Ehm Proper scoring rules can be constructed using consistent scoring functions. In particular, if $s$ is a consistent scoring function for a functional $\text{T}$ (relative to $\mathcal{F}$), then -\[ +$$ S(F, y) = s(\text{T}(F), y) -\] +$$ is a proper scoring rule (relative to $\mathcal{F}$). Hence, the squared error, quantile loss, and absolute error above all induce proper scoring rules. @@ -558,6 +558,6 @@ This framework can additionally be used to evaluate interval forecasts, or predi Given a central interval forecast in the form of a lower and upper value $L, U \in \mathbb{R}$ with $L < U$, the interval score is defined as -\[ +$$ \mathrm{IS}([L, U], y) = |U - L| + \frac{2}{\alpha} (y - u) \mathbf{1} \{ y > u \} + \frac{2}{\alpha} (l - y) \mathbf{1} \{ y < l \}. -\] +$$ From 84d4c0fe66e58f2c1bc93395a9219fae44f88c15 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Thu, 16 Jan 2025 16:50:02 +0100 Subject: [PATCH 04/79] delete obsolete files --- docs/scripts/katex.js | 38 -------------------------- docs/stylesheets/extra.css | 56 -------------------------------------- 2 files changed, 94 deletions(-) delete mode 100644 docs/scripts/katex.js delete mode 100644 docs/stylesheets/extra.css diff --git a/docs/scripts/katex.js b/docs/scripts/katex.js deleted file mode 100644 index 35c15f1..0000000 --- a/docs/scripts/katex.js +++ /dev/null @@ -1,38 +0,0 @@ -(function () { - 'use strict'; - - var katexMath = (function () { - var maths = document.querySelectorAll('.arithmatex'), - tex; - - for (var i = 0; i < maths.length; i++) { - tex = maths[i].textContent || maths[i].innerText; - if (tex.startsWith('\\(') && tex.endsWith('\\)')) { - katex.render(tex.slice(2, -2), maths[i], {'displayMode': false}); - } else if (tex.startsWith('\\[') && tex.endsWith('\\]')) { - katex.render(tex.slice(2, -2), maths[i], {'displayMode': true}); - } - } - }); - - (function () { - var onReady = function onReady(fn) { - if (document.addEventListener) { - document.addEventListener("DOMContentLoaded", fn); - } else { - document.attachEvent("onreadystatechange", function () { - if (document.readyState === "interactive") { - fn(); - } - }); - } - }; - - onReady(function () { - if (typeof katex !== "undefined") { - katexMath(); - } - }); - })(); - - }()); diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css deleted file mode 100644 index 9dea9ae..0000000 --- a/docs/stylesheets/extra.css +++ /dev/null @@ -1,56 +0,0 @@ -:root { - --md-primary-fg-color: #d84a4a; - --md-primary-fg-color--light: #262680; - --md-primary-fg-color--dark: #0f3c23; -} - - -div.doc-contents { - padding-left: 25px; - border-left: 4px solid rgba(230, 230, 230); -} - - - -/* Highlight functions, classes etc. type signatures. Really helps to make clear where - one item ends and another begins. */ - -[data-md-color-scheme="default"] { - --doc-heading-color: #DDD; - --doc-heading-border-color: #CCC; - --doc-heading-color-alt: #F0F0F0; -} - -[data-md-color-scheme="slate"] { - --doc-heading-color: rgb(25, 25, 33); - --doc-heading-border-color: rgb(25, 25, 33); - --doc-heading-color-alt: rgb(33, 33, 44); - --md-code-bg-color: rgb(38, 38, 50); -} - -h4.doc-heading { - /* NOT var(--md-code-bg-color) as that's not visually distinct from other code blocks.*/ - background-color: var(--doc-heading-color); - border: solid var(--doc-heading-border-color); - border-width: 1.5pt; - border-radius: 2pt; - padding: 0pt 5pt 2pt 5pt; -} - -h5.doc-heading, -h6.heading { - background-color: var(--doc-heading-color-alt); - border-radius: 2pt; - padding: 0pt 5pt 2pt 5pt; -} - - -/* Maximum space for text block */ -.md-grid { - max-width: 85%; /* or 100%, if you want to stretch to full-width */ -} - - -.highlight .gp, .highlight .go { /* Generic.Prompt, Generic.Output */ - user-select: none; -} From e409203b74c3167c4fe3649f81e66116496dd5aa Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Thu, 16 Jan 2025 16:52:17 +0100 Subject: [PATCH 05/79] update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f3160e1..1332dc7 100644 --- a/.gitignore +++ b/.gitignore @@ -151,3 +151,4 @@ scratch_nbs/ _devlog/ tests/output .devcontainer/ +docs/generated From c5179afc109ec85675468576b268ca8f8339fcd0 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Thu, 16 Jan 2025 16:58:24 +0100 Subject: [PATCH 06/79] update docs requirements --- docs/requirements.txt | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 271dba0..f0694bd 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1 @@ -mkdocs -mkdocs-bibtex -mkdocs-material -mkdocs-section-index -mkdocstrings-python -black +myst-parser From 18b8859dc765448f0cc960d4ebcff793abe8a1f4 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Thu, 16 Jan 2025 16:59:54 +0100 Subject: [PATCH 07/79] use docs requirements file --- .readthedocs.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f72c324..22cfb7e 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -17,6 +17,6 @@ sphinx: # Optionally, but recommended, # declare the Python requirements required to build your documentation # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html -# python: -# install: -# - requirements: docs/requirements.txt +python: + install: + - requirements: docs/requirements.txt From d47f5144f44dce672499f3d85de89e2e5064f6c9 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Thu, 16 Jan 2025 17:05:20 +0100 Subject: [PATCH 08/79] update docs requirements --- docs/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index f0694bd..07f9045 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1 +1,3 @@ myst-parser +numpydoc +sphinx-book-theme From b1eaf7a5332cc3477802ae6b80380b12a750cdf4 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Thu, 16 Jan 2025 17:07:03 +0100 Subject: [PATCH 09/79] update docs requirements --- docs/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 07f9045..0166889 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ myst-parser numpydoc sphinx-book-theme +scoringrules From af0ec144f267ddb2dd10f8da243f0d000e2f14a5 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Fri, 17 Jan 2025 13:47:00 +0100 Subject: [PATCH 10/79] misc fixes and test cross-references --- docs/api/categorical.md | 28 ----------- docs/api/crps.md | 102 -------------------------------------- docs/api/energy.md | 87 -------------------------------- docs/api/error_spread.md | 14 ------ docs/api/interval.md | 5 -- docs/api/kernels.md | 17 ------- docs/api/logarithmic.md | 66 ------------------------ docs/api/quantile.md | 3 -- docs/api/variogram.md | 91 ---------------------------------- docs/api/visualization.md | 3 -- docs/conf.py | 3 -- docs/crps_estimators.md | 45 +++++++++++++++++ docs/index.md | 2 + docs/theory.md | 5 ++ scoringrules/_crps.py | 33 +++++++++--- 15 files changed, 78 insertions(+), 426 deletions(-) delete mode 100644 docs/api/categorical.md delete mode 100644 docs/api/crps.md delete mode 100644 docs/api/energy.md delete mode 100644 docs/api/error_spread.md delete mode 100644 docs/api/interval.md delete mode 100644 docs/api/kernels.md delete mode 100644 docs/api/logarithmic.md delete mode 100644 docs/api/quantile.md delete mode 100644 docs/api/variogram.md delete mode 100644 docs/api/visualization.md create mode 100644 docs/crps_estimators.md diff --git a/docs/api/categorical.md b/docs/api/categorical.md deleted file mode 100644 index f0dfe0a..0000000 --- a/docs/api/categorical.md +++ /dev/null @@ -1,28 +0,0 @@ -# Scoring rules for categorical outcomes - -Suppose that the outcome $Y \in \{1, 2, \dots, K\}$ is one of $K$ possible categories. -Then, a probabilistic forecast $F$ for $Y$ is a vector $F = (F_{1}, \dots, F_{K})$ -with $\sum_{i=1}^{K} F_{i} = 1$, containing the forecast probabilities that $Y = 1, \dots, Y = K$. - -When $K = 2$, it is common to instead consider a binary outcome $Y \in \{0, 1\}$, which -represents an event that either occurs $(Y = 1)$ or does not $(Y = 0)$. The forecast in this -case is typically represented by a single probability $F \in [0, 1]$ that $Y = 1$, rather than -the vector $(F, 1 - F)$. However, evaluating these probability forecasts is a particular case of -evaluation methods for the more general categorical case described above. - - -## Brier Score - -::: scoringrules.brier_score - -## Log Score - -::: scoringrules.log_score - -## Ranked Probability Score - -::: scoringrules.rps_score - -## Ranked Log Score - -::: scoringrules.rls_score diff --git a/docs/api/crps.md b/docs/api/crps.md deleted file mode 100644 index 75dac96..0000000 --- a/docs/api/crps.md +++ /dev/null @@ -1,102 +0,0 @@ -# Continuous Ranked Probability Score - -The CRPS is defined as - -\begin{align} - \text{CRPS}(F, y) &= \int_{\mathbb{R}}[F(x)-\mathbbm{1}\{y \le x\}]^2 dx \\ - &= \mathbb{E} | X - y | - \frac{1}{2} \mathbb{E} | X - X^{\prime} | -\end{align} - -where $F$ is the forecast cumulative distribution function (CDF), $\mathbb{1}$ is the indicator function, -and $X, X^{\prime} \sim F$ are independent. The second expression is only valid when the -forecast distribution $F$ has a finite expectation, $\mathbb{E}|X| < \infty$. - -The CRPS can be interpreted as the Brier score between $F(x)$ and $\mathbb{1}\{y \le x\}$, -integrated over all real values $x$. When $F$ is a discrete forecast distribution, such -as an ensemble forecast, the CRPS can be calculated easily be replacing the expectations in -the second expression with sample means over the ensemble members. If $F$ is a point forecast -at some $x \in \mathbb{R}$, then the CRPS reduces the Mean Absolute Error between $x$ and $y$. - - - - -## Analytical formulations - -::: scoringrules.crps_beta - -::: scoringrules.crps_binomial - -::: scoringrules.crps_exponential - -::: scoringrules.crps_exponentialM - -::: scoringrules.crps_2pexponential - -::: scoringrules.crps_gamma - -::: scoringrules.crps_gev - -::: scoringrules.crps_gpd - -::: scoringrules.crps_gtclogistic - -::: scoringrules.crps_tlogistic - -::: scoringrules.crps_clogistic - -::: scoringrules.crps_gtcnormal - -::: scoringrules.crps_tnormal - -::: scoringrules.crps_cnormal - -::: scoringrules.crps_gtct - -::: scoringrules.crps_tt - -::: scoringrules.crps_ct - -::: scoringrules.crps_hypergeometric - -::: scoringrules.crps_laplace - -::: scoringrules.crps_logistic - -::: scoringrules.crps_loglaplace - -::: scoringrules.crps_loglogistic - -::: scoringrules.crps_lognormal - -::: scoringrules.crps_mixnorm - -::: scoringrules.crps_negbinom - -::: scoringrules.crps_normal - -::: scoringrules.crps_2pnormal - -::: scoringrules.crps_poisson - -::: scoringrules.crps_t - -::: scoringrules.crps_uniform - -::: scoringrules.crps_normal - -## Ensemble-based estimators - -::: scoringrules.crps_ensemble - -::: scoringrules.twcrps_ensemble - -::: scoringrules.owcrps_ensemble - -::: scoringrules.vrcrps_ensemble - -## Quantile-based estimators - -::: scoringrules.crps_quantile - - -

diff --git a/docs/api/energy.md b/docs/api/energy.md deleted file mode 100644 index 9a91cab..0000000 --- a/docs/api/energy.md +++ /dev/null @@ -1,87 +0,0 @@ -# Energy Score - -The energy score (ES) is a scoring rule for evaluating multivariate probabilistic forecasts. -It is defined as - -$$\text{ES}(F, \mathbf{y})= \mathbb{E} \| \mathbf{X} - \mathbf{y} \| - \frac{1}{2} \mathbb{E} \| \mathbf{X} - \mathbf{X}^{\prime} \|, $$ - -where $\mathbf{y} \in \mathbb{R}^{d}$ is the multivariate observation ($d > 1$), and -$\mathbf{X}$ and $\mathbf{X}^{\prime}$ are independent random variables that follow the -multivariate forecast distribution $F$ (Gneiting and Raftery, 2007)[@gneiting_strictly_2007]. -If the dimension $d$ were equal to one, the energy score would reduce to the continuous ranked probability score (CRPS). - -While multivariate probabilistic forecasts could belong to a parametric family of -distributions, such as a multivariate normal distribution, it is more common in practice -that these forecasts are ensemble forecasts; that is, the forecast is comprised of a -predictive sample $\mathbf{x}_{1}, \dots, \mathbf{x}_{M}$, -where each ensemble member $\mathbf{x}_{1}, \dots, \mathbf{x}_{M} \in \R^{d}$. - -In this case, the expectations in the definition of the energy score can be replaced by -sample means over the ensemble members, yielding the following representation of the energy -score when evaluating an ensemble forecast $F_{ens}$ with $M$ members. - - -::: scoringrules.energy_score - -

Weighted versions

- -The energy score provides a measure of overall forecast performance. However, it is often -the case that certain outcomes are of more interest than others, making it desirable to -assign more weight to these outcomes when evaluating forecast performance. This can be -achieved using weighted scoring rules. Weighted scoring rules typically introduce a -weight function into conventional scoring rules, and users can choose the weight function -depending on what outcomes they want to emphasise. Allen et al. (2022)[@allen2022evaluating] -discuss three weighted versions of the energy score. These are all available in `scoringrules`. - -Firstly, the outcome-weighted energy score (originally introduced by Holzmann and Klar (2014)[@holzmann2017focusing]) -is defined as - -$$\text{owES}(F, \mathbf{y}; w)= \frac{1}{\bar{w}} \mathbb{E} \| \mathbf{X} - \mathbf{y} \| w(\mathbf{X}) w(\mathbf{y}) - \frac{1}{2 \bar{w}^{2}} \mathbb{E} \| \mathbf{X} - \mathbf{X}^{\prime} \| w(\mathbf{X})w(\mathbf{X}^{\prime})w(\mathbf{y}), $$ - -where $w : \mathbb{R}^{d} \to [0, \infty)$ is the non-negative weight function used to -target particular multivariate outcomes, and $\bar{w} = \mathbb{E}[w(X)]$. -As before, $\mathbf{X}, \mathbf{X}^{\prime} \sim F$ are independent. - -::: scoringrules.owenergy_score - -

- -Secondly, Allen et al. (2022) introduced the threshold-weighted energy score as - -$$\text{twES}(F, \mathbf{y}; v)= \mathbb{E} \| v(\mathbf{X}) - v(\mathbf{y}) \| - \frac{1}{2} \mathbb{E} \| v(\mathbf{X}) - v(\mathbf{X}^{\prime}) \|, $$ - -where $v : \mathbb{R}^{d} \to \mathbb{R}^{d}$ is a so-called chaining function. -The threshold-weighted energy score transforms the forecasts and observations according -to the chaining function $v$, prior to calculating the unweighted energy score. Choosing -a chaining function is generally more difficult than choosing a weight function when -emphasising particular outcomes. - -::: scoringrules.twenergy_score - -

- -As an alternative, the vertically re-scaled energy score is defined as - -$$ -\begin{split} - \text{vrES}(F, \mathbf{y}; w, \mathbf{x}_{0}) = & \mathbb{E} \| \mathbf{X} - \mathbf{y} \| w(\mathbf{X}) w(\mathbf{y}) \\ & - \frac{1}{2} \mathbb{E} \| \mathbf{X} - \mathbf{X}^{\prime} \| w(\mathbf{X})w(\mathbf{X}^{\prime}) \\ - & + \left( \mathbb{E} \| \mathbf{X} - \mathbf{x}_{0} \| w(\mathbf{X}) - \| \mathbf{y} - \mathbf{x}_{0} \| w(\mathbf{y}) \right) \left(\mathbb{E}[w(\mathbf{X})] - w(\mathbf{y}) \right), -\end{split} -$$ - -where $w : \mathbb{R}^{d} \to [0, \infty)$ is the non-negative weight function used to -target particular multivariate outcomes, and $\mathbf{x}_{0} \in \mathbb{R}^{d}$. Typically, -$\mathbf{x}_{0}$ is chosen to be zero. - - -::: scoringrules.vrenergy_score - -

- -Each of these weighted energy scores targets particular outcomes in a different way. -Further details regarding the differences between these scoring rules, as well as choices -for the weight and chaining functions, can be found in Allen et al. (2022). The weighted -energy scores can easily be computed for ensemble forecasts by -replacing the expectations with sample means over the ensemble members. - -

diff --git a/docs/api/error_spread.md b/docs/api/error_spread.md deleted file mode 100644 index a14e2e7..0000000 --- a/docs/api/error_spread.md +++ /dev/null @@ -1,14 +0,0 @@ -# Error Spread Score - -The error spread score [(Christensen et al., 2015)](https://doi.org/10.1002/qj.2375) is given by: - -$$ESS = \left(s^2 - e^2 - e \cdot s \cdot g\right)^2$$ - -where the mean $m$, variance $s^2$, and skewness $g$ of the ensemble forecast of size $F$ are computed as follows: - -$$m = \frac{1}{F} \sum_{f=1}^{F} X_f, \quad s^2 = \frac{1}{F-1} \sum_{f=1}^{F} (X_f - m)^2, \quad g = \frac{F}{(F-1)(F-2)} \sum_{f=1}^{F} \left(\frac{X_f - m}{s}\right)^3$$ - -The error in the ensemble mean $e$ is calculated as $e = m - y$, where $y$ is the observed value. - - -::: scoringrules.error_spread_score diff --git a/docs/api/interval.md b/docs/api/interval.md deleted file mode 100644 index afe6d10..0000000 --- a/docs/api/interval.md +++ /dev/null @@ -1,5 +0,0 @@ -# Interval Score - -::: scoringrules.interval_score - -::: scoringrules.weighted_interval_score diff --git a/docs/api/kernels.md b/docs/api/kernels.md deleted file mode 100644 index cea97fa..0000000 --- a/docs/api/kernels.md +++ /dev/null @@ -1,17 +0,0 @@ -# Kernel scores - -::: scoringrules.gksuv_ensemble - -::: scoringrules.twgksuv_ensemble - -::: scoringrules.owgksuv_ensemble - -::: scoringrules.vrgksuv_ensemble - -::: scoringrules.gksmv_ensemble - -::: scoringrules.twgksmv_ensemble - -::: scoringrules.owgksmv_ensemble - -::: scoringrules.vrgksmv_ensemble diff --git a/docs/api/logarithmic.md b/docs/api/logarithmic.md deleted file mode 100644 index 607b626..0000000 --- a/docs/api/logarithmic.md +++ /dev/null @@ -1,66 +0,0 @@ -# Logarithmic Score - -The Log score is defined as - -$$\text{LS}(F, y) = - \log f(y)$$, - -where $f$ is the density function associated with $F$. The Log score is only defined for -forecasts that have a density function. This means the Log score cannot be calculated -when evaluating discrete forecast distributions, such as ensemble forecasts. - - -## Analytical formulations - -::: scoringrules.logs_beta - -::: scoringrules.logs_binomial - -::: scoringrules.logs_ensemble - -::: scoringrules.logs_exponential - -::: scoringrules.logs_exponential2 - -::: scoringrules.logs_2pexponential - -::: scoringrules.logs_gamma - -::: scoringrules.logs_gev - -::: scoringrules.logs_gpd - -::: scoringrules.logs_hypergeometric - -::: scoringrules.logs_laplace - -::: scoringrules.logs_loglaplace - -::: scoringrules.logs_logistic - -::: scoringrules.logs_loglogistic - -::: scoringrules.logs_lognormal - -::: scoringrules.logs_mixnorm - -::: scoringrules.logs_negbinom - -::: scoringrules.logs_normal - -::: scoringrules.logs_2pnormal - -::: scoringrules.logs_poisson - -::: scoringrules.logs_t - -::: scoringrules.logs_tlogistic - -::: scoringrules.logs_tnormal - -::: scoringrules.logs_tt - -::: scoringrules.logs_uniform - -## Conditional and Censored Likelihood Score - -::: scoringrules.clogs_ensemble diff --git a/docs/api/quantile.md b/docs/api/quantile.md deleted file mode 100644 index d0d6b23..0000000 --- a/docs/api/quantile.md +++ /dev/null @@ -1,3 +0,0 @@ -# Quantile score - -::: scoringrules.quantile_score diff --git a/docs/api/variogram.md b/docs/api/variogram.md deleted file mode 100644 index 0d9f8cc..0000000 --- a/docs/api/variogram.md +++ /dev/null @@ -1,91 +0,0 @@ -# Variogram Score - -The varigoram score (VS) is a scoring rule for evaluating multivariate probabilistic forecasts. -It is defined as - -$$\text{VS}_{p}(F, \mathbf{y})= \sum_{i=1}^{d} \sum_{j=1}^{d} \left( \mathbb{E} | X_{i} - X_{j} |^{p} - | y_{i} - y_{j} |^{p} \right)^{2}, $$ - -where $p > 0$, $\mathbf{y} = (y_{1}, \dots, y_{d}) \in \mathbb{R}^{d}$ is the multivariate observation ($d > 1$), and -$\mathbf{X} = (X_{1}, \dots, X_{d})$ is a random vector that follows the -multivariate forecast distribution $F$ (Scheuerer and Hamill, 2015)[@scheuerer_variogram-based_2015]. -The exponent $p$ is typically chosen to be 0.5 or 1. - -The variogram score is less sensitive to marginal forecast performance than the energy score, -and Scheuerer and Hamill (2015) argue that it should therefore be more sensitive to errors in the -forecast's dependence structure. - -While multivariate probabilistic forecasts could belong to a parametric family of -distributions, such as a multivariate normal distribution, it is more common in practice -that these forecasts are ensemble forecasts; that is, the forecast is comprised of a -predictive sample $\mathbf{x}_{1}, \dots, \mathbf{x}_{M}$, -where each ensemble member $\mathbf{x}_{i} = (x_{i, 1}, \dots, x_{i, d}) \in \R^{d}$ for -$i = 1, \dots, M$. - -In this case, the expectation in the definition of the variogram score can be replaced by -a sample mean over the ensemble members, yielding the following representation of the variogram -score when evaluating an ensemble forecast $F_{ens}$ with $M$ members. - -::: scoringrules.variogram_score - -

- -

Weighted versions

- -It is often the case that certain outcomes are of more interest than others when evaluating -forecast performance. These outcomes can be emphasised by employing weighted scoring rules. -Weighted scoring rules typically introduce a weight function into conventional scoring rules, -and users can choose the weight function depending on what outcomes they want to emphasise. -Allen et al. (2022)[@allen2022evaluating] introduced three weighted versions of the variogram score. -These are all available in `scoringrules`. - -Firstly, the outcome-weighted variogram score (see also Holzmann and Klar (2014)[@holzmann2017focusing]) -is defined as - -$$\text{owVS}_{p}(F, \mathbf{y}; w) = \frac{1}{\bar{w}} \mathbb{E} [ \rho_{p}(\mathbf{X}, \mathbf{y}) w(\mathbf{X}) w(\mathbf{y}) ] - \frac{1}{2 \bar{w}^{2}} \mathbb{E} [ \rho_{p}(\mathbf{X}, \mathbf{X}^{\prime}) w(\mathbf{X}) w(\mathbf{X}^{\prime}) w(\mathbf{y}) ], $$ - -where - -$$ \rho_{p}(\mathbf{x}, \mathbf{z}) = \sum_{i=1}^{d} \sum_{j=1}^{d} \left( |x_{i} - x_{j}|^{p} - |z_{i} - z_{j}|^{p} \right)^{2}, $$ - -for $\mathbf{x} = (x_{1}, \dots, x_{d}) \in \mathbb{R}^{d}$ and $\mathbf{z} = (z_{1}, \dots, z_{d}) \in \mathbb{R}^{d}$. - -Here, $w : \mathbb{R}^{d} \to [0, \infty)$ is the non-negative weight function used to -target particular multivariate outcomes, and $\bar{w} = \mathbb{E}[w(X)]$. -As before, $\mathbf{X}, \mathbf{X}^{\prime} \sim F$ are independent. - -::: scoringrules.owvariogram_score - -

- -Secondly, Allen et al. (2022) introduced the threshold-weighted variogram score as - -$$\text{twVS}_{p}(F, \mathbf{y}; v)= \sum_{i=1}^{d} \sum_{j=1}^{d} \left( \mathbb{E} | v(\mathbf{X})_{i} - v(\mathbf{X})_{j} |^{p} - | v(\mathbf{y})_{i} - v(\mathbf{y})_{j} |^{p} \right)^{2}, $$ - -where $v : \mathbb{R}^{d} \to \mathbb{R}^{d}$ is a so-called chaining function, so that -$v(\mathbf{X}) = (v(\mathbf{X})_{1}, \dots, v(\mathbf{X})_{d}) \in \mathbb{R}^{d}$. -The threshold-weighted variogram score transforms the forecasts and observations according -to the chaining function $v$, prior to calculating the unweighted variogram score. Choosing -a chaining function is generally more difficult than choosing a weight function when -emphasising particular outcomes. - -::: scoringrules.twvariogram_score - -

- -As an alternative, the vertically re-scaled variogram score is defined as - -$$\text{vrVS}_{p}(F, \mathbf{y}; w) = \mathbb{E} [ \rho_{p}(\mathbf{X}, \mathbf{y}) w(\mathbf{X}) w(\mathbf{y}) ] - \frac{1}{2} \mathbb{E} [ \rho_{p}(\mathbf{X}, \mathbf{X}^{\prime}) w(\mathbf{X}) w(\mathbf{X}^{\prime}) ] + \left( \mathbb{E} [ \rho_{p} ( \mathbf{X}, \mathbf{x}_{0} ) w(\mathbf{X}) ] - \rho_{p} ( \mathbf{y}, \mathbf{x}_{0}) w(\mathbf{y}) \right) \left(\mathbb{E}[w(\mathbf{X})] - w(\mathbf{y}) \right), $$ - -where $w$ and $\rho_{p}$ are as defined above, and $\mathbf{x}_{0} \in \mathbb{R}^{d}$. -Typically, $\mathbf{x}_{0}$ is chosen to be the zero vector. - -::: scoringrules.vrvariogram_score - -

- - -Each of these weighted variogram scores targets particular outcomes in a different way. -Further details regarding the differences between these scoring rules, as well as choices -for the weight and chaining functions, can be found in Allen et al. (2022). The weighted -variogram scores can easily be computed for ensemble forecasts by -replacing the expectations with sample means over the ensemble members. diff --git a/docs/api/visualization.md b/docs/api/visualization.md deleted file mode 100644 index 981453e..0000000 --- a/docs/api/visualization.md +++ /dev/null @@ -1,3 +0,0 @@ -Scoring rules alone are not enough for a thorough evaluation of probabilistic forecasts. Visualizations can be used as a complement. - -::: scoringrules.visualization.reliability_diagram diff --git a/docs/conf.py b/docs/conf.py index 2732bb6..7b08fb8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -47,10 +47,7 @@ "pygments_dark_style": "monokai", # toc "collapse_navigation": True, - "sticky_navigation": True, "navigation_depth": 4, - "includehidden": False, - "titles_only": False, } html_title = "scoringrules" html_favicon = "_static/favicon.ico" diff --git a/docs/crps_estimators.md b/docs/crps_estimators.md new file mode 100644 index 0000000..7f52787 --- /dev/null +++ b/docs/crps_estimators.md @@ -0,0 +1,45 @@ +(crps-estimators)= +# CRPS estimators + +## Integral form (INT) + +The numerical approximation of the cumulative integral over the finite ensemble. + +$$ +\text{CRPS}_{\text{INT}}(M, y) = \int_{\mathbb{R}} \left[ \frac{1}{M} + \sum_{i=1}^M \mathbb{1}\{x_i \le x \} - \mathbb{1}\{y \le x\} \right] ^2 dx +$$ + +Runs with $O(m\cdot\mathrm{log}m)$ complexity, including the sorting of the ensemble. + +## Energy form (NRG) + +Introduced by Gneiting and Raftery (2007): + +$$ +\text{CRPS}_{\text{NRG}}(M, y) = \frac{1}{M} \sum_{i=1}^{M}|x_i - y| - \frac{1}{2 M^2}\sum_{i,j=1}^{M}|x_i - x_j| +$$ + +It is called the "energy form" because it is the one-dimensional case of the Energy Score. + +Runs with $O(m^2)$ complexity. + +## Quantile decomposition form (QD) + +Introduced by Jordan (2016): + +$$ +\mathrm{CRPS}_{\mathrm{QD}}(M, y) = \frac{2}{M^2} \sum_{i=1}^{M}(x_i - y)\left[M\mathbb{1}\{y \le x_i\} - i + \frac{1}{2} \right] +$$ + +Runs with $O(m\cdot\mathrm{log}m)$ complexity, including the sorting of the ensemble. + +## Probability weighted moment form (PWM) + +Introduced by Taillardat et al. (2016): + +$$ +\mathrm{CRPS}_{\mathrm{NRG}}(M, y) = \frac{1}{M} \sum_{i=1}^{M}|x_i - y| + \hat{\beta_0} - 2\hat{\beta_1}, +$$ + +where $\hat{\beta_0} = \frac{1}{M} \sum_{i=1}^{M}x_i$ and $\hat{\beta_1} = \frac{1}{M(M-1)} \sum_{i=1}^{M}(i - 1)x_i$. Runs with $O(m\cdot\mathrm{log}m)$ complexity, including the sorting of the ensemble. diff --git a/docs/index.md b/docs/index.md index 9c58622..80ac08c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -34,6 +34,7 @@ The scoring rules available in `scoringrules` include, but are not limited to, t :caption: Background theory.md +crps_estimators.md ``` ```{toctree} @@ -43,6 +44,7 @@ theory.md user_guide.md contributing.md reference.md +prova.rst ``` diff --git a/docs/theory.md b/docs/theory.md index 1ae12df..64e1c98 100644 --- a/docs/theory.md +++ b/docs/theory.md @@ -1,5 +1,7 @@ +(theory)= # The theory of proper scoring rules +(theory.definitions)= ## Definitions Suppose we issue a probabilistic forecast $F$ for an outcome variable $Y$ that takes values @@ -435,6 +437,7 @@ introduce efficient methods when pooling probabilistic forecasts. +(theory.weighted)= ### Weighted scoring rules Often, some outcomes lead to larger impacts than others, making accurate forecasts for these outcomes @@ -448,10 +451,12 @@ term of the summation in their definition, thereby assigning more weight to part For continuous outcomes, this can be extended to the CRPS by incorporating a weight function into the integral, yielding the *threshold-weighted CRPS* +$$ \begin{align*} \mathrm{twCRPS}(F, y; w) &= \int_{\mathbb{R}} (F(x) - \mathbb{1}\{y \leq x\})^{2} w(x) dx \\ &= \mathbb{E} | v(X) - v(y) | - \frac{1}{2} \mathbb{E} | v(X) - v(X^{\prime}) |, \end{align*} +$$ where $X, X^{\prime} \sim F$ are independent, and $v$ is such that $v(x) - v(x^{\prime}) = \int_{x^{\prime}}^{x} w(z) dz$ for any $x, x^{\prime} \in \mathbb{R}$. Allen et al. (2022) refer to $v$ as the *chaining function*. diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index c02ed61..370913d 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -82,16 +82,18 @@ def twcrps_ensemble( sorted_ensemble: bool = False, backend: "Backend" = None, ) -> "Array": - r"""Estimate the Threshold-Weighted Continuous Ranked Probability Score (twCRPS) for a finite ensemble. + r"""Estimate the threshold-weighted CRPS (twCRPS) for a finite ensemble. - Computation is performed using the ensemble representation of the twCRPS in - [Allen et al. (2022)](https://arxiv.org/abs/2202.12732): + Computation is performed using the ensemble representation of the twCRPS in [1]_. - $$ \mathrm{twCRPS}(F_{ens}, y) = \frac{1}{M} \sum_{m = 1}^{M} |v(x_{m}) - v(y)| - \frac{1}{2 M^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} |v(x_{m}) - v(x_{j})|,$$ + .. math:: + \mathrm{twCRPS}(F_{ens}, y) = \frac{1}{M} \sum_{m = 1}^{M} |v(x_{m}) - v(y)| + - \frac{1}{2 M^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} |v(x_{m}) - v(x_{j})|, + + where :math:`F_{ens}(x) = \sum_{m=1}^{M} 1 \{ x_{m} \leq x \}/M` is the empirical + distribution function associated with an ensemble forecast :math:`x_{1}, \dots, x_{M}` with + :math:`M` members, and :math:`v` is the chaining function used to target particular outcomes. - where $F_{ens}(x) = \sum_{m=1}^{M} 1 \{ x_{m} \leq x \}/M$ is the empirical - distribution function associated with an ensemble forecast $x_{1}, \dots, x_{M}$ with - $M$ members, and $v$ is the chaining function used to target particular outcomes. Parameters ---------- @@ -114,6 +116,23 @@ def twcrps_ensemble( twcrps: ArrayLike The twCRPS between the forecast ensemble and obs for the chosen chaining function. + See Also + -------- + crps_ensemble : Compute the CRPS for a finite ensemble. + + Notes + ----- + :ref:`theory.weighted` + Some theoretical background on weighted versions of scoring rules, needed when + one wants to assign more weight to outcomes that are higher impact. + + + References + ---------- + .. [1] Allen, S., Ginsbourger, D., & Ziegel, J. (2023). + Evaluating forecasts for high-impact events using transformed kernel scores. + SIAM/ASA Journal on Uncertainty Quantification, 11(3), 906-940. + Examples -------- >>> import numpy as np From 74d0c9008d16c3a56a2ff27fea9fd4ccd1026c73 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Fri, 17 Jan 2025 13:52:14 +0100 Subject: [PATCH 11/79] remove old file from index --- docs/index.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 80ac08c..4d5c484 100644 --- a/docs/index.md +++ b/docs/index.md @@ -44,7 +44,6 @@ crps_estimators.md user_guide.md contributing.md reference.md -prova.rst ``` From 71bc6712061cb1c6011db6d0b617d43afa668d24 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Fri, 17 Jan 2025 14:21:52 +0100 Subject: [PATCH 12/79] install the local package code for the build --- docs/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 0166889..f4b1cf0 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,5 @@ myst-parser numpydoc +sphinx sphinx-book-theme -scoringrules +-e . From 141da5727ebc91782bd1a516d6f11871e1f2d4e5 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Fri, 17 Jan 2025 14:56:37 +0100 Subject: [PATCH 13/79] expand API reference documentation --- docs/reference.md | 215 +++++++++++++++++++++++----------------------- 1 file changed, 107 insertions(+), 108 deletions(-) diff --git a/docs/reference.md b/docs/reference.md index d70c0c2..39ec733 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -24,114 +24,113 @@ Univariate twgksuv_ensemble owgksuv_ensemble vrgksuv_ensemble - error_spread_score -.. Multivariate -.. -------------------- - -.. .. autosummary:: -.. :toctree: generated - -.. energy_score -.. twenergy_score -.. owenergy_score -.. vrenergy_score -.. variogram_score -.. owvariogram_score -.. twvariogram_score -.. vrvariogram_score -.. gksmv_ensemble -.. twgksmv_ensemble -.. owgksmv_ensemble -.. vrgksmv_ensemble - -.. Parametric distributions forecasts -.. ==================================== -.. .. autosummary:: -.. :toctree: generated - -.. crps_beta -.. crps_binomial -.. crps_exponential -.. crps_exponentialM -.. crps_2pexponential -.. crps_gamma -.. crps_gev -.. crps_gpd -.. crps_gtclogistic -.. crps_tlogistic -.. crps_clogistic -.. crps_gtcnormal -.. crps_tnormal -.. crps_cnormal -.. crps_gtct -.. crps_tt -.. crps_ct -.. crps_hypergeometric -.. crps_laplace -.. crps_logistic -.. crps_loglaplace -.. crps_loglogistic -.. crps_lognormal -.. crps_mixnorm -.. crps_negbinom -.. crps_normal -.. crps_2pnormal -.. crps_poisson -.. crps_quantile -.. crps_t -.. crps_uniform -.. logs_beta -.. logs_binomial -.. logs_ensemble -.. logs_exponential -.. logs_exponential2 -.. logs_2pexponential -.. logs_gamma -.. logs_gev -.. logs_gpd -.. logs_hypergeometric -.. logs_laplace -.. logs_loglaplace -.. logs_logistic -.. logs_loglogistic -.. logs_lognormal -.. logs_mixnorm -.. logs_negbinom -.. logs_normal -.. logs_2pnormal -.. logs_poisson -.. logs_t -.. logs_tlogistic -.. logs_tnormal -.. logs_tt -.. logs_uniform - -.. Functions of distributions -.. ========================== -.. .. autosummary:: -.. :toctree: generated - -.. interval_score -.. weighted_interval_score - - -.. Categorical forecasts -.. ================================= -.. .. autosummary:: -.. :toctree: generated - -.. brier_score -.. rps_score -.. log_score -.. rls_score - - -.. Backends -.. ======== -.. .. autosummary:: -.. :toctree: generated - -.. register_backend +Multivariate +-------------------- + +.. autosummary:: + :toctree: generated + + energy_score + twenergy_score + owenergy_score + vrenergy_score + variogram_score + owvariogram_score + twvariogram_score + vrvariogram_score + gksmv_ensemble + twgksmv_ensemble + owgksmv_ensemble + vrgksmv_ensemble + +Parametric distributions forecasts +==================================== +.. autosummary:: + :toctree: generated + + crps_beta + crps_binomial + crps_exponential + crps_exponentialM + crps_2pexponential + crps_gamma + crps_gev + crps_gpd + crps_gtclogistic + crps_tlogistic + crps_clogistic + crps_gtcnormal + crps_tnormal + crps_cnormal + crps_gtct + crps_tt + crps_ct + crps_hypergeometric + crps_laplace + crps_logistic + crps_loglaplace + crps_loglogistic + crps_lognormal + crps_mixnorm + crps_negbinom + crps_normal + crps_2pnormal + crps_poisson + crps_quantile + crps_t + crps_uniform + logs_beta + logs_binomial + logs_ensemble + logs_exponential + logs_exponential2 + logs_2pexponential + logs_gamma + logs_gev + logs_gpd + logs_hypergeometric + logs_laplace + logs_loglaplace + logs_logistic + logs_loglogistic + logs_lognormal + logs_mixnorm + logs_negbinom + logs_normal + logs_2pnormal + logs_poisson + logs_t + logs_tlogistic + logs_tnormal + logs_tt + logs_uniform + +Functions of distributions +========================== +.. autosummary:: + :toctree: generated + + interval_score + weighted_interval_score + + +Categorical forecasts +================================= +.. autosummary:: + :toctree: generated + + brier_score + rps_score + log_score + rls_score + + +Backends +======== +.. autosummary:: + :toctree: generated + + register_backend ``` From 0a61e5fbbc7cb31b381e9d4539889982d98e3884 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Sat, 18 Jan 2025 14:18:14 +0100 Subject: [PATCH 14/79] delete mkdocs config file --- mkdocs.yaml | 92 ----------------------------------------------------- 1 file changed, 92 deletions(-) delete mode 100644 mkdocs.yaml diff --git a/mkdocs.yaml b/mkdocs.yaml deleted file mode 100644 index 97c8769..0000000 --- a/mkdocs.yaml +++ /dev/null @@ -1,92 +0,0 @@ -site_name: Scoringrules -site_description: Documentation for scoringrules, a python library for probabilistic forecast evaluation. -repo_name: frazane/scoringrules -repo_url: https://github.com/frazane/scoringrules - -nav: - - Home: index.md - - User guide: user_guide.md - - Contributing: contributing.md - - Theory: theory.md - - API reference: - - Categorical Outcomes: api/categorical.md - - Continuous Univariate Outcomes: - - Continuous Ranked Probability Score: api/crps.md - - Logarithmic Score: api/logarithmic.md - - Error Spread Score: api/error_spread.md - - Continuous Multivariate Outcomes: - - Energy Score: api/energy.md - - Variogram Score: api/variogram.md - - Gaussian Kernel Score: api/kernels.md - - Functionals: - - Interval Score: api/interval.md - - Quantile Score: api/quantile.md - - Visualization: api/visualization.md - - -theme: - name: material - font: - text: "Roboto" - - features: - - navigation.sections - - toc.follow - - header.autohide - - - palette: - - scheme: default - primary: white - accent: amber - toggle: - icon: material/weather-night - name: Switch to dark mode - - scheme: slate - primary: white - # accent: amber - toggle: - icon: material/weather-sunny - name: Switch to light mode - icon: - repo: fontawesome/brands/github - - - logo: "assets/images/logo.svg" - - -plugins: - - search - - mkdocstrings: - handlers: - python: - options: - docstring_style: "numpy" - heading_level: 3 - show_source: false - show_root_heading: true - separate_signature: true - show_signature_annotations: true - - bibtex: - bib_file: "docs/refs.bib" - - section-index - -markdown_extensions: - - pymdownx.arithmatex: - generic: true - - pymdownx.highlight: - anchor_linenums: true - line_spans: __span - pygments_lang_class: true - - pymdownx.inlinehilite - - pymdownx.snippets - - pymdownx.superfences - - footnotes - -extra_css: - - stylesheets/extra.css - - https://cdn.jsdelivr.net/npm/katex@0.16.7/dist/katex.css - -extra_javascript: - - scripts/katex.js - - https://cdn.jsdelivr.net/npm/katex@0.16.7/dist/katex.js From 6b33a4b687c73a311bed8050d7a7d9af6e0ae678 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Sat, 1 Feb 2025 09:24:23 +0100 Subject: [PATCH 15/79] wip on crps docstrings --- docs/reference.md | 4 +- scoringrules/_crps.py | 797 ++++++++++++++++++++++++------------------ 2 files changed, 465 insertions(+), 336 deletions(-) diff --git a/docs/reference.md b/docs/reference.md index 39ec733..00edc03 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -24,7 +24,7 @@ Univariate twgksuv_ensemble owgksuv_ensemble vrgksuv_ensemble - error_spread_score + crps_quantile Multivariate -------------------- @@ -107,7 +107,7 @@ Parametric distributions forecasts logs_tt logs_uniform -Functions of distributions +Consistent scoring functions ========================== .. autosummary:: :toctree: generated diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index 370913d..6ae1b41 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -19,11 +19,33 @@ def crps_ensemble( ) -> "Array": r"""Estimate the Continuous Ranked Probability Score (CRPS) for a finite ensemble. + For a forecast :math:`F` and observation :math:`y`, the CRPS is formally defined as: + + .. math:: + \begin{align*} + \mathrm{CRPS}(F, y) &= \int_{-\infty}^{\infty} (F(x) + - \mathbf{1}\{y \le x\})^{2} dx \\ + &= \mathbb{E} | X - y | - \frac{1}{2} \mathbb{E} | X - X^{\prime} |, + \end{align*} + + where :math:`X, X^{\prime} \sim F` are independent. When :math:`F` is the empirical + distribution function of an ensemble forecast :math:`x_{1}, \dots, x_{M}`, the CRPS + can be estimated in several ways. Currently, scoringrules supports several + alternatives for ``estimator``: + + - the energy estimator (``"nrg"``), + - the fair estimator (``"fair"``), + - the probability weighted moment estimator (``"pwm"``), + - the approximate kernel representation estimator (``"akr"``). + - the AKR with circular permutation estimator (``"akr_circperm"``). + - the integral estimator (``"int"``). + - the quantile decomposition estimator (``"qd"``). + Parameters ---------- - observations: ArrayLike + observations: array_like The observed values. - forecasts: ArrayLike + forecasts: array_like, shape (..., m) The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. axis: int @@ -33,18 +55,43 @@ def crps_ensemble( Default is False. estimator: str Indicates the CRPS estimator to be used. - backend: str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - crps: ArrayLike + crps: array_like The CRPS between the forecast ensemble and obs. + See Also + -------- + twcrps_ensemble, owcrps_ensemble, vrcrps_ensemble + Weighted variants of the CRPS. + crps_quantile + CRPS for quantile forecasts. + + Notes + ----- + :ref:`crps-estimators` + Information on the different estimators available for the CRPS. + + References + ---------- + .. [1] Matheson JE, Winkler RL (1976). “Scoring rules for continuous + probability distributions.” Management Science, 22(10), 1087-1096. + doi:10.1287/mnsc.22.10.1087. + .. [2] Gneiting T, Raftery AE (2007). “Strictly proper scoring rules, + prediction, and estimation.” Journal of the American Statistical + Association, 102(477), 359-378. doi:10.1198/016214506000001437. + Examples -------- + >>> import numpy as np >>> import scoringrules as sr + >>> obs = np.array([1.0, 2.0, 3.0]) + >>> pred = np.array([[1.1, 1.2, 1.3], [2.1, 2.2, 2.3], [3.1, 3.2, 3.3]]) >>> sr.crps_ensemble(obs, pred) + 0.0 """ B = backends.active if backend is None else backends[backend] observations, forecasts = map(B.asarray, (observations, forecasts)) @@ -97,23 +144,23 @@ def twcrps_ensemble( Parameters ---------- - observations: ArrayLike + observations: array_like The observed values. - forecasts: ArrayLike + forecasts: array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - v_func: tp.Callable + v_func: callable, array_like -> array_like Chaining function used to emphasise particular outcomes. For example, a function that only considers values above a certain threshold $t$ by projecting forecasts and observations to $[t, \inf)$. axis: int The axis corresponding to the ensemble. Default is the last axis. - backend: str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - twcrps: ArrayLike + twcrps: array_like The twCRPS between the forecast ensemble and obs for the chosen chaining function. See Also @@ -123,15 +170,14 @@ def twcrps_ensemble( Notes ----- :ref:`theory.weighted` - Some theoretical background on weighted versions of scoring rules, needed when - one wants to assign more weight to outcomes that are higher impact. - + Some theoretical background on weighted versions of scoring rules. References ---------- .. [1] Allen, S., Ginsbourger, D., & Ziegel, J. (2023). Evaluating forecasts for high-impact events using transformed kernel scores. SIAM/ASA Journal on Uncertainty Quantification, 11(3), 906-940. + Available at https://arxiv.org/abs/2202.12732. Examples -------- @@ -154,71 +200,102 @@ def twcrps_ensemble( ) -def crps_quantile( +def owcrps_ensemble( observations: "ArrayLike", forecasts: "Array", - alpha: "Array", + w_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, axis: int = -1, *, + estimator: tp.Literal["nrg"] = "nrg", backend: "Backend" = None, ) -> "Array": - r"""Approximate the CRPS from quantile predictions via the Pinball Loss. + r"""Estimate the outcome-weighted CRPS (owCRPS) for a finite ensemble. - It is based on the notation in [Berrisch & Ziel, 2022](https://arxiv.org/pdf/2102.00968) + Computation is performed using the ensemble representation of the owCRPS in [1]_. - The CRPS can be approximated as the mean pinball loss for all - quantile forecasts $F_q$ with level $q \in Q$: + .. math:: + \begin{aligned} + \mathrm{owCRPS}(F_{ens}, y) + &= \frac{1}{M \bar{w}} \sum_{m = 1}^{M} |x_{m} - y|\,w(x_{m})\,w(y)\\ + &\quad - \frac{1}{2 M^{2} \bar{w}^{2}} + \sum_{m = 1}^{M} \sum_{j = 1}^{M} |x_{m} - x_{j}|\, + w(x_{m})\,w(x_{j})\,w(y). + \end{aligned} - $$\text{quantileCRPS} = \frac{2}{|Q|} \sum_{q \in Q} PB_q$$ - where the pinball loss is defined as: - - $$\text{PB}_q = \begin{cases} - q(y - F_q) &\text{if} & y \geq F_q \\ - (1-q)(F_q - y) &\text{else.} & \\ - \end{cases} $$ + where :math:`F_{ens}(x) = \sum_{m=1}^{M} 1\{ x_{m} \leq x \}/M` is the empirical + distribution function associated with an ensemble forecast + :math:`x_{1}, \dots, x_{M}` with :math:`M` members, :math:`w` is the chosen weight + function, and :math:`\bar{w} = \sum_{m=1}^{M}w(x_{m})/M` is the average weight. Parameters ---------- - observations: ArrayLike + observations: array_like The observed values. - forecasts: Array + forecasts: array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - alpha: Array - The percentile levels. We expect the quantile array to match the axis (see below) of the forecast array. + w_func: callable, array_like -> array_like + Weight function used to emphasise particular outcomes. axis: int The axis corresponding to the ensemble. Default is the last axis. - backend: str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - qcrps: Array - An array of CRPS scores for each forecast, which should be averaged to get meaningful values. + owcrps: array_like + The owCRPS between the forecast ensemble and obs for the chosen weight function. + + Notes + ----- + :ref:`theory.weighted` + Some theoretical background on weighted versions of scoring rules. + + References + ---------- + .. [1] Allen, S., Ginsbourger, D., & Ziegel, J. (2023). + Evaluating forecasts for high-impact events using transformed kernel scores. + SIAM/ASA Journal on Uncertainty Quantification, 11(3), 906-940. + Available at https://arxiv.org/abs/2202.12732. Examples -------- + >>> import numpy as np >>> import scoringrules as sr - >>> sr.crps_quantile(obs, fct, alpha) + >>> + >>> def w_func(x): + >>> return (x > -1).astype(float) + >>> + >>> sr.owcrps_ensemble(obs, pred, w_func) """ B = backends.active if backend is None else backends[backend] - observations, forecasts, alpha = map(B.asarray, (observations, forecasts, alpha)) + if estimator != "nrg": + raise ValueError( + "Only the energy form of the estimator is available " + "for the outcome-weighted CRPS." + ) if axis != -1: forecasts = B.moveaxis(forecasts, axis, -1) - if not forecasts.shape[-1] == alpha.shape[-1]: - raise ValueError("Expected matching length of forecasts and alpha values.") + obs_weights, fct_weights = map(w_func, (observations, forecasts)) - if B.name == "numba": - return crps.quantile_pinball_gufunc(observations, forecasts, alpha) + if backend == "numba": + return crps.estimator_gufuncs["ow" + estimator]( + observations, forecasts, obs_weights, fct_weights + ) - return crps.quantile_pinball(observations, forecasts, alpha, backend=backend) + observations, forecasts, obs_weights, fct_weights = map( + B.asarray, (observations, forecasts, obs_weights, fct_weights) + ) + return crps.ow_ensemble( + observations, forecasts, obs_weights, fct_weights, backend=backend + ) -def owcrps_ensemble( +def vrcrps_ensemble( observations: "ArrayLike", forecasts: "Array", w_func: tp.Callable[["ArrayLike"], "ArrayLike"], @@ -228,35 +305,53 @@ def owcrps_ensemble( estimator: tp.Literal["nrg"] = "nrg", backend: "Backend" = None, ) -> "Array": - r"""Estimate the Outcome-Weighted Continuous Ranked Probability Score (owCRPS) for a finite ensemble. + r"""Estimate the vertically re-scaled CRPS (vrCRPS) for a finite ensemble. - Computation is performed using the ensemble representation of the owCRPS in - [Allen et al. (2022)](https://arxiv.org/abs/2202.12732): + Computation is performed using the ensemble representation of the vrCRPS in [1]_. - $$ \mathrm{owCRPS}(F_{ens}, y) = \frac{1}{M \bar{w}} \sum_{m = 1}^{M} |x_{m} - y|w(x_{m})w(y) - \frac{1}{2 M^{2} \bar{w}^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} |x_{m} - x_{j}|w(x_{m})w(x_{j})w(y),$$ + .. math:: + \begin{split} + \mathrm{vrCRPS}(F_{ens}, y) = & \frac{1}{M} \sum_{m = 1}^{M} |x_{m} + - y|w(x_{m})w(y) - \frac{1}{2 M^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} |x_{m} + - x_{j}|w(x_{m})w(x_{j}) \\ + & + \left( \frac{1}{M} \sum_{m = 1}^{M} |x_{m}| w(x_{m}) + - |y| w(y) \right) \left( \frac{1}{M} \sum_{m = 1}^{M} w(x_{m}) - w(y) \right), + \end{split} - where $F_{ens}(x) = \sum_{m=1}^{M} 1\{ x_{m} \leq x \}/M$ is the empirical - distribution function associated with an ensemble forecast $x_{1}, \dots, x_{M}$ with - $M$ members, $w$ is the chosen weight function, and $\bar{w} = \sum_{m=1}^{M}w(x_{m})/M$. + where :math:`F_{ens}(x) = \sum_{m=1}^{M} 1 \{ x_{m} \leq x \}/M` is the empirical + distribution function associated with an ensemble forecast :math:`x_{1}, \dots, x_{M}` with + :math:`M` members, :math:`w` is the chosen weight function, and :math:`\bar{w} = \sum_{m=1}^{M}w(x_{m})/M`. Parameters ---------- - observations: ArrayLike + observations: array_like The observed values. - forecasts: ArrayLike + forecasts: array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - w_func: tp.Callable + w_func: callable, array_like -> array_like Weight function used to emphasise particular outcomes. axis: int The axis corresponding to the ensemble. Default is the last axis. - backend: str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - owcrps: ArrayLike - The owCRPS between the forecast ensemble and obs for the chosen weight function. + vrcrps: array_like + The vrCRPS between the forecast ensemble and obs for the chosen weight function. + + Notes + ----- + :ref:`theory.weighted` + Some theoretical background on weighted versions of scoring rules. + + References + ---------- + .. [1] Allen, S., Ginsbourger, D., & Ziegel, J. (2023). + Evaluating forecasts for high-impact events using transformed kernel scores. + SIAM/ASA Journal on Uncertainty Quantification, 11(3), 906-940. + Available at https://arxiv.org/abs/2202.12732. Examples -------- @@ -266,7 +361,7 @@ def owcrps_ensemble( >>> def w_func(x): >>> return (x > -1).astype(float) >>> - >>> sr.owcrps_ensemble(obs, pred, w_func) + >>> sr.vrcrps_ensemble(obs, pred, w_func) """ B = backends.active if backend is None else backends[backend] @@ -281,96 +376,92 @@ def owcrps_ensemble( obs_weights, fct_weights = map(w_func, (observations, forecasts)) if backend == "numba": - return crps.estimator_gufuncs["ow" + estimator]( + return crps.estimator_gufuncs["vr" + estimator]( observations, forecasts, obs_weights, fct_weights ) observations, forecasts, obs_weights, fct_weights = map( B.asarray, (observations, forecasts, obs_weights, fct_weights) ) - return crps.ow_ensemble( + return crps.vr_ensemble( observations, forecasts, obs_weights, fct_weights, backend=backend ) -def vrcrps_ensemble( +def crps_quantile( observations: "ArrayLike", forecasts: "Array", - w_func: tp.Callable[["ArrayLike"], "ArrayLike"], + alpha: "Array", /, axis: int = -1, *, - estimator: tp.Literal["nrg"] = "nrg", backend: "Backend" = None, ) -> "Array": - r"""Estimate the Vertically Re-scaled Continuous Ranked Probability Score (vrCRPS) for a finite ensemble. + r"""Approximate the CRPS from quantile predictions via the Pinball Loss. - Computation is performed using the ensemble representation of the vrCRPS in - [Allen et al. (2022)](https://arxiv.org/abs/2202.12732): + It is based on the notation in [1]_. - $$ - \begin{split} - \mathrm{vrCRPS}(F_{ens}, y) = & \frac{1}{M} \sum_{m = 1}^{M} |x_{m} - y|w(x_{m})w(y) - \frac{1}{2 M^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} |x_{m} - x_{j}|w(x_{m})w(x_{j}) \\ - & + \left( \frac{1}{M} \sum_{m = 1}^{M} |x_{m}| w(x_{m}) - |y| w(y) \right) \left( \frac{1}{M} \sum_{m = 1}^{M} w(x_{m}) - w(y) \right), - \end{split} - $$ + The CRPS can be approximated as the mean pinball loss for all + quantile forecasts :math:`F_q` with level :math:`q \in Q`: + + .. math:: + \text{quantileCRPS} = \frac{2}{|Q|} \sum_{q \in Q} PB_q - where $F_{ens}(x) = \sum_{m=1}^{M} 1 \{ x_{m} \leq x \}/M$ is the empirical - distribution function associated with an ensemble forecast $x_{1}, \dots, x_{M}$ with - $M$ members, $w$ is the chosen weight function, and $\bar{w} = \sum_{m=1}^{M}w(x_{m})/M$. + where the pinball loss is defined as: + + .. math:: + \text{PB}_q = \begin{cases} + q(y - F_q) &\text{if} & y \geq F_q \\ + (1-q)(F_q - y) &\text{else.} & \\ + \end{cases} Parameters ---------- - observations: ArrayLike + observations: array_like The observed values. - forecasts: ArrayLike + forecasts: Array The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - w_func: tp.Callable - Weight function used to emphasise particular outcomes. + alpha: Array + The percentile levels. We expect the quantile array to match the axis (see below) of the forecast array. axis: int The axis corresponding to the ensemble. Default is the last axis. - backend: str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - vrcrps: ArrayLike - The vrCRPS between the forecast ensemble and obs for the chosen weight function. + qcrps: Array + An array of CRPS scores for each forecast, which should be averaged to get meaningful values. + + References + ---------- + .. [1] Berrisch, J., & Ziel, F. (2023). CRPS learning. + Journal of Econometrics, 237(2), 105221. + Available at https://arxiv.org/abs/2102.00968. Examples -------- >>> import numpy as np >>> import scoringrules as sr - >>> - >>> def w_func(x): - >>> return (x > -1).astype(float) - >>> - >>> sr.vrcrps_ensemble(obs, pred, w_func) + >>> obs = np.array([1.0, 2.0, 3.0]) + >>> fct = np.array([[1.1, 1.2, 1.3], [2.1, 2.2, 2.3], [3.1, 3.2, 3.3]]) + >>> sr.crps_quantile(obs, fct, alpha) + 0.0 """ B = backends.active if backend is None else backends[backend] + observations, forecasts, alpha = map(B.asarray, (observations, forecasts, alpha)) - if estimator != "nrg": - raise ValueError( - "Only the energy form of the estimator is available " - "for the outcome-weighted CRPS." - ) if axis != -1: forecasts = B.moveaxis(forecasts, axis, -1) - obs_weights, fct_weights = map(w_func, (observations, forecasts)) + if not forecasts.shape[-1] == alpha.shape[-1]: + raise ValueError("Expected matching length of forecasts and alpha values.") - if backend == "numba": - return crps.estimator_gufuncs["vr" + estimator]( - observations, forecasts, obs_weights, fct_weights - ) + if B.name == "numba": + return crps.quantile_pinball_gufunc(observations, forecasts, alpha) - observations, forecasts, obs_weights, fct_weights = map( - B.asarray, (observations, forecasts, obs_weights, fct_weights) - ) - return crps.vr_ensemble( - observations, forecasts, obs_weights, fct_weights, backend=backend - ) + return crps.quantile_pinball(observations, forecasts, alpha, backend=backend) def crps_beta( @@ -385,40 +476,49 @@ def crps_beta( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the beta distribution. - It is based on the following formulation from - [Jordan et al. (2019)](https://www.jstatsoft.org/article/view/v090i12): + It is based on the following formulation from [1]_: + + .. math:: + \begin{split} + \mathrm{CRPS}(F_{\alpha, \beta}, y) = & (u - l)\left\{ \frac{y - l}{u - l} + \left( 2F_{\alpha, \beta} \left( \frac{y - l}{u - l} \right) - 1 \right) \right. \\ + & \left. + \frac{\alpha}{\alpha + \beta} \left( 1 - 2F_{\alpha + 1, \beta} + \left( \frac{y - l}{u - l} \right) + - \frac{2B(2\alpha, 2\beta)}{\alpha B(\alpha, \beta)^{2}} \right) \right\} + \end{split} - $$ - \mathrm{CRPS}(F_{\alpha, \beta}, y) = (u - l)\left\{ \frac{y - l}{u - l} - \left( 2F_{\alpha, \beta} \left( \frac{y - l}{u - l} \right) - 1 \right) - + \frac{\alpha}{\alpha + \beta} \left( 1 - 2F_{\alpha + 1, \beta} - \left( \frac{y - l}{u - l} \right) - - \frac{2B(2\alpha, 2\beta)}{\alpha B(\alpha, \beta)^{2}} \right) \right\} - $$ - where $F_{\alpha, \beta}$ is the beta distribution function with shape parameters - $\alpha, \beta > 0$, and lower and upper bounds $l, u \in \R$, $l < u$. + where :math:`F_{\alpha, \beta}` is the beta distribution function with + shape parameters :math:`\alpha, \beta > 0`, and lower and upper bounds + :math:`l, u \in \mathbb{R}`, :math:`l < u`. Parameters ---------- - observation: + observation: array_like The observed values. - a: + a: array_like First shape parameter of the forecast beta distribution. - b: + b: array_like Second shape parameter of the forecast beta distribution. - lower: + lower: array_like Lower bound of the forecast beta distribution. - upper: + upper: array_like Upper bound of the forecast beta distribution. - backend: - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - score: + crps: The CRPS between Beta(a, b) and obs. + References + ---------- + .. [1] Jordan, A., Krüger, F., & Lerch, S. (2019). + Evaluating Probabilistic Forecasts with scoringRules. + Journal of Statistical Software, 90(12), 1-37. + https://doi.org/10.18637/jss.v090.i12 + Examples -------- >>> import scoringrules as sr @@ -451,18 +551,18 @@ def crps_binomial( Parameters ---------- - observation: + observation: array_like The observed values as an integer or array of integers. - n: + n: array_like Size parameter of the forecast binomial distribution as an integer or array of integers. - prob: + prob: array_like Probability parameter of the forecast binomial distribution as a float or array of floats. - backend: - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - score: + crps: The CRPS between Binomial(n, prob) and obs. Examples @@ -492,28 +592,26 @@ def crps_exponential( Parameters ---------- - observation: + observation: array_like The observed values. - rate: + rate: array_like Rate parameter of the forecast exponential distribution. - backend: - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - score: + crps: The CRPS between Exp(rate) and obs. Examples -------- - ```pycon >>> import scoringrules as sr >>> import numpy as np >>> sr.crps_exponential(0.8, 3.0) 0.360478635526275 >>> sr.crps_exponential(np.array([0.8, 0.9]), np.array([3.0, 2.0])) array([0.36047864, 0.24071795]) - ``` """ return crps.exponential(observation, rate, backend=backend) @@ -547,20 +645,20 @@ def crps_exponentialM( Parameters ---------- - observation: + observation: array_like The observed values. - mass: + mass: array_like Mass parameter of the forecast exponential distribution. - location: + location: array_like Location parameter of the forecast exponential distribution. - scale: + scale: array_like Scale parameter of the forecast exponential distribution. - backend: - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - score: + crps: array_like The CRPS between obs and ExpM(mass, location, scale). Examples @@ -595,20 +693,20 @@ def crps_2pexponential( Parameters ---------- - observation: + observation: array_like The observed values. - scale1: + scale1: array_like First scale parameter of the forecast two-piece exponential distribution. - scale2: + scale2: array_like Second scale parameter of the forecast two-piece exponential distribution. - location: + location: array_like Location parameter of the forecast two-piece exponential distribution. - backend: - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - score: + crps: array_like The CRPS between 2pExp(sigma1, sigma2, location) and obs. Examples @@ -632,35 +730,44 @@ def crps_gamma( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the gamma distribution. - It is based on the following formulation from - [Scheuerer and Möller (2015)](https://doi.org/10.1214/15-AOAS843): + It is based on the following formulation from [1]_: - $$ \mathrm{CRPS}(F_{\alpha, \beta}, y) = y(2F_{\alpha, \beta}(y) - 1) - - \frac{\alpha}{\beta} (2 F_{\alpha + 1, \beta}(y) - 1) - - \frac{1}{\beta B(1/2, \alpha)}. $$ + .. math:: + \mathrm{CRPS}(F_{\alpha, \beta}, y) = y(2F_{\alpha, \beta}(y) - 1) + - \frac{\alpha}{\beta} (2 F_{\alpha + 1, \beta}(y) - 1) + - \frac{1}{\beta B(1/2, \alpha)}, - where $F_{\alpha, \beta}$ is gamma distribution function with shape - parameter $\alpha > 0$ and rate parameter $\beta > 0$ (equivalently, - with scale parameter $1/\beta$). + where :math:`F_{\alpha, \beta}` is gamma distribution function with shape + parameter :math:`\alpha > 0` and rate parameter :math:`\beta > 0` (equivalently, + with scale parameter :math:`1/\beta`). Parameters ---------- - observation: + observation: array_like The observed values. - shape: + shape: array_like Shape parameter of the forecast gamma distribution. - rate: + rate: array_like, optional Rate parameter of the forecast rate distribution. - scale: - Scale parameter of the forecast scale distribution, where `scale = 1 / rate`. - backend: - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + Either ``rate`` or ``scale`` must be provided. + scale: array_like, optional + Scale parameter of the forecast scale distribution, where ``scale = 1 / rate``. + Either ``rate`` or ``scale`` must be provided. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - score: + crps: The CRPS between obs and Gamma(shape, rate). + References + ---------- + .. [1] Michael Scheuerer. David Möller. "Probabilistic wind speed forecasting + on a grid based on ensemble model output statistics." + Ann. Appl. Stat. 9 (3) 1328 - 1349, September 2015. + https://doi.org/10.1214/15-AOAS843 + Examples -------- >>> import scoringrules as sr @@ -670,11 +777,11 @@ def crps_gamma( Raises ------ ValueError - If both `rate` and `scale` are provided, or if neither is provided. + If both ``rate`` and ``scale`` are provided, or if neither is provided. """ if (scale is None and rate is None) or (scale is not None and rate is not None): raise ValueError( - "Either `rate` or `scale` must be provided, but not both or neither." + "Either ``rate`` or ``scale`` must be provided, but not both or neither." ) if rate is None: @@ -694,56 +801,13 @@ def crps_gev( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the generalised extreme value (GEV) distribution. - It is based on the following formulation from - [Friederichs and Thorarinsdottir (2012)](https://doi.org/10.1002/env.2176): - - $$ - \text{CRPS}(F_{\xi, \mu, \sigma}, y) = - \sigma \cdot \text{CRPS}(F_{\xi}, \frac{y - \mu}{\sigma}) - $$ - - Special cases are handled as follows: - - - For $\xi = 0$: - - $$ - \text{CRPS}(F_{\xi}, y) = -y - 2\text{Ei}(\log F_{\xi}(y)) + C - \log 2 - $$ - - - For $\xi \neq 0$: - - $$ - \text{CRPS}(F_{\xi}, y) = y(2F_{\xi}(y) - 1) - 2G_{\xi}(y) - - \frac{1 - (2 - 2^{\xi}) \Gamma(1 - \xi)}{\xi} - $$ - - where $C$ is the Euler-Mascheroni constant, $\text{Ei}$ is the exponential - integral, and $\Gamma$ is the gamma function. The GEV cumulative distribution - function $F_{\xi}$ and the auxiliary function $G_{\xi}$ are defined as: - - - For $\xi = 0$: - - $$ - F_{\xi}(x) = \exp(-\exp(-x)) - $$ + It is based on the following formulation from [1]_: - - For $\xi \neq 0$: + .. math:: + \text{CRPS}(F_{\xi, \mu, \sigma}, y) = + \sigma \cdot \text{CRPS}(F_{\xi}, \frac{y - \mu}{\sigma}) - $$ - F_{\xi}(x) = - \begin{cases} - 0, & x \leq \frac{1}{\xi} \\ - \exp(-(1 + \xi x)^{-1/\xi}), & x > \frac{1}{\xi} - \end{cases} - $$ - - $$ - G_{\xi}(x) = - \begin{cases} - 0, & x \leq \frac{1}{\xi} \\ - \frac{F_{\xi}(x)}{\xi} + \frac{\Gamma_u(1-\xi, -\log F_{\xi}(x))}{\xi}, & x > \frac{1}{\xi} - \end{cases} - $$ + see Notes below for special cases. Parameters ---------- @@ -755,15 +819,63 @@ def crps_gev( Location parameter of the forecast GEV distribution. scale: Scale parameter of the forecast GEV distribution. - backend: - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. - + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - score: + crps: The CRPS between obs and GEV(shape, location, scale). + Notes + ----- + + Special cases are handled as follows: + + - For :math:`\xi = 0`: + + .. math:: + \text{CRPS}(F_{\xi}, y) = -y - 2\text{Ei}(\log F_{\xi}(y)) + C - \log 2 + + - For :math:`\xi \neq 0`: + + .. math:: + \text{CRPS}(F_{\xi}, y) = y(2F_{\xi}(y) - 1) - 2G_{\xi}(y) + - \frac{1 - (2 - 2^{\xi}) \Gamma(1 - \xi)}{\xi} + + where :math:`C` is the Euler-Mascheroni constant, :math:`\text{Ei}` is the exponential + integral, and :math:`\Gamma` is the gamma function. The GEV cumulative distribution + function :math:`F_{\xi}` and the auxiliary function :math:`G_{\xi}` are defined as: + + - For :math:`\xi = 0`: + + .. math:: + F_{\xi}(x) = \exp(-\exp(-x)) + + - For :math:`\xi \neq 0`: + + .. math:: + F_{\xi}(x) = + \begin{cases} + 0, & x \leq \frac{1}{\xi} \\ + \exp(-(1 + \xi x)^{-1/\xi}), & x > \frac{1}{\xi} + \end{cases} + + .. math:: + G_{\xi}(x) = + \begin{cases} + 0, & x \leq \frac{1}{\xi} \\ + \frac{F_{\xi}(x)}{\xi} + \frac{\Gamma_u(1-\xi, -\log F_{\xi}(x))}{\xi}, & x > \frac{1}{\xi} + \end{cases} + + References + ---------- + .. [1] Friederichs, P., & Thorarinsdottir, T. L. (2012). + A comparison of parametric and non-parametric methods for + forecasting extreme events. Environmetrics, 23(7), 595-611. + https://doi.org/10.1002/env.2176 + + Examples -------- >>> import scoringrules as sr @@ -785,24 +897,20 @@ def crps_gpd( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the generalised pareto distribution (GPD). - It is based on the following formulation from - [Jordan et al. (2019)](https://www.jstatsoft.org/article/view/v090i12): + It is based on the following formulation from [1]_: + .. math:: + \mathrm{CRPS}(F_{M, \xi}, y) = + |y| - \frac{2 (1 - M)}{1 - \xi} \left( 1 - (1 - F_{\xi}(y))^{1 - \xi} \right) + + \frac{(1 - M)^{2}}{2 - \xi}, - $$ - \mathrm{CRPS}(F_{M, \xi}, y) = - |y| - \frac{2 (1 - M)}{1 - \xi} \left( 1 - (1 - F_{\xi}(y))^{1 - \xi} \right) - + \frac{(1 - M)^{2}}{2 - \xi}, - $$ - - $$ - \mathrm{CRPS}(F_{M, \xi, \mu, \sigma}, y) = - \sigma \mathrm{CRPS} \left( F_{M, \xi}, \frac{y - \mu}{\sigma} \right), - $$ + .. math:: + \mathrm{CRPS}(F_{M, \xi, \mu, \sigma}, y) = + \sigma \mathrm{CRPS} \left( F_{M, \xi}, \frac{y - \mu}{\sigma} \right), - where $F_{M, \xi, \mu, \sigma}$ is the GPD distribution function with shape - parameter $\xi < 1$, location parameter $\mu$, scale parameter $\sigma > 0$, - and point mass $M \in [0, 1]$ at the lower boundary. $F_{M, \xi} = F_{M, \xi, 0, 1}$. + where :math:`F_{M, \xi, \mu, \sigma}` is the GPD distribution function with shape + parameter :math:`\xi < 1`, location parameter :math:`\mu`, scale parameter :math:`\sigma > 0`, + and point mass :math:`M \in [0, 1]` at the lower boundary. :math:`F_{M, \xi} = F_{M, \xi, 0, 1}`. Parameters ---------- @@ -816,14 +924,21 @@ def crps_gpd( Scale parameter of the forecast GPD distribution. mass: Mass parameter at the lower boundary of the forecast GPD distribution. - backend: - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - score: + crps: The CRPS between obs and GPD(shape, location, scale, mass). + References + ---------- + .. [1] Jordan, A., Krüger, F., & Lerch, S. (2019). + Evaluating Probabilistic Forecasts with scoringRules. + Journal of Statistical Software, 90(12), 1-37. + https://doi.org/10.18637/jss.v090.i12 + Examples -------- >>> import scoringrules as sr @@ -847,13 +962,27 @@ def crps_gtclogistic( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the generalised truncated and censored logistic distribution. - $$ \mathrm{CRPS}(F_{l, L}^{u, U}, y) = |y - z| + uU^{2} - lL^{2} - \left( \frac{1 - L - U}{F(u) - F(l)} \right) z \left( \frac{(1 - 2L) F(u) + (1 - 2U) F(l)}{1 - L - U} \right) - \left( \frac{1 - L - U}{F(u) - F(l)} \right) \left( 2 \log F(-z) - 2G(u)U - 2 G(l)L \right) - \left( \frac{1 - L - U}{F(u) - F(l)} \right)^{2} \left( H(u) - H(l) \right), $$ + .. math:: + \begin{aligned} + \mathrm{CRPS}(F_{l, L}^{u, U}, y) = & \, |y - z| + uU^{2} - lL^{2} \\ + & - \left( \frac{1 - L - U}{F(u) - F(l)} \right) z + \left( \frac{(1 - 2L) F(u) + (1 - 2U) F(l)}{1 - L - U} \right) \\ + & - \left( \frac{1 - L - U}{F(u) - F(l)} \right) + \left( 2 \log F(-z) - 2G(u)U - 2 G(l)L \right) \\ + & - \left( \frac{1 - L - U}{F(u) - F(l)} \right)^{2} \left( H(u) - H(l) \right), + \end{aligned} - $$ \mathrm{CRPS}(F_{l, L, \mu, \sigma}^{u, U}, y) = \sigma \mathrm{CRPS}(F_{(l - \mu)/\sigma, L}^{(u - \mu)/\sigma, U}, \frac{y - \mu}{\sigma}), $$ + .. math:: + \begin{aligned} + \mathrm{CRPS}(F_{l, L, \mu, \sigma}^{u, U}, y) = & \, \sigma + \mathrm{CRPS}(F_{(l - \mu)/\sigma, L}^{(u - \mu)/\sigma, U}, \frac{y - \mu}{\sigma}), + \end{aligned} - $$G(x) = xF(x) + \log F(-x),$$ + .. math:: + G(x) = xF(x) + \log F(-x), - $$H(x) = F(x) - xF(x)^{2} + (1 - 2F(x))\log F(-x),$$ + ..math:: + H(x) = F(x) - xF(x)^{2} + (1 - 2F(x))\log F(-x), where $F$ is the CDF of the standard logistic distribution, $F_{l, L, \mu, \sigma}^{u, U}$ is the CDF of the logistic distribution truncated below at $l$ and above at $u$, @@ -862,19 +991,19 @@ def crps_gtclogistic( Parameters ---------- - observation: ArrayLike + observation: array_like The observed values. - location: ArrayLike + location: array_like Location parameter of the forecast distribution. - scale: ArrayLike + scale: array_like Scale parameter of the forecast distribution. - lower: ArrayLike + lower: array_like Lower boundary of the truncated forecast distribution. - upper: ArrayLike + upper: array_like Upper boundary of the truncated forecast distribution. - lmass: ArrayLike + lmass: array_like Point mass assigned to the lower boundary of the forecast distribution. - umass: ArrayLike + umass: array_like Point mass assigned to the upper boundary of the forecast distribution. Returns @@ -916,15 +1045,15 @@ def crps_tlogistic( Parameters ---------- - observation: ArrayLike + observation: array_like The observed values. - location: ArrayLike + location: array_like Location parameter of the forecast distribution. - scale: ArrayLike + scale: array_like Scale parameter of the forecast distribution. - lower: ArrayLike + lower: array_like Lower boundary of the truncated forecast distribution. - upper: ArrayLike + upper: array_like Upper boundary of the truncated forecast distribution. Returns @@ -959,15 +1088,15 @@ def crps_clogistic( Parameters ---------- - observation: ArrayLike + observation: array_like The observed values. - location: ArrayLike + location: array_like Location parameter of the forecast distribution. - scale: ArrayLike + scale: array_like Scale parameter of the forecast distribution. - lower: ArrayLike + lower: array_like Lower boundary of the truncated forecast distribution. - upper: ArrayLike + upper: array_like Upper boundary of the truncated forecast distribution. Returns @@ -1055,15 +1184,15 @@ def crps_tnormal( Parameters ---------- - observation: ArrayLike + observation: array_like The observed values. - location: ArrayLike + location: array_like Location parameter of the forecast distribution. - scale: ArrayLike + scale: array_like Scale parameter of the forecast distribution. - lower: ArrayLike + lower: array_like Lower boundary of the truncated forecast distribution. - upper: ArrayLike + upper: array_like Upper boundary of the truncated forecast distribution. Returns @@ -1098,15 +1227,15 @@ def crps_cnormal( Parameters ---------- - observation: ArrayLike + observation: array_like The observed values. - location: ArrayLike + location: array_like Location parameter of the forecast distribution. - scale: ArrayLike + scale: array_like Scale parameter of the forecast distribution. - lower: ArrayLike + lower: array_like Lower boundary of the truncated forecast distribution. - upper: ArrayLike + upper: array_like Upper boundary of the truncated forecast distribution. Returns @@ -1169,21 +1298,21 @@ def crps_gtct( Parameters ---------- - observation: ArrayLike + observation: array_like The observed values. - df: ArrayLike + df: array_like Degrees of freedom parameter of the forecast distribution. - location: ArrayLike + location: array_like Location parameter of the forecast distribution. - scale: ArrayLike + scale: array_like Scale parameter of the forecast distribution. - lower: ArrayLike + lower: array_like Lower boundary of the truncated forecast distribution. - upper: ArrayLike + upper: array_like Upper boundary of the truncated forecast distribution. - lmass: ArrayLike + lmass: array_like Point mass assigned to the lower boundary of the forecast distribution. - umass: ArrayLike + umass: array_like Point mass assigned to the upper boundary of the forecast distribution. Returns @@ -1227,17 +1356,17 @@ def crps_tt( Parameters ---------- - observation: ArrayLike + observation: array_like The observed values. - df: ArrayLike + df: array_like Degrees of freedom parameter of the forecast distribution. - location: ArrayLike + location: array_like Location parameter of the forecast distribution. - scale: ArrayLike + scale: array_like Scale parameter of the forecast distribution. - lower: ArrayLike + lower: array_like Lower boundary of the truncated forecast distribution. - upper: ArrayLike + upper: array_like Upper boundary of the truncated forecast distribution. Returns @@ -1281,17 +1410,17 @@ def crps_ct( Parameters ---------- - observation: ArrayLike + observation: array_like The observed values. - df: ArrayLike + df: array_like Degrees of freedom parameter of the forecast distribution. - location: ArrayLike + location: array_like Location parameter of the forecast distribution. - scale: ArrayLike + scale: array_like Scale parameter of the forecast distribution. - lower: ArrayLike + lower: array_like Lower boundary of the truncated forecast distribution. - upper: ArrayLike + upper: array_like Upper boundary of the truncated forecast distribution. Returns @@ -1352,12 +1481,12 @@ def crps_hypergeometric( Number of failure states in the population. k: Number of draws, without replacement. Must be in 0, 1, ..., m + n. - backend: - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - score: + crps: The CRPS between obs and Hypergeometric(m, n, k). Examples @@ -1398,12 +1527,12 @@ def crps_laplace( Location parameter of the forecast laplace distribution. scale: Scale parameter of the forecast laplace distribution. - backend: - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - score: + crps: The CRPS between obs and Laplace(location, scale). >>> sr.crps_laplace(0.3, 0.1, 0.2) @@ -1432,11 +1561,11 @@ def crps_logistic( Parameters ---------- - observations: ArrayLike + observations: array_like Observed values. - mu: ArrayLike + mu: array_like Location parameter of the forecast logistic distribution. - sigma: ArrayLike + sigma: array_like Scale parameter of the forecast logistic distribution. Returns @@ -1489,12 +1618,12 @@ def crps_loglaplace( Location parameter of the forecast log-laplace distribution. scalelog: Scale parameter of the forecast log-laplace distribution. - backend: - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - score: + crps: The CRPS between obs and Loglaplace(locationlog, scalelog). Examples @@ -1544,13 +1673,13 @@ def crps_loglogistic( Location parameter of the log-logistic distribution. sigmalog: Scale parameter of the log-logistic distribution. - backend: - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - score: + crps: The CRPS between obs and Loglogis(mulog, sigmalog). Examples @@ -1594,7 +1723,7 @@ def crps_lognormal( Returns ------- - crps: ArrayLike + crps: array_like The CRPS between Lognormal(mu, sigma) and obs. Examples @@ -1627,22 +1756,22 @@ def crps_mixnorm( Parameters ---------- - observation: ArrayLike + observation: array_like The observed values. - m: ArrayLike + m: array_like Means of the component normal distributions. - s: ArrayLike + s: array_like Standard deviations of the component normal distributions. - w: ArrayLike + w: array_like Non-negative weights assigned to each component. axis: int The axis corresponding to the mixture components. Default is the last axis. - backend: - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend: str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - score: + crps: The CRPS between MixNormal(m, s) and obs. Examples @@ -1689,13 +1818,13 @@ def crps_negbinom( Parameters ---------- - observation: ArrayLike + observation: array_like The observed values. - n: ArrayLike + n: array_like Size parameter of the forecast negative binomial distribution. - prob: ArrayLike + prob: array_like Probability parameter of the forecast negative binomial distribution. - mu: ArrayLike + mu: array_like Mean of the forecast negative binomial distribution. Returns @@ -1744,11 +1873,11 @@ def crps_normal( Parameters ---------- - observations: ArrayLike + observations: array_like The observed values. - mu: ArrayLike + mu: array_like Mean of the forecast normal distribution. - sigma: ArrayLike + sigma: array_like Standard deviation of the forecast normal distribution. Returns @@ -1788,13 +1917,13 @@ def crps_2pnormal( Parameters ---------- - observations: ArrayLike + observations: array_like The observed values. - scale1: ArrayLike + scale1: array_like Scale parameter of the lower half of the forecast two-piece normal distribution. - scale2: ArrayLike + scale2: array_like Scale parameter of the upper half of the forecast two-piece normal distribution. - mu: ArrayLike + mu: array_like Location parameter of the forecast two-piece normal distribution. Returns @@ -1849,9 +1978,9 @@ def crps_poisson( Parameters ---------- - observation: ArrayLike + observation: array_like The observed values. - mean: ArrayLike + mean: array_like Mean parameter of the forecast poisson distribution. Returns @@ -1895,13 +2024,13 @@ def crps_t( Parameters ---------- - observation: ArrayLike + observation: array_like The observed values. - df: ArrayLike + df: array_like Degrees of freedom parameter of the forecast t distribution. - location: ArrayLike + location: array_like Location parameter of the forecast t distribution. - sigma: ArrayLike + sigma: array_like Scale parameter of the forecast t distribution. Returns @@ -1940,15 +2069,15 @@ def crps_uniform( Parameters ---------- - observation: ArrayLike + observation: array_like The observed values. - min: ArrayLike + min: array_like Lower bound of the forecast uniform distribution. - max: ArrayLike + max: array_like Upper bound of the forecast uniform distribution. - lmass: ArrayLike + lmass: array_like Point mass on the lower bound of the forecast uniform distribution. - umass: ArrayLike + umass: array_like Point mass on the upper bound of the forecast uniform distribution. Returns From f248be4ac24336ff840b9b2b61f37709c4fb76b5 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Sat, 1 Feb 2025 10:34:14 +0100 Subject: [PATCH 16/79] more wip on crps docstrings --- scoringrules/_crps.py | 957 +++++++++++++++++++++++------------------- 1 file changed, 528 insertions(+), 429 deletions(-) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index 6ae1b41..b4fc4df 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -8,8 +8,8 @@ def crps_ensemble( - observations: "ArrayLike", - forecasts: "Array", + obs: "ArrayLike", + fct: "Array", /, axis: int = -1, *, @@ -43,24 +43,24 @@ def crps_ensemble( Parameters ---------- - observations: array_like + obs : array_like The observed values. - forecasts: array_like, shape (..., m) + fct : array_like, shape (..., m) The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - axis: int + axis : int The axis corresponding to the ensemble. Default is the last axis. - sorted_ensemble: bool + sorted_ensemble : bool Boolean indicating whether the ensemble members are already in ascending order. Default is False. - estimator: str + estimator : str Indicates the CRPS estimator to be used. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - crps: array_like + crps : array_like The CRPS between the forecast ensemble and obs. See Also @@ -94,10 +94,10 @@ def crps_ensemble( 0.0 """ B = backends.active if backend is None else backends[backend] - observations, forecasts = map(B.asarray, (observations, forecasts)) + obs, fct = map(B.asarray, (obs, fct)) if axis != -1: - forecasts = B.moveaxis(forecasts, axis, -1) + fct = B.moveaxis(fct, axis, -1) if not sorted_ensemble and estimator not in [ "nrg", @@ -105,7 +105,7 @@ def crps_ensemble( "akr_circperm", "fair", ]: - forecasts = B.sort(forecasts, axis=-1) + fct = B.sort(fct, axis=-1) if backend == "numba": if estimator not in crps.estimator_gufuncs: @@ -113,14 +113,14 @@ def crps_ensemble( f"{estimator} is not a valid estimator. " f"Must be one of {crps.estimator_gufuncs.keys()}" ) - return crps.estimator_gufuncs[estimator](observations, forecasts) + return crps.estimator_gufuncs[estimator](obs, fct) - return crps.ensemble(observations, forecasts, estimator, backend=backend) + return crps.ensemble(obs, fct, estimator, backend=backend) def twcrps_ensemble( - observations: "ArrayLike", - forecasts: "Array", + obs: "ArrayLike", + fct: "Array", v_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, axis: int = -1, @@ -144,23 +144,23 @@ def twcrps_ensemble( Parameters ---------- - observations: array_like + obs : array_like The observed values. - forecasts: array_like + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - v_func: callable, array_like -> array_like + v_func : callable, array_like -> array_like Chaining function used to emphasise particular outcomes. For example, a function that - only considers values above a certain threshold $t$ by projecting forecasts and observations - to $[t, \inf)$. - axis: int + only considers values above a certain threshold :math:`t` by projecting forecasts and observations + to :math:`[t, \inf)`. + axis : int The axis corresponding to the ensemble. Default is the last axis. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - twcrps: array_like + twcrps : array_like The twCRPS between the forecast ensemble and obs for the chosen chaining function. See Also @@ -189,10 +189,10 @@ def twcrps_ensemble( >>> >>> sr.twcrps_ensemble(obs, pred, v_func) """ - observations, forecasts = map(v_func, (observations, forecasts)) + obs, fct = map(v_func, (obs, fct)) return crps_ensemble( - observations, - forecasts, + obs, + fct, axis=axis, sorted_ensemble=sorted_ensemble, estimator=estimator, @@ -201,8 +201,8 @@ def twcrps_ensemble( def owcrps_ensemble( - observations: "ArrayLike", - forecasts: "Array", + obs: "ArrayLike", + fct: "Array", w_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, axis: int = -1, @@ -231,21 +231,21 @@ def owcrps_ensemble( Parameters ---------- - observations: array_like + obs : array_like The observed values. - forecasts: array_like + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - w_func: callable, array_like -> array_like + w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. - axis: int + axis : int The axis corresponding to the ensemble. Default is the last axis. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - owcrps: array_like + owcrps : array_like The owCRPS between the forecast ensemble and obs for the chosen weight function. Notes @@ -278,26 +278,24 @@ def owcrps_ensemble( "for the outcome-weighted CRPS." ) if axis != -1: - forecasts = B.moveaxis(forecasts, axis, -1) + fct = B.moveaxis(fct, axis, -1) - obs_weights, fct_weights = map(w_func, (observations, forecasts)) + obs_weights, fct_weights = map(w_func, (obs, fct)) if backend == "numba": return crps.estimator_gufuncs["ow" + estimator]( - observations, forecasts, obs_weights, fct_weights + obs, fct, obs_weights, fct_weights ) - observations, forecasts, obs_weights, fct_weights = map( - B.asarray, (observations, forecasts, obs_weights, fct_weights) - ) - return crps.ow_ensemble( - observations, forecasts, obs_weights, fct_weights, backend=backend + obs, fct, obs_weights, fct_weights = map( + B.asarray, (obs, fct, obs_weights, fct_weights) ) + return crps.ow_ensemble(obs, fct, obs_weights, fct_weights, backend=backend) def vrcrps_ensemble( - observations: "ArrayLike", - forecasts: "Array", + obs: "ArrayLike", + fct: "Array", w_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, axis: int = -1, @@ -324,21 +322,21 @@ def vrcrps_ensemble( Parameters ---------- - observations: array_like + obs : array_like The observed values. - forecasts: array_like + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - w_func: callable, array_like -> array_like + w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. - axis: int + axis : int The axis corresponding to the ensemble. Default is the last axis. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - vrcrps: array_like + vrcrps : array_like The vrCRPS between the forecast ensemble and obs for the chosen weight function. Notes @@ -371,26 +369,24 @@ def vrcrps_ensemble( "for the outcome-weighted CRPS." ) if axis != -1: - forecasts = B.moveaxis(forecasts, axis, -1) + fct = B.moveaxis(fct, axis, -1) - obs_weights, fct_weights = map(w_func, (observations, forecasts)) + obs_weights, fct_weights = map(w_func, (obs, fct)) if backend == "numba": return crps.estimator_gufuncs["vr" + estimator]( - observations, forecasts, obs_weights, fct_weights + obs, fct, obs_weights, fct_weights ) - observations, forecasts, obs_weights, fct_weights = map( - B.asarray, (observations, forecasts, obs_weights, fct_weights) - ) - return crps.vr_ensemble( - observations, forecasts, obs_weights, fct_weights, backend=backend + obs, fct, obs_weights, fct_weights = map( + B.asarray, (obs, fct, obs_weights, fct_weights) ) + return crps.vr_ensemble(obs, fct, obs_weights, fct_weights, backend=backend) def crps_quantile( - observations: "ArrayLike", - forecasts: "Array", + obs: "ArrayLike", + fct: "Array", alpha: "Array", /, axis: int = -1, @@ -417,21 +413,21 @@ def crps_quantile( Parameters ---------- - observations: array_like + obs : array_like The observed values. - forecasts: Array + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - alpha: Array + alpha : array_like The percentile levels. We expect the quantile array to match the axis (see below) of the forecast array. - axis: int + axis : int The axis corresponding to the ensemble. Default is the last axis. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - qcrps: Array + qcrps : array_like An array of CRPS scores for each forecast, which should be averaged to get meaningful values. References @@ -450,22 +446,22 @@ def crps_quantile( 0.0 """ B = backends.active if backend is None else backends[backend] - observations, forecasts, alpha = map(B.asarray, (observations, forecasts, alpha)) + obs, fct, alpha = map(B.asarray, (obs, fct, alpha)) if axis != -1: - forecasts = B.moveaxis(forecasts, axis, -1) + fct = B.moveaxis(fct, axis, -1) - if not forecasts.shape[-1] == alpha.shape[-1]: - raise ValueError("Expected matching length of forecasts and alpha values.") + if not fct.shape[-1] == alpha.shape[-1]: + raise ValueError("Expected matching length of `fct` and `alpha` values.") if B.name == "numba": - return crps.quantile_pinball_gufunc(observations, forecasts, alpha) + return crps.quantile_pinball_gufunc(obs, fct, alpha) - return crps.quantile_pinball(observations, forecasts, alpha, backend=backend) + return crps.quantile_pinball(obs, fct, alpha, backend=backend) def crps_beta( - observation: "ArrayLike", + obs: "ArrayLike", a: "ArrayLike", b: "ArrayLike", /, @@ -494,22 +490,22 @@ def crps_beta( Parameters ---------- - observation: array_like + obs : array_like The observed values. a: array_like First shape parameter of the forecast beta distribution. b: array_like Second shape parameter of the forecast beta distribution. - lower: array_like + lower : array_like Lower bound of the forecast beta distribution. - upper: array_like + upper : array_like Upper bound of the forecast beta distribution. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - crps: + crps : array_like The CRPS between Beta(a, b) and obs. References @@ -525,11 +521,11 @@ def crps_beta( >>> sr.crps_beta(0.3, 0.7, 1.1) 0.0850102437 """ - return crps.beta(observation, a, b, lower, upper, backend=backend) + return crps.beta(obs, a, b, lower, upper, backend=backend) def crps_binomial( - observation: "ArrayLike", + obs: "ArrayLike", n: "ArrayLike", prob: "ArrayLike", /, @@ -538,26 +534,24 @@ def crps_binomial( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the binomial distribution. - It is based on the following formulation from - [Jordan et al. (2019)](https://www.jstatsoft.org/article/view/v090i12): + It is based on the following formulation from [1]_: - $$ - \mathrm{CRPS}(F_{n, p}, y) = 2 \sum_{x = 0}^{n} f_{n,p}(x) (1\{y < x\} - - F_{n,p}(x) + f_{n,p}(x)/2) (x - y), - $$ + .. math:: + \mathrm{CRPS}(F_{n, p}, y) = 2 \sum_{x = 0}^{n} f_{n,p}(x) (1\{y < x\} + - F_{n,p}(x) + f_{n,p}(x)/2) (x - y), - where $f_{n, p}$ and $F_{n, p}$ are the PDF and CDF of the binomial distribution - with size parameter $n = 0, 1, 2, ...$ and probability parameter $p \in [0, 1]$. + where :math:`f_{n, p}` and :math:`F_{n, p}` are the PDF and CDF of the binomial distribution + with size parameter :math:`n = 0, 1, 2, ...` and probability parameter :math:`p \in [0, 1]`. Parameters ---------- - observation: array_like + obs : array_like The observed values as an integer or array of integers. n: array_like Size parameter of the forecast binomial distribution as an integer or array of integers. - prob: array_like + prob : array_like Probability parameter of the forecast binomial distribution as a float or array of floats. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns @@ -565,17 +559,24 @@ def crps_binomial( crps: The CRPS between Binomial(n, prob) and obs. + References + ---------- + .. [1] Jordan, A., Krüger, F., & Lerch, S. (2019). + Evaluating Probabilistic Forecasts with scoringRules. + Journal of Statistical Software, 90(12), 1-37. + https://doi.org/10.18637/jss.v090.i12 + Examples -------- >>> import scoringrules as sr >>> sr.crps_binomial(4, 10, 0.5) 0.5955715179443359 """ - return crps.binomial(observation, n, prob, backend=backend) + return crps.binomial(obs, n, prob, backend=backend) def crps_exponential( - observation: "ArrayLike", + obs: "ArrayLike", rate: "ArrayLike", /, *, @@ -583,20 +584,20 @@ def crps_exponential( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the exponential distribution. - It is based on the following formulation from - [Jordan et al. (2019)](https://www.jstatsoft.org/article/view/v090i12): + It is based on the following formulation from [1]_: - $$\mathrm{CRPS}(F_{\lambda}, y) = |y| - \frac{2F_{\lambda}(y)}{\lambda} + \frac{1}{2 \lambda},$$ + .. math:: + \mathrm{CRPS}(F_{\lambda}, y) = |y| - \frac{2F_{\lambda}(y)}{\lambda} + \frac{1}{2 \lambda}, - where $F_{\lambda}$ is exponential distribution function with rate parameter $\lambda > 0$. + where :math:`F_{\lambda}` is exponential distribution function with rate parameter :math:`\lambda > 0`. Parameters ---------- - observation: array_like + obs : array_like The observed values. - rate: array_like + rate : array_like Rate parameter of the forecast exponential distribution. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns @@ -604,6 +605,13 @@ def crps_exponential( crps: The CRPS between Exp(rate) and obs. + References + ---------- + .. [1] Jordan, A., Krüger, F., & Lerch, S. (2019). + Evaluating Probabilistic Forecasts with scoringRules. + Journal of Statistical Software, 90(12), 1-37. + https://doi.org/10.18637/jss.v090.i12 + Examples -------- >>> import scoringrules as sr @@ -613,11 +621,11 @@ def crps_exponential( >>> sr.crps_exponential(np.array([0.8, 0.9]), np.array([3.0, 2.0])) array([0.36047864, 0.24071795]) """ - return crps.exponential(observation, rate, backend=backend) + return crps.exponential(obs, rate, backend=backend) def crps_exponentialM( - observation: "ArrayLike", + obs: "ArrayLike", /, mass: "ArrayLike" = 0.0, location: "ArrayLike" = 0.0, @@ -627,50 +635,59 @@ def crps_exponentialM( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the standard exponential distribution with a point mass at the boundary. - It is based on the following formulation from - [Jordan et al. (2019)](https://www.jstatsoft.org/article/view/v090i12): + It is based on the following formulation from [1]_: - $$ \mathrm{CRPS}(F_{M}, y) = |y| - 2 (1 - M) F(y) + \frac{(1 - M)**2}{2}, $$ + .. math:: + \mathrm{CRPS}(F_{M}, y) = |y| - 2 (1 - M) F(y) + \frac{(1 - M)**2}{2}, - $$ \mathrm{CRPS}(F_{M, \mu, \sigma}, y) = - \sigma \mathrm{CRPS} \left( F_{M}, \frac{y - \mu}{\sigma} \right), $$ + .. math:: + \mathrm{CRPS}(F_{M, \mu, \sigma}, y) = + \sigma \mathrm{CRPS} \left( F_{M}, \frac{y - \mu}{\sigma} \right), - where $F_{M, \mu, \sigma}$ is standard exponential distribution function - generalised using a location parameter $\mu$ and scale parameter $\sigma < 0$ - and a point mass $M \in [0, 1]$ at $\mu$, $F_{M} = F_{M, 0, 1}$, and + where :math:`F_{M, \mu, \sigma}` is standard exponential distribution function + generalised using a location parameter :math:`\mu` and scale parameter :math:`\sigma < 0` + and a point mass :math:`M \in [0, 1]` at :math:`\mu`, :math:`F_{M} = F_{M, 0, 1}`, and - $$ F(y) = 1 - \exp(-y) $$ + .. math:: + F(y) = 1 - \exp(-y) - for $y \geq 0$, and 0 otherwise. + for :math:`y \geq 0`, and 0 otherwise. Parameters ---------- - observation: array_like + obs : array_like The observed values. - mass: array_like + mass : array_like Mass parameter of the forecast exponential distribution. - location: array_like + location : array_like Location parameter of the forecast exponential distribution. - scale: array_like + scale : array_like Scale parameter of the forecast exponential distribution. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - crps: array_like + crps : array_like The CRPS between obs and ExpM(mass, location, scale). + References + ---------- + .. [1] Jordan, A., Krüger, F., & Lerch, S. (2019). + Evaluating Probabilistic Forecasts with scoringRules. + Journal of Statistical Software, 90(12), 1-37. + https://doi.org/10.18637/jss.v090.i12 + Examples -------- >>> import scoringrules as sr >>> sr.crps_exponentialM(0.4, 0.2, 0.0, 1.0) """ - return crps.exponentialM(observation, mass, location, scale, backend=backend) + return crps.exponentialM(obs, mass, location, scale, backend=backend) def crps_2pexponential( - observation: "ArrayLike", + obs: "ArrayLike", scale1: "ArrayLike", scale2: "ArrayLike", location: "ArrayLike", @@ -680,47 +697,52 @@ def crps_2pexponential( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the two-piece exponential distribution. - It is based on the following formulation from - [Jordan et al. (2019)](https://www.jstatsoft.org/article/view/v090i12): + It is based on the following formulation from [1]_: - $$\mathrm{CRPS}(F_{\sigma_{1}, \sigma_{2}, \mu}, y) = |y - \mu| + - \frac{2\sigma_{\pm}^{2}}{\sigma_{1} + \sigma_{2}} \exp \left( - \frac{|y - \mu|}{\sigma_{\pm}} \right) - - \frac{2\sigma_{\pm}^{2}}{\sigma_{1} + \sigma_{2}} + \frac{\sigma_{1}^{3} + \sigma_{2}^{3}}{2(\sigma_{1} + \sigma_{2})^2} $$ + .. math:: + \mathrm{CRPS}(F_{\sigma_{1}, \sigma_{2}, \mu}, y) = |y - \mu| + + \frac{2\sigma_{\pm}^{2}}{\sigma_{1} + \sigma_{2}} \exp \left( - \frac{|y - \mu|}{\sigma_{\pm}} \right) - + \frac{2\sigma_{\pm}^{2}}{\sigma_{1} + \sigma_{2}} + \frac{\sigma_{1}^{3} + \sigma_{2}^{3}}{2(\sigma_{1} + \sigma_{2})^2}, - where $F_{\sigma_{1}, \sigma_{2}, \mu}$ is the two-piece exponential distribution function - with scale parameters $\sigma_{1}, \sigma_{2} > 0$ and location parameter $\mu$. The - parameter $\sigma_{\pm}$ is equal to $\sigma_{1}$ if $y < 0$ and $\sigma_{2}$ if $y \geq 0$. + where :math:`F_{\sigma_{1}, \sigma_{2}, \mu}` is the two-piece exponential distribution function + with scale parameters :math:`\sigma_{1}, \sigma_{2} > 0` and location parameter :math:`\mu`. The + parameter :math:`\sigma_{\pm}` is equal to :math:`\sigma_{1}` if :math:`y < 0` and :math:`\sigma_{2}` if :math:`y \geq 0`. Parameters ---------- - observation: array_like + obs : array_like The observed values. - scale1: array_like + scale1 : array_like First scale parameter of the forecast two-piece exponential distribution. - scale2: array_like + scale2 : array_like Second scale parameter of the forecast two-piece exponential distribution. - location: array_like + location : array_like Location parameter of the forecast two-piece exponential distribution. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - crps: array_like + crps : array_like The CRPS between 2pExp(sigma1, sigma2, location) and obs. + References + ---------- + .. [1] Jordan, A., Krüger, F., & Lerch, S. (2019). + Evaluating Probabilistic Forecasts with scoringRules. + Journal of Statistical Software, 90(12), 1-37. + https://doi.org/10.18637/jss.v090.i12 + Examples -------- - ```pycon >>> import scoringrules as sr >>> sr.crps_2pexponential(0.8, 3.0, 1.4, 0.0) - ``` """ - return crps.twopexponential(observation, scale1, scale2, location, backend=backend) + return crps.twopexponential(obs, scale1, scale2, location, backend=backend) def crps_gamma( - observation: "ArrayLike", + obs: "ArrayLike", shape: "ArrayLike", /, rate: "ArrayLike | None" = None, @@ -743,17 +765,17 @@ def crps_gamma( Parameters ---------- - observation: array_like + obs : array_like The observed values. shape: array_like Shape parameter of the forecast gamma distribution. - rate: array_like, optional + rate : array_like, optional Rate parameter of the forecast rate distribution. Either ``rate`` or ``scale`` must be provided. - scale: array_like, optional + scale : array_like, optional Scale parameter of the forecast scale distribution, where ``scale = 1 / rate``. Either ``rate`` or ``scale`` must be provided. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns @@ -787,11 +809,11 @@ def crps_gamma( if rate is None: rate = 1.0 / scale - return crps.gamma(observation, shape, rate, backend=backend) + return crps.gamma(obs, shape, rate, backend=backend) def crps_gev( - observation: "ArrayLike", + obs: "ArrayLike", shape: "ArrayLike", /, location: "ArrayLike" = 0.0, @@ -811,20 +833,20 @@ def crps_gev( Parameters ---------- - observation: + obs : array_like The observed values. - shape: + shape : array_like Shape parameter of the forecast GEV distribution. - location: + location : array_like, optional Location parameter of the forecast GEV distribution. - scale: + scale : array_like, optional Scale parameter of the forecast GEV distribution. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - crps: + crps : array_like The CRPS between obs and GEV(shape, location, scale). Notes @@ -882,11 +904,11 @@ def crps_gev( >>> sr.crps_gev(0.3, 0.1) 0.2924712413052034 """ - return crps.gev(observation, shape, location, scale, backend=backend) + return crps.gev(obs, shape, location, scale, backend=backend) def crps_gpd( - observation: "ArrayLike", + obs: "ArrayLike", shape: "ArrayLike", /, location: "ArrayLike" = 0.0, @@ -914,22 +936,22 @@ def crps_gpd( Parameters ---------- - observation: + obs : array_like The observed values. - shape: + shape : array_like Shape parameter of the forecast GPD distribution. - location: + location : array_like Location parameter of the forecast GPD distribution. - scale: + scale : array_like Scale parameter of the forecast GPD distribution. - mass: + mass : array_like Mass parameter at the lower boundary of the forecast GPD distribution. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - crps: + crps : array_like The CRPS between obs and GPD(shape, location, scale, mass). References @@ -945,11 +967,11 @@ def crps_gpd( >>> sr.crps_gpd(0.3, 0.9) 0.6849331901197213 """ - return crps.gpd(observation, shape, location, scale, mass, backend=backend) + return crps.gpd(obs, shape, location, scale, mass, backend=backend) def crps_gtclogistic( - observation: "ArrayLike", + obs: "ArrayLike", location: "ArrayLike", scale: "ArrayLike", /, @@ -984,31 +1006,31 @@ def crps_gtclogistic( ..math:: H(x) = F(x) - xF(x)^{2} + (1 - 2F(x))\log F(-x), - where $F$ is the CDF of the standard logistic distribution, $F_{l, L, \mu, \sigma}^{u, U}$ - is the CDF of the logistic distribution truncated below at $l$ and above at $u$, - with point masses $L, U > 0$ at the lower and upper boundaries, respectively, and - location and scale parameters $\mu$ and $\sigma > 0$. + where :math:`F` is the CDF of the standard logistic distribution, :math:`F_{l, L, \mu, \sigma}^{u, U}` + is the CDF of the logistic distribution truncated below at :math:`l` and above at :math:`u`, + with point masses :math:`L, U > 0` at the lower and upper boundaries, respectively, and + location and scale parameters :math:`\mu` and :math:`\sigma > 0`. Parameters ---------- - observation: array_like + obs : array_like The observed values. - location: array_like + location : array_like Location parameter of the forecast distribution. - scale: array_like + scale : array_like Scale parameter of the forecast distribution. - lower: array_like + lower : array_like Lower boundary of the truncated forecast distribution. - upper: array_like + upper : array_like Upper boundary of the truncated forecast distribution. - lmass: array_like + lmass : array_like Point mass assigned to the lower boundary of the forecast distribution. - umass: array_like + umass : array_like Point mass assigned to the upper boundary of the forecast distribution. Returns ------- - crps: array_like + crps : array_like The CRPS between gtcLogistic(location, scale, lower, upper, lmass, umass) and obs. Examples @@ -1017,7 +1039,7 @@ def crps_gtclogistic( >>> sr.crps_gtclogistic(0.0, 0.1, 0.4, -1.0, 1.0, 0.1, 0.1) """ return crps.gtclogistic( - observation, + obs, location, scale, lower, @@ -1029,7 +1051,7 @@ def crps_gtclogistic( def crps_tlogistic( - observation: "ArrayLike", + obs: "ArrayLike", location: "ArrayLike", scale: "ArrayLike", /, @@ -1045,20 +1067,20 @@ def crps_tlogistic( Parameters ---------- - observation: array_like + obs : array_like The observed values. - location: array_like + location : array_like Location parameter of the forecast distribution. - scale: array_like + scale : array_like Scale parameter of the forecast distribution. - lower: array_like + lower : array_like Lower boundary of the truncated forecast distribution. - upper: array_like + upper : array_like Upper boundary of the truncated forecast distribution. Returns ------- - crps: array_like + crps : array_like The CRPS between tLogistic(location, scale, lower, upper) and obs. Examples @@ -1067,12 +1089,12 @@ def crps_tlogistic( >>> sr.crps_tlogistic(0.0, 0.1, 0.4, -1.0, 1.0) """ return crps.gtclogistic( - observation, location, scale, lower, upper, 0.0, 0.0, backend=backend + obs, location, scale, lower, upper, 0.0, 0.0, backend=backend ) def crps_clogistic( - observation: "ArrayLike", + obs: "ArrayLike", location: "ArrayLike", scale: "ArrayLike", /, @@ -1088,20 +1110,20 @@ def crps_clogistic( Parameters ---------- - observation: array_like + obs : array_like The observed values. - location: array_like + location : array_like Location parameter of the forecast distribution. - scale: array_like + scale : array_like Scale parameter of the forecast distribution. - lower: array_like + lower : array_like Lower boundary of the truncated forecast distribution. - upper: array_like + upper : array_like Upper boundary of the truncated forecast distribution. Returns ------- - crps: array_like + crps : array_like The CRPS between cLogistic(location, scale, lower, upper) and obs. Examples @@ -1112,7 +1134,7 @@ def crps_clogistic( lmass = stats._logis_cdf((lower - location) / scale) umass = 1 - stats._logis_cdf((upper - location) / scale) return crps.gtclogistic( - observation, + obs, location, scale, lower, @@ -1124,7 +1146,7 @@ def crps_clogistic( def crps_gtcnormal( - observation: "ArrayLike", + obs: "ArrayLike", location: "ArrayLike", scale: "ArrayLike", /, @@ -1137,18 +1159,29 @@ def crps_gtcnormal( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the generalised truncated and censored normal distribution. - It is based on the following formulation from - [Jordan et al. (2019)](https://www.jstatsoft.org/article/view/v090i12): + It is based on the following formulation from [1]_: - $$ \mathrm{CRPS}(F_{l, L}^{u, U}, y) = |y - z| + uU^{2} - lL^{2} + \left( \frac{1 - L - U}{\Phi(u) - \Phi(l)} \right) z \left( 2 \Phi(z) - \frac{(1 - 2L) \Phi(u) + (1 - 2U) \Phi(l)}{1 - L - U} \right) + \left( \frac{1 - L - U}{\Phi(u) - \Phi(l)} \right) \left( 2 \phi(z) - 2 \phi(u)U - 2 \phi(l)L \right) - \left( \frac{1 - L - U}{\Phi(u) - \Phi(l)} \right)^{2} \left( \frac{1}{\sqrt{\pi}} \right) \left( \Phi(u \sqrt{2}) - \Phi(l \sqrt{2}) \right), $$ + .. math:: + \mathrm{CRPS}(F_{l, L}^{u, U}, y) = |y - z| + uU^{2} - lL^{2} + + \left( \frac{1 - L - U}{\Phi(u) - \Phi(l)} \right) z \left( 2 \Phi(z) - \frac{(1 - 2L) \Phi(u) + (1 - 2U) \Phi(l)}{1 - L - U} \right) + + \left( \frac{1 - L - U}{\Phi(u) - \Phi(l)} \right) \left( 2 \phi(z) - 2 \phi(u)U - 2 \phi(l)L \right) + - \left( \frac{1 - L - U}{\Phi(u) - \Phi(l)} \right)^{2} \left( \frac{1}{\sqrt{\pi}} \right) \left( \Phi(u \sqrt{2}) - \Phi(l \sqrt{2}) \right), - $$ \mathrm{CRPS}(F_{l, L, \mu, \sigma}^{u, U}, y) = \sigma \mathrm{CRPS}(F_{(l - \mu)/\sigma, L}^{(u - \mu)/\sigma, U}, \frac{y - \mu}{\sigma}), $$ + .. math:: + \mathrm{CRPS}(F_{l, L, \mu, \sigma}^{u, U}, y) = \sigma \mathrm{CRPS}(F_{(l - \mu)/\sigma, L}^{(u - \mu)/\sigma, U}, \frac{y - \mu}{\sigma}), - where $\Phi$ and $\phi$ are respectively the CDF and PDF of the standard normal - distribution, $F_{l, L, \mu, \sigma}^{u, U}$ is the CDF of the normal distribution - truncated below at $l$ and above at $u$, with point masses $L, U > 0$ at the lower and upper - boundaries, respectively, and location and scale parameters $\mu$ and $\sigma > 0$. - $F_{l, L}^{u, U} = F_{l, L, 0, 1}^{u, U}$. + where :math:`\Phi:math:` and :math:`\phi` are respectively the CDF and PDF of the standard normal + distribution, :math:`F_{l, L, \mu, \sigma}^{u, U}` is the CDF of the normal distribution + truncated below at :math:`l` and above at :math:`u`, with point masses :math:`L, U > 0` at the + lower and upper boundaries, respectively, and location and scale parameters :math:`\mu` and :math:`\sigma > 0`. + :math:`F_{l, L}^{u, U} = F_{l, L, 0, 1}^{u, U}`. + + References + ---------- + .. [1] Jordan, A., Krüger, F., & Lerch, S. (2019). + Evaluating Probabilistic Forecasts with scoringRules. + Journal of Statistical Software, 90(12), 1-37. + https://doi.org/10.18637/jss.v090.i12 Examples -------- @@ -1156,7 +1189,7 @@ def crps_gtcnormal( >>> sr.crps_gtcnormal(0.0, 0.1, 0.4, -1.0, 1.0, 0.1, 0.1) """ return crps.gtcnormal( - observation, + obs, location, scale, lower, @@ -1168,7 +1201,7 @@ def crps_gtcnormal( def crps_tnormal( - observation: "ArrayLike", + obs: "ArrayLike", location: "ArrayLike", scale: "ArrayLike", /, @@ -1184,20 +1217,20 @@ def crps_tnormal( Parameters ---------- - observation: array_like + obs : array_like The observed values. - location: array_like + location : array_like Location parameter of the forecast distribution. - scale: array_like + scale : array_like Scale parameter of the forecast distribution. - lower: array_like + lower : array_like Lower boundary of the truncated forecast distribution. - upper: array_like + upper : array_like Upper boundary of the truncated forecast distribution. Returns ------- - crps: array_like + crps : array_like The CRPS between tNormal(location, scale, lower, upper) and obs. Examples @@ -1205,13 +1238,11 @@ def crps_tnormal( >>> import scoringrules as sr >>> sr.crps_tnormal(0.0, 0.1, 0.4, -1.0, 1.0) """ - return crps.gtcnormal( - observation, location, scale, lower, upper, 0.0, 0.0, backend=backend - ) + return crps.gtcnormal(obs, location, scale, lower, upper, 0.0, 0.0, backend=backend) def crps_cnormal( - observation: "ArrayLike", + obs: "ArrayLike", location: "ArrayLike", scale: "ArrayLike", /, @@ -1227,20 +1258,20 @@ def crps_cnormal( Parameters ---------- - observation: array_like + obs : array_like The observed values. - location: array_like + location : array_like Location parameter of the forecast distribution. - scale: array_like + scale : array_like Scale parameter of the forecast distribution. - lower: array_like + lower : array_like Lower boundary of the truncated forecast distribution. - upper: array_like + upper : array_like Upper boundary of the truncated forecast distribution. Returns ------- - crps: array_like + crps : array_like The CRPS between cNormal(location, scale, lower, upper) and obs. Examples @@ -1251,7 +1282,7 @@ def crps_cnormal( lmass = stats._norm_cdf((lower - location) / scale) umass = 1 - stats._norm_cdf((upper - location) / scale) return crps.gtcnormal( - observation, + obs, location, scale, lower, @@ -1263,7 +1294,7 @@ def crps_cnormal( def crps_gtct( - observation: "ArrayLike", + obs: "ArrayLike", df: "ArrayLike", /, location: "ArrayLike" = 0.0, @@ -1277,56 +1308,67 @@ def crps_gtct( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the generalised truncated and censored t distribution. - It is based on the following formulation from - [Jordan et al. (2019)](https://www.jstatsoft.org/article/view/v090i12): + It is based on the following formulation from [1]_: - $$ \mathrm{CRPS}(F_{l, L, \nu}^{u, U}, y) = |y - z| + uU^{2} - lL^{2} + \left( \frac{1 - L - U}{F_{\nu}(u) - F_{\nu}(l)} \right) z \left( 2 F_{\nu}(z) - \frac{(1 - 2L) F_{\nu}(u) + (1 - 2U) F_{\nu}(l)}{1 - L - U} \right) - \left( \frac{1 - L - U}{F_{\nu}(u) - F_{\nu}(l)} \right) \left( 2 G_{\nu}(z) - 2 G_{\nu}(u)U - 2 G_{\nu}(l)L \right) - \left( \frac{1 - L - U}{F_{\nu}(u) - F_{\nu}(l)} \right)^{2} \bar{B}_{\nu} \left( H_{\nu}(u) - H_{\nu}(l) \right), $$ + .. math:: + \mathrm{CRPS}(F_{l, L, \nu}^{u, U}, y) = |y - z| + uU^{2} - lL^{2} + \left( \frac{1 - L - U}{F_{\nu}(u) - F_{\nu}(l)} \right) z \left( 2 F_{\nu}(z) - \frac{(1 - 2L) F_{\nu}(u) + (1 - 2U) F_{\nu}(l)}{1 - L - U} \right) - \left( \frac{1 - L - U}{F_{\nu}(u) - F_{\nu}(l)} \right) \left( 2 G_{\nu}(z) - 2 G_{\nu}(u)U - 2 G_{\nu}(l)L \right) - \left( \frac{1 - L - U}{F_{\nu}(u) - F_{\nu}(l)} \right)^{2} \bar{B}_{\nu} \left( H_{\nu}(u) - H_{\nu}(l) \right), - $$ \mathrm{CRPS}(F_{l, L, \nu, \mu, \sigma}^{u, U}, y) = \sigma \mathrm{CRPS}(F_{(l - \mu)/\sigma, L, \nu}^{(u - \mu)/\sigma, U}, \frac{y - \mu}{\sigma}), $$ + .. math:: + \mathrm{CRPS}(F_{l, L, \nu, \mu, \sigma}^{u, U}, y) = \sigma \mathrm{CRPS}(F_{(l - \mu)/\sigma, L, \nu}^{(u - \mu)/\sigma, U}, \frac{y - \mu}{\sigma}), - $$ G_{\nu}(x) = - \left( \frac{\nu + x^{2}}{\nu - 1} \right) f_{\nu}(x), $$ + .. math:: + G_{\nu}(x) = - \left( \frac{\nu + x^{2}}{\nu - 1} \right) f_{\nu}(x), - $$ H_{\nu}(x) = \frac{1}{2} + \frac{1}{2} \mathrm{sgn}(x) I \left( \frac{1}{2}, \nu - \frac{1}{2}, \frac{x^{2}}{\nu + x^{2}} \right), $$ + .. math:: + H_{\nu}(x) = \frac{1}{2} + \frac{1}{2} \mathrm{sgn}(x) I \left( \frac{1}{2}, \nu - \frac{1}{2}, \frac{x^{2}}{\nu + x^{2}} \right), - $$ \bar{B}_{\nu} = \left( \frac{2 \sqrt{\nu}}{\nu - 1} \right) \frac{B(\frac{1}{2}, \nu - \frac{1}{2})}{B(\frac{1}{2}, \frac{\nu}{2})^{2}}, $$ + .. math:: + \bar{B}_{\nu} = \left( \frac{2 \sqrt{\nu}}{\nu - 1} \right) \frac{B(\frac{1}{2}, \nu - \frac{1}{2})}{B(\frac{1}{2}, \frac{\nu}{2})^{2}}, - where $F_{\nu}$ is the CDF of the standard t distribution with $\nu > 1$ degrees of freedom, - distribution, $F_{l, L, \nu, \mu, \sigma}^{u, U}$ is the CDF of the t distribution - truncated below at $l$ and above at $u$, with point masses $L, U > 0$ at the lower and upper - boundaries, respectively, and degrees of freedom, location and scale parameters $\nu > 1$, $\mu$ and $\sigma > 0$. - $F_{l, L, \nu}^{u, U} = F_{l, L, \nu, 0, 1}^{u, U}$. + where :math:`F_{\nu}` is the CDF of the standard t distribution with :math:`\nu > 1` degrees of freedom, + distribution, :math:`F_{l, L, \nu, \mu, \sigma}^{u, U}` is the CDF of the t distribution + truncated below at :math:`l` and above at :math:`u`, with point masses :math:`L, U > 0` at the lower and upper + boundaries, respectively, and degrees of freedom, location and scale parameters :math:`\nu > 1`, :math:`\mu` and :math:`\sigma > 0`. + :math:`F_{l, L, \nu}^{u, U} = F_{l, L, \nu, 0, 1}^{u, U}`. Parameters ---------- - observation: array_like + obs : array_like The observed values. - df: array_like + df : array_like Degrees of freedom parameter of the forecast distribution. - location: array_like + location : array_like Location parameter of the forecast distribution. - scale: array_like + scale : array_like Scale parameter of the forecast distribution. - lower: array_like + lower : array_like Lower boundary of the truncated forecast distribution. - upper: array_like + upper : array_like Upper boundary of the truncated forecast distribution. - lmass: array_like + lmass : array_like Point mass assigned to the lower boundary of the forecast distribution. - umass: array_like + umass : array_like Point mass assigned to the upper boundary of the forecast distribution. Returns ------- - crps: array_like + crps : array_like The CRPS between gtct(df, location, scale, lower, upper, lmass, umass) and obs. + References + ---------- + .. [1] Jordan, A., Krüger, F., & Lerch, S. (2019). + Evaluating Probabilistic Forecasts with scoringRules. + Journal of Statistical Software, 90(12), 1-37. + https://doi.org/10.18637/jss.v090.i12 + Examples -------- >>> import scoringrules as sr >>> sr.crps_gtct(0.0, 2.0, 0.1, 0.4, -1.0, 1.0, 0.1, 0.1) """ return crps.gtct( - observation, + obs, df, location, scale, @@ -1339,7 +1381,7 @@ def crps_gtct( def crps_tt( - observation: "ArrayLike", + obs: "ArrayLike", df: "ArrayLike", /, location: "ArrayLike" = 0.0, @@ -1356,22 +1398,22 @@ def crps_tt( Parameters ---------- - observation: array_like + obs : array_like The observed values. - df: array_like + df : array_like Degrees of freedom parameter of the forecast distribution. - location: array_like + location : array_like Location parameter of the forecast distribution. - scale: array_like + scale : array_like Scale parameter of the forecast distribution. - lower: array_like + lower : array_like Lower boundary of the truncated forecast distribution. - upper: array_like + upper : array_like Upper boundary of the truncated forecast distribution. Returns ------- - crps: array_like + crps : array_like The CRPS between tt(df, location, scale, lower, upper) and obs. Examples @@ -1380,7 +1422,7 @@ def crps_tt( >>> sr.crps_tt(0.0, 2.0, 0.1, 0.4, -1.0, 1.0) """ return crps.gtct( - observation, + obs, df, location, scale, @@ -1393,7 +1435,7 @@ def crps_tt( def crps_ct( - observation: "ArrayLike", + obs: "ArrayLike", df: "ArrayLike", /, location: "ArrayLike" = 0.0, @@ -1410,22 +1452,22 @@ def crps_ct( Parameters ---------- - observation: array_like + obs : array_like The observed values. - df: array_like + df : array_like Degrees of freedom parameter of the forecast distribution. - location: array_like + location : array_like Location parameter of the forecast distribution. - scale: array_like + scale : array_like Scale parameter of the forecast distribution. - lower: array_like + lower : array_like Lower boundary of the truncated forecast distribution. - upper: array_like + upper : array_like Upper boundary of the truncated forecast distribution. Returns ------- - crps: array_like + crps : array_like The CRPS between ct(df, location, scale, lower, upper) and obs. Examples @@ -1436,7 +1478,7 @@ def crps_ct( lmass = stats._t_cdf((lower - location) / scale, df) umass = 1 - stats._t_cdf((upper - location) / scale, df) return crps.gtct( - observation, + obs, df, location, scale, @@ -1449,7 +1491,7 @@ def crps_ct( def crps_hypergeometric( - observation: "ArrayLike", + obs: "ArrayLike", m: "ArrayLike", n: "ArrayLike", k: "ArrayLike", @@ -1459,29 +1501,27 @@ def crps_hypergeometric( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the hypergeometric distribution. - It is based on the following formulation from - [Jordan et al. (2019)](https://www.jstatsoft.org/article/view/v090i12): + It is based on the following formulation from [1]_: - $$ - \mathrm{CRPS}(F_{m, n, k}, y) = 2 \sum_{x = 0}^{n} f_{m,n,k}(x) (1\{y < x\} - - F_{m,n,k}(x) + f_{m,n,k}(x)/2) (x - y), - $$ + .. math:: + \mathrm{CRPS}(F_{m, n, k}, y) = 2 \sum_{x = 0}^{n} f_{m,n,k}(x) (1\{y < x\} + - F_{m,n,k}(x) + f_{m,n,k}(x)/2) (x - y), - where $f_{m, n, k}$ and $F_{m, n, k}$ are the PDF and CDF of the hypergeometric - distribution with population parameters $m,n = 0, 1, 2, ...$ and size parameter - $k = 0, ..., m + n$. + where :math:`f_{m, n, k}` and :math:`F_{m, n, k}` are the PDF and CDF of the hypergeometric + distribution with population parameters :math:`m,n = 0, 1, 2, ...` and size parameter + :math:`k = 0, ..., m + n`. Parameters ---------- - observation: + obs : array_like The observed values. - m: + m : array_like Number of success states in the population. - n: + n : array_like Number of failure states in the population. - k: + k : array_like Number of draws, without replacement. Must be in 0, 1, ..., m + n. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns @@ -1489,17 +1529,24 @@ def crps_hypergeometric( crps: The CRPS between obs and Hypergeometric(m, n, k). + References + ---------- + .. [1] Jordan, A., Krüger, F., & Lerch, S. (2019). + Evaluating Probabilistic Forecasts with scoringRules. + Journal of Statistical Software, 90(12), 1-37. + https://doi.org/10.18637/jss.v090.i12 + Examples -------- >>> import scoringrules as sr >>> sr.crps_hypergeometric(5, 7, 13, 12) 0.44697415547610597 """ - return crps.hypergeometric(observation, m, n, k, backend=backend) + return crps.hypergeometric(obs, m, n, k, backend=backend) def crps_laplace( - observation: "ArrayLike", + obs: "ArrayLike", /, location: "ArrayLike" = 0.0, scale: "ArrayLike" = 1.0, @@ -1508,26 +1555,24 @@ def crps_laplace( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the laplace distribution. - It is based on the following formulation from - [Jordan et al. (2019)](https://www.jstatsoft.org/article/view/v090i12): + It is based on the following formulation from [1]_: - $$ - \mathrm{CRPS}(F, y) = |y - \mu| - + \sigma \exp ( -| y - \mu| / \sigma) - \frac{3\sigma}{4}, - $$ + .. math:: + \mathrm{CRPS}(F, y) = |y - \mu| + + \sigma \exp ( -| y - \mu| / \sigma) - \frac{3\sigma}{4}, - where $\mu$ and $\sigma > 0$ are the location and scale parameters + where :math:`\mu` and :math:`\sigma > 0` are the location and scale parameters of the Laplace distribution. Parameters ---------- - observation: + obs : array_like Observed values. - location: + location : array_like Location parameter of the forecast laplace distribution. - scale: + scale : array_like Scale parameter of the forecast laplace distribution. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns @@ -1535,14 +1580,23 @@ def crps_laplace( crps: The CRPS between obs and Laplace(location, scale). + References + ---------- + .. [1] Jordan, A., Krüger, F., & Lerch, S. (2019). + Evaluating Probabilistic Forecasts with scoringRules. + Journal of Statistical Software, 90(12), 1-37. + https://doi.org/10.18637/jss.v090.i12 + + Examples + -------- >>> sr.crps_laplace(0.3, 0.1, 0.2) 0.12357588823428847 """ - return crps.laplace(observation, location, scale, backend=backend) + return crps.laplace(obs, location, scale, backend=backend) def crps_logistic( - observation: "ArrayLike", + obs: "ArrayLike", mu: "ArrayLike", sigma: "ArrayLike", /, @@ -1551,17 +1605,17 @@ def crps_logistic( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the logistic distribution. - It is based on the following formulation from - [Jordan et al. (2019)](https://www.jstatsoft.org/article/view/v090i12): + It is based on the following formulation from [1]_: - $$ \mathrm{CRPS}(\mathcal{L}(\mu, \sigma), y) = \sigma \left\{ \omega - 2 \log F(\omega) - 1 \right\}, $$ + .. math:: + \mathrm{CRPS}(\mathcal{L}(\mu, \sigma), y) = \sigma \left\{ \omega - 2 \log F(\omega) - 1 \right\}, - where $F(\omega)$ is the CDF of the standard logistic distribution at the - normalized prediction error $\omega = \frac{y - \mu}{\sigma}$. + where :math:`F(\omega)` is the CDF of the standard logistic distribution at the + normalized prediction error :math:`\omega = \frac{y - \mu}{\sigma}`. Parameters ---------- - observations: array_like + obs : array_like Observed values. mu: array_like Location parameter of the forecast logistic distribution. @@ -1570,20 +1624,27 @@ def crps_logistic( Returns ------- - crps: array_like + crps : array_like The CRPS for the Logistic(mu, sigma) forecasts given the observations. + References + ---------- + .. [1] Jordan, A., Krüger, F., & Lerch, S. (2019). + Evaluating Probabilistic Forecasts with scoringRules. + Journal of Statistical Software, 90(12), 1-37. + https://doi.org/10.18637/jss.v090.i12 + Examples -------- >>> import scoringrules as sr >>> sr.crps_logistic(0.0, 0.4, 0.1) 0.30363 """ - return crps.logistic(observation, mu, sigma, backend=backend) + return crps.logistic(obs, mu, sigma, backend=backend) def crps_loglaplace( - observation: "ArrayLike", + obs: "ArrayLike", locationlog: "ArrayLike", scalelog: "ArrayLike", *, @@ -1591,108 +1652,119 @@ def crps_loglaplace( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the log-Laplace distribution. - It is based on the following formulation from - [Jordan et al. (2019)](https://www.jstatsoft.org/article/view/v090i12): + It is based on the following formulation from [1]_: - $$ - \mathrm{CRPS}(F_{\mu, \sigma}, y) = y (2 F_{\mu, \sigma}(y) - 1) - + \exp(\mu) \left( \frac{\sigma}{4 - \sigma^{2}} + A(y) \right), - $$ + .. math:: + \mathrm{CRPS}(F_{\mu, \sigma}, y) = y (2 F_{\mu, \sigma}(y) - 1) + + \exp(\mu) \left( \frac{\sigma}{4 - \sigma^{2}} + A(y) \right), - where $F_{\mu, \sigma}$ is the CDF of the log-laplace distribution with location - parameter $\mu$ and scale parameter $\sigma \in (0, 1)$, and + where :math:`F_{\mu, \sigma}` is the CDF of the log-laplace distribution with location + parameter :math:`\mu` and scale parameter :math:`\sigma \in (0, 1)`, and - $$ A(y) = \frac{1}{1 + \sigma} \left( 1 - (2 F_{\mu, \sigma}(y) - 1)^{1 + \sigma} \right), $$ + .. math:: + A(y) = \frac{1}{1 + \sigma} \left( 1 - (2 F_{\mu, \sigma}(y) - 1)^{1 + \sigma} \right), - if $y < \exp{\mu}$, and + if :math:`y < \exp{\mu}`, and - $$ A(y) = \frac{-1}{1 - \sigma} \left( 1 - (2 (1 - F_{\mu, \sigma}(y)))^{1 - \sigma} \right), $$ + .. math:: + A(y) = \frac{-1}{1 - \sigma} \left( 1 - (2 (1 - F_{\mu, \sigma}(y)))^{1 - \sigma} \right), - if $y \ge \exp{\mu}$. + if :math:`y \ge \exp{\mu}`. Parameters ---------- - observation: + obs : array_like Observed values. - locationlog: + locationlog : array_like Location parameter of the forecast log-laplace distribution. - scalelog: + scalelog : array_like Scale parameter of the forecast log-laplace distribution. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - crps: + crps : array_like The CRPS between obs and Loglaplace(locationlog, scalelog). + References + ---------- + .. [1] Jordan, A., Krüger, F., & Lerch, S. (2019). + Evaluating Probabilistic Forecasts with scoringRules. + Journal of Statistical Software, 90(12), 1-37. + https://doi.org/10.18637/jss.v090.i12 + Examples -------- >>> import scoringrules as sr >>> sr.crps_loglaplace(3.0, 0.1, 0.9) 1.162020513653791 """ - return crps.loglaplace(observation, locationlog, scalelog, backend=backend) + return crps.loglaplace(obs, locationlog, scalelog, backend=backend) def crps_loglogistic( - observation: "ArrayLike", + obs: "ArrayLike", mulog: "ArrayLike", sigmalog: "ArrayLike", backend: "Backend" = None, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the log-logistic distribution. - It is based on the following formulation from - [Jordan et al. (2019)](https://www.jstatsoft.org/article/view/v090i12): + It is based on the following formulation from [1]_: - $$ - \text{CRPS}(y, F_{\mu,\sigma}) = y \left( 2F_{\mu,\sigma}(y) - 1 \right) - - 2 \exp(\mu) I\left(F_{\mu,\sigma}(y); 1 + \sigma, 1 - \sigma\right) \\ - + \exp(\mu)(1 - \sigma) B(1 + \sigma, 1 - \sigma) - $$ + .. math:: + \text{CRPS}(y, F_{\mu,\sigma}) = y \left( 2F_{\mu,\sigma}(y) - 1 \right) + - 2 \exp(\mu) I\left(F_{\mu,\sigma}(y); 1 + \sigma, 1 - \sigma\right) \\ + + \exp(\mu)(1 - \sigma) B(1 + \sigma, 1 - \sigma) where \( F_{\mu,\sigma}(x) \) is the cumulative distribution function (CDF) of the log-logistic distribution, defined as: - $$ - F_{\mu,\sigma}(x) = - \begin{cases} - 0, & x \leq 0 \\ - \left( 1 + \exp\left(-\frac{\log x - \mu}{\sigma}\right) \right)^{-1}, & x > 0 - \end{cases} - $$ + .. math:: + F_{\mu,\sigma}(x) = + \begin{cases} + 0, & x \leq 0 \\ + \left( 1 + \exp\left(-\frac{\log x - \mu}{\sigma}\right) \right)^{-1}, & x > 0 + \end{cases} - $B$ is the beta function, and $I$ is the regularised incomplete beta function. + :math:`B` is the beta function, and :math:`I` is the regularised incomplete beta function. Parameters ---------- - observation: + obs : array_like The observed values. - mulog: + mulog : array_like Location parameter of the log-logistic distribution. - sigmalog: + sigmalog : array_like Scale parameter of the log-logistic distribution. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - crps: + crps : array_like The CRPS between obs and Loglogis(mulog, sigmalog). + References + ---------- + .. [1] Jordan, A., Krüger, F., & Lerch, S. (2019). + Evaluating Probabilistic Forecasts with scoringRules. + Journal of Statistical Software, 90(12), 1-37. + https://doi.org/10.18637/jss.v090.i12 + Examples -------- >>> import scoringrules as sr >>> sr.crps_loglogistic(3.0, 0.1, 0.9) 1.1329527730161177 """ - return crps.loglogistic(observation, mulog, sigmalog, backend=backend) + return crps.loglogistic(obs, mulog, sigmalog, backend=backend) def crps_lognormal( - observation: "ArrayLike", + obs: "ArrayLike", mulog: "ArrayLike", sigmalog: "ArrayLike", backend: "Backend" = None, @@ -1702,28 +1774,29 @@ def crps_lognormal( It is based on the formulation introduced by [Baran and Lerch (2015)](https://rmets.onlinelibrary.wiley.com/doi/full/10.1002/qj.2521) - $$ \mathrm{CRPS}(\mathrm{log}\mathcal{N}(\mu, \sigma), y) = - y [2 \Phi(y) - 1] - 2 \mathrm{exp}(\mu + \frac{\sigma^2}{2}) - \left[ \Phi(\omega - \sigma) + \Phi(\frac{\sigma}{\sqrt{2}}) \right]$$ + .. math:: + \mathrm{CRPS}(\mathrm{log}\mathcal{N}(\mu, \sigma), y) = + y [2 \Phi(y) - 1] - 2 \mathrm{exp}(\mu + \frac{\sigma^2}{2}) + \left[ \Phi(\omega - \sigma) + \Phi(\frac{\sigma}{\sqrt{2}}) \right], - where $\Phi$ is the CDF of the standard normal distribution and - $\omega = \frac{\mathrm{log}y - \mu}{\sigma}$. + where :math:`\Phi` is the CDF of the standard normal distribution and + :math:`\omega = \frac{\mathrm{log}y - \mu}{\sigma}`. Note that mean and standard deviation are not the values for the distribution itself, but of the underlying normal distribution it is derived from. Parameters ---------- - observation: + obs : array_like The observed values. - mulog: + mulog : array_like Mean of the normal underlying distribution. - sigmalog: + sigmalog : array_like Standard deviation of the underlying normal distribution. Returns ------- - crps: array_like + crps : array_like The CRPS between Lognormal(mu, sigma) and obs. Examples @@ -1731,11 +1804,11 @@ def crps_lognormal( >>> import scoringrules as sr >>> sr.crps_lognormal(0.1, 0.4, 0.0) """ - return crps.lognormal(observation, mulog, sigmalog, backend=backend) + return crps.lognormal(obs, mulog, sigmalog, backend=backend) def crps_mixnorm( - observation: "ArrayLike", + obs: "ArrayLike", m: "ArrayLike", s: "ArrayLike", /, @@ -1749,14 +1822,15 @@ def crps_mixnorm( It is based on the following formulation from [Grimit et al. (2006)](https://doi.org/10.1256/qj.05.235): - $$ \mathrm{CRPS}(F, y) = \sum_{i=1}^{M} w_{i} A(y - \mu_{i}, \sigma_{i}^{2}) - \frac{1}{2} \sum_{i=1}^{M} \sum_{j=1}^{M} w_{i} w_{j} A(\mu_{i} - \mu_{j}, \sigma_{i}^{2} + \sigma_{j}^{2}), $$ + .. math:: + \mathrm{CRPS}(F, y) = \sum_{i=1}^{M} w_{i} A(y - \mu_{i}, \sigma_{i}^{2}) - \frac{1}{2} \sum_{i=1}^{M} \sum_{j=1}^{M} w_{i} w_{j} A(\mu_{i} - \mu_{j}, \sigma_{i}^{2} + \sigma_{j}^{2}), - where $F(x) = \sum_{i=1}^{M} w_{i} \Phi \left( \frac{x - \mu_{i}}{\sigma_{i}} \right)$, - and $A(\mu, \sigma^{2}) = \mu (2 \Phi(\frac{\mu}{\sigma}) - 1) + 2\sigma \phi(\frac{\mu}{\sigma}).$ + where :math:`F(x) = \sum_{i=1}^{M} w_{i} \Phi \left( \frac{x - \mu_{i}}{\sigma_{i}} \right)`, + and :math:`A(\mu, \sigma^{2}) = \mu (2 \Phi(\frac{\mu}{\sigma}) - 1) + 2\sigma \phi(\frac{\mu}{\sigma}).` Parameters ---------- - observation: array_like + obs : array_like The observed values. m: array_like Means of the component normal distributions. @@ -1764,14 +1838,14 @@ def crps_mixnorm( Standard deviations of the component normal distributions. w: array_like Non-negative weights assigned to each component. - axis: int + axis : int The axis corresponding to the mixture components. Default is the last axis. - backend: str, optional + backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. Returns ------- - crps: + crps : array_like The CRPS between MixNormal(m, s) and obs. Examples @@ -1780,7 +1854,7 @@ def crps_mixnorm( >>> sr.crps_mixnormal(0.0, [0.1, -0.3, 1.0], [0.4, 2.1, 0.7], [0.1, 0.2, 0.7]) """ B = backends.active if backend is None else backends[backend] - observation, m, s = map(B.asarray, (observation, m, s)) + observation, m, s = map(B.asarray, (obs, m, s)) if w is None: M: int = m.shape[axis] @@ -1793,11 +1867,11 @@ def crps_mixnorm( s = B.moveaxis(s, axis, -1) w = B.moveaxis(w, axis, -1) - return crps.mixnorm(observation, m, s, w, backend=backend) + return crps.mixnorm(obs, m, s, w, backend=backend) def crps_negbinom( - observation: "ArrayLike", + obs: "ArrayLike", n: "ArrayLike", /, prob: "ArrayLike | None" = None, @@ -1810,26 +1884,27 @@ def crps_negbinom( It is based on the following formulation from [Wei and Held (2014)](https://link.springer.com/article/10.1007/s11749-014-0380-8): - $$ \mathrm{CRPS}(F_{n, p}, y) = y (2 F_{n, p}(y) - 1) - \frac{n(1 - p)}{p^{2}} \left( p (2 F_{n+1, p}(y - 1) - 1) + _{2} F_{1} \left( n + 1, \frac{1}{2}; 2; -\frac{4(1 - p)}{p^{2}} \right) \right), $$ + .. math:: + \mathrm{CRPS}(F_{n, p}, y) = y (2 F_{n, p}(y) - 1) - \frac{n(1 - p)}{p^{2}} \left( p (2 F_{n+1, p}(y - 1) - 1) + _{2} F_{1} \left( n + 1, \frac{1}{2}; 2; -\frac{4(1 - p)}{p^{2}} \right) \right), - where $F_{n, p}$ is the CDF of the negative binomial distribution with - size parameter $n > 0$ and probability parameter $p \in (0, 1]$. The mean - of the negative binomial distribution is $\mu = n (1 - p)/p$. + where :math:`F_{n, p}` is the CDF of the negative binomial distribution with + size parameter :math:`n > 0` and probability parameter :math:`p \in (0, 1]`. The mean + of the negative binomial distribution is :math:`\mu = n (1 - p)/p`. Parameters ---------- - observation: array_like + obs : array_like The observed values. n: array_like Size parameter of the forecast negative binomial distribution. - prob: array_like + prob : array_like Probability parameter of the forecast negative binomial distribution. mu: array_like Mean of the forecast negative binomial distribution. Returns ------- - crps: array_like + crps : array_like The CRPS between NegBinomial(n, prob) and obs. Examples @@ -1850,11 +1925,11 @@ def crps_negbinom( if prob is None: prob = n / (n + mu) - return crps.negbinom(observation, n, prob, backend=backend) + return crps.negbinom(obs, n, prob, backend=backend) def crps_normal( - observation: "ArrayLike", + obs: "ArrayLike", mu: "ArrayLike", sigma: "ArrayLike", /, @@ -1866,14 +1941,15 @@ def crps_normal( It is based on the following formulation from [Geiting et al. (2005)](https://journals.ametsoc.org/view/journals/mwre/133/5/mwr2904.1.xml): - $$ \mathrm{CRPS}(\mathcal{N}(\mu, \sigma), y) = \sigma \Bigl\{ \omega [\Phi(ω) - 1] + 2 \phi(\omega) - \frac{1}{\sqrt{\pi}} \Bigl\},$$ + .. math:: + \mathrm{CRPS}(\mathcal{N}(\mu, \sigma), y) = \sigma \Bigl\{ \omega [\Phi(ω) - 1] + 2 \phi(\omega) - \frac{1}{\sqrt{\pi}} \Bigl\} - where $\Phi(ω)$ and $\phi(ω)$ are respectively the CDF and PDF of the standard normal - distribution at the normalized prediction error $\omega = \frac{y - \mu}{\sigma}$. + where :math:`\Phi(ω)` and :math:`\phi(ω)` are respectively the CDF and PDF of the standard normal + distribution at the normalized prediction error :math:`\omega = \frac{y - \mu}{\sigma}`. Parameters ---------- - observations: array_like + obs : array_like The observed values. mu: array_like Mean of the forecast normal distribution. @@ -1882,7 +1958,7 @@ def crps_normal( Returns ------- - crps: array_like + crps : array_like The CRPS between Normal(mu, sigma) and obs. Examples @@ -1890,11 +1966,11 @@ def crps_normal( >>> import scoringrules as sr >>> sr.crps_normal(0.0, 0.1, 0.4) """ - return crps.normal(observation, mu, sigma, backend=backend) + return crps.normal(obs, mu, sigma, backend=backend) def crps_2pnormal( - observation: "ArrayLike", + obs: "ArrayLike", scale1: "ArrayLike", scale2: "ArrayLike", location: "ArrayLike", @@ -1904,42 +1980,47 @@ def crps_2pnormal( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the two-piece normal distribution. - It is based on the following relationship given in - [Jordan et al. (2019)](https://www.jstatsoft.org/article/view/v090i12): + It is based on the following relationship given in [1]_: - $$ \mathrm{CRPS}(F_{\sigma_{1}, \sigma_{2}, \mu}, y) = - \sigma_{1} \mathrm{CRPS} \left( F_{-\infty,0}^{0, \sigma_{2}/(\sigma_{1} + \sigma_{2})}, \frac{\min(0, y - \mu)}{\sigma_{1}} \right) + - \sigma_{2} \mathrm{CRPS} \left( F_{0, \sigma_{1}/(\sigma_{1} + \sigma_{2})}^{\infty, 0}, \frac{\min(0, y - \mu)}{\sigma_{2}} \right), $$ + .. math:: + \mathrm{CRPS}(F_{\sigma_{1}, \sigma_{2}, \mu}, y) = + \sigma_{1} \mathrm{CRPS} \left( F_{-\infty,0}^{0, \sigma_{2}/(\sigma_{1} + \sigma_{2})}, \frac{\min(0, y - \mu)}{\sigma_{1}} \right) + + \sigma_{2} \mathrm{CRPS} \left( F_{0, \sigma_{1}/(\sigma_{1} + \sigma_{2})}^{\infty, 0}, \frac{\min(0, y - \mu)}{\sigma_{2}} \right), - where $F_{\sigma_{1}, \sigma_{2}, \mu}$ is the two-piece normal distribution with - scale1 and scale2 parameters $\sigma_{1}, \sigma_{2} > 0$ and location parameter $\mu$, - and $F_{l, L}^{u, U}$ is the CDF of the generalised truncated and censored normal distribution. + where :math:`F_{\sigma_{1}, \sigma_{2}, \mu}` is the two-piece normal distribution with + scale1 and scale2 parameters :math:`\sigma_{1}, \sigma_{2} > 0` and location parameter :math:`\mu`, + and :math:`F_{l, L}^{u, U}` is the CDF of the generalised truncated and censored normal distribution. Parameters ---------- - observations: array_like + obs : array_like The observed values. - scale1: array_like + scale1 : array_like Scale parameter of the lower half of the forecast two-piece normal distribution. - scale2: array_like + scale2 : array_like Scale parameter of the upper half of the forecast two-piece normal distribution. mu: array_like Location parameter of the forecast two-piece normal distribution. Returns ------- - crps: array_like + crps : array_like The CRPS between 2pNormal(scale1, scale2, mu) and obs. + References + ---------- + .. [1] Jordan, A., Krüger, F., & Lerch, S. (2019). + Evaluating Probabilistic Forecasts with scoringRules. + Journal of Statistical Software, 90(12), 1-37. + https://doi.org/10.18637/jss.v090.i12 + Examples -------- >>> import scoringrules as sr >>> sr.crps_2pnormal(0.0, 0.4, 2.0, 0.1) """ B = backends.active if backend is None else backends[backend] - obs, scale1, scale2, location = map( - B.asarray, (observation, scale1, scale2, location) - ) + obs, scale1, scale2, location = map(B.asarray, (obs, scale1, scale2, location)) lower = float("-inf") upper = 0.0 lmass = 0.0 @@ -1960,7 +2041,7 @@ def crps_2pnormal( def crps_poisson( - observation: "ArrayLike", + obs: "ArrayLike", mean: "ArrayLike", /, *, @@ -1971,21 +2052,24 @@ def crps_poisson( It is based on the following formulation from [Wei and Held (2014)](https://link.springer.com/article/10.1007/s11749-014-0380-8): - $$ \mathrm{CRPS}(F_{\lambda}, y) = (y - \lambda) (2F_{\lambda}(y) - 1) + 2 \lambda f_{\lambda}(\lfloor y \rfloor ) - \lambda \exp (-2 \lambda) (I_{0} (2 \lambda) + I_{1} (2 \lambda))..$$ + .. math:: + \mathrm{CRPS}(F_{\lambda}, y) = (y - \lambda) (2F_{\lambda}(y) - 1) + + 2 \lambda f_{\lambda}(\lfloor y \rfloor ) + - \lambda \exp (-2 \lambda) (I_{0} (2 \lambda) + I_{1} (2 \lambda)), - where $F_{\lambda}$ is Poisson distribution function with mean parameter $\lambda > 0$, - and $I_{0}$ and $I_{1}$ are modified Bessel functions of the first kind. + where :math:`F_{\lambda}` is Poisson distribution function with mean parameter :math:`\lambda > 0`, + and :math:`I_{0}` and :math:`I_{1}` are modified Bessel functions of the first kind. Parameters ---------- - observation: array_like + obs : array_like The observed values. mean: array_like Mean parameter of the forecast poisson distribution. Returns ------- - crps: array_like + crps : array_like The CRPS between Pois(mean) and obs. Examples @@ -1993,11 +2077,11 @@ def crps_poisson( >>> import scoringrules as sr >>> sr.crps_poisson(1, 2) """ - return crps.poisson(observation, mean, backend=backend) + return crps.poisson(obs, mean, backend=backend) def crps_t( - observation: "ArrayLike", + obs: "ArrayLike", df: "ArrayLike", /, location: "ArrayLike" = 0.0, @@ -2007,47 +2091,52 @@ def crps_t( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the student's t distribution. - It is based on the following formulation from - [Jordan et al. (2019)](https://www.jstatsoft.org/article/view/v090i12): + It is based on the following formulation from [1]_: - $$ - \mathrm{CRPS}(F, y) = \sigma \left\{ \omega (2 F_{\nu} (\omega) - 1) - + 2 f_{\nu} \left( \frac{\nu + \omega^{2}}{\nu - 1} \right) - - \frac{2 \sqrt{\nu}}{\nu - 1} \frac{B(\frac{1}{2}, - \nu - \frac{1}{2})}{B(\frac{1}{2}, \frac{\nu}{2}^{2})} \right\}, - $$ + .. math:: + \mathrm{CRPS}(F, y) = \sigma \left\{ \omega (2 F_{\nu} (\omega) - 1) + + 2 f_{\nu} \left( \frac{\nu + \omega^{2}}{\nu - 1} \right) + - \frac{2 \sqrt{\nu}}{\nu - 1} \frac{B(\frac{1}{2}, + \nu - \frac{1}{2})}{B(\frac{1}{2}, \frac{\nu}{2}^{2})} \right\}, - where $\omega = (y - \mu)/\sigma$, where $\nu > 1, \mu$, and $\sigma > 0$ are the + where :math:`\omega = (y - \mu)/\sigma`, where :math:`\nu > 1, \mu`, and :math:`\sigma > 0` are the degrees of freedom, location, and scale parameters respectively of the Student's t - distribution, and $f_{\nu}$ and $F_{\nu}$ are the PDF and CDF of the standard Student's - t distribution with $\nu$ degrees of freedom. + distribution, and :math:`f_{\nu}` and :math:`F_{\nu}` are the PDF and CDF of the standard Student's + t distribution with :math:`\nu` degrees of freedom. Parameters ---------- - observation: array_like + obs : array_like The observed values. - df: array_like + df : array_like Degrees of freedom parameter of the forecast t distribution. - location: array_like + location : array_like Location parameter of the forecast t distribution. sigma: array_like Scale parameter of the forecast t distribution. Returns ------- - crps: array_like + crps : array_like The CRPS between t(df, location, scale) and obs. + References + ---------- + .. [1] Jordan, A., Krüger, F., & Lerch, S. (2019). + Evaluating Probabilistic Forecasts with scoringRules. + Journal of Statistical Software, 90(12), 1-37. + https://doi.org/10.18637/jss.v090.i12 + Examples -------- >>> import scoringrules as sr >>> sr.crps_t(0.0, 0.1, 0.4, 0.1) """ - return crps.t(observation, df, location, scale, backend=backend) + return crps.t(obs, df, location, scale, backend=backend) def crps_uniform( - observation: "ArrayLike", + obs: "ArrayLike", min: "ArrayLike", max: "ArrayLike", /, @@ -2058,39 +2147,49 @@ def crps_uniform( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the uniform distribution. - It is based on the following formulation from - [Jordan et al. (2019)](https://www.jstatsoft.org/article/view/v090i12): + It is based on the following formulation from [1]_: - $$ \mathrm{CRPS}(\mathcal{U}_{L}^{U}(l, u), y) = (u - l) \left\{ | \frac{y - l}{u - l} - F \left( \frac{y - l}{u - l} \right) | + F \left( \frac{y - l}{u - l} \right)^{2} (1 - L - U) - F \left( \frac{y - l}{u - l} \right) (1 - 2L) + \frac{(1 - L - U)^{2}}{3} + (1 - L)U \right\},$$ + .. math:: + \mathrm{CRPS}(\mathcal{U}_{L}^{U}(l, u), y) = (u - l) \left\{ | \frac{y - l}{u - l} + - F \left( \frac{y - l}{u - l} \right) | + + F \left( \frac{y - l}{u - l} \right)^{2} (1 - L - U) + - F \left( \frac{y - l}{u - l} \right) (1 - 2L) + \frac{(1 - L - U)^{2}}{3} + (1 - L)U \right\} - where $\mathcal{U}_{L}^{U}(l, u)$ is the uniform distribution with lower bound $l$, - upper bound $u > l$, point mass $L$ on the lower bound, and point mass $U$ on the upper bound. - We must have that $L, U \ge 0, L + U < 1$. + where :math:`\mathcal{U}_{L}^{U}(l, u)` is the uniform distribution with lower bound :math:`l`, + upper bound :math:`u > l`, point mass :math:`L` on the lower bound, and point mass :math:`U` on the upper bound. + We must have that :math:`L, U \ge 0, L + U < 1`. Parameters ---------- - observation: array_like + obs : array_like The observed values. min: array_like Lower bound of the forecast uniform distribution. max: array_like Upper bound of the forecast uniform distribution. - lmass: array_like + lmass : array_like Point mass on the lower bound of the forecast uniform distribution. - umass: array_like + umass : array_like Point mass on the upper bound of the forecast uniform distribution. Returns ------- - crps: array_like + crps : array_like The CRPS between U(min, max, lmass, umass) and obs. + References + ---------- + .. [1] Jordan, A., Krüger, F., & Lerch, S. (2019). + Evaluating Probabilistic Forecasts with scoringRules. + Journal of Statistical Software, 90(12), 1-37. + https://doi.org/10.18637/jss.v090.i12 + Examples -------- >>> import scoringrules as sr >>> sr.crps_uniform(0.4, 0.0, 1.0, 0.0, 0.0) """ - return crps.uniform(observation, min, max, lmass, umass, backend=backend) + return crps.uniform(obs, min, max, lmass, umass, backend=backend) __all__ = [ From ca58cea3fe768b130de32f74ffcb76cb3504831d Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Sat, 1 Feb 2025 11:32:23 +0100 Subject: [PATCH 17/79] pass doctest for examples in _crps.py --- scoringrules/_crps.py | 136 +++++++++++++++++++++++++++++------------- 1 file changed, 94 insertions(+), 42 deletions(-) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index b4fc4df..8b099c9 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -88,10 +88,11 @@ def crps_ensemble( -------- >>> import numpy as np >>> import scoringrules as sr - >>> obs = np.array([1.0, 2.0, 3.0]) - >>> pred = np.array([[1.1, 1.2, 1.3], [2.1, 2.2, 2.3], [3.1, 3.2, 3.3]]) + >>> rng = np.random.default_rng(123) + >>> obs = rng.normal(size=3) + >>> pred = rng.normal(size=(3, 10)) >>> sr.crps_ensemble(obs, pred) - 0.0 + array([0.69605316, 0.32865417, 0.39048665]) """ B = backends.active if backend is None else backends[backend] obs, fct = map(B.asarray, (obs, fct)) @@ -183,11 +184,15 @@ def twcrps_ensemble( -------- >>> import numpy as np >>> import scoringrules as sr - >>> + >>> rng = np.random.default_rng(123) + ... >>> def v_func(x): - >>> return np.maximum(x, -1.0) - >>> - >>> sr.twcrps_ensemble(obs, pred, v_func) + ... return np.maximum(x, -1.0) + ... + >>> obs = rng.normal(size=3) + >>> fct = rng.normal(size=(3, 10)) + >>> sr.twcrps_ensemble(obs, fct, v_func) + array([0.69605316, 0.32865417, 0.39048665]) """ obs, fct = map(v_func, (obs, fct)) return crps_ensemble( @@ -264,11 +269,15 @@ def owcrps_ensemble( -------- >>> import numpy as np >>> import scoringrules as sr - >>> + >>> rng = np.random.default_rng(123) + ... >>> def w_func(x): - >>> return (x > -1).astype(float) - >>> - >>> sr.owcrps_ensemble(obs, pred, w_func) + ... return (x > -1).astype(float) + ... + >>> obs = rng.normal(size=3) + >>> fct = rng.normal(size=(3, 10)) + >>> sr.owcrps_ensemble(obs, fct, w_func) + array([0.91103733, 0.45212402, 0.35686667]) """ B = backends.active if backend is None else backends[backend] @@ -355,11 +364,15 @@ def vrcrps_ensemble( -------- >>> import numpy as np >>> import scoringrules as sr - >>> + >>> rng = np.random.default_rng(123) + ... >>> def w_func(x): - >>> return (x > -1).astype(float) - >>> - >>> sr.vrcrps_ensemble(obs, pred, w_func) + ... return (x > -1).astype(float) + ... + >>> obs = rng.normal(size=3) + >>> fct = rng.normal(size=(3, 10)) + >>> sr.vrcrps_ensemble(obs, fct, w_func) + array([0.90036433, 0.41515255, 0.41653833]) """ B = backends.active if backend is None else backends[backend] @@ -436,14 +449,7 @@ def crps_quantile( Journal of Econometrics, 237(2), 105221. Available at https://arxiv.org/abs/2102.00968. - Examples - -------- - >>> import numpy as np - >>> import scoringrules as sr - >>> obs = np.array([1.0, 2.0, 3.0]) - >>> fct = np.array([[1.1, 1.2, 1.3], [2.1, 2.2, 2.3], [3.1, 3.2, 3.3]]) - >>> sr.crps_quantile(obs, fct, alpha) - 0.0 + # TODO: add example """ B = backends.active if backend is None else backends[backend] obs, fct, alpha = map(B.asarray, (obs, fct, alpha)) @@ -519,7 +525,7 @@ def crps_beta( -------- >>> import scoringrules as sr >>> sr.crps_beta(0.3, 0.7, 1.1) - 0.0850102437 + 0.08501024366637236 """ return crps.beta(obs, a, b, lower, upper, backend=backend) @@ -570,7 +576,7 @@ def crps_binomial( -------- >>> import scoringrules as sr >>> sr.crps_binomial(4, 10, 0.5) - 0.5955715179443359 + 0.5955772399902344 """ return crps.binomial(obs, n, prob, backend=backend) @@ -619,7 +625,7 @@ def crps_exponential( >>> sr.crps_exponential(0.8, 3.0) 0.360478635526275 >>> sr.crps_exponential(np.array([0.8, 0.9]), np.array([3.0, 2.0])) - array([0.36047864, 0.24071795]) + array([0.36047864, 0.31529889]) """ return crps.exponential(obs, rate, backend=backend) @@ -682,6 +688,7 @@ def crps_exponentialM( -------- >>> import scoringrules as sr >>> sr.crps_exponentialM(0.4, 0.2, 0.0, 1.0) + 0.19251207365702294 """ return crps.exponentialM(obs, mass, location, scale, backend=backend) @@ -737,6 +744,7 @@ def crps_2pexponential( -------- >>> import scoringrules as sr >>> sr.crps_2pexponential(0.8, 3.0, 1.4, 0.0) + array(1.18038524) """ return crps.twopexponential(obs, scale1, scale2, location, backend=backend) @@ -1037,6 +1045,7 @@ def crps_gtclogistic( -------- >>> import scoringrules as sr >>> sr.crps_gtclogistic(0.0, 0.1, 0.4, -1.0, 1.0, 0.1, 0.1) + 0.1658713056903939 """ return crps.gtclogistic( obs, @@ -1087,6 +1096,7 @@ def crps_tlogistic( -------- >>> import scoringrules as sr >>> sr.crps_tlogistic(0.0, 0.1, 0.4, -1.0, 1.0) + 0.12714830546327846 """ return crps.gtclogistic( obs, location, scale, lower, upper, 0.0, 0.0, backend=backend @@ -1130,6 +1140,7 @@ def crps_clogistic( -------- >>> import scoringrules as sr >>> sr.crps_clogistic(0.0, 0.1, 0.4, -1.0, 1.0) + 0.15805632276434345 """ lmass = stats._logis_cdf((lower - location) / scale) umass = 1 - stats._logis_cdf((upper - location) / scale) @@ -1185,8 +1196,9 @@ def crps_gtcnormal( Examples -------- - >>> import scoring rules as sr + >>> import scoringrules as sr >>> sr.crps_gtcnormal(0.0, 0.1, 0.4, -1.0, 1.0, 0.1, 0.1) + 0.1351100832878575 """ return crps.gtcnormal( obs, @@ -1237,6 +1249,7 @@ def crps_tnormal( -------- >>> import scoringrules as sr >>> sr.crps_tnormal(0.0, 0.1, 0.4, -1.0, 1.0) + 0.10070146718008832 """ return crps.gtcnormal(obs, location, scale, lower, upper, 0.0, 0.0, backend=backend) @@ -1278,6 +1291,7 @@ def crps_cnormal( -------- >>> import scoringrules as sr >>> sr.crps_cnormal(0.0, 0.1, 0.4, -1.0, 1.0) + 0.10338851213123085 """ lmass = stats._norm_cdf((lower - location) / scale) umass = 1 - stats._norm_cdf((upper - location) / scale) @@ -1366,6 +1380,7 @@ def crps_gtct( -------- >>> import scoringrules as sr >>> sr.crps_gtct(0.0, 2.0, 0.1, 0.4, -1.0, 1.0, 0.1, 0.1) + 0.13997789333289662 """ return crps.gtct( obs, @@ -1420,6 +1435,7 @@ def crps_tt( -------- >>> import scoringrules as sr >>> sr.crps_tt(0.0, 2.0, 0.1, 0.4, -1.0, 1.0) + 0.10323007471747117 """ return crps.gtct( obs, @@ -1474,6 +1490,7 @@ def crps_ct( -------- >>> import scoringrules as sr >>> sr.crps_ct(0.0, 2.0, 0.1, 0.4, -1.0, 1.0) + 0.12672580744453948 """ lmass = stats._t_cdf((lower - location) / scale, df) umass = 1 - stats._t_cdf((upper - location) / scale, df) @@ -1589,6 +1606,7 @@ def crps_laplace( Examples -------- + >>> import scoringrules as sr >>> sr.crps_laplace(0.3, 0.1, 0.2) 0.12357588823428847 """ @@ -1638,7 +1656,7 @@ def crps_logistic( -------- >>> import scoringrules as sr >>> sr.crps_logistic(0.0, 0.4, 0.1) - 0.30363 + 0.3036299855835619 """ return crps.logistic(obs, mu, sigma, backend=backend) @@ -1771,8 +1789,7 @@ def crps_lognormal( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the lognormal distribution. - It is based on the formulation introduced by - [Baran and Lerch (2015)](https://rmets.onlinelibrary.wiley.com/doi/full/10.1002/qj.2521) + It is based on the formulation introduced by [1]_: .. math:: \mathrm{CRPS}(\mathrm{log}\mathcal{N}(\mu, \sigma), y) = @@ -1799,10 +1816,17 @@ def crps_lognormal( crps : array_like The CRPS between Lognormal(mu, sigma) and obs. + References + ---------- + .. [1] Baran, S. and Lerch, S. (2015), Log-normal distribution based + Ensemble Model Output Statistics models for probabilistic wind-speed forecasting. + Q.J.R. Meteorol. Soc., 141: 2289-2299. https://doi.org/10.1002/qj.2521 + Examples -------- >>> import scoringrules as sr >>> sr.crps_lognormal(0.1, 0.4, 0.0) + 1.3918246976412703 """ return crps.lognormal(obs, mulog, sigmalog, backend=backend) @@ -1819,8 +1843,7 @@ def crps_mixnorm( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for a mixture of normal distributions. - It is based on the following formulation from - [Grimit et al. (2006)](https://doi.org/10.1256/qj.05.235): + It is based on the following formulation from [1]_: .. math:: \mathrm{CRPS}(F, y) = \sum_{i=1}^{M} w_{i} A(y - \mu_{i}, \sigma_{i}^{2}) - \frac{1}{2} \sum_{i=1}^{M} \sum_{j=1}^{M} w_{i} w_{j} A(\mu_{i} - \mu_{j}, \sigma_{i}^{2} + \sigma_{j}^{2}), @@ -1848,13 +1871,21 @@ def crps_mixnorm( crps : array_like The CRPS between MixNormal(m, s) and obs. + References + ---------- + .. [1] Grimit, E.P., Gneiting, T., Berrocal, V.J. and Johnson, N.A. (2006), + The continuous ranked probability score for circular variables and its + application to mesoscale forecast ensemble verification. + Q.J.R. Meteorol. Soc., 132: 2925-2942. https://doi.org/10.1256/qj.05.235 + Examples -------- >>> import scoringrules as sr - >>> sr.crps_mixnormal(0.0, [0.1, -0.3, 1.0], [0.4, 2.1, 0.7], [0.1, 0.2, 0.7]) + >>> sr.crps_mixnorm(0.0, [0.1, -0.3, 1.0], [0.4, 2.1, 0.7], [0.1, 0.2, 0.7]) + 0.46806866729387275 """ B = backends.active if backend is None else backends[backend] - observation, m, s = map(B.asarray, (obs, m, s)) + obs, m, s = map(B.asarray, (obs, m, s)) if w is None: M: int = m.shape[axis] @@ -1881,8 +1912,7 @@ def crps_negbinom( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the negative binomial distribution. - It is based on the following formulation from - [Wei and Held (2014)](https://link.springer.com/article/10.1007/s11749-014-0380-8): + It is based on the following formulation from [1]_: .. math:: \mathrm{CRPS}(F_{n, p}, y) = y (2 F_{n, p}(y) - 1) - \frac{n(1 - p)}{p^{2}} \left( p (2 F_{n+1, p}(y - 1) - 1) + _{2} F_{1} \left( n + 1, \frac{1}{2}; 2; -\frac{4(1 - p)}{p^{2}} \right) \right), @@ -1907,10 +1937,17 @@ def crps_negbinom( crps : array_like The CRPS between NegBinomial(n, prob) and obs. + References + ---------- + .. [1] Wei, W., Held, L. (2014), + Calibration tests for count data. + TEST 23, 787-805. https://doi.org/10.1007/s11749-014-0380-8 + Examples -------- >>> import scoringrules as sr >>> sr.crps_negbinom(2, 5, 0.5) + 1.5533629909058577 Raises ------ @@ -1938,8 +1975,7 @@ def crps_normal( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the normal distribution. - It is based on the following formulation from - [Geiting et al. (2005)](https://journals.ametsoc.org/view/journals/mwre/133/5/mwr2904.1.xml): + It is based on the following formulation from [1]_: .. math:: \mathrm{CRPS}(\mathcal{N}(\mu, \sigma), y) = \sigma \Bigl\{ \omega [\Phi(ω) - 1] + 2 \phi(\omega) - \frac{1}{\sqrt{\pi}} \Bigl\} @@ -1961,10 +1997,17 @@ def crps_normal( crps : array_like The CRPS between Normal(mu, sigma) and obs. + References + ---------- + .. [1] Gneiting, T., A. E. Raftery, A. H. Westveld, and T. Goldman (2005), + Calibrated Probabilistic Forecasting Using Ensemble Model Output Statistics and Minimum CRPS Estimation. + Mon. Wea. Rev., 133, 1098-1118, https://doi.org/10.1175/MWR2904.1. + Examples -------- >>> import scoringrules as sr >>> sr.crps_normal(0.0, 0.1, 0.4) + 0.10339992515976162 """ return crps.normal(obs, mu, sigma, backend=backend) @@ -2018,6 +2061,7 @@ def crps_2pnormal( -------- >>> import scoringrules as sr >>> sr.crps_2pnormal(0.0, 0.4, 2.0, 0.1) + 0.7243199144002115 """ B = backends.active if backend is None else backends[backend] obs, scale1, scale2, location = map(B.asarray, (obs, scale1, scale2, location)) @@ -2049,8 +2093,7 @@ def crps_poisson( ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the Poisson distribution. - It is based on the following formulation from - [Wei and Held (2014)](https://link.springer.com/article/10.1007/s11749-014-0380-8): + It is based on the following formulation from [1]_: .. math:: \mathrm{CRPS}(F_{\lambda}, y) = (y - \lambda) (2F_{\lambda}(y) - 1) @@ -2064,7 +2107,7 @@ def crps_poisson( ---------- obs : array_like The observed values. - mean: array_like + mean : array_like Mean parameter of the forecast poisson distribution. Returns @@ -2072,10 +2115,17 @@ def crps_poisson( crps : array_like The CRPS between Pois(mean) and obs. + References + ---------- + .. [1] Wei, W., Held, L. (2014), + Calibration tests for count data. + TEST 23, 787-805. https://doi.org/10.1007/s11749-014-0380-8 + Examples -------- >>> import scoringrules as sr >>> sr.crps_poisson(1, 2) + 0.4991650450203817 """ return crps.poisson(obs, mean, backend=backend) @@ -2112,7 +2162,7 @@ def crps_t( Degrees of freedom parameter of the forecast t distribution. location : array_like Location parameter of the forecast t distribution. - sigma: array_like + sigma : array_like Scale parameter of the forecast t distribution. Returns @@ -2131,6 +2181,7 @@ def crps_t( -------- >>> import scoringrules as sr >>> sr.crps_t(0.0, 0.1, 0.4, 0.1) + 0.07687151141732129 """ return crps.t(obs, df, location, scale, backend=backend) @@ -2188,6 +2239,7 @@ def crps_uniform( -------- >>> import scoringrules as sr >>> sr.crps_uniform(0.4, 0.0, 1.0, 0.0, 0.0) + 0.09333333333333332 """ return crps.uniform(obs, min, max, lmass, umass, backend=backend) From b7740f282f5cdfde317ec372ac90c64556d79091 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Sat, 1 Feb 2025 13:36:36 +0100 Subject: [PATCH 18/79] more work on docstrings - math blocks, args, formatting, etc. --- .gitignore | 1 + docs/theory.md | 2 +- scoringrules/_brier.py | 98 +++++---- scoringrules/_crps.py | 2 + scoringrules/_energy.py | 179 +++++++-------- scoringrules/_interval.py | 65 +++--- scoringrules/_kernels.py | 231 ++++++++++--------- scoringrules/_logs.py | 438 ++++++++++++++++++------------------- scoringrules/_quantile.py | 15 +- scoringrules/_variogram.py | 182 ++++++++------- 10 files changed, 617 insertions(+), 596 deletions(-) diff --git a/.gitignore b/.gitignore index 1332dc7..be1ebf5 100644 --- a/.gitignore +++ b/.gitignore @@ -152,3 +152,4 @@ _devlog/ tests/output .devcontainer/ docs/generated +scoringrules/vendored diff --git a/docs/theory.md b/docs/theory.md index 64e1c98..eb0b8a7 100644 --- a/docs/theory.md +++ b/docs/theory.md @@ -190,7 +190,7 @@ mean and variance. The Dawid-Sebastiani score evaluates forecasts only via their deviation, making it easy to implement in practice, but insensitive to higher moments of the predictive distribution. - +(theory.multivariate)= ### Multivariate outcomes: Let $\boldsymbol{Y} \in \mathbb{R}^{d}$, with $d > 1$, and suppose $F$ is a multivariate probability distribution. diff --git a/scoringrules/_brier.py b/scoringrules/_brier.py index e7152ba..5230346 100644 --- a/scoringrules/_brier.py +++ b/scoringrules/_brier.py @@ -8,8 +8,8 @@ def brier_score( - observations: "ArrayLike", - forecasts: "ArrayLike", + obs: "ArrayLike", + fct: "ArrayLike", /, *, backend: "Backend" = None, @@ -19,31 +19,32 @@ def brier_score( The BS is formulated as - $$ BS(f, y) = (f - y)^2, $$ + .. math:: + BS(f, y) = (f - y)^2, - where $f \in [0, 1]$ is the predicted probability of an event and $y \in \{0, 1\}$ the actual outcome. + where :math:`f \in [0, 1]` is the predicted probability of an event and :math:`y \in \{0, 1\}` the actual outcome. Parameters ---------- - observations: NDArray + obs : array_like Observed outcome, either 0 or 1. - forecasts : NDArray + fct : array_like Forecasted probabilities between 0 and 1. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numpy'. Returns ------- - brier_score : NDArray + brier_score : array_like The computed Brier Score. """ - return brier.brier_score(obs=observations, fct=forecasts, backend=backend) + return brier.brier_score(obs=obs, fct=fct, backend=backend) def rps_score( - observations: "ArrayLike", - forecasts: "ArrayLike", + obs: "ArrayLike", + fct: "ArrayLike", /, axis: int = -1, *, @@ -52,25 +53,26 @@ def rps_score( r""" Compute the (Discrete) Ranked Probability Score (RPS). - Suppose the outcome corresponds to one of $K$ ordered categories. The RPS is defined as + Suppose the outcome corresponds to one of :math:`K` ordered categories. The RPS is defined as - $$ RPS(f, y) = \sum_{k=1}^{K}(\tilde{f}_{k} - \tilde{y}_{k})^2, $$ + .. math:: + RPS(f, y) = \sum_{k=1}^{K}(\tilde{f}_{k} - \tilde{y}_{k})^2, - where $f \in [0, 1]^{K}$ is a vector of length $K$ containing forecast probabilities - that each of the $K$ categories will occur, and $y \in \{0, 1\}^{K}$ is a vector of - length $K$, with the $k$-th element equal to one if the $k$-th category occurs. We - have $\sum_{k=1}^{K} y_{k} = \sum_{k=1}^{K} f_{k} = 1$, and, for $k = 1, \dots, K$, - $\tilde{y}_{k} = \sum_{i=1}^{k} y_{i}$ and $\tilde{f}_{k} = \sum_{i=1}^{k} f_{i}$. + where :math:`f \in [0, 1]^{K}` is a vector of length :math:`K` containing forecast probabilities + that each of the :math:`K` categories will occur, and :math:`y \in \{0, 1\}^{K}` is a vector of + length :math:`K`, with the :math:`k`-th element equal to one if the :math:`k`-th category occurs. We + have :math:`\sum_{k=1}^{K} y_{k} = \sum_{k=1}^{K} f_{k} = 1`, and, for :math:`k = 1, \dots, K`, + :math:`\tilde{y}_{k} = \sum_{i=1}^{k} y_{i}` and :math:`\tilde{f}_{k} = \sum_{i=1}^{k} f_{i}`. Parameters ---------- - observations: + obs : array_like Array of 0's and 1's corresponding to unobserved and observed categories forecasts : Array of forecast probabilities for each category. axis: int The axis corresponding to the categories. Default is the last axis. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numpy'. Returns @@ -80,17 +82,17 @@ def rps_score( """ B = backends.active if backend is None else backends[backend] - forecasts = B.asarray(forecasts) + fct = B.asarray(fct) if axis != -1: - forecasts = B.moveaxis(forecasts, axis, -1) + fct = B.moveaxis(fct, axis, -1) - return brier.rps_score(obs=observations, fct=forecasts, backend=backend) + return brier.rps_score(obs=obs, fct=fct, backend=backend) def log_score( - observations: "ArrayLike", - forecasts: "ArrayLike", + obs: "ArrayLike", + fct: "ArrayLike", /, *, backend: "Backend" = None, @@ -100,17 +102,18 @@ def log_score( The LS is formulated as - $$ LS(f, y) = -\log|f + y - 1|, $$ + .. math:: + LS(f, y) = -\log|f + y - 1|, - where $f \in [0, 1]$ is the predicted probability of an event and $y \in \{0, 1\}$ the actual outcome. + where :math:`f \in [0, 1]` is the predicted probability of an event and :math:`y \in \{0, 1\}` the actual outcome. Parameters ---------- - observations: NDArray + obs : array_like Observed outcome, either 0 or 1. - forecasts : NDArray + fct : array_like Forecasted probabilities between 0 and 1. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numpy'. Returns @@ -119,12 +122,12 @@ def log_score( The computed Log Score. """ - return brier.log_score(obs=observations, fct=forecasts, backend=backend) + return brier.log_score(obs=obs, fct=fct, backend=backend) def rls_score( - observations: "ArrayLike", - forecasts: "ArrayLike", + obs: "ArrayLike", + fct: "ArrayLike", /, axis: int = -1, *, @@ -133,38 +136,39 @@ def rls_score( r""" Compute the (Discrete) Ranked Logarithmic Score (RLS). - Suppose the outcome corresponds to one of $K$ ordered categories. The RLS is defined as + Suppose the outcome corresponds to one of :math:`K` ordered categories. The RLS is defined as - $$ RPS(f, y) = -\sum_{k=1}^{K} \log|\tilde{f}_{k} + \tilde{y}_{k} - 1|, $$ + .. math:: + RPS(f, y) = -\sum_{k=1}^{K} \log|\tilde{f}_{k} + \tilde{y}_{k} - 1|, - where $f \in [0, 1]^{K}$ is a vector of length $K$ containing forecast probabilities - that each of the $K$ categories will occur, and $y \in \{0, 1\}^{K}$ is a vector of - length $K$, with the $k$-th element equal to one if the $k$-th category occurs. We - have $\sum_{k=1}^{K} y_{k} = \sum_{k=1}^{K} f_{k} = 1$, and, for $k = 1, \dots, K$, - $\tilde{y}_{k} = \sum_{i=1}^{k} y_{i}$ and $\tilde{f}_{k} = \sum_{i=1}^{k} f_{i}$. + where :math:`f \in [0, 1]^{K}` is a vector of length :math:`K` containing forecast probabilities + that each of the :math:`K` categories will occur, and :math:`y \in \{0, 1\}^{K}` is a vector of + length :math:`K`, with the :math:`k`-th element equal to one if the :math:`k`-th category occurs. We + have :math:`\sum_{k=1}^{K} y_{k} = \sum_{k=1}^{K} f_{k} = 1`, and, for :math:`k = 1, \dots, K`, + :math:`\tilde{y}_{k} = \sum_{i=1}^{k} y_{i}` and :math:`\tilde{f}_{k} = \sum_{i=1}^{k} f_{i}`. Parameters ---------- - observations: NDArray + obs : array_like Observed outcome, either 0 or 1. - forecasts : NDArray + fct : array_like Forecasted probabilities between 0 and 1. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numpy'. Returns ------- - score: + score : array_like The computed Ranked Logarithmic Score. """ B = backends.active if backend is None else backends[backend] - forecasts = B.asarray(forecasts) + fct = B.asarray(fct) if axis != -1: - forecasts = B.moveaxis(forecasts, axis, -1) + fct = B.moveaxis(fct, axis, -1) - return brier.rls_score(obs=observations, fct=forecasts, backend=backend) + return brier.rls_score(obs=obs, fct=fct, backend=backend) __all__ = [ diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index 8b099c9..633ab53 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -69,6 +69,8 @@ def crps_ensemble( Weighted variants of the CRPS. crps_quantile CRPS for quantile forecasts. + energy_score + The multivariate equivalent of the CRPS. Notes ----- diff --git a/scoringrules/_energy.py b/scoringrules/_energy.py index a1e35b5..f96f20b 100644 --- a/scoringrules/_energy.py +++ b/scoringrules/_energy.py @@ -9,8 +9,8 @@ def energy_score( - observations: "Array", - forecasts: "Array", + obs: "Array", + fct: "Array", /, m_axis: int = -2, v_axis: int = -1, @@ -21,46 +21,56 @@ def energy_score( The Energy Score is a multivariate scoring rule expressed as - $$\text{ES}(F_{ens}, \mathbf{y})= \frac{1}{M} \sum_{m=1}^{M} \| \mathbf{x}_{m} - - \mathbf{y} \| - \frac{1}{2 M^{2}} \sum_{m=1}^{M} \sum_{j=1}^{M} \| \mathbf{x}_{m} - \mathbf{x}_{j} \| $$ - - where $||\cdot||$ is the euclidean norm over the input dimensions (the variables). + .. math:: + \text{ES}(F_{ens}, \mathbf{y})= \frac{1}{M} \sum_{m=1}^{M} \| \mathbf{x}_{m} - + \mathbf{y} \| - \frac{1}{2 M^{2}} \sum_{m=1}^{M} \sum_{j=1}^{M} \| \mathbf{x}_{m} - \mathbf{x}_{j} \|, + where :math:`||\cdot||` is the euclidean norm over the input dimensions (the variables). Parameters ---------- - observations: Array + obs : array_like The observed values, where the variables dimension is by default the last axis. - forecasts: Array + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. - m_axis: int + m_axis : int The axis corresponding to the ensemble dimension on the forecasts array. Defaults to -2. - v_axis: int + v_axis : int The axis corresponding to the variables dimension on the forecasts array (or the observations array with an extra dimension on `m_axis`). Defaults to -1. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - energy_score: Array of shape (...) + energy_score : array_like The computed Energy Score. + + See Also + -------- + twenergy_score, owenergy_score, vrenergy_score + Weighted variants of the Energy Score. + crps_ensemble + The univariate equivalent of the Energy Score. + + Notes + ----- + :ref:`theory.multivariate` + Some theoretical background on scoring rules for multivariate forecasts. """ backend = backend if backend is not None else backends._active - observations, forecasts = multivariate_array_check( - observations, forecasts, m_axis, v_axis, backend=backend - ) + obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) if backend == "numba": - return energy._energy_score_gufunc(observations, forecasts) + return energy._energy_score_gufunc(obs, fct) - return energy.nrg(observations, forecasts, backend=backend) + return energy.nrg(obs, fct, backend=backend) def twenergy_score( - observations: "Array", - forecasts: "Array", + obs: "Array", + fct: "Array", v_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, m_axis: int = -2, @@ -73,45 +83,43 @@ def twenergy_score( Computation is performed using the ensemble representation of the twES in [Allen et al. (2022)](https://arxiv.org/abs/2202.12732): - \[ - \mathrm{twES}(F_{ens}, \mathbf{y}) = \frac{1}{M} \sum_{m = 1}^{M} \| v(\mathbf{x}_{m}) - v(\mathbf{y}) \| - \frac{1}{2 M^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} \| v(\mathbf{x}_{m}) - v(\mathbf{x}_{j}) \|, - \] + .. math:: + \mathrm{twES}(F_{ens}, \mathbf{y}) = \frac{1}{M} \sum_{m = 1}^{M} \| v(\mathbf{x}_{m}) - v(\mathbf{y}) \| + - \frac{1}{2 M^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} \| v(\mathbf{x}_{m}) - v(\mathbf{x}_{j}) \|, - where $F_{ens}$ is the ensemble forecast $\mathbf{x}_{1}, \dots, \mathbf{x}_{M}$ with - $M$ members, $\| \cdotp \|$ is the Euclidean distance, and $v$ is the chaining function + where :math:`F_{ens}` is the ensemble forecast :math:`\mathbf{x}_{1}, \dots, \mathbf{x}_{M}` with + :math:`M` members, :math:`\| \cdotp \|` is the Euclidean distance, and :math:`v` is the chaining function used to target particular outcomes. Parameters ---------- - observations: ArrayLike of shape (...,D) + obs : array_like The observed values, where the variables dimension is by default the last axis. - forecasts: ArrayLike of shape (..., M, D) + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. - v_func: tp.Callable + v_func : callable, array_like -> array_like Chaining function used to emphasise particular outcomes. - m_axis: int + m_axis : int The axis corresponding to the ensemble dimension. Defaults to -2. - v_axis: int or tuple(int) + v_axis : int or tuple of int The axis corresponding to the variables dimension. Defaults to -1. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - twenergy_score: ArrayLike of shape (...) + twenergy_score : array_like The computed Threshold-Weighted Energy Score. """ - observations, forecasts = map(v_func, (observations, forecasts)) - return energy_score( - observations, forecasts, m_axis=m_axis, v_axis=v_axis, backend=backend - ) + obs, fct = map(v_func, (obs, fct)) + return energy_score(obs, fct, m_axis=m_axis, v_axis=v_axis, backend=backend) def owenergy_score( - observations: "Array", - forecasts: "Array", + obs: "Array", + fct: "Array", w_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, m_axis: int = -2, @@ -124,58 +132,55 @@ def owenergy_score( Computation is performed using the ensemble representation of the owES in [Allen et al. (2022)](https://arxiv.org/abs/2202.12732): - \[ - \mathrm{owES}(F_{ens}, \mathbf{y}) = \frac{1}{M \bar{w}} \sum_{m = 1}^{M} \| \mathbf{x}_{m} - \mathbf{y} \| w(\mathbf{x}_{m}) w(\mathbf{y}) - \frac{1}{2 M^{2} \bar{w}^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} \| \mathbf{x}_{m} - \mathbf{x}_{j} \| w(\mathbf{x}_{m}) w(\mathbf{x}_{j}) w(\mathbf{y}), - \] + .. math:: + \mathrm{owES}(F_{ens}, \mathbf{y}) = \frac{1}{M \bar{w}} \sum_{m = 1}^{M} \| \mathbf{x}_{m} + - \mathbf{y} \| w(\mathbf{x}_{m}) w(\mathbf{y}) + - \frac{1}{2 M^{2} \bar{w}^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} \| \mathbf{x}_{m} + - \mathbf{x}_{j} \| w(\mathbf{x}_{m}) w(\mathbf{x}_{j}) w(\mathbf{y}), + - where $F_{ens}$ is the ensemble forecast $\mathbf{x}_{1}, \dots, \mathbf{x}_{M}$ with - $M$ members, $\| \cdotp \|$ is the Euclidean distance, $w$ is the chosen weight function, - and $\bar{w} = \sum_{m=1}^{M}w(\mathbf{x}_{m})/M$. + where :math:`F_{ens}` is the ensemble forecast :math:`\mathbf{x}_{1}, \dots, \mathbf{x}_{M}` with + :math:`M` members, :math:`\| \cdotp \|` is the Euclidean distance, :math:`w` is the chosen weight function, + and :math:`\bar{w} = \sum_{m=1}^{M}w(\mathbf{x}_{m})/M`. Parameters ---------- - observations: ArrayLike of shape (...,D) + obs : array_like The observed values, where the variables dimension is by default the last axis. - forecasts: ArrayLike of shape (..., M, D) + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. - w_func: tp.Callable + w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. - m_axis: int + m_axis : int The axis corresponding to the ensemble dimension. Defaults to -2. - v_axis: int or tuple(int) + v_axis : int or tuple of ints The axis corresponding to the variables dimension. Defaults to -1. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - owenergy_score: ArrayLike of shape (...) + owenergy_score : array_like The computed Outcome-Weighted Energy Score. """ B = backends.active if backend is None else backends[backend] - observations, forecasts = multivariate_array_check( - observations, forecasts, m_axis, v_axis, backend=backend - ) + obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) - fct_weights = B.apply_along_axis(w_func, forecasts, -1) - obs_weights = B.apply_along_axis(w_func, observations, -1) + fct_weights = B.apply_along_axis(w_func, fct, -1) + obs_weights = B.apply_along_axis(w_func, obs, -1) if B.name == "numba": - return energy._owenergy_score_gufunc( - observations, forecasts, obs_weights, fct_weights - ) + return energy._owenergy_score_gufunc(obs, fct, obs_weights, fct_weights) - return energy.ownrg( - observations, forecasts, obs_weights, fct_weights, backend=backend - ) + return energy.ownrg(obs, fct, obs_weights, fct_weights, backend=backend) def vrenergy_score( - observations: "Array", - forecasts: "Array", + obs: "Array", + fct: "Array", w_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, *, @@ -188,52 +193,48 @@ def vrenergy_score( Computation is performed using the ensemble representation of the vrES in [Allen et al. (2022)](https://arxiv.org/abs/2202.12732): - \[ - \begin{split} - \mathrm{vrES}(F_{ens}, \mathbf{y}) = & \frac{1}{M} \sum_{m = 1}^{M} \| \mathbf{x}_{m} - \mathbf{y} \| w(\mathbf{x}_{m}) w(\mathbf{y}) - \frac{1}{2 M^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} \| \mathbf{x}_{m} - \mathbf{x}_{j} \| w(\mathbf{x}_{m}) w(\mathbf{x_{j}}) \\ - & + \left( \frac{1}{M} \sum_{m = 1}^{M} \| \mathbf{x}_{m} \| w(\mathbf{x}_{m}) - \| \mathbf{y} \| w(\mathbf{y}) \right) \left( \frac{1}{M} \sum_{m = 1}^{M} w(\mathbf{x}_{m}) - w(\mathbf{y}) \right), - \end{split} - \] + .. math:: + \begin{split} + \mathrm{vrES}(F_{ens}, \mathbf{y}) = & \frac{1}{M} \sum_{m = 1}^{M} \| \mathbf{x}_{m} + - \mathbf{y} \| w(\mathbf{x}_{m}) w(\mathbf{y}) - \frac{1}{2 M^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} \| \mathbf{x}_{m} + - \mathbf{x}_{j} \| w(\mathbf{x}_{m}) w(\mathbf{x_{j}}) \\ + & + \left( \frac{1}{M} \sum_{m = 1}^{M} \| \mathbf{x}_{m} \| w(\mathbf{x}_{m}) + - \| \mathbf{y} \| w(\mathbf{y}) \right) \left( \frac{1}{M} \sum_{m = 1}^{M} w(\mathbf{x}_{m}) - w(\mathbf{y}) \right), + \end{split} - where $F_{ens}$ is the ensemble forecast $\mathbf{x}_{1}, \dots, \mathbf{x}_{M}$ with - $M$ members, and $w$ is the weight function used to target particular outcomes. + where :math:`F_{ens}` is the ensemble forecast :math:`\mathbf{x}_{1}, \dots, \mathbf{x}_{M}` with + :math:`M` members, and :math:`w` is the weight function used to target particular outcomes. Parameters ---------- - observations: ArrayLike of shape (...,D) + obs : array_like The observed values, where the variables dimension is by default the last axis. - forecasts: ArrayLike of shape (..., M, D) + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. - w_func: tp.Callable + w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. - m_axis: int + m_axis : int The axis corresponding to the ensemble dimension. Defaults to -2. - v_axis: int or tuple(int) + v_axis : int or tuple of int The axis corresponding to the variables dimension. Defaults to -1. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - vrenergy_score: ArrayLike of shape (...) + vrenergy_score : array_like The computed Vertically Re-scaled Energy Score. """ B = backends.active if backend is None else backends[backend] - observations, forecasts = multivariate_array_check( - observations, forecasts, m_axis, v_axis, backend=backend - ) + obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) - fct_weights = B.apply_along_axis(w_func, forecasts, -1) - obs_weights = B.apply_along_axis(w_func, observations, -1) + fct_weights = B.apply_along_axis(w_func, fct, -1) + obs_weights = B.apply_along_axis(w_func, obs, -1) if backend == "numba": - return energy._vrenergy_score_gufunc( - observations, forecasts, obs_weights, fct_weights - ) + return energy._vrenergy_score_gufunc(obs, fct, obs_weights, fct_weights) - return energy.vrnrg( - observations, forecasts, obs_weights, fct_weights, backend=backend - ) + return energy.vrnrg(obs, fct, obs_weights, fct_weights, backend=backend) diff --git a/scoringrules/_interval.py b/scoringrules/_interval.py index 749f21d..ce742fb 100644 --- a/scoringrules/_interval.py +++ b/scoringrules/_interval.py @@ -22,33 +22,32 @@ def interval_score( [(Gneiting & Raftery, 2012)](https://doi.org/10.1198/016214506000001437) is defined as - $$ - \text{IS} = - \begin{cases} - (u - l) + \frac{2}{\alpha}(l - y) & \text{for } y < l \\ - (u - l) & \text{for } l \leq y \leq u \\ - (u - l) + \frac{2}{\alpha}(y - u) & \text{for } y > u. \\ - \end{cases} - $$ + .. math:: + \text{IS} = + \begin{cases} + (u - l) + \frac{2}{\alpha}(l - y) & \text{for } y < l \\ + (u - l) & \text{for } l \leq y \leq u \\ + (u - l) + \frac{2}{\alpha}(y - u) & \text{for } y > u. \\ + \end{cases} - for an $1 - \alpha$ prediction interval of $[l, u]$ and the true value $y$. + for an :math:`1 - \alpha` prediction interval of :math:`[l, u]` and the true value :math:`y`. Parameters ---------- - obs: + obs : array_like The observations as a scalar or array of values. - lower: + lower : array_like The predicted lower bound of the PI as a scalar or array of values. - upper: + upper : array_like The predicted upper bound of the PI as a scalar or array of values. - alpha: + alpha : array_like The 1 - alpha level for the PI as a scalar or array of values. - backend: + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: + interval_score : array_like Array with the interval score for the input values. Raises @@ -125,42 +124,40 @@ def weighted_interval_score( The WIS [(Bracher et al., 2022)](https://doi.org/10.1371/journal.pcbi.1008618) is defined as - $$ - \text{WIS}_{\alpha_{0:K}}(F, y) = \frac{1}{K+0.5}(w_0 \times |y - m| - + \sum_{k=1}^K (w_k \times IS_{\alpha_k}(F, y))) - $$ + .. math:: + \text{WIS}_{\alpha_{0:K}}(F, y) = \frac{1}{K+0.5}(w_0 \times |y - m| + + \sum_{k=1}^K (w_k \times IS_{\alpha_k}(F, y))) - where $m$ denotes the median prediction, $w_0$ denotes the weight of the - median prediction, $IS_{\alpha_k}(F, y)$ denotes the interval score for the - $1 - \alpha$ prediction interval and $w_k$ is the according weight. + where :math:`m` denotes the median prediction, :math:`w_0` denotes the weight of the + median prediction, :math:`IS_{\alpha_k}(F, y)` denotes the interval score for the + :math:`1 - \alpha` prediction interval and :math:`w_k` is the according weight. The WIS is calculated for a set of (central) PIs and the predictive median. The weights are an optional parameter and default weight is the canonical - weight $w_k = \frac{2}{\alpha_k}$ and $w_0 = 0.5$. + weight :math:`w_k = \frac{2}{\alpha_k}` and :math:`w_0 = 0.5`. For these weights, it holds that: - $$ - \text{WIS}_{\alpha_{0:K}}(F, y) \approx \text{CRPS}(F, y). - $$ + .. math:: + \text{WIS_{\alpha_{0:K}}(F, y) \approx \text{CRPS}(F, y). Parameters ---------- - obs: + obs : array_like The observations as a scalar or array of shape `(...,)`. - median: + median : array_like The predicted median of the distribution as a scalar or array of shape `(...,)`. - lower: + lower : array_like The predicted lower bound of the PI. If `alpha` is an array of shape `(K,)`, `lower` must have shape `(...,K)`. - upper: + upper : array_like The predicted upper bound of the PI. If `alpha` is an array of shape `(K,)`, `upper` must have shape `(...,K)`. - alpha: + alpha : array_like The 1 - alpha level for the prediction intervals as an array of shape `(K,)`. - w_median: + w_median : float The weight for the median prediction. Defaults to 0.5. - w_alpha: + w_alpha : array_like The weights for the PI. Defaults to `2/alpha`. - backend: + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns diff --git a/scoringrules/_kernels.py b/scoringrules/_kernels.py index dad1794..b4da612 100644 --- a/scoringrules/_kernels.py +++ b/scoringrules/_kernels.py @@ -21,28 +21,30 @@ def gksuv_ensemble( The GKS is the kernel score associated with the Gaussian kernel - $$ k(x_{1}, x_{2}) = \exp \left(- \frac{(x_{1} - x_{2})^{2}}{2} \right). $$ + .. math:: + k(x_{1}, x_{2}) = \exp \left(- \frac{(x_{1} - x_{2})^{2}}{2} \right). - Given an ensemble forecast $F_{ens}$ comprised of members $x_{1}, \dots, x_{M}$, + Given an ensemble forecast :math:`F_{ens}` comprised of members :math:`x_{1}, \dots, x_{M}`, the GKS is - $$\text{GKS}(F_{ens}, y)= - \frac{1}{M} \sum_{m=1}^{M} k(x_{m}, y) + \frac{1}{2 M^{2}} \sum_{m=1}^{M} \sum_{j=1}^{M} k(x_{m}, x_{j}) + \frac{1}{2}k(y, y) $$ + .. math:: + \text{GKS}(F_{ens}, y)= - \frac{1}{M} \sum_{m=1}^{M} k(x_{m}, y) + \frac{1}{2 M^{2}} \sum_{m=1}^{M} \sum_{j=1}^{M} k(x_{m}, x_{j}) + \frac{1}{2}k(y, y) - If the fair estimator is to be used, then $M^{2}$ in the second component of the right-hand-side - is replaced with $M(M - 1)$. + If the fair estimator is to be used, then :math:`M^{2}` in the second component of the right-hand-side + is replaced with :math:`M(M - 1)`. Parameters ---------- - obs: ArrayLike + obs : array_like The observed values. - fct: ArrayLike + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - axis: int + axis : int The axis corresponding to the ensemble. Default is the last axis. - estimator: str + estimator : str Indicates the estimator to be used. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns @@ -95,35 +97,37 @@ def twgksuv_ensemble( Computation is performed using the ensemble representation of threshold-weighted kernel scores in [Allen et al. (2022)](https://arxiv.org/abs/2202.12732). - $$ \mathrm{twGKS}(F_{ens}, y) = - \frac{1}{M} \sum_{m = 1}^{M} k(v(x_{m}), v(y)) + \frac{1}{2 M^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} k(v(x_{m}), v(x_{j})) + \frac{1}{2} k(v(y), v(y)), $$ + .. math:: + \mathrm{twGKS}(F_{ens}, y) = - \frac{1}{M} \sum_{m = 1}^{M} k(v(x_{m}), v(y)) + \frac{1}{2 M^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} k(v(x_{m}), v(x_{j})) + \frac{1}{2} k(v(y), v(y)), - where $F_{ens}(x) = \sum_{m=1}^{M} 1 \{ x_{m} \leq x \}/M$ is the empirical - distribution function associated with an ensemble forecast $x_{1}, \dots, x_{M}$ with - $M$ members, $v$ is the chaining function used to target particular outcomes, and + where :math:`F_{ens}(x) = \sum_{m=1}^{M} 1 \{ x_{m} \leq x \}/M` is the empirical + distribution function associated with an ensemble forecast :math:`x_{1}, \dots, x_{M}` with + :math:`M` members, :math:`v` is the chaining function used to target particular outcomes, and - $$ k(x_{1}, x_{2}) = \exp \left(- \frac{(x_{1} - x_{2})^{2}}{2} \right) $$ + .. math:: + k(x_{1}, x_{2}) = \exp \left(- \frac{(x_{1} - x_{2})^{2}}{2} \right) is the Gaussian kernel. Parameters ---------- - obs: ArrayLike + obs : array_like The observed values. - fct: ArrayLike + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - v_func: tp.Callable + v_func : callable, array_like -> array_like Chaining function used to emphasise particular outcomes. For example, a function that - only considers values above a certain threshold $t$ by projecting forecasts and observations - to $[t, \inf)$. - axis: int + only considers values above a certain threshold :math:`t` by projecting forecasts and observations + to :math:`[t, \inf)`. + axis : int The axis corresponding to the ensemble. Default is the last axis. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - twgks: ArrayLike + twgks : array_like The twGKS between the forecast ensemble and obs for the chosen chaining function. Examples @@ -160,34 +164,36 @@ def owgksuv_ensemble( Computation is performed using the ensemble representation of the owCRPS in [Allen et al. (2022)](https://arxiv.org/abs/2202.12732): - $$ \mathrm{owGKS}(F_{ens}, y) = -\frac{1}{M \bar{w}} \sum_{m = 1}^{M} k(x_{m}, y)w(x_{m})w(y) - \frac{1}{2 M^{2} \bar{w}^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} k(x_{m}, x_{j})w(x_{m})w(x_{j})w(y),$$ + .. math:: + \mathrm{owGKS}(F_{ens}, y) = -\frac{1}{M \bar{w}} \sum_{m = 1}^{M} k(x_{m}, y)w(x_{m})w(y) - \frac{1}{2 M^{2} \bar{w}^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} k(x_{m}, x_{j})w(x_{m})w(x_{j})w(y), - where $F_{ens}(x) = \sum_{m=1}^{M} 1\{ x_{m} \leq x \}/M$ is the empirical - distribution function associated with an ensemble forecast $x_{1}, \dots, x_{M}$ with - $M$ members, $w$ is the chosen weight function, $\bar{w} = \sum_{m=1}^{M}w(x_{m})/M$, + where :math:`F_{ens}(x) = \sum_{m=1}^{M} 1\{ x_{m} \leq x \}/M` is the empirical + distribution function associated with an ensemble forecast :math:`x_{1}, \dots, x_{M}` with + :math:`M` members, :math:`w` is the chosen weight function, :math:`\bar{w} = \sum_{m=1}^{M}w(x_{m})/M`, and - $$ k(x_{1}, x_{2}) = \exp \left(- \frac{(x_{1} - x_{2})^{2}}{2} \right) $$ + .. math:: + k(x_{1}, x_{2}) = \exp \left(- \frac{(x_{1} - x_{2})^{2}}{2} \right) is the Gaussian kernel. Parameters ---------- - obs: ArrayLike + obs : array_like The observed values. - fct: ArrayLike + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - w_func: tp.Callable + w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. - axis: int + axis : int The axis corresponding to the ensemble. Default is the last axis. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: ArrayLike + score : array_like The owGKS between the forecast ensemble and obs for the chosen weight function. Examples @@ -232,33 +238,35 @@ def vrgksuv_ensemble( Computation is performed using the ensemble representation of vertically re-scaled kernel scores in [Allen et al. (2022)](https://arxiv.org/abs/2202.12732): - $$ \mathrm{vrGKS}(F_{ens}, y) = - \frac{1}{M} \sum_{m = 1}^{M} k(x_{m}, y)w(x_{m})w(y) + \frac{1}{2 M^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} k(x_{m}, x_{j})w(x_{m})w(x_{j}) + \frac{1}{2} k(y, y)w(y)w(y), $$ + .. math:: + \mathrm{vrGKS}(F_{ens}, y) = - \frac{1}{M} \sum_{m = 1}^{M} k(x_{m}, y)w(x_{m})w(y) + \frac{1}{2 M^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} k(x_{m}, x_{j})w(x_{m})w(x_{j}) + \frac{1}{2} k(y, y)w(y)w(y), - where $F_{ens}(x) = \sum_{m=1}^{M} 1 \{ x_{m} \leq x \}/M$ is the empirical - distribution function associated with an ensemble forecast $x_{1}, \dots, x_{M}$ with - $M$ members, $w$ is the chosen weight function, $\bar{w} = \sum_{m=1}^{M}w(x_{m})/M$, and + where :math:`F_{ens}(x) = \sum_{m=1}^{M} 1 \{ x_{m} \leq x \}/M` is the empirical + distribution function associated with an ensemble forecast :math:`x_{1}, \dots, x_{M}` with + :math:`M` members, :math:`w` is the chosen weight function, :math:`\bar{w} = \sum_{m=1}^{M}w(x_{m})/M`, and - $$ k(x_{1}, x_{2}) = \exp \left(- \frac{(x_{1} - x_{2})^{2}}{2} \right) $$ + .. math:: + k(x_{1}, x_{2}) = \exp \left(- \frac{(x_{1} - x_{2})^{2}}{2} \right) is the Gaussian kernel. Parameters ---------- - obs: ArrayLike + obs : array_like The observed values. - fct: ArrayLike + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - w_func: tp.Callable + w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. - axis: int + axis : int The axis corresponding to the ensemble. Default is the last axis. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: ArrayLike + score : array_like The vrGKS between the forecast ensemble and obs for the chosen weight function. Examples @@ -303,39 +311,41 @@ def gksmv_ensemble( The GKS is the kernel score associated with the Gaussian kernel - $$ k(x_{1}, x_{2}) = \exp \left(- \frac{ \| x_{1} - x_{2} \| ^{2}}{2} \right), $$ + .. math:: + k(x_{1}, x_{2}) = \exp \left(- \frac{ \| x_{1} - x_{2} \| ^{2}}{2} \right), - where $ \| \cdot \|$ is the euclidean norm. + where :math:` \| \cdot \|` is the euclidean norm. - Given an ensemble forecast $F_{ens}$ comprised of multivariate members $\mathbf{x}_{1}, \dots, \mathbf{x}_{M}$, + Given an ensemble forecast :math:`F_{ens}` comprised of multivariate members :math:`\mathbf{x}_{1}, \dots, \mathbf{x}_{M}`, the GKS is - $$\text{GKS}(F_{ens}, y)= - \frac{1}{M} \sum_{m=1}^{M} k(\mathbf{x}_{m}, \mathbf{y}) + \frac{1}{2 M^{2}} \sum_{m=1}^{M} \sum_{j=1}^{M} k(\mathbf{x}_{m}, \mathbf{x}_{j}) + \frac{1}{2}k(y, y) $$ + .. math:: + \text{GKS}(F_{ens}, y)= - \frac{1}{M} \sum_{m=1}^{M} k(\mathbf{x}_{m}, \mathbf{y}) + \frac{1}{2 M^{2}} \sum_{m=1}^{M} \sum_{j=1}^{M} k(\mathbf{x}_{m}, \mathbf{x}_{j}) + \frac{1}{2}k(y, y) - If the fair estimator is to be used, then $M^{2}$ in the second component of the right-hand-side - is replaced with $M(M - 1)$. + If the fair estimator is to be used, then :math:`M^{2}` in the second component of the right-hand-side + is replaced with :math:`M(M - 1)`. Parameters ---------- - obs: Array + obs : array_like The observed values, where the variables dimension is by default the last axis. - fct: Array + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. - m_axis: int + m_axis : int The axis corresponding to the ensemble dimension on the forecasts array. Defaults to -2. - v_axis: int + v_axis : int The axis corresponding to the variables dimension on the forecasts array (or the observations array with an extra dimension on `m_axis`). Defaults to -1. - estimator: str + estimator : str Indicates the estimator to be used. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: + score : array_like The GKS between the forecast ensemble and obs. """ backend = backend if backend is not None else backends._active @@ -368,35 +378,37 @@ def twgksmv_ensemble( Computation is performed using the ensemble representation of threshold-weighted kernel scores in [Allen et al. (2022)](https://arxiv.org/abs/2202.12732): - $$ \mathrm{twGKS}(F_{ens}, y) = - \frac{1}{M} \sum_{m = 1}^{M} k(v(x_{m}), v(y)) + \frac{1}{2 M^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} k(v(x_{m}), v(x_{j})) + \frac{1}{2} k(v(y), v(y)), $$ + .. math:: + \mathrm{twGKS}(F_{ens}, y) = - \frac{1}{M} \sum_{m = 1}^{M} k(v(x_{m}), v(y)) + \frac{1}{2 M^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} k(v(x_{m}), v(x_{j})) + \frac{1}{2} k(v(y), v(y)), - where $F_{ens}$ is the ensemble forecast $\mathbf{x}_{1}, \dots, \mathbf{x}_{M}$ with - $M$ members, $\| \cdotp \|$ is the Euclidean distance, $v$ is the chaining function + where :math:`F_{ens}` is the ensemble forecast :math:`\mathbf{x}_{1}, \dots, \mathbf{x}_{M}` with + :math:`M` members, :math:`\| \cdotp \|` is the Euclidean distance, :math:`v` is the chaining function used to target particular outcomes, and - $$ k(x_{1}, x_{2}) = \exp \left(- \frac{(x_{1} - x_{2})^{2}}{2} \right) $$ + .. math:: + k(x_{1}, x_{2}) = \exp \left(- \frac{(x_{1} - x_{2})^{2}}{2} \right) is the Gaussian kernel. Parameters ---------- - obs: ArrayLike of shape (...,D) + obs : array_like The observed values, where the variables dimension is by default the last axis. - fct: ArrayLike of shape (..., M, D) + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. - v_func: tp.Callable + v_func : callable, array_like -> array_like Chaining function used to emphasise particular outcomes. - m_axis: int + m_axis : int The axis corresponding to the ensemble dimension. Defaults to -2. - v_axis: int or tuple(int) + v_axis : int or tuple of ints The axis corresponding to the variables dimension. Defaults to -1. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: ArrayLike of shape (...) + score : array_like The computed Threshold-Weighted Gaussian Kernel Score. """ obs, fct = map(v_func, (obs, fct)) @@ -417,49 +429,53 @@ def owgksmv_ensemble( - Given an ensemble forecast $F_{ens}$ comprised of multivariate members $\mathbf{x}_{1}, \dots, \mathbf{x}_{M}$, + Given an ensemble forecast :math:`F_{ens}` comprised of multivariate members :math:`\mathbf{x}_{1}, \dots, \mathbf{x}_{M}`, the GKS is - $$\text{GKS}(F_{ens}, y)= - \frac{1}{M} \sum_{m=1}^{M} k(\mathbf{x}_{m}, \mathbf{y}) + \frac{1}{2 M^{2}} \sum_{m=1}^{M} \sum_{j=1}^{M} k(\mathbf{x}_{m}, \mathbf{x}_{j}) + \frac{1}{2}k(y, y) $$ + .. math:: + \text{GKS}(F_{ens}, y)= - \frac{1}{M} \sum_{m=1}^{M} k(\mathbf{x}_{m}, \mathbf{y}) + + \frac{1}{2 M^{2}} \sum_{m=1}^{M} \sum_{j=1}^{M} k(\mathbf{x}_{m}, \mathbf{x}_{j}) + \frac{1}{2}k(y, y) - If the fair estimator is to be used, then $M^{2}$ in the second component of the right-hand-side - is replaced with $M(M - 1)$. + If the fair estimator is to be used, then :math:`M^{2}` in the second component of the right-hand-side + is replaced with :math:`M(M - 1)`. Computation is performed using the ensemble representation of outcome-weighted kernel scores in [Allen et al. (2022)](https://arxiv.org/abs/2202.12732): - \[ - \mathrm{owGKS}(F_{ens}, \mathbf{y}) = - \frac{1}{M \bar{w}} \sum_{m = 1}^{M} k(\mathbf{x}_{m}, \mathbf{y}) w(\mathbf{x}_{m}) w(\mathbf{y}) + \frac{1}{2 M^{2} \bar{w}^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} k(\mathbf{x}_{m}, \mathbf{x}_{j}) w(\mathbf{x}_{m}) w(\mathbf{x}_{j}) w(\mathbf{y}) + \frac{1}{2}k(\mathbf{y}, \mathbf{y})w(\mathbf{y}), - \] + .. math:: + \mathrm{owGKS}(F_{ens}, \mathbf{y}) = - \frac{1}{M \bar{w}} \sum_{m = 1}^{M} k(\mathbf{x}_{m}, \mathbf{y}) w(\mathbf{x}_{m}) w(\mathbf{y}) + + \frac{1}{2 M^{2} \bar{w}^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} k(\mathbf{x}_{m}, \mathbf{x}_{j}) w(\mathbf{x}_{m}) w(\mathbf{x}_{j}) w(\mathbf{y}) + + \frac{1}{2}k(\mathbf{y}, \mathbf{y})w(\mathbf{y}), - where $F_{ens}$ is the ensemble forecast $\mathbf{x}_{1}, \dots, \mathbf{x}_{M}$ with - $M$ members, $\| \cdotp \|$ is the Euclidean distance, $w$ is the chosen weight function, - $\bar{w} = \sum_{m=1}^{M}w(\mathbf{x}_{m})/M$, and + where :math:`F_{ens}` is the ensemble forecast :math:`\mathbf{x}_{1}, \dots, \mathbf{x}_{M}` with + :math:`M` members, :math:`\| \cdotp \|` is the Euclidean distance, :math:`w` is the chosen weight function, + :math:`\bar{w} = \sum_{m=1}^{M}w(\mathbf{x}_{m})/M`, and - $$ k(x_{1}, x_{2}) = \exp \left(- \frac{ \| x_{1} - x_{2} \| ^{2}}{2} \right), $$ + .. math:: + k(x_{1}, x_{2}) = \exp \left(- \frac{ \| x_{1} - x_{2} \| ^{2}}{2} \right), - is the multivariate Gaussian kernel, with $ \| \cdot \|$ the Euclidean norm. + is the multivariate Gaussian kernel, with :math:` \| \cdot \|` the Euclidean norm. Parameters ---------- - obs: ArrayLike of shape (...,D) + obs : array_like The observed values, where the variables dimension is by default the last axis. - fct: ArrayLike of shape (..., M, D) + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. - w_func: tp.Callable + w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. - m_axis: int + m_axis : int The axis corresponding to the ensemble dimension. Defaults to -2. - v_axis: int or tuple(int) + v_axis : int or tuple of ints The axis corresponding to the variables dimension. Defaults to -1. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: ArrayLike of shape (...) + score : array_like The computed Outcome-Weighted GKS. """ B = backends.active if backend is None else backends[backend] @@ -490,37 +506,38 @@ def vrgksmv_ensemble( Computation is performed using the ensemble representation of vertically re-scaled kernel scores in [Allen et al. (2022)](https://arxiv.org/abs/2202.12732): - \[ - \mathrm{vrGKS}(F_{ens}, \mathbf{y}) = & - \frac{1}{M} \sum_{m = 1}^{M} k(\mathbf{x}_{m}, \mathbf{y}) w(\mathbf{x}_{m}) w(\mathbf{y}) + \frac{1}{2 M^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} k(\mathbf{x}_{m}, \mathbf{x}_{j}) w(\mathbf{x}_{m}) w(\mathbf{x_{j}}) + \frac{1}{2} k(y, y)w(y)w(y), - \] + .. math:: + \mathrm{vrGKS}(F_{ens}, \mathbf{y}) = & - \frac{1}{M} \sum_{m = 1}^{M} k(\mathbf{x}_{m}, \mathbf{y}) w(\mathbf{x}_{m}) w(\mathbf{y}) + + \frac{1}{2 M^{2}} \sum_{m = 1}^{M} \sum_{j = 1}^{M} k(\mathbf{x}_{m}, \mathbf{x}_{j}) w(\mathbf{x}_{m}) w(\mathbf{x_{j}}) + \frac{1}{2} k(y, y)w(y)w(y), - where $F_{ens}$ is the ensemble forecast $\mathbf{x}_{1}, \dots, \mathbf{x}_{M}$ with - $M$ members, $w$ is the weight function used to target particular outcomes, and + where :math:`F_{ens}` is the ensemble forecast :math:`\mathbf{x}_{1}, \dots, \mathbf{x}_{M}` with + :math:`M` members, :math:`w` is the weight function used to target particular outcomes, and - $$ k(x_{1}, x_{2}) = \exp \left(- \frac{ \| x_{1} - x_{2} \| ^{2}}{2} \right), $$ + .. math:: + k(x_{1}, x_{2}) = \exp \left(- \frac{ \| x_{1} - x_{2} \| ^{2}}{2} \right), - is the multivariate Gaussian kernel, with $ \| \cdot \|$ the Euclidean norm. + is the multivariate Gaussian kernel, with :math:` \| \cdot \|` the Euclidean norm. Parameters ---------- - obs: ArrayLike of shape (...,D) + obs : array_like The observed values, where the variables dimension is by default the last axis. - fct: ArrayLike of shape (..., M, D) + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. - w_func: tp.Callable + w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. - m_axis: int + m_axis : int The axis corresponding to the ensemble dimension. Defaults to -2. - v_axis: int or tuple(int) + v_axis : int or tuple of ints The axis corresponding to the variables dimension. Defaults to -1. backend: str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: ArrayLike of shape (...) + score : array_like The computed Vertically Re-scaled Gaussian Kernel Score. """ B = backends.active if backend is None else backends[backend] @@ -534,3 +551,15 @@ def vrgksmv_ensemble( return kernels.estimator_gufuncs_mv["vr"](obs, fct, obs_weights, fct_weights) return kernels.vr_ensemble_mv(obs, fct, obs_weights, fct_weights, backend=backend) + + +__all__ = [ + "gksuv_ensemble", + "twgksuv_ensemble", + "owgksuv_ensemble", + "vrgksuv_ensemble", + "gksmv_ensemble", + "twgksmv_ensemble", + "owgksmv_ensemble", + "vrgksmv_ensemble", +] diff --git a/scoringrules/_logs.py b/scoringrules/_logs.py index 6a675bb..deeb319 100644 --- a/scoringrules/_logs.py +++ b/scoringrules/_logs.py @@ -8,8 +8,8 @@ def logs_ensemble( - observations: "ArrayLike", - forecasts: "Array", + obs: "ArrayLike", + fct: "Array", /, axis: int = -1, *, @@ -27,22 +27,22 @@ def logs_ensemble( Parameters ---------- - observations: ArrayLike + obs : array_like The observed values. - forecasts: ArrayLike + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - axis: int + axis : int The axis corresponding to the ensemble. Default is the last axis. - bw : ArrayLike + bw : array_like The bandwidth parameter for each forecast ensemble. If not given, estimated using Silverman's rule of thumb. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: + score : array_like The LS between the forecast ensemble and obs. Examples @@ -51,30 +51,30 @@ def logs_ensemble( >>> sr.logs_ensemble(obs, pred) """ B = backends.active if backend is None else backends[backend] - observations, forecasts = map(B.asarray, (observations, forecasts)) + obs, fct = map(B.asarray, (obs, fct)) if axis != -1: - forecasts = B.moveaxis(forecasts, axis, -1) + fct = B.moveaxis(fct, axis, -1) - M = forecasts.shape[-1] + M = fct.shape[-1] # Silverman's rule of thumb for estimating the bandwidth parameter if bw is None: - sigmahat = B.std(forecasts, axis=-1) - q75 = B.quantile(forecasts, 0.75, axis=-1) - q25 = B.quantile(forecasts, 0.25, axis=-1) + sigmahat = B.std(fct, axis=-1) + q75 = B.quantile(fct, 0.75, axis=-1) + q25 = B.quantile(fct, 0.25, axis=-1) iqr = q75 - q25 bw = 1.06 * B.minimum(sigmahat, iqr / 1.34) * (M ** (-1 / 5)) bw = B.stack([bw] * M, axis=-1) - w = B.zeros(forecasts.shape) + 1 / M + w = B.zeros(fct.shape) + 1 / M - return logarithmic.mixnorm(observations, forecasts, bw, w, backend=backend) + return logarithmic.mixnorm(obs, fct, bw, w, backend=backend) def clogs_ensemble( - observations: "ArrayLike", - forecasts: "Array", + obs: "ArrayLike", + fct: "Array", /, a: "ArrayLike" = float("-inf"), b: "ArrayLike" = float("inf"), @@ -89,36 +89,36 @@ def clogs_ensemble( The conditional and censored likelihood scores are introduced by [Diks et al. (2011)](https://doi.org/10.1016/j.jeconom.2011.04.001): - The weight function is an indicator function of the form $w(z) = 1\{a < z < b\}$. + The weight function is an indicator function of the form :math:`w(z) = 1\{a < z < b\}`. The ensemble forecast is converted to a mixture of normal distributions using Gaussian kernel density estimation. The score is then calculated for this smoothed distribution. Parameters ---------- - observations: ArrayLike + obs : array_like The observed values. - forecasts: ArrayLike + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - a: ArrayLike + a : array_like The lower bound in the weight function. - b: ArrayLike + b : array_like The upper bound in the weight function. - axis: int + axis : int The axis corresponding to the ensemble. Default is the last axis. - bw : ArrayLike + bw : array_like The bandwidth parameter for each forecast ensemble. If not given, estimated using Silverman's rule of thumb. cens : Boolean Boolean specifying whether to return the conditional ('cens = False') or the censored likelihood score ('cens = True'). - backend: str + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: + score : array_like The CoLS or CeLS between the forecast ensemble and obs for the chosen weight parameters. Examples @@ -127,25 +127,25 @@ def clogs_ensemble( >>> sr.clogs_ensemble(obs, pred, -1.0, 1.0) """ B = backends.active if backend is None else backends[backend] - forecasts = B.asarray(forecasts) + fct = B.asarray(fct) if axis != -1: - forecasts = B.moveaxis(forecasts, axis, -1) + fct = B.moveaxis(fct, axis, -1) - M = forecasts.shape[-1] + M = fct.shape[-1] # Silverman's rule of thumb for estimating the bandwidth parameter if bw is None: - sigmahat = B.std(forecasts, axis=-1) - q75 = B.quantile(forecasts, 0.75, axis=-1) - q25 = B.quantile(forecasts, 0.25, axis=-1) + sigmahat = B.std(fct, axis=-1) + q75 = B.quantile(fct, 0.75, axis=-1) + q25 = B.quantile(fct, 0.25, axis=-1) iqr = q75 - q25 bw = 1.06 * B.minimum(sigmahat, iqr / 1.34) * (M ** (-1 / 5)) bw = B.stack([bw] * M, axis=-1) return logarithmic.clogs_ensemble( - observations, - forecasts, + obs, + fct, a=a, b=b, bw=bw, @@ -155,7 +155,7 @@ def clogs_ensemble( def logs_beta( - observation: "ArrayLike", + obs: "ArrayLike", a: "ArrayLike", b: "ArrayLike", /, @@ -170,22 +170,22 @@ def logs_beta( Parameters ---------- - observation: + obs : array_like The observed values. - a: + a : array_like First shape parameter of the forecast beta distribution. - b: + b : array_like Second shape parameter of the forecast beta distribution. - lower: + lower : array_like Lower bound of the forecast beta distribution. - upper: + upper : array_like Upper bound of the forecast beta distribution. - backend: + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: + score : array_like The LS between Beta(a, b) and obs. Examples @@ -193,11 +193,11 @@ def logs_beta( >>> import scoringrules as sr >>> sr.logs_beta(0.3, 0.7, 1.1) """ - return logarithmic.beta(observation, a, b, lower, upper, backend=backend) + return logarithmic.beta(obs, a, b, lower, upper, backend=backend) def logs_binomial( - observation: "ArrayLike", + obs: "ArrayLike", n: "ArrayLike", prob: "ArrayLike", /, @@ -210,18 +210,18 @@ def logs_binomial( Parameters ---------- - observation: + obs : array_like The observed values. - n: + n : array_like Size parameter of the forecast binomial distribution as an integer or array of integers. - prob: + prob : array_like Probability parameter of the forecast binomial distribution as a float or array of floats. - backend: + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: + score : array_like The LS between Binomial(n, prob) and obs. Examples @@ -229,11 +229,11 @@ def logs_binomial( >>> import scoringrules as sr >>> sr.logs_binomial(4, 10, 0.5) """ - return logarithmic.binomial(observation, n, prob, backend=backend) + return logarithmic.binomial(obs, n, prob, backend=backend) def logs_exponential( - observation: "ArrayLike", + obs: "ArrayLike", rate: "ArrayLike", /, *, @@ -245,16 +245,16 @@ def logs_exponential( Parameters ---------- - observation: + obs : array_like The observed values. - rate: + rate : array_like Rate parameter of the forecast exponential distribution. - backend: + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: + score : array_like The LS between Exp(rate) and obs. Examples @@ -262,11 +262,11 @@ def logs_exponential( >>> import scoringrules as sr >>> sr.logs_exponential(0.8, 3.0) """ - return logarithmic.exponential(observation, rate, backend=backend) + return logarithmic.exponential(obs, rate, backend=backend) def logs_exponential2( - observation: "ArrayLike", + obs: "ArrayLike", /, location: "ArrayLike" = 0.0, scale: "ArrayLike" = 1.0, @@ -279,18 +279,18 @@ def logs_exponential2( Parameters ---------- - observation: + obs : array_like The observed values. - location: + location : array_like Location parameter of the forecast exponential distribution. - scale: + scale : array_like Scale parameter of the forecast exponential distribution. - backend: + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: + score : array_like The LS between obs and Exp2(location, scale). Examples @@ -298,11 +298,11 @@ def logs_exponential2( >>> import scoringrules as sr >>> sr.logs_exponential2(0.2, 0.0, 1.0) """ - return logarithmic.exponential2(observation, location, scale, backend=backend) + return logarithmic.exponential2(obs, location, scale, backend=backend) def logs_2pexponential( - observation: "ArrayLike", + obs: "ArrayLike", scale1: "ArrayLike", scale2: "ArrayLike", location: "ArrayLike", @@ -316,20 +316,20 @@ def logs_2pexponential( Parameters ---------- - observation: + obs : array_like The observed values. - scale1: + scale1 : array_like First scale parameter of the forecast two-piece exponential distribution. - scale2: + scale2 : array_like Second scale parameter of the forecast two-piece exponential distribution. - location: + location : array_like Location parameter of the forecast two-piece exponential distribution. - backend: + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: + score : array_like The LS between 2pExp(sigma1, sigma2, location) and obs. Examples @@ -337,13 +337,11 @@ def logs_2pexponential( >>> import scoringrules as sr >>> sr.logs_2pexponential(0.8, 3.0, 1.4, 0.0) """ - return logarithmic.twopexponential( - observation, scale1, scale2, location, backend=backend - ) + return logarithmic.twopexponential(obs, scale1, scale2, location, backend=backend) def logs_gamma( - observation: "ArrayLike", + obs: "ArrayLike", shape: "ArrayLike", /, rate: "ArrayLike | None" = None, @@ -357,18 +355,18 @@ def logs_gamma( Parameters ---------- - observation: + obs : array_like The observed values. - shape: + shape : array_like Shape parameter of the forecast gamma distribution. - rate: + rate : array_like Rate parameter of the forecast gamma distribution. - scale: + scale : array_like Scale parameter of the forecast gamma distribution, where `scale = 1 / rate`. Returns ------- - score: + score : array_like The LS between obs and Gamma(shape, rate). Examples @@ -389,11 +387,11 @@ def logs_gamma( if rate is None: rate = 1.0 / scale - return logarithmic.gamma(observation, shape, rate, backend=backend) + return logarithmic.gamma(obs, shape, rate, backend=backend) def logs_gev( - observation: "ArrayLike", + obs: "ArrayLike", shape: "ArrayLike", /, location: "ArrayLike" = 0.0, @@ -407,20 +405,20 @@ def logs_gev( Parameters ---------- - observation: + obs : array_like The observed values. - shape: + shape : array_like Shape parameter of the forecast GEV distribution. - location: + location : array_like Location parameter of the forecast GEV distribution. - scale: + scale : array_like Scale parameter of the forecast GEV distribution. - backend: + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: + score : array_like The LS between obs and GEV(shape, location, scale). Examples @@ -428,11 +426,11 @@ def logs_gev( >>> import scoringrules as sr >>> sr.logs_gev(0.3, 0.1) """ - return logarithmic.gev(observation, shape, location, scale, backend=backend) + return logarithmic.gev(obs, shape, location, scale, backend=backend) def logs_gpd( - observation: "ArrayLike", + obs: "ArrayLike", shape: "ArrayLike", /, location: "ArrayLike" = 0.0, @@ -447,20 +445,20 @@ def logs_gpd( Parameters ---------- - observation: + obs : array_like The observed values. - shape: + shape : array_like Shape parameter of the forecast GPD distribution. - location: + location : array_like Location parameter of the forecast GPD distribution. - scale: + scale : array_like Scale parameter of the forecast GPD distribution. - backend: + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: + score : array_like The LS between obs and GPD(shape, location, scale). Examples @@ -468,11 +466,11 @@ def logs_gpd( >>> import scoringrules as sr >>> sr.logs_gpd(0.3, 0.9) """ - return logarithmic.gpd(observation, shape, location, scale, backend=backend) + return logarithmic.gpd(obs, shape, location, scale, backend=backend) def logs_hypergeometric( - observation: "ArrayLike", + obs: "ArrayLike", m: "ArrayLike", n: "ArrayLike", k: "ArrayLike", @@ -486,20 +484,20 @@ def logs_hypergeometric( Parameters ---------- - observation: + obs : array_like The observed values. - m: + m : array_like Number of success states in the population. - n: + n : array_like Number of failure states in the population. - k: + k : array_like Number of draws, without replacement. Must be in 0, 1, ..., m + n. - backend: + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: + score : array_like The LS between obs and Hypergeometric(m, n, k). Examples @@ -507,11 +505,11 @@ def logs_hypergeometric( >>> import scoringrules as sr >>> sr.logs_hypergeometric(5, 7, 13, 12) """ - return logarithmic.hypergeometric(observation, m, n, k, backend=backend) + return logarithmic.hypergeometric(obs, m, n, k, backend=backend) def logs_laplace( - observation: "ArrayLike", + obs: "ArrayLike", location: "ArrayLike" = 0.0, scale: "ArrayLike" = 1.0, /, @@ -525,21 +523,29 @@ def logs_laplace( Parameters ---------- - observation: + obs : array_like Observed values. - location: + location : array_like Location parameter of the forecast laplace distribution. - scale: + scale : array_like Scale parameter of the forecast laplace distribution. The LS between obs and Laplace(location, scale). + Returns + ------- + score : array_like + The LS between obs and Laplace(location, scale). + + Examples + -------- + >>> import scoringrules as sr >>> sr.logs_laplace(0.3, 0.1, 0.2) """ - return logarithmic.laplace(observation, location, scale, backend=backend) + return logarithmic.laplace(obs, location, scale, backend=backend) def logs_loglaplace( - observation: "ArrayLike", + obs: "ArrayLike", locationlog: "ArrayLike", scalelog: "ArrayLike", *, @@ -551,16 +557,16 @@ def logs_loglaplace( Parameters ---------- - observation: + obs : array_like Observed values. - locationlog: + locationlog : array_like Location parameter of the forecast log-laplace distribution. - scalelog: + scalelog : array_like Scale parameter of the forecast log-laplace distribution. Returns ------- - score: + score : array_like The LS between obs and Loglaplace(locationlog, scalelog). Examples @@ -568,11 +574,11 @@ def logs_loglaplace( >>> import scoringrules as sr >>> sr.logs_loglaplace(3.0, 0.1, 0.9) """ - return logarithmic.loglaplace(observation, locationlog, scalelog, backend=backend) + return logarithmic.loglaplace(obs, locationlog, scalelog, backend=backend) def logs_logistic( - observation: "ArrayLike", + obs: "ArrayLike", mu: "ArrayLike", sigma: "ArrayLike", /, @@ -585,16 +591,16 @@ def logs_logistic( Parameters ---------- - observations: ArrayLike + obs : array_like Observed values. - mu: ArrayLike + mu : array_like Location parameter of the forecast logistic distribution. - sigma: ArrayLike + sigma : array_like Scale parameter of the forecast logistic distribution. Returns ------- - score: + score : array_like The LS for the Logistic(mu, sigma) forecasts given the observations. Examples @@ -602,11 +608,11 @@ def logs_logistic( >>> import scoringrules as sr >>> sr.logs_logistic(0.0, 0.4, 0.1) """ - return logarithmic.logistic(observation, mu, sigma, backend=backend) + return logarithmic.logistic(obs, mu, sigma, backend=backend) def logs_loglogistic( - observation: "ArrayLike", + obs: "ArrayLike", mulog: "ArrayLike", sigmalog: "ArrayLike", backend: "Backend" = None, @@ -617,18 +623,18 @@ def logs_loglogistic( Parameters ---------- - observation: + obs : array_like The observed values. - mulog: + mulog : array_like Location parameter of the log-logistic distribution. - sigmalog: + sigmalog : array_like Scale parameter of the log-logistic distribution. - backend: + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: + score : array_like The LS between obs and Loglogis(mulog, sigmalog). Examples @@ -636,11 +642,11 @@ def logs_loglogistic( >>> import scoringrules as sr >>> sr.logs_loglogistic(3.0, 0.1, 0.9) """ - return logarithmic.loglogistic(observation, mulog, sigmalog, backend=backend) + return logarithmic.loglogistic(obs, mulog, sigmalog, backend=backend) def logs_lognormal( - observation: "ArrayLike", + obs: "ArrayLike", mulog: "ArrayLike", sigmalog: "ArrayLike", backend: "Backend" = None, @@ -651,16 +657,16 @@ def logs_lognormal( Parameters ---------- - observation: + obs : array_like The observed values. - mulog: + mulog : array_like Mean of the normal underlying distribution. - sigmalog: + sigmalog : array_like Standard deviation of the underlying normal distribution. Returns ------- - score: + score : array_like The LS between Lognormal(mu, sigma) and obs. Examples @@ -668,11 +674,11 @@ def logs_lognormal( >>> import scoringrules as sr >>> sr.logs_lognormal(0.0, 0.4, 0.1) """ - return logarithmic.lognormal(observation, mulog, sigmalog, backend=backend) + return logarithmic.lognormal(obs, mulog, sigmalog, backend=backend) def logs_mixnorm( - observation: "ArrayLike", + obs: "ArrayLike", m: "ArrayLike", s: "ArrayLike", /, @@ -687,22 +693,22 @@ def logs_mixnorm( Parameters ---------- - observation: ArrayLike + obs : array_like The observed values. - m: ArrayLike + m : array_like Means of the component normal distributions. - s: ArrayLike + s : array_like Standard deviations of the component normal distributions. - w: ArrayLike + w : array_like Non-negative weights assigned to each component. - axis: int + axis : int The axis corresponding to the mixture components. Default is the last axis. - backend: + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: + score : array_like The LS between MixNormal(m, s) and obs. Examples @@ -711,7 +717,7 @@ def logs_mixnorm( >>> sr.logs_mixnormal(0.0, [0.1, -0.3, 1.0], [0.4, 2.1, 0.7], [0.1, 0.2, 0.7]) """ B = backends.active if backend is None else backends[backend] - observation, m, s = map(B.asarray, (observation, m, s)) + obs, m, s = map(B.asarray, (obs, m, s)) if w is None: M: int = m.shape[axis] @@ -724,11 +730,11 @@ def logs_mixnorm( s = B.moveaxis(s, axis, -1) w = B.moveaxis(w, axis, -1) - return logarithmic.mixnorm(observation, m, s, w, backend=backend) + return logarithmic.mixnorm(obs, m, s, w, backend=backend) def logs_negbinom( - observation: "ArrayLike", + obs: "ArrayLike", n: "ArrayLike", /, prob: "ArrayLike | None" = None, @@ -742,18 +748,18 @@ def logs_negbinom( Parameters ---------- - observation: ArrayLike + obs : array_like The observed values. - n: ArrayLike + n : array_like Size parameter of the forecast negative binomial distribution. - prob: ArrayLike + prob : array_like Probability parameter of the forecast negative binomial distribution. - mu: ArrayLike + mu : array_like Mean of the forecast negative binomial distribution. Returns ------- - score: + score : array_like The LS between NegBinomial(n, prob) and obs. Examples @@ -774,11 +780,11 @@ def logs_negbinom( if prob is None: prob = n / (n + mu) - return logarithmic.negbinom(observation, n, prob, backend=backend) + return logarithmic.negbinom(obs, n, prob, backend=backend) def logs_normal( - observation: "ArrayLike", + obs: "ArrayLike", mu: "ArrayLike", sigma: "ArrayLike", /, @@ -792,18 +798,18 @@ def logs_normal( Parameters ---------- - observation: ArrayLike + obs : array_like The observed values. - mu: ArrayLike + mu : array_like Mean of the forecast normal distribution. - sigma: ArrayLike + sigma : array_like Standard deviation of the forecast normal distribution. - backend: str, optional + backend : str, optional The backend used for computations. Returns ------- - score: + score : array_like The LS between Normal(mu, sigma) and obs. Examples @@ -811,13 +817,11 @@ def logs_normal( >>> import scoringrules as sr >>> sr.logs_normal(0.0, 0.4, 0.1) """ - return logarithmic.normal( - observation, mu, sigma, negative=negative, backend=backend - ) + return logarithmic.normal(obs, mu, sigma, negative=negative, backend=backend) def logs_2pnormal( - observation: "ArrayLike", + obs: "ArrayLike", scale1: "ArrayLike", scale2: "ArrayLike", location: "ArrayLike", @@ -831,33 +835,31 @@ def logs_2pnormal( Parameters ---------- - observations: ArrayLike + obs : array_like The observed values. - scale1: ArrayLike + scale1 : array_like Scale parameter of the lower half of the forecast two-piece normal distribution. - scale2: ArrayLike + scale2 : array_like Scale parameter of the upper half of the forecast two-piece normal distribution. - location: ArrayLike + location : array_like Location parameter of the forecast two-piece normal distribution. - backend: + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: + score : array_like The LS between 2pNormal(scale1, scale2, location) and obs. Examples -------- >>> import scoringrules as sr >>> sr.logs_2pnormal(0.0, 0.4, 2.0, 0.1) """ - return logarithmic.twopnormal( - observation, scale1, scale2, location, backend=backend - ) + return logarithmic.twopnormal(obs, scale1, scale2, location, backend=backend) def logs_poisson( - observation: "ArrayLike", + obs: "ArrayLike", mean: "ArrayLike", /, *, @@ -869,16 +871,16 @@ def logs_poisson( Parameters ---------- - observation: ArrayLike + obs : array_like The observed values. - mean: ArrayLike + mean : array_like Mean parameter of the forecast poisson distribution. - backend: + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - score: + score : array_like The LS between Pois(mean) and obs. Examples @@ -886,11 +888,11 @@ def logs_poisson( >>> import scoringrules as sr >>> sr.logs_poisson(1, 2) """ - return logarithmic.poisson(observation, mean, backend=backend) + return logarithmic.poisson(obs, mean, backend=backend) def logs_t( - observation: "ArrayLike", + obs: "ArrayLike", df: "ArrayLike", /, location: "ArrayLike" = 0.0, @@ -904,18 +906,18 @@ def logs_t( Parameters ---------- - observation: ArrayLike + obs : array_like The observed values. - df: ArrayLike + df : array_like Degrees of freedom parameter of the forecast t distribution. - location: ArrayLike + location : array_like Location parameter of the forecast t distribution. - sigma: ArrayLike + sigma : array_like Scale parameter of the forecast t distribution. Returns ------- - score: + score : array_like The LS between t(df, location, scale) and obs. Examples @@ -923,11 +925,11 @@ def logs_t( >>> import scoringrules as sr >>> sr.logs_t(0.0, 0.1, 0.4, 0.1) """ - return logarithmic.t(observation, df, location, scale, backend=backend) + return logarithmic.t(obs, df, location, scale, backend=backend) def logs_tlogistic( - observation: "ArrayLike", + obs: "ArrayLike", location: "ArrayLike", scale: "ArrayLike", /, @@ -942,20 +944,20 @@ def logs_tlogistic( Parameters ---------- - observation: ArrayLike + obs : array_like The observed values. - location: ArrayLike + location : array_like Location parameter of the forecast distribution. - scale: ArrayLike + scale : array_like Scale parameter of the forecast distribution. - lower: ArrayLike + lower : array_like Lower boundary of the truncated forecast distribution. - upper: ArrayLike + upper : array_like Upper boundary of the truncated forecast distribution. Returns ------- - score: + score : array_like The LS between tLogistic(location, scale, lower, upper) and obs. Examples @@ -963,13 +965,11 @@ def logs_tlogistic( >>> import scoringrules as sr >>> sr.logs_tlogistic(0.0, 0.1, 0.4, -1.0, 1.0) """ - return logarithmic.tlogistic( - observation, location, scale, lower, upper, backend=backend - ) + return logarithmic.tlogistic(obs, location, scale, lower, upper, backend=backend) def logs_tnormal( - observation: "ArrayLike", + obs: "ArrayLike", location: "ArrayLike", scale: "ArrayLike", /, @@ -984,20 +984,20 @@ def logs_tnormal( Parameters ---------- - observation: ArrayLike + obs : array_like The observed values. - location: ArrayLike + location : array_like Location parameter of the forecast distribution. - scale: ArrayLike + scale : array_like Scale parameter of the forecast distribution. - lower: ArrayLike + lower : array_like Lower boundary of the truncated forecast distribution. - upper: ArrayLike + upper : array_like Upper boundary of the truncated forecast distribution. Returns ------- - score: + score : array_like The LS between tNormal(location, scale, lower, upper) and obs. Examples @@ -1005,13 +1005,11 @@ def logs_tnormal( >>> import scoringrules as sr >>> sr.logs_tnormal(0.0, 0.1, 0.4, -1.0, 1.0) """ - return logarithmic.tnormal( - observation, location, scale, lower, upper, backend=backend - ) + return logarithmic.tnormal(obs, location, scale, lower, upper, backend=backend) def logs_tt( - observation: "ArrayLike", + obs: "ArrayLike", df: "ArrayLike", /, location: "ArrayLike" = 0.0, @@ -1027,22 +1025,22 @@ def logs_tt( Parameters ---------- - observation: ArrayLike + obs : array_like The observed values. - df: ArrayLike + df : array_like Degrees of freedom parameter of the forecast distribution. - location: ArrayLike + location : array_like Location parameter of the forecast distribution. - scale: ArrayLike + scale : array_like Scale parameter of the forecast distribution. - lower: ArrayLike + lower : array_like Lower boundary of the truncated forecast distribution. - upper: ArrayLike + upper : array_like Upper boundary of the truncated forecast distribution. Returns ------- - score: + score : array_like The LS between tt(df, location, scale, lower, upper) and obs. Examples @@ -1050,13 +1048,11 @@ def logs_tt( >>> import scoringrules as sr >>> sr.logs_tt(0.0, 2.0, 0.1, 0.4, -1.0, 1.0) """ - return logarithmic.tt( - observation, df, location, scale, lower, upper, backend=backend - ) + return logarithmic.tt(obs, df, location, scale, lower, upper, backend=backend) def logs_uniform( - observation: "ArrayLike", + obs: "ArrayLike", min: "ArrayLike", max: "ArrayLike", /, @@ -1069,16 +1065,16 @@ def logs_uniform( Parameters ---------- - observation: ArrayLike + obs : array_like The observed values. - min: ArrayLike + min : array_like Lower bound of the forecast uniform distribution. - max: ArrayLike + max : array_like Upper bound of the forecast uniform distribution. Returns ------- - score: + score : array_like The LS between U(min, max, lmass, umass) and obs. Examples @@ -1086,7 +1082,7 @@ def logs_uniform( >>> import scoringrules as sr >>> sr.logs_uniform(0.4, 0.0, 1.0) """ - return logarithmic.uniform(observation, min, max, backend=backend) + return logarithmic.uniform(obs, min, max, backend=backend) __all__ = [ diff --git a/scoringrules/_quantile.py b/scoringrules/_quantile.py index 8b76c10..ed324d7 100644 --- a/scoringrules/_quantile.py +++ b/scoringrules/_quantile.py @@ -16,28 +16,27 @@ def quantile_score( The quantile score (Koenker, R. and G. Bassett, 1978) is defined as - $$ + .. math:: S_{\alpha}(q_{\alpha}, y) = \begin{cases} (1 - \alpha) (q_{\alpha} - y), & \text{if } y \leq q_{\alpha}, \\ \alpha (y - q_{\alpha}), & \text{if } y > q_{\alpha}. \end{cases} - $$ - where $y$ is the observed value and $q_{\alpha}$ is the predicted value at the - $\alpha$ quantile level. + where :math:`y` is the observed value and :math:`q_{\alpha}` is the predicted value at the + :math:`\alpha` quantile level. Parameters ---------- - obs: + obs : array_like The observed values. - fct: + fct : array_like The forecast values. - alpha: + alpha : array_like The quantile level. Returns ------- - score: + score : array_like The quantile score. Examples diff --git a/scoringrules/_variogram.py b/scoringrules/_variogram.py index c24eb95..ef16bc0 100644 --- a/scoringrules/_variogram.py +++ b/scoringrules/_variogram.py @@ -9,8 +9,8 @@ def variogram_score( - observations: "Array", - forecasts: "Array", + obs: "Array", + fct: "Array", /, m_axis: int = -2, v_axis: int = -1, @@ -20,50 +20,49 @@ def variogram_score( ) -> "Array": r"""Compute the Variogram Score for a finite multivariate ensemble. - For a $D$-variate ensemble the Variogram Score + For a :math:`D`-variate ensemble the Variogram Score [(Sheuerer and Hamill, 2015)](https://journals.ametsoc.org/view/journals/mwre/143/4/mwr-d-14-00269.1.xml#bib9) - of order $p$ is expressed as + of order :math:`p` is expressed as - $$\text{VS}_{p}(F_{ens}, \mathbf{y})= \sum_{i=1}^{d} \sum_{j=1}^{d} - \left( \frac{1}{M} \sum_{m=1}^{M} | x_{m,i} - x_{m,j} |^{p} - | y_{i} - y_{j} |^{p} \right)^{2}. $$ + .. math:: + \text{VS}_{p}(F_{ens}, \mathbf{y})= \sum_{i=1}^{d} \sum_{j=1}^{d} + \left( \frac{1}{M} \sum_{m=1}^{M} | x_{m,i} - x_{m,j} |^{p} - | y_{i} - y_{j} |^{p} \right)^{2}, - where $\mathbf{X}$ and $\mathbf{X'}$ are independently sampled ensembles from from $F$. + where :math:`\mathbf{X}` and :math:`\mathbf{X'}` are independently sampled ensembles from from :math:`F`. Parameters ---------- - forecasts: Array + obs : array_like + The observed values, where the variables dimension is by default the last axis. + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. - observations: Array - The observed values, where the variables dimension is by default the last axis. - p: float + p : float The order of the Variogram Score. Typical values are 0.5, 1.0 or 2.0. Defaults to 1.0. - m_axis: int + m_axis : int The axis corresponding to the ensemble dimension. Defaults to -2. - v_axis: int + v_axis : int The axis corresponding to the variables dimension. Defaults to -1. backend: str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - variogram_score: Array + variogram_score : array_like The computed Variogram Score. """ - observations, forecasts = multivariate_array_check( - observations, forecasts, m_axis, v_axis, backend=backend - ) + obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) if backend == "numba": - return variogram._variogram_score_gufunc(observations, forecasts, p) + return variogram._variogram_score_gufunc(obs, fct, p) - return variogram.vs(observations, forecasts, p, backend=backend) + return variogram.vs(obs, fct, p, backend=backend) def twvariogram_score( - observations: "Array", - forecasts: "Array", + obs: "Array", + fct: "Array", v_func: tp.Callable, /, m_axis: int = -2, @@ -77,46 +76,44 @@ def twvariogram_score( Computation is performed using the ensemble representation of the twVS in [Allen et al. (2022)](https://arxiv.org/abs/2202.12732): - \[ - \mathrm{twVS}(F_{ens}, \mathbf{y}) = \sum_{i,j=1}^{D}(|v(\mathbf{y})_i - v(\mathbf{y})_{j}|^{p} - \frac{1}{M} \sum_{m=1}^{M}|v(\mathbf{x}_{m})_{i} - v(\mathbf{x}_{m})_{j}|^{p})^{2}, - \] + .. math:: + \mathrm{twVS}(F_{ens}, \mathbf{y}) = \sum_{i,j=1}^{D}(|v(\mathbf{y})_i - v(\mathbf{y})_{j}|^{p} + - \frac{1}{M} \sum_{m=1}^{M}|v(\mathbf{x}_{m})_{i} - v(\mathbf{x}_{m})_{j}|^{p})^{2}, - where $F_{ens}$ is the ensemble forecast $\mathbf{x}_{1}, \dots, \mathbf{x}_{M}$ with - $M$ members, and $v$ is the chaining function used to target particular outcomes. + where :math:`F_{ens}` is the ensemble forecast :math:`\mathbf{x}_{1}, \dots, \mathbf{x}_{M}` with + :math:`M` members, and :math:`v` is the chaining function used to target particular outcomes. Parameters ---------- - forecasts: Array + obs : array_like + The observed values, where the variables dimension is by default the last axis. + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. - observations: Array - The observed values, where the variables dimension is by default the last axis. - p: float + p : float The order of the Variogram Score. Typical values are 0.5, 1.0 or 2.0. Defaults to 1.0. - v_func: tp.Callable + v_func : callable, array_like -> array_like Chaining function used to emphasise particular outcomes. - m_axis: int + m_axis : int The axis corresponding to the ensemble dimension. Defaults to -2. - v_axis: int + v_axis : int The axis corresponding to the variables dimension. Defaults to -1. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - twvariogram_score: ArrayLike of shape (...) + twvariogram_score : array_like The computed Threshold-Weighted Variogram Score. """ - observations, forecasts = map(v_func, (observations, forecasts)) - return variogram_score( - observations, forecasts, m_axis, v_axis, p=p, backend=backend - ) + obs, fct = map(v_func, (obs, fct)) + return variogram_score(obs, fct, m_axis, v_axis, p=p, backend=backend) def owvariogram_score( - observations: "Array", - forecasts: "Array", + obs: "Array", + fct: "Array", w_func: tp.Callable, /, m_axis: int = -2, @@ -130,61 +127,58 @@ def owvariogram_score( Computation is performed using the ensemble representation of the owVS in [Allen et al. (2022)](https://arxiv.org/abs/2202.12732): - \[ - \begin{split} - \mathrm{owVS}(F_{ens}, \mathbf{y}) = & \frac{1}{M \bar{w}} \sum_{m=1}^{M} \sum_{i,j=1}^{D}(|y_{i} - y_{j}|^{p} - |x_{m,i} - x_{m,j}|^{p})^{2} w(\mathbf{x}_{m}) w(\mathbf{y}) \\ - & - \frac{1}{2 M^{2} \bar{w}^{2}} \sum_{k,m=1}^{M} \sum_{i,j=1}^{D} \left( |x_{k,i} - x_{k,j}|^{p} - |x_{m,i} - x_{m,j}|^{p} \right)^{2} w(\mathbf{x}_{k}) w(\mathbf{x}_{m}) w(\mathbf{y}), - \end{split} - \] + .. math:: + \begin{split} + \mathrm{owVS}(F_{ens}, \mathbf{y}) = & \frac{1}{M \bar{w}} \sum_{m=1}^{M} \sum_{i,j=1}^{D}(|y_{i} - y_{j}|^{p} + - |x_{m,i} - x_{m,j}|^{p})^{2} w(\mathbf{x}_{m}) w(\mathbf{y}) \\ + & - \frac{1}{2 M^{2} \bar{w}^{2}} \sum_{k,m=1}^{M} \sum_{i,j=1}^{D} \left( |x_{k,i} + - x_{k,j}|^{p} - |x_{m,i} - x_{m,j}|^{p} \right)^{2} w(\mathbf{x}_{k}) w(\mathbf{x}_{m}) w(\mathbf{y}), + \end{split} - where $F_{ens}$ is the ensemble forecast $\mathbf{x}_{1}, \dots, \mathbf{x}_{M}$ with - $M$ members, $w$ is the chosen weight function, and $\bar{w} = \sum_{m=1}^{M}w(\mathbf{x}_{m})/M$. + where :math:`F_{ens}` is the ensemble forecast :math:`\mathbf{x}_{1}, \dots, \mathbf{x}_{M}` with + :math:`M` members, :math:`w` is the chosen weight function, and :math:`\bar{w} = \sum_{m=1}^{M}w(\mathbf{x}_{m})/M`. Parameters ---------- - forecasts: Array + obs : array_like + The observed values, where the variables dimension is by default the last axis. + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. - observations: Array - The observed values, where the variables dimension is by default the last axis. - p: float + p : float The order of the Variogram Score. Typical values are 0.5, 1.0 or 2.0. Defaults to 1.0. - w_func: tp.Callable + w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. - m_axis: int + m_axis : int The axis corresponding to the ensemble dimension. Defaults to -2. - v_axis: int + v_axis : int The axis corresponding to the variables dimension. Defaults to -1. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - owvariogram_score: ArrayLike of shape (...) + owvariogram_score : array_like The computed Outcome-Weighted Variogram Score. """ B = backends.active if backend is None else backends[backend] - observations, forecasts = multivariate_array_check( - observations, forecasts, m_axis, v_axis, backend=backend - ) + obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) - obs_weights = B.apply_along_axis(w_func, observations, -1) - fct_weights = B.apply_along_axis(w_func, forecasts, -1) + obs_weights = B.apply_along_axis(w_func, obs, -1) + fct_weights = B.apply_along_axis(w_func, fct, -1) if backend == "numba": return variogram._owvariogram_score_gufunc( - observations, forecasts, p, obs_weights, fct_weights + obs, fct, p, obs_weights, fct_weights ) - return variogram.owvs( - observations, forecasts, obs_weights, fct_weights, p=p, backend=backend - ) + return variogram.owvs(obs, fct, obs_weights, fct_weights, p=p, backend=backend) def vrvariogram_score( - observations: "Array", - forecasts: "Array", + obs: "Array", + fct: "Array", w_func: tp.Callable, /, m_axis: int = -2, @@ -198,54 +192,52 @@ def vrvariogram_score( Computation is performed using the ensemble representation of the vrVS in [Allen et al. (2022)](https://arxiv.org/abs/2202.12732): - \[ - \begin{split} - \mathrm{vrVS}(F_{ens}, \mathbf{y}) = & \frac{1}{M} \sum_{m=1}^{M} \sum_{i,j=1}^{D}(|y_{i} - y_{j}|^{p} - |x_{m,i} - x_{m,j}|^{p})^{2} w(\mathbf{x}_{m}) w(\mathbf{y}) \\ - & - \frac{1}{2 M^{2}} \sum_{k,m=1}^{M} \sum_{i,j=1}^{D} \left( |x_{k,i} - x_{k,j}|^{p} - |x_{m,i} - x_{m,j}|^{p} \right)^{2} w(\mathbf{x}_{k}) w(\mathbf{x}_{m})) \\ - & + \left( \frac{1}{M} \sum_{m = 1}^{M} \sum_{i,j=1}^{D}(|x_{m,i} - x_{m,j}|^{p} w(\mathbf{x}_{m}) - \sum_{i,j=1}^{D}(|y_{i} - y_{j}|^{p} w(\mathbf{y}) \right) \left( \frac{1}{M} \sum_{m = 1}^{M} w(\mathbf{x}_{m}) - w(\mathbf{y}) \right), - \end{split} - \] + .. math:: + \begin{split} + \mathrm{vrVS}(F_{ens}, \mathbf{y}) = & \frac{1}{M} \sum_{m=1}^{M} \sum_{i,j=1}^{D}(|y_{i} - y_{j}|^{p} + - |x_{m,i} - x_{m,j}|^{p})^{2} w(\mathbf{x}_{m}) w(\mathbf{y}) \\ + & - \frac{1}{2 M^{2}} \sum_{k,m=1}^{M} \sum_{i,j=1}^{D} \left( |x_{k,i} - x_{k,j}|^{p} - |x_{m,i} + - x_{m,j}|^{p} \right)^{2} w(\mathbf{x}_{k}) w(\mathbf{x}_{m})) \\ + & + \left( \frac{1}{M} \sum_{m = 1}^{M} \sum_{i,j=1}^{D}(|x_{m,i} - x_{m,j}|^{p} w(\mathbf{x}_{m}) + - \sum_{i,j=1}^{D}(|y_{i} - y_{j}|^{p} w(\mathbf{y}) \right) \left( \frac{1}{M} \sum_{m = 1}^{M} w(\mathbf{x}_{m}) - w(\mathbf{y}) \right), + \end{split} - where $F_{ens}$ is the ensemble forecast $\mathbf{x}_{1}, \dots, \mathbf{x}_{M}$ with - $M$ members, $w$ is the chosen weight function, and $\bar{w} = \sum_{m=1}^{M}w(\mathbf{x}_{m})/M$. + where :math:`F_{ens}` is the ensemble forecast :math:`\mathbf{x}_{1}, \dots, \mathbf{x}_{M}` with + :math:`M` members, :math:`w` is the chosen weight function, and :math:`\bar{w} = \sum_{m=1}^{M}w(\mathbf{x}_{m})/M`. Parameters ---------- - observations: Array + obs : array_like The observed values, where the variables dimension is by default the last axis. - forecasts: Array + fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. - p: float + p : float The order of the Variogram Score. Typical values are 0.5, 1.0 or 2.0. Defaults to 1.0. - w_func: tp.Callable + w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. - m_axis: int + m_axis : int The axis corresponding to the ensemble dimension. Defaults to -2. - v_axis: int + v_axis : int The axis corresponding to the variables dimension. Defaults to -1. - backend: str + backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. Returns ------- - vrvariogram_score: ArrayLike of shape (...) + vrvariogram_score : array_like The computed Vertically Re-scaled Variogram Score. """ B = backends.active if backend is None else backends[backend] - observations, forecasts = multivariate_array_check( - observations, forecasts, m_axis, v_axis, backend=backend - ) + obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) - obs_weights = B.apply_along_axis(w_func, observations, -1) - fct_weights = B.apply_along_axis(w_func, forecasts, -1) + obs_weights = B.apply_along_axis(w_func, obs, -1) + fct_weights = B.apply_along_axis(w_func, fct, -1) if backend == "numba": return variogram._vrvariogram_score_gufunc( - observations, forecasts, p, obs_weights, fct_weights + obs, fct, p, obs_weights, fct_weights ) - return variogram.vrvs( - observations, forecasts, obs_weights, fct_weights, p=p, backend=backend - ) + return variogram.vrvs(obs, fct, obs_weights, fct_weights, p=p, backend=backend) From 66563afdd4e9fce0cad3286a5cde5ebf72b8eed6 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Sat, 1 Feb 2025 14:12:22 +0100 Subject: [PATCH 19/79] add examples and pass doctest for variogram scores --- scoringrules/_variogram.py | 61 ++++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/scoringrules/_variogram.py b/scoringrules/_variogram.py index ef16bc0..1ec34d3 100644 --- a/scoringrules/_variogram.py +++ b/scoringrules/_variogram.py @@ -20,9 +20,7 @@ def variogram_score( ) -> "Array": r"""Compute the Variogram Score for a finite multivariate ensemble. - For a :math:`D`-variate ensemble the Variogram Score - [(Sheuerer and Hamill, 2015)](https://journals.ametsoc.org/view/journals/mwre/143/4/mwr-d-14-00269.1.xml#bib9) - of order :math:`p` is expressed as + For a :math:`D`-variate ensemble the Variogram Score [1]_ is defined as: .. math:: \text{VS}_{p}(F_{ens}, \mathbf{y})= \sum_{i=1}^{d} \sum_{j=1}^{d} @@ -51,6 +49,22 @@ def variogram_score( ------- variogram_score : array_like The computed Variogram Score. + + References + ---------- + .. [1] Scheuerer, M., and T. M. Hamill (2015), + Variogram-Based Proper Scoring Rules for Probabilistic Forecasts of Multivariate Quantities. + Mon. Wea. Rev., 143, 1321-1334, https://doi.org/10.1175/MWR-D-14-00269.1. + + Examples + -------- + >>> import numpy as np + >>> import scoringrules as sr + >>> rng = np.random.default_rng(123) + >>> obs = rng.normal(size=(3, 5)) + >>> fct = rng.normal(size=(3, 10, 5)) + >>> sr.variogram_score(obs, fct) + array([ 8.65630139, 6.84693866, 19.52993307]) """ obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) @@ -73,8 +87,7 @@ def twvariogram_score( ) -> "Array": r"""Compute the Threshold-Weighted Variogram Score (twVS) for a finite multivariate ensemble. - Computation is performed using the ensemble representation of the twVS in - [Allen et al. (2022)](https://arxiv.org/abs/2202.12732): + Computation is performed using the ensemble representation of the twVS in [1]_, .. math:: \mathrm{twVS}(F_{ens}, \mathbf{y}) = \sum_{i,j=1}^{D}(|v(\mathbf{y})_i - v(\mathbf{y})_{j}|^{p} @@ -101,11 +114,27 @@ def twvariogram_score( backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. - Returns ------- twvariogram_score : array_like The computed Threshold-Weighted Variogram Score. + + References + ---------- + .. [1] Allen, S., Ginsbourger, D., & Ziegel, J. (2023). + Evaluating forecasts for high-impact events using transformed kernel scores. + SIAM/ASA Journal on Uncertainty Quantification, 11(3), 906-940. + Available at https://arxiv.org/abs/2202.12732. + + Examples + -------- + >>> import numpy as np + >>> import scoringrules as sr + >>> rng = np.random.default_rng(123) + >>> obs = rng.normal(size=(3, 5)) + >>> fct = rng.normal(size=(3, 10, 5)) + >>> sr.twvariogram_score(obs, fct, lambda x: np.maximum(x, -0.2)) + array([5.94996894, 4.72029765, 6.08947229]) """ obs, fct = map(v_func, (obs, fct)) return variogram_score(obs, fct, m_axis, v_axis, p=p, backend=backend) @@ -160,6 +189,16 @@ def owvariogram_score( ------- owvariogram_score : array_like The computed Outcome-Weighted Variogram Score. + + Examples + -------- + >>> import numpy as np + >>> import scoringrules as sr + >>> rng = np.random.default_rng(123) + >>> obs = rng.normal(size=(3, 5)) + >>> fct = rng.normal(size=(3, 10, 5)) + >>> sr.owvariogram_score(obs, fct, lambda x: x.mean() + 1.0) + array([ 9.86816636, 6.75532522, 19.59353723]) """ B = backends.active if backend is None else backends[backend] @@ -227,6 +266,16 @@ def vrvariogram_score( ------- vrvariogram_score : array_like The computed Vertically Re-scaled Variogram Score. + + Examples + -------- + >>> import numpy as np + >>> import scoringrules as sr + >>> rng = np.random.default_rng(123) + >>> obs = rng.normal(size=(3, 5)) + >>> fct = rng.normal(size=(3, 10, 5)) + >>> sr.vrvariogram_score(obs, fct, lambda x: x.max() + 1.0) + array([46.48256493, 57.90759816, 92.37153472]) """ B = backends.active if backend is None else backends[backend] From 2c1b2f64fa2121517014d61d48d877551138b8d3 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Fri, 4 Apr 2025 17:55:15 +0200 Subject: [PATCH 20/79] comment out latex math hook --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7e5a51a..a35e73d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,8 +26,8 @@ repos: hooks: - id: codespell - - repo: https://github.com/frazane/check-latex-math - rev: v0.2.2 - hooks: - - id: latex-math-validation - args: ["scoringrules/", "docs/", "*.py", "*.md"] + #- repo: https://github.com/frazane/check-latex-math + # rev: v0.2.2 + # hooks: + # - id: latex-math-validation + # args: ["scoringrules/", "docs/", "*.py", "*.md"] From e6b0cf90500f548c323297c0974524d0819f7bf6 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Fri, 4 Apr 2025 17:58:02 +0200 Subject: [PATCH 21/79] update theory docs --- docs/theory.md | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/docs/theory.md b/docs/theory.md index eb0b8a7..81ccf6f 100644 --- a/docs/theory.md +++ b/docs/theory.md @@ -35,6 +35,19 @@ our expected score is minimised by issuing $G$ as our forecast; proper scoring r encourage honest predictions, and discourage hedging. A scoring rule is *strictly proper* if the above inequality holds with equality if and only if $F = G$. +Proper scoring rules allow us to compare competing forecasters (or prediction strategies) +via their average score. In practice, for a given forecaster, we typically have access to +multiple forecasts $F_{1}, F_{2}, \dots, F_{n}$ and observations $y_{1}, \dots, y_{n}$, +corresponding to different points in time, for example. The forecaster with the lowest average score, + +$$ + \frac{1}{n} \sum_{i=1}^{n} S(F_{i}, y_{i}), +$$ + +is then considered to be the most accurate. However, the ordering of competing forecasters will +generally depend on the chosen scoring rule. It is therefore important to employ a scoring rule +that encompasses our beliefs about what makes a forecast good. + ## Examples @@ -61,8 +74,9 @@ $$ \end{cases} $$ -Other popular binary scoring rules include the spherical score, power score, and pseudo-spherical -score. +The Brier score is also often referred to as the quadratic score, while the Log score forms +the basis of the Cross-entropy loss in classifiaction tasks. Other popular scoring rules +for binary outcomes include the *spherical score*, *power score*, and *pseudo-spherical score*. ### Categorical outcomes: @@ -75,7 +89,7 @@ $$ F = (F_{1}, \dots, F_{K}) \in [0, 1]^{K} \quad \text{such that} \quad \sum_{i=1}^{K} F_{i} = 1. $$ -The $i$-th element of $F$ represents the probability that $Y = i$, for $i = 1, \dots, K$. +The $i$-th element of $F$ represents the forecast probability that $Y = i$, for $i = 1, \dots, K$. Proper scoring rules for binary outcomes can readily be used to evaluate probabilistic forecasts for categorical outcomes, by applying the score separately for each category, @@ -375,7 +389,7 @@ be loosely interpreted as a measure of similarity between its two inputs. The *kernel score* corresponding to the positive definite kernel $k$ is defined as $$ - S_{k}(F, y) = \frac{1}{2} \mathbb{E} k(y, y) + \frac{1}{2} \mathbb{E} k(X, X^{\prime}) - \mathbb{E} k(X, y), + S_{k}(F, y) = \frac{1}{2} k(y, y) + \frac{1}{2} \mathbb{E} k(X, X^{\prime}) - \mathbb{E} k(X, y), $$ where $X, X^{\prime} \sim F$ are independent. The first term on the right-hand-side does not From 516ff25cff3a037718306fdb48d2b7c320f5d309 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Fri, 4 Apr 2025 17:58:32 +0200 Subject: [PATCH 22/79] add documentation page for ensemble forecasts --- docs/crps_estimators.md | 407 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 388 insertions(+), 19 deletions(-) diff --git a/docs/crps_estimators.md b/docs/crps_estimators.md index 7f52787..26e7b1a 100644 --- a/docs/crps_estimators.md +++ b/docs/crps_estimators.md @@ -1,45 +1,414 @@ (crps-estimators)= -# CRPS estimators +# Ensemble forecasts -## Integral form (INT) +Suppose the forecast is an *ensemble forecast*. That is, the forecast is +available via $M$ values, $x_{1}, \dots, x_{M} \in \Omega$, called *ensemble members*. +Each ensemble member can be interpreted as a possible outcome, so that the distribution of the +members represents the range of outcomes that could occur. +Ensemble forecasts can be generated, for example, from generative machine learning models, +Markov chain Monte Carlo (MCMC) methods, or from the output of physical weather or climate models. -The numerical approximation of the cumulative integral over the finite ensemble. +Note that in some fields, the term ensemble forecast refers to a (linear) aggregation of different +forecasts. This differs from the nomenclature employed here, where we instead refer to an +ensemble forecast as a discrete predictive distribution defined by the empirical distribution +of the values $x_{1}, \dots, x_{M}$. For example, when $\Omega \subseteq \mathbb{R}$, +the ensemble forecast is $$ -\text{CRPS}_{\text{INT}}(M, y) = \int_{\mathbb{R}} \left[ \frac{1}{M} - \sum_{i=1}^M \mathbb{1}\{x_i \le x \} - \mathbb{1}\{y \le x\} \right] ^2 dx +F_{M}(x) = \frac{1}{M} \sum_{i=1}^{M} \mathbb{1}\{x_{i} \le x\}, $$ -Runs with $O(m\cdot\mathrm{log}m)$ complexity, including the sorting of the ensemble. +where $\mathbb{1}\{\cdotp\}$ is the indicator function. -## Energy form (NRG) +This forecast can be evaluated by plugging $F_{M}$ into the definition of scoring rules. +Since this empirical distribution does not emit a density function, ensemble forecasts can +generally not be evaluated using the Log score without making additional assumptions about +the shape of the distribution at unobserved values. -Introduced by Gneiting and Raftery (2007): +In the following, we discuss different +ways to calculate the Continuous Ranked Probability Score (CRPS) when the forecast is an +ensemble forecast, and also how the interpretation of the ensemble members changes the way +scores should be calculated; in particular, when a *fair* scoring rule should be employed. + + +## CRPS calculation + +Suppose $\Omega \subseteq \mathbb{R}$, so that $F$ is a predictive distribution function. +The CRPS can be expressed in several ways: + +$$ +\begin{align} +\mathrm{CRPS}(F, y) &= \int_{-\infty}^{\infty} (F(z) - \mathbb{1}\{y \le z\})^{2} dz \\ +&= \int_{0}^{1} (\mathbb{1}\{F^{-1}(\alpha) \ge y\} - \alpha)(F^{-1}(\alpha) - y) d\alpha \\ +&= \mathbb{E}|X - y| - \frac{1}{2} \mathbb{E}|X - X^{\prime}|, \\ +\end{align} +$$ + +where $F^{-1}$ is the generalised inverse of $F$, and $X, X^{\prime} \sim F$ are independent +(Matheson and Winkler, 1976; Laio and Tamea, 2007; Gneiting and Raftery, 2007). + +The different representations of the CRPS lead to many different ways to calculate the score. +While these all yield the same score, the different representations can often be calculated with +different computational complexity, meaning some are faster to implement than others. This is particularly +relevant when the forecast is an ensemble. +Some common options to calculate the CRPS for ensemble forecasts are given below. + + +### Energy form (NRG) + +Grimit et al. (2006) showed that the CRPS of the empirical distribution $F_{M}$ is equal to $$ -\text{CRPS}_{\text{NRG}}(M, y) = \frac{1}{M} \sum_{i=1}^{M}|x_i - y| - \frac{1}{2 M^2}\sum_{i,j=1}^{M}|x_i - x_j| +\mathrm{CRPS}_{\text{NRG}}(F_{M}, y) = \frac{1}{M} \sum_{i=1}^{M}|x_i - y| - \frac{1}{2 M^2}\sum_{i=1}^{M} \sum_{j=1}^{M}|x_i - x_j|, $$ -It is called the "energy form" because it is the one-dimensional case of the Energy Score. +which follows from the kernel score representation of the CRPS given by Gneiting and Raftery (2007). +This has been referred to as the *energy form* of the CRPS, and can be calculated with $O(M^2)$ complexity. -Runs with $O(m^2)$ complexity. -## Quantile decomposition form (QD) +### Quantile decomposition form (QD) -Introduced by Jordan (2016): +Jordan (2016) introduced a *quantile decomposition form* of the CRPS, $$ -\mathrm{CRPS}_{\mathrm{QD}}(M, y) = \frac{2}{M^2} \sum_{i=1}^{M}(x_i - y)\left[M\mathbb{1}\{y \le x_i\} - i + \frac{1}{2} \right] +\mathrm{CRPS}_{\mathrm{QD}}(F_{M}, y) = \frac{2}{M} \sum_{i=1}^{M}\left[\mathbb{1}\{y \le x_{(i)}\} - \frac{2i - 1}{2M} \right] (x_{(i)} - y), +$$ + +where $x_{(i)}$ denotes the $i$-th order statistic of the ensemble members, +so that $x_{(1)} \le x_{(2)} \le \dots \le x_{(M)}$. This can be calculated +with $O(M\cdot\mathrm{log}M)$ complexity, including the sorting of the ensemble members. + + +### Probability weighted moment form (PWM) + +Taillardat et al. (2016) proposed an expression of the CRPS based on probability weighted moments, + +$$ +\mathrm{CRPS}_{\mathrm{PWM}}(F_{M}, y) = \frac{1}{M} \sum_{i=1}^{M}|x_{(i)} - y| + \frac{M - 1}{M} \left(\hat{\beta_0} - 2\hat{\beta_1}\right), +$$ + +where $\hat{\beta_0} = \frac{1}{M} \sum_{i=1}^{M}x_{(i)}$ and $\hat{\beta_1} = \frac{1}{M(M-1)} \sum_{i=1}^{M}(i - 1)x_{(i)}$. +This *probability weighted moment form* of the CRPS also requires sorting the ensemble, +and in total runs with $O(M\cdot\mathrm{log}M)$ complexity. + +The above expression differs slightly from that initially introduced by Taillardat et al. (2016), in that +$\hat{\beta_{0}}$ and $\hat{\beta_{1}}$ have been scaled by $(M - 1)/M$. This ensures that the CRPS is +equivalent to the other forms listed in this section. Without rescaling by $(M - 1)/M$, +we recover the *fair* version of the CRPS, which is discussed in more detail in the following +sections. + + +### Integral form (INT) + +Alternatively, the *integral form* of the CRPS, + +$$ +\mathrm{CRPS}_{\text{INT}}(F_{M}, y) = \int_{\mathbb{R}} \left[ \frac{1}{M} + \sum_{i=1}^M \mathbb{1}\{x_i \le x \} - \mathbb{1}\{y \le x\} \right] ^2 dx, +$$ + +can be numerically approximated. This also runs with $O(M\cdot\mathrm{log}M)$ complexity, +but introduces small errors due to the numerical approximation. + + +## Fair scoring rules + +The above expressions of the CRPS assume that we wish to assess the empirical distribution defined by +the ensemble members $x_{1}, \dots, x_{M}$. However, one could argue that the ensemble forecast +should not be treated as a forecast distribution in itself, but rather a random sample of values from +an underlying (unknown) distribution, say $F$. We may wish to evaluate this underlying distribution +rather than the empirical distribution of the available sample ($F_{M}$). + +The choice to interpret $x_{1}, \dots, x_{M}$ as a sample or as a distribution in itself is +not trivial. For example, by treating the ensemble forecast as an empirical distribution, we implicitly +assume that the forecast probability that the outcome exceeds all ensemble members is zero, +which may not be how we interpret the ensemble. However, if the forecast is to be used for +decision making, then we generally have to make decisions based only on the information available +from $x_{1}, \dots, x_{M}$, and not the underlying, unknown distribution. For this reason, +`scoringrules` treats the ensemble forecast as an empirical distribution by default. + +If we wish to evaluate the underlying distribution $F$ rather than the empirical distribution +defined by the ensemble members, we must adapt the scoring rule to account for the fact that +$x_{1}, \dots, x_{M}$ are only a selection of possible values that could have been drawn from $F$. +For this purpose, Ferro (2014) introduced *fair* scoring rules. + +A scoring rule for ensemble forecasts is fair if the expected score is minimised when the ensemble members +are random draws from the distribution of $Y$. That is, given a class of probability distributions +$\mathcal{F}$ on $\Omega$, a scoring rule $S$ is fair (with respect to $\mathcal{F}$) if, +when $Y \sim G \in \mathcal{F}$, + +$$ +\mathbb{E}S(G_{M}, Y) \le \mathbb{E}S(F_{M}, Y), +$$ + +for all $F, G \in \mathcal{F}$ , where $F_{M}$ represents a (random) sample of $M$ independent values +drawn from $F$, i.e. $x_{1}, \dots, x_{M} \sim F$. Note that the expectation is taken over both +the outcome $Y$ and the sample values. + +Given a proper scoring rule $S$, a fair version of the score (denoted $S^{f}$) can be attained +by ensuring that $\mathbb{E}S(F_{M}, Y) = \mathbb{E}S(F, Y)$, for all $F \in \mathcal{F}$, +in which case the score assigned to $F_{M}$ represents an unbiased estimator of the score +assigned to the underlying distribution $F$. Some examples of fair versions of popular +scoring rules are provided below. + + +### Kernel scores + +A very general class of scoring rules is the class of kernel scores. +Recall that the *kernel score* corresponding to a positive definite kernel $k$ on $\Omega$ is defined as + $$ + S_{k}(F, y) = \frac{1}{2} k(y, y) + \frac{1}{2} \mathbb{E} k(X, X^{\prime}) - \mathbb{E} k(X, y), +$$ + +where $X, X^{\prime} \sim F$ are independent. + +Since kernel scores are defined in terms of expectations, they can be calculated easily +for ensemble forecasts by replacing the expectations with empirical means over the ensemble members. +That is, + +$$ + S_{k}(F_{M}, y) = \frac{1}{2} k(y, y) + \frac{1}{2 M^{2}} \sum_{i=1}^{M} \sum_{j=1}^{M} k(x_{i}, x_{j}) - \frac{1}{M} \sum_{i=1}^{M} k(x_{i}, y). +$$ + +Many kernel scores used in practice are constructed using a kernel of the form + +$$ k(x, x^{\prime}) = \rho(x, x_{0}) + \rho(x^{\prime}, x_{0}) - \rho(x, x^{\prime}) $$ + +where $x_{0} \in \Omega$, and $\rho : \Omega \to \Omega \to [0, \infty)$ is a symmetric, +conditionally negative definite function with $\rho(x, x) = 0$ for all $x \in \Omega$. +In this case, for any $x_{0} \in \Omega$, the kernel score simplifies to + +$$ + S_{k}(F, y) = \mathbb{E} \rho(X, y) - \frac{1}{2} \mathbb{E} \rho(X, X^{\prime}), +$$ + +where $X, X^{\prime} \sim F$ are independent. Note that this holds only up to integrability conditions +that are not relevant for the discussion here, see Sejdinovic et al. (2013). + +Kernel scores of this form include: + +- The CRPS: $\rho(x, x^{\prime}) = |x - x^{\prime}|$ +- The energy score: $\rho(x, x^{\prime}) = \|x - x^{\prime}\|$, where $\| \cdot \|$ is the Euclidean distance on $\mathbb{R}^{d}$ +- The variogram score: $\rho(x, x^{\prime}) = \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i, j} \left( |x_{i} - x_{j}|^{p} - |x_{i}^{\prime} - x_{j}^{\prime}|^{p} \right)^{2}$ for some $p > 0$, $h_{i, j} > 0$ +- The threshold-weighted CRPS: $\rho(x, x^{\prime}) = |v(x) - v(x^{\prime})|$ for some $v : \mathbb{R} \to \mathbb{R}$ +- The threshold-weighted energy score : $\rho(x, x^{\prime}) = \|v(x) - v(x^{\prime})\|$ for some $v : \mathbb{R}^{d} \to \mathbb{R}^{d}$ + +When the forecast is an ensemble forecast, these kernel scores simplify to + +$$ + S_{k}(F_{M}, y) = \frac{1}{M} \sum_{i=1}^{M} \rho(x_{i}, y) - \frac{1}{2 M^{2}} \sum_{i=1}^{M} \sum_{j=1}^{M} \rho(x_{i}, x_{j}). +$$ + +However, these expressions of kernel scores are generally not fair. +In particular, assuming $x_{1}, \dots, x_{M}$ are independent random draws from $F$, + +$$ \mathbb{E} S_{k}(F_{M}, y) = \mathbb{E} S_{k}(F, y) + \frac{1}{2M} \mathbb{E} \rho(x_{1}, x_{2}). $$ + +This was shown for the CRPS by Ferro et al. (2008), and extends trivially to any kernel score of this form. +However, fair versions of these kernel scores can be obtained by replacing the double expectation +term with an unbiased sample mean: + +$$ + S_{k}^{f}(F_{M}, y) = \frac{1}{M} \sum_{i=1}^{M} \rho(x_{i}, y) - \frac{1}{2 M(M - 1)} \sum_{i=1}^{M} \sum_{j=1}^{M} \rho(x_{i}, x_{j}). +$$ + +Some examples are given below. + + +#### CRPS + +Plugging $\rho(x, x^{\prime}) = |x - x^{\prime}|$ into the expressions above yields + +$$ +\mathrm{CRPS}(F_{M}, y) = \frac{1}{M} \sum_{i=1}^{M}|x_i - y| - \frac{1}{2 M^2}\sum_{i=1}^{M} \sum_{j=1}^{M}|x_i - x_j| +$$ + +(Grimit et al., 2006). + +If $x_{1}, \dots, x_{M}$ are instead to be interpreted as a sample from an underlying distribution, +then the forecast can be evaluated with the fair version of the CRPS, + +$$ +\mathrm{CRPS}^{f}(F_{M}, y) = \frac{1}{M} \sum_{i=1}^{M}|x_i - y| - \frac{1}{2 M (M - 1)}\sum_{i=1}^{M} \sum_{j=1}^{M}|x_i - x_j| +$$ + +(Ferro et al., 2008). While this expression corresponds to the energy form of the CRPS, +fair representations of the other forms of the CRPS are provided in the following sections. + + + +#### Energy score + +The CRPS is one example of a kernel score, and the energy score provides a generalisation of +this to the multivariate case. The energy score is defined as + +$$ +\mathrm{ES}(F, \boldsymbol{y}) = \mathbb{E} \| \boldsymbol{X} - \boldsymbol{y} \| - \frac{1}{2} \mathbb{E} \| \boldsymbol{X} - \boldsymbol{X}^{\prime} \|, +$$ + +where $\| \cdot \|$ is the Euclidean distance on $\mathbb{R}^{d}$, +$\boldsymbol{y} \in \Omega \subseteq \mathbb{R}^{d}$, and $\boldsymbol{X}, \boldsymbol{X}^{\prime} \sim F$ are independent, +with $F$ a multivariate predictive distribution on $\Omega$ (Gneiting and Raftery, 2007). + +Using the general representations for kernel scores given above, when the forecast is an ensemble +forecast with members $\boldsymbol{x}_{1}, \dots, \boldsymbol{x}_{M} \in \mathbb{R}^{d}$, the energy score becomes + +$$ +\mathrm{ES}(F_{M}, \boldsymbol{y}) = \frac{1}{M} \sum_{i=1}^{M} \| \boldsymbol{x}_{i} - \boldsymbol{y} \| - \frac{1}{2M^{2}} \sum_{i=1}^{M} \sum_{j=1}^{M} \| \boldsymbol{x}_{i} - \boldsymbol{x}_{j} \|. +$$ + +The fair version of the energy score is then + +$$ +\mathrm{ES}^{f}(F_{M}, \boldsymbol{y}) = \frac{1}{M} \sum_{i=1}^{M} \| \boldsymbol{x}_{i} - \boldsymbol{y} \| - \frac{1}{2M(M - 1)} \sum_{i=1}^{M} \sum_{j=1}^{M} \| \boldsymbol{x}_{i} - \boldsymbol{x}_{j} \|. +$$ + + + +#### Variogram score + +The Variogram score is an alternative multivariate scoring rule that quantifies the difference +between the variogram of the forecast and that of the observation. It is defined as + +$$ + \mathrm{VS}_{p}(F, \boldsymbol{y}) = \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left( \mathbb{E} | X_{i} - X_{j} |^{p} - | y_{i} - y_{j} |^{p} \right)^{2}, +$$ + +where $p > 0$, $\boldsymbol{y} = (y_{1}, \dots, y_{d}) \in \Omega \subseteq \mathbb{R}^{d}$, +$\boldsymbol{X} = (X_{1}, \dots, X_{d}) \sim F$, with $F$ a multivariate predictive distribution on $\Omega$, +and $h_{i,j} \ge 0$ are weights assigned to different pairs of dimensions (Scheuerer and Hamill, 2015). + +Allen et al. (2024) show that the Variogram score is also a kernel score. +In particular, the Variogram score can alternatively be expressed as + +$$ + \mathrm{VS}_{p}(F, \boldsymbol{y}) = \mathbb{E} \left[ \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left(| X_{i} - X_{j} |^{p} - | y_{i} - y_{j} |^{p} \right)^{2} \right] - \frac{1}{2} \mathbb{E} \left[ \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left(| X_{i} - X_{j} |^{p} - | X_{i}^{\prime} - X_{j}^{\prime} |^{p} \right)^{2} \right], +$$ + +where $\boldsymbol{X} = (X_{1}, \dots, X_{d}), \boldsymbol{X}^{\prime} = (X^{\prime}_{1}, \dots, X^{\prime}_{d}) \sim F$ are independent. + +Using, this, when $F$ is an ensemble forecast with members $\boldsymbol{x}_{1}, \dots, \boldsymbol{x}_{M} \in \mathbb{R}^{d}$, +the Variogram score becomes + +$$ +\begin{align} + \mathrm{VS}_{p}(F_{M}, \boldsymbol{y}) &= \frac{1}{M} \sum_{m=1}^{M} \left[ \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left(| x_{m,i} - x_{m,j} |^{p} - | y_{i} - y_{j} |^{p} \right)^{2} \right] - \frac{1}{2M^{2}} \sum_{m=1}^{M} \sum_{k=1}^{M} \left[ \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left(| x_{m,i} - x_{m,j} |^{p} - | x_{k,i} - x_{k,j} |^{p} \right)^{2} \right] \\ + &= \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left( \frac{1}{M}\sum_{m=1}^{M} | x_{m,i} - x_{m,j} |^{p} - | y_{i} - y_{j} |^{p} \right)^{2}, \\ +\end{align} +$$ + +which is given by Scheuerer and Hamill (2015), while its fair version is + +$$ +\begin{align} + \mathrm{VS}_{p}^{f}(F_{M}, \boldsymbol{y}) &= \frac{1}{M} \sum_{m=1}^{M} \left[ \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left(| x_{m,i} - x_{m,j} |^{p} - | y_{i} - y_{j} |^{p} \right)^{2} \right] - \frac{1}{2M(M - 1)} \sum_{m=1}^{M} \sum_{k=1}^{M} \left[ \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left(| x_{m,i} - x_{m,j} |^{p} - | x_{k,i} - x_{k,j} |^{p} \right)^{2} \right] \\ + &= \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left[|y_{i} - y_{j} |^{2p} + \frac{2}{M(M - 1)} \sum_{m=1}^{M - 1} \sum_{k=m+1}^{M} | x_{m,i} - x_{m,j} |^{p} | x_{k,i} - x_{k,j} |^{p} - \frac{2}{M} | y_{i} - y_{j} |^{p} \sum_{m=1}^{M} |x_{m, i} - x_{m, j} |^{p} \right]. +\end{align} +$$ + + +#### Threshold-weighted CRPS + +The threshold-weighted CRPS is a weighted scoring rule that allows more weight to be assigned +to particular outcomes (Matheson and Winkler, 1976; Gneiting and Ranjan, 2011). +The threshold-weighted CRPS can also be expressed as a kernel score of the above form. In particular, + +$$ +\begin{align} +\mathrm{twCRPS}(F, y; w) &= \int_{-\infty}^{\infty} (F(x) - 1\{y \le x\})^{2} w(x) dx \\ +&= \mathbb{E} | v(X) - v(y) | - \frac{1}{2} \mathbb{E} |v(X) - v(X^{\prime}) |, +\end{align} +$$ + +where $y \in \Omega \subseteq \mathbb{R}$, $X, X^{\prime} \sim F$ are independent, +$w : \mathbb{R} \to [0, \infty)$ is a non-negative weight function, and +$v : \mathbb{R} \to \mathbb{R}$ is an anti-derivative of the weight function $w$. +That is, $v(x) - v(x^{\prime}) = \int_{x^{\prime}}^{x} w(z) dz$ for all $x, x^{\prime} \in \mathbb{R}$. + +Using the second expression above, the threshold-weighted CRPS for an ensemble forecast +with members $x_{1}, \dots, x_{M} \in \mathbb{R}$ is + +$$ +\mathrm{twCRPS}(F_{M}, y; w) = \frac{1}{M} \sum_{i=1}^{M} |v(x_{i}) - v(y)| - \frac{1}{2M^{2}} \sum_{i=1}^{M} \sum_{j=1}^{M} |v(x_{i}) - v(x_{j})| +$$ + +(Allen et al., 2024), and the fair version of the threshold-weighted CRPS becomes + +$$ +\mathrm{twCRPS}^{f}(F_{M}, y; w) = \frac{1}{M} \sum_{i=1}^{M} |v(x_{i}) - v(y)| - \frac{1}{2M(M - 1)} \sum_{i=1}^{M} \sum_{j=1}^{M} |v(x_{i}) - v(x_{j})|. +$$ + + +#### Threshold-weighted energy score + +Similarly, the threshold-weighted energy score is a kernel score, defined as + +$$ +\mathrm{twES}(F, \boldsymbol{y}; v) = \mathbb{E} \| v(\boldsymbol{X}) - v(\boldsymbol{y}) \| - \frac{1}{2} \mathbb{E} \|v(\boldsymbol{X}) - v(\boldsymbol{X}^{\prime}) \|, +$$ +where $\boldsymbol{y} \in \Omega \subseteq \mathbb{R}^{d}$, $\boldsymbol{X}, \boldsymbol{X}^{\prime} \sim F$ are independent, and $v : \Omega \to \Omega$ (Allen et al., 2024). + +The threshold-weighted energy score for an ensemble forecast with members $\boldsymbol{x}_{1}, \dots, \boldsymbol{x}_{M} \in \mathbb{R}^{d}$ is then + +$$ +\mathrm{twES}(F_{M}, \boldsymbol{y}; v) = \frac{1}{M} \sum_{i=1}^{M} \|v(\boldsymbol{x}_{i}) - v(\boldsymbol{y})\| - \frac{1}{2M^{2}} \sum_{i=1}^{M} \sum_{j=1}^{M} \|v(\boldsymbol{x}_{i}) - v(\boldsymbol{x}_{j})\|. +$$ + +The fair version of the threshold-weighted energy score becomes + +$$ +\mathrm{twES}^{f}(F_{M}, \boldsymbol{y}; v) = \frac{1}{M} \sum_{i=1}^{M} \|v(\boldsymbol{x}_{i}) - v(\boldsymbol{y})\| - \frac{1}{2M(M - 1)} \sum_{i=1}^{M} \sum_{j=1}^{M} \|v(\boldsymbol{x}_{i}) - v(\boldsymbol{x}_{j})\|. +$$ + + +### Fair versions of CRPS forms + +The various representations of the CRPS also lead to multiple ways of calculating the fair CRPS. +These representations all yield the same score, but again differ in their computational complexity; +the fair forms all have the same complexity as the original forms given previously. + + +#### Energy form (NRG) + +The energy form of the fair CRPS, given already above, is + +$$ +\mathrm{CRPS}_{\text{NRG}}^{f}(F_{M}, y) = \frac{1}{M} \sum_{i=1}^{M}|x_i - y| - \frac{1}{2 M(M - 1)}\sum_{i=1}^{M} \sum_{j=1}^{M}|x_i - x_j|. +$$ + + +#### Quantile decomposition form (QD) + +A quantile decomposition form of the fair CRPS is + +$$ +\mathrm{CRPS}_{\mathrm{QD}}^{f}(F_{M}, y) = \frac{2}{M} \sum_{i=1}^{M}\left[\mathbb{1}\{y \le x_{(i)}\} - \frac{i - 1}{M - 1} \right] (x_{(i)} - y). +$$ + + +#### Probability weighted moment form (PWM) + +A probability weighted moment form of the fair CRPS is + +$$ +\mathrm{CRPS}_{\mathrm{PWM}}^{f}(F_{M}, y) = \frac{1}{M} \sum_{i=1}^{M}|x_{(i)} - y| + \hat{\beta_0} - 2\hat{\beta_1}, +$$ + +where $\hat{\beta_0} = \frac{1}{M} \sum_{i=1}^{M}x_{(i)}$ and $\hat{\beta_1} = \frac{1}{M(M-1)} \sum_{i=1}^{M}(i - 1)x_{(i)}$. +This was proposed by Taillardat et al. (2016). -Runs with $O(m\cdot\mathrm{log}m)$ complexity, including the sorting of the ensemble. -## Probability weighted moment form (PWM) +#### Integral form (INT) -Introduced by Taillardat et al. (2016): +The integral form of the CRPS can also be adapted to recover the fair version, $$ -\mathrm{CRPS}_{\mathrm{NRG}}(M, y) = \frac{1}{M} \sum_{i=1}^{M}|x_i - y| + \hat{\beta_0} - 2\hat{\beta_1}, +\mathrm{CRPS}_{\text{INT}}^{f}(F_{M}, y) = \int_{\mathbb{R}} \left[ \mathbb{1}\{y \le x\} + +\frac{2}{M(M - 1)} \sum_{i=1}^{M - 1} \sum_{j=i+1}^{M} \mathbb{1}\{x_i \le x \}\mathbb{1}\{x_j \le x \} - +\frac{2}{M} \mathbb{1}\{y \le x\} \sum_{i=1}^{M} \mathbb{1}\{x_i \le x \} \right] dx. $$ -where $\hat{\beta_0} = \frac{1}{M} \sum_{i=1}^{M}x_i$ and $\hat{\beta_1} = \frac{1}{M(M-1)} \sum_{i=1}^{M}(i - 1)x_i$. Runs with $O(m\cdot\mathrm{log}m)$ complexity, including the sorting of the ensemble. +This corresponds to the original CRPS with the integrand expanded out and with the components +in the double summation removed when $i = j$. This also requires numerical approximation of +the integral, so can again introduce small errors to the score. From 6f6a08ddaa42af070858ee1321167ab12796693a Mon Sep 17 00:00:00 2001 From: sallen12 Date: Fri, 4 Apr 2025 17:59:34 +0200 Subject: [PATCH 23/79] add documentation page for ensemble forecasts --- docs/forecast_dists.md | 872 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 872 insertions(+) create mode 100644 docs/forecast_dists.md diff --git a/docs/forecast_dists.md b/docs/forecast_dists.md new file mode 100644 index 0000000..a8e3d94 --- /dev/null +++ b/docs/forecast_dists.md @@ -0,0 +1,872 @@ +(forecast-dists)= +# Parametric forecast distributions + +Probabilistic forecasts take the form of probability distributions over the set of possible outcomes. +When the outcome is real-valued ($Y \in \mathbb{R}$), probabilistic forecasts often come in the +form of parametric distributions. The `scoringrules` package provides the functionality to calculate +the CRPS and Log score for a range of parametric distributions. In this document, we list the +distributions that are available, along with their probability density/mass function (pdf/pmf), +cumulative distribution function (cdf), and an analytical formula for their CRPS. The Log score +is simply the negative logarithm of the pdf/pmf, when this exists. The analytical +formulae for the CRPS are all taken from Appendix A of the scoringRules R package (Jordan et al., 2017). + + +## Discrete outcomes + +### Binomial distribution + +The *binomial distribution* has two parameters that describe the number of trials, $n = 0, 1, \dots, $ +and the probability of a success, $p \in [0, 1]$. Its pmf, cdf, and CRPS are + +$$ +f_{n, p}(x) = +\begin{cases} + {n \choose x} p^{x} (1 - p)^{n - x}, & x = 0, 1, \dots, n, \\ + 0, & \text{otherwise,} +\end{cases} +$$ + +$$ +F_{n, p}(x) = +\begin{cases} + I(n - \lfloor x \rfloor, \lfloor x \rfloor + 1, 1 - p), & x \ge 0, \\ + 0, & x < 0, +\end{cases} +$$ + +$$ +\mathrm{CRPS}(F_{n, p}, y) = 2 \sum_{x = 0}^{n} f_{n,p}(x) (1\{y < x\} + - F_{n,p}(x) + f_{n,p}(x)/2) (x - y), +$$ + +where $I(a, b, x)$ is the regularised incomplete beta function. + + +### Hypergeometric distribution + +The *hypergeometric distribution* has three parameters that describe the number of objects with +the relevant feature, $m = 0, 1, \dots, $ the number of objects without this feature, $n = 0, 1, \dots $, +and the size of the sample to be drawn without replacement from these objects, $k = 0, 1, \dots, m + n$. +Its pmf, cdf, and CRPS are + +$$ +f_{m, n, k}(x) = +\begin{cases} + \frac{{m \choose x} {n \choose k-x}}{m + n \choose k}, & x = \max{0, k - n}, \dots, \min{k, m}, \\ + 0, & \text{otherwise,} +\end{cases} +$$ + +$$ +F_{m, n, k}(x) = +\begin{cases} + \sum_{i = 0}^{\lfloor x \rfloor} f_{m, n, k}(i), & x \ge 0, \\ + 0, & x < 0, +\end{cases} +$$ + +$$ +\mathrm{CRPS}(F_{m, n, k}, y) = 2 \sum_{x = 0}^{n} f_{m,n,k}(x) (1\{y < x\} + - F_{m,n,k}(x) + f_{m,n,k}(x)/2) (x - y). +$$ + + +### Negative binomial distribution + +The *negative binomial distribution* has two parameters that describe the number of successes, +$n > 0$, and the probability of a success, $p \in [0, 1]$. Its pmf, cdf, and CRPS are + +$$ +f_{n, p}(x) = +\begin{cases} + \frac{\Gamma(x + n)}{\Gamma(n) x!} p^{n} (1 - p)^{x} , & x = 0, 1, 2, \dots, \\ + 0, & \text{otherwise,} +\end{cases} +$$ + +$$ +F_{n, p}(x) = +\begin{cases} + I(n, \lfloor x + 1 \rfloor, p), & x \ge 0, \\ + 0, & x < 0, +\end{cases} +$$ + +$$ +\mathrm{CRPS}(F_{n, p}, y) = y (2 F_{n, p}(y) - 1) - \frac{n(1 - p)}{p^{2}} \left( p (2 F_{n+1, p}(y - 1) - 1) + + _{2} F_{1} \left( n + 1, \frac{1}{2}; 2; -\frac{4(1 - p)}{p^{2}} \right) \right), +$$ + +where $I(a, b, x)$ is the regularised incomplete beta function, and $_2 F_{1}(a, b; c; x)$ is the hypergeometric function. + + +### Poisson distribution + +The *Poisson distribution* has one mean parameter, $\lambda > 0$. Its pmf, cdf, and CRPS are + +$$ +f_{\lambda}(x) = +\begin{cases} + \frac{\lambda^{x}}{x!}e^{-\lambda}, & x = 0, 1, 2, \dots, \\ + 0, & \text{otherwise,} +\end{cases} +$$ + +$$ +F_{\lambda}(x) = +\begin{cases} + \frac{\Gamma_{u}(\lfloor x + 1 \rfloor, \lambda)}{\Gamma(\lfloor x + 1 \rfloor)}, & x \ge 0, \\ + 0, & x < 0, +\end{cases} +$$ + +$$ +\mathrm{CRPS}(F_{\lambda}, y) = (y - \lambda) (2F_{\lambda}(y) - 1) + + 2 \lambda f_{\lambda}(\lfloor y \rfloor ) + - \lambda e^{-2 \lambda} (I_{0} (2 \lambda) + I_{1} (2 \lambda)), +$$ + +where $\Gamma_{u}(a, x)$ is the upper incomplete gamma function, and $I_{m}(x)$ is the +modified Bessel function of the first kind. + + + +## Continuous real-valued outcomes + +### Laplace distribution + +The *Laplace distribution* has one location parameter, $\mu \in \mathbb{R}$, and one scale parameter $\sigma > 0$. +Its pdf, cdf, and CRPS are + +$$ +f_{\mu, \sigma}(x) = \frac{1}{2\sigma} \exp \left( - \frac{|x - \mu|}{\sigma} \right) +$$ + +$$ +F_{\mu, \sigma}(x) = +\begin{cases} + \frac{1}{2} \exp \left( \frac{x - \mu}{\sigma} \right), & x \le \mu, \\ + 1 - \frac{1}{2} \exp \left( - \frac{x - \mu}{\sigma} \right), & x \ge \mu, \\ +\end{cases} +$$ + +$$ +\begin{align} + \mathrm{CRPS}(F_{0, 1}, y) &= |y| + \exp(-|y|) - \frac{3}{4}, \\ + \mathrm{CRPS}(F_{\mu, \sigma}, y) &= \sigma \mathrm{CRPS} \left( F_{0, 1}, \frac{y - \mu}{\sigma} \right). +\end{align} +$$ + + +### Logistic distribution + +The *Logistic distribution* has one location parameter, $\mu \in \mathbb{R}$, and one scale parameter, $\sigma > 0$. +Its pdf, cdf, and CRPS are + +$$ +f_{\mu, \sigma}(x) = \frac{\exp \left( - \frac{x - \mu}{\sigma} \right)}{\sigma \left( 1 + \exp \left( - \frac{x - \mu}{\sigma} \right) \right)^{2}}, +$$ + +$$ +F_{\mu, \sigma}(x) = \frac{1}{1 + \exp \left( - \frac{x - \mu}{\sigma} \right)}, +$$ + +$$ +\begin{align} + \mathrm{CRPS}(F_{0, 1}, y) &= y - 2 \log \left( F_{0, 1}(y) \right) - 1, \\ + \mathrm{CRPS}(F_{\mu, \sigma}, y) &= \sigma \mathrm{CRPS} \left( F_{0, 1}, \frac{y - \mu}{\sigma} \right). +\end{align} +$$ + + +### Normal distribution + +The *normal distribution* has one location parameter, $\mu \in \mathbb{R}$, and one scale parameter, $\sigma > 0$. +Its pdf, cdf, and CRPS are + +$$ +f_{\mu, \sigma}(x) = \frac{1}{\sigma} \phi \left( \frac{x - \mu}{\sigma} \right) = \frac{1}{\sqrt{2 \pi} \sigma} \exp \left( - \frac{(x - \mu)^{2}}{2\sigma^{2}} \right), +$$ + +$$ +F_{\mu, \sigma}(x) = \Phi \left( \frac{x - \mu}{\sigma} \right) = \frac{1}{2} \left[ 1 + \text{erf} \left( \frac{x - \mu}{\sigma \sqrt{2}} \right) \right], +$$ + +$$ +\begin{align} + \mathrm{CRPS}(F_{0, 1}, y) &= y (2\Phi(y) - 1) + 2 \phi(y) - \frac{1}{\sqrt{\pi}}, \\ + \mathrm{CRPS}(F_{\mu, \sigma}, y) &= \sigma \mathrm{CRPS} \left( F_{0, 1}, \frac{y - \mu}{\sigma} \right), +\end{align} +$$ + +where $\phi$ and $\Phi$ represent the standard normal pdf and cdf, respectively, and +$\text{erf}$ is the error function. + + +### Mixture of normal distributions + +A *mixture of normal distributions* has $M$ location parameters, $\mu_{1}, \dots, \mu_{M} \in \mathbb{R}$, +$M$ scale parameters, $\sigma_{1}, \dots, \sigma_{M} > 0$, and $M$ weight parameters, $w_{1}, \dots, w_{M} \ge 0$ +that sum to one. Its pdf, cdf, and CRPS are + +$$ +f(x) = \sum_{m=1}^{M} \frac{w_{m}}{\sigma_{m}} \phi \left( \frac{x - \mu_{m}}{\sigma_{m}} \right), +$$ + +$$ +F(x) = \sum_{m=1}^{M} w_{m} \Phi \left( \frac{x - \mu_{m}}{\sigma_{m}} \right), +$$ + +$$ +\mathrm{CRPS}(F, y) = \sum_{m=1}^{M} w_{m} A(y - \mu_{m}, \sigma_{m}^{2}) - \frac{1}{2} \sum_{m=1}^{M} \sum_{k=1}^{M} w_{m} w_{k} A(\mu_{m} - \mu_{k}, \sigma_{m}^{2} + \sigma_{k}^{2}), +$$ + +where $A(\mu, \sigma^{2}) = \mu (2 \Phi(\frac{\mu}{\sigma}) - 1) + 2\sigma \phi(\frac{\mu}{\sigma})$, +with $\phi$ and $\Phi$ the standard normal pdf and cdf, respectively. + + +### Student's t distribution + +The (generalised) *Student's t distribution* has one degrees of freedom parameter, $\nu > 0$, +one location parameter, $\mu \in \mathbb{R}$, and one scale parameter, $\sigma > 0$. +Its pdf, cdf, and CRPS are + +$$ +f_{\nu, \mu, \sigma}(x) = \frac{1}{\sigma \sqrt{\nu} B(\frac{1}{2}, \frac{\nu}{2})} \left( 1 + \frac{(x - \mu)^{2}}{\sigma^{2} \nu} \right)^{- \frac{\nu + 1}{2}}, +$$ + +$$ +F_{\nu, \mu, \sigma}(x) = \frac{1}{2} + \left( \frac{x - \mu}{\sigma} \right) \frac{_{2} F_{1} (\frac{1}{2}, \frac{\nu + 1}{2} ; \frac{3}{2} ; - \frac{(x - \mu)^{2}}{\sigma^{2} \nu})}{ \sqrt{\nu} B(\frac{1}{2}, \frac{\nu}{2}) }, +$$ + +$$ +\begin{align} + \mathrm{CRPS}(F_{\nu, 0, 1}, y) &= y (2F_{\nu, 0, 1}(y) - 1) + 2 f_{\nu, 0, 1}(y) \left( \frac{\nu + y^{2}}{\nu - 1} \right) - \frac{2 \sqrt{\nu}}{\nu - 1} \frac{B(\frac{1}{2}, \nu - \frac{1}{2})}{B(\frac{1}{2}, \frac{\nu}{2})^{2}}, \\ + \mathrm{CRPS}(F_{\nu, \mu, \sigma}, y) &= \sigma \mathrm{CRPS} \left( F_{\nu, 0, 1}, \frac{y - \mu}{\sigma} \right). +\end{align} +$$ + + +### Two-piece exponential distribution + +The *two-piece exponential distribution* has one location parameter, $\mu \in \mathbb{R}$, and +two scale parameters, $\sigma_{1}, \sigma_{2} > 0$. Its pdf, cdf, and CRPS are + +$$ +f_{\mu, \sigma_{1}, \sigma_{2}}(x) = +\begin{cases} +\frac{1}{\sigma_{1} + \sigma_{2}} \exp \left( - \frac{\mu - x}{\sigma_{1}} \right), & x \le \mu, \\ +\frac{1}{\sigma_{1} + \sigma_{2}} \exp \left( - \frac{x - \mu}{\sigma_{2}} \right), & x \ge \mu, \\ +\end{cases} +$$ + +$$ +F_{\mu, \sigma_{1}, \sigma_{2}}(x) = +\begin{cases} + \frac{\sigma_{1}}{\sigma_{1} + \sigma_{2}} \exp \left( - \frac{\mu - x}{\sigma_{1}} \right), & x \le \mu, \\ + 1 - \frac{\sigma_{2}}{\sigma_{1} + \sigma_{2}} \exp \left( - \frac{x - \mu}{\sigma_{2}} \right), & x \ge \mu, \\ +\end{cases} +$$ + +$$ +\mathrm{CRPS}(F_{\mu, \sigma_{1}, \sigma_{2}}, y) = +\begin{cases} + |y - \mu| + \frac{2 \sigma_{1}^{2}}{\sigma_{1} + \sigma_{2}} \left[ \exp \left( - \frac{|y - \mu|}{\sigma_{1}} \right) - 1 \right] + \frac{\sigma_{1}^{3} + \sigma_{2}^{3}}{2(\sigma_{1} + \sigma_{2})^{2}}, & y \le \mu, \\ + |y - \mu| + \frac{2 \sigma_{2}^{2}}{\sigma_{1} + \sigma_{2}} \left[ \exp \left( - \frac{|y - \mu|}{\sigma_{2}} \right) - 1 \right] + \frac{\sigma_{1}^{3} + \sigma_{2}^{3}}{2(\sigma_{1} + \sigma_{2})^{2}}, & y \ge \mu. +\end{cases} +$$ + + +### Two-piece normal distribution + +The *two-piece normal distribution* has one location parameter, $\mu \in \mathbb{R}$, and +two scale parameters, $\sigma_{1}, \sigma_{2} > 0$. Its pdf, cdf, and CRPS are + +$$ +f_{\mu, \sigma_{1}, \sigma_{2}}(x) = +\begin{cases} +\frac{2}{\sigma_{1} + \sigma_{2}} \phi \left( \frac{x - \mu}{\sigma_{1}} \right), & x \le \mu, \\ +\frac{2}{\sigma_{1} + \sigma_{2}} \phi \left( \frac{x - \mu}{\sigma_{2}} \right), & x \ge \mu, \\ +\end{cases} +$$ + +$$ +F_{\mu, \sigma_{1}, \sigma_{2}}(x) = +\begin{cases} + \frac{2\sigma_{1}}{\sigma_{1} + \sigma_{2}} \Phi \left( \frac{x - \mu}{\sigma_{1}} \right), & x \le \mu, \\ + \frac{\sigma_{1} - \sigma_{2}}{\sigma_{1} + \sigma_{2}} + \frac{2\sigma_{2}}{\sigma_{1} + \sigma_{2}} \Phi \left( \frac{x - \mu}{\sigma_{2}} \right), & x \ge \mu, \\ +\end{cases} +$$ + +$$ +\begin{align} +\mathrm{CRPS}(F_{\mu, \sigma_{1}, \sigma_{2}}, y) = & \sigma_{1} \mathrm{CRPS} \left( F_{-\infty, 0}^{0, \sigma_{2}/(\sigma_{1} + \sigma_{2})}, \min \left(0, \frac{y - \mu}{\sigma_{1}} \right) \right) \\ + & + \sigma_{2} \mathrm{CRPS} \left( F^{\infty, 0}_{0, \sigma_{1}/(\sigma_{1} + \sigma_{2})}, \max \left(0, \frac{y - \mu}{\sigma_{2}} \right) \right), +\end{align} +$$ + +where $\phi$ and $\Phi$ the standard normal pdf and cdf, respectively, and $F_{l, L}^{u, U}$ +is the cdf of the truncated and censored normal distribution. + + +## Continuous positive outcomes + +### Exponential distribution + +The *exponential distribution* has one rate parameter, $\lambda > 0$. Its pdf, cdf, and CRPS are + +$$ +f_{\lambda}(x) = +\begin{cases} + \lambda e^{-\lambda x}, & x \ge 0, \\ + 0, & x < 0, \\ +\end{cases} +$$ + +$$ +F_{\lambda}(x) = +\begin{cases} + 1 - e^{-\lambda x}, & x \ge 0, \\ + 0, & x < 0, \\ +\end{cases} +$$ + +$$ +\mathrm{CRPS}(F_{\lambda}, y) = |y| - \frac{2 F_{\lambda}(y)}{\lambda} + \frac{1}{2\lambda}. +$$ + + +### Gamma distribution + +The *gamma distribution* has one shape parameter, $\alpha > 0$, and one rate parameter, $\beta > 0$. Its pdf, cdf, and CRPS are + +$$ +f_{\alpha, \beta}(x) = +\begin{cases} + \frac{\beta^{\alpha}}{\Gamma(\alpha)} x^{\alpha-1} e^{-\beta x}, & x \ge 0, \\ + 0, & x < 0, \\ +\end{cases} +$$ + +$$ +F_{\alpha, \beta}(x) = +\begin{cases} + \frac{\Gamma_{l}(\alpha, \beta x)}{\Gamma(\alpha)} , & x \ge 0, \\ + 0, & x < 0, \\ +\end{cases} +$$ + +$$ +\mathrm{CRPS}(F_{\alpha, \beta}, y) = y (2 F_{\alpha, \beta}(y) - 1) - \frac{\alpha}{\beta} (2 F_{\alpha + 1, \beta}(y) - 1) - \frac{1}{\beta B(\frac{1}{2}, \alpha)}, +$$ + +where $\Gamma_{l}$ is the lower incomplete gamma function, and $B$ is the beta function. + + +### Log-Laplace distribution + +The *log-Laplace distribution* has one location parameter, $\mu \in \mathbb{R}$, and one scale parameter, $\sigma > 0$. +Its pdf, cdf, and CRPS are + +$$ +f_{\mu, \sigma}(x) = +\begin{cases} + \frac{1}{2 \sigma x} \exp \left( - \frac{| \log x - \mu |}{\sigma} \right), & x > 0, \\ + 0, & x \le 0, \\ +\end{cases} +$$ + +$$ +F_{\mu, \sigma}(x) = +\begin{cases} + 0, & x \le 0, \\ + \frac{1}{2} \exp \left( \frac{\log x - \mu}{\sigma} \right) , & 0 < \log x < \mu, \\ + 1 - \frac{1}{2} \exp \left( - \frac{\log x - \mu}{\sigma} \right), & \log x \ge \mu, \\ +\end{cases} +$$ + +$$ +\mathrm{CRPS}(F_{\mu, \sigma}, y) = y (2 F_{\mu, \sigma}(y) - 1) + e^{\mu} \left( \frac{\sigma}{4 - \sigma^{2}} + A(y) \right), +$$ + +where + +$$ +A(x) = +\begin{cases} + \frac{1}{1 + \sigma} \left( 1 - (2 F_{\mu, \sigma}(x))^{1 + \sigma} \right), & \log x < \mu, \\ + -\frac{1}{1 - \sigma} \left( 1 - (2 (1 - F_{\mu, \sigma}(x)))^{1 - \sigma} \right), & \log x \ge \mu. \\ +\end{cases} +$$ + + +### Log-logistic distribution + +The *log-logistic distribution* has one location parameter, $\mu \in \mathbb{R}$, and one scale parameter, $\sigma > 0$. +Its pdf, cdf, and CRPS are + +$$ +f_{\mu, \sigma}(x) = +\begin{cases} + \frac{\exp \left(\frac{\log x - \mu}{\sigma} \right)}{\sigma x \left(1 + \exp \left(\frac{\log x - \mu}{\sigma} \right) \right)^{2}}, & x > 0, \\ + 0, & x \le 0, \\ +\end{cases} +$$ + +$$ +F_{\mu, \sigma}(x) = +\begin{cases} + \left(1 + \exp \left( - \frac{\log x - \mu}{\sigma} \right) \right)^{-1}, & x > 0, \\ + 0, & x \le 0, \\ +\end{cases} +$$ + +$$ +\mathrm{CRPS}(F_{\mu, \sigma}, y) = y (2 F_{\mu, \sigma}(y) - 1) - e^{\mu} B(1 + \sigma, 1 - \sigma) ( 2I(1 + \sigma, 1 - \sigma, F_{\mu, \sigma}(y)) + \sigma - 1 ), +$$ + +where $B$ is the beta function, and $I$ is the regularised incomplete beta function. + + +### Log-normal distribution + +The *log-normal distribution* has one location parameter, $\mu \in \mathbb{R}$, and one scale parameter, $\sigma > 0$. +Its pdf, cdf, and CRPS are + +$$ +f_{\mu, \sigma}(x) = +\begin{cases} + \frac{1}{\sigma x} \phi \left( \frac{\log x - \mu}{\sigma} \right) = \frac{1}{\sigma x \sqrt{2 \pi}} \exp \left( - \frac{(\log x - \mu)^{2}}{2\sigma^{2}} \right), & x > 0, \\ + 0, & x \le 0, \\ +\end{cases} +$$ + +$$ +F_{\mu, \sigma}(x) = +\begin{cases} + \Phi \left( \frac{\log x - \mu}{\sigma} \right) = \frac{1}{2} \left[ 1 + \text{erf} \left( \frac{\log x - \mu}{\sigma \sqrt{2}} \right) \right], & x > 0, \\ + 0, & x \le 0, \\ +\end{cases} +$$ + +$$ +\mathrm{CRPS}(F_{\mu, \sigma}, y) = y (2 F_{\mu, \sigma}(y) - 1) - 2 \exp \left( \mu + \frac{\sigma^{2}}{2} \right) \left( \Phi \left( \frac{\log y - \mu - \sigma^{2}}{\sigma} \right) + \Phi \left( \frac{\sigma}{\sqrt{2}} \right) - 1 \right), +$$ + +where $\phi$ and $\Phi$ represent the standard normal pdf and cdf, respectively, and $\text{erf}$ is the error function. + + +## Bounded or censored outcomes + +### Beta distribution + +The (generalised) *beta distribution* has two shape parameters, $\alpha, \beta > 0$, and +lower and upper bound parameters $l,u \in \mathbb{R}$, such that $l < u$. Its pdf, cdf, and CRPS are + +$$ +f_{\alpha, \beta}^{l, u}(x) = +\begin{cases} + \frac{1}{B(\alpha, \beta)} \left( \frac{x - l}{u - l} \right)^{\alpha - 1} \left(\frac{u - x}{u - l} \right)^{\beta - 1} , & l \le x \le u, \\ + 0, & x < l \quad \text{or} \quad x > u, \\ +\end{cases} +$$ + +$$ +F_{\alpha, \beta}^{l, u}(x) = +\begin{cases} + 0, & x < l, \\ + I \left( \alpha, \beta, \frac{x - l}{u - l} \right), & l \le x \le u, \\ + 1, & x > u, \\ +\end{cases} +$$ + +$$ +\begin{align} + \mathrm{CRPS}(F_{\alpha, \beta}^{0, 1}, y) &= y (2F_{\alpha, \beta}^{0, 1}(y) - 1) + \frac{\alpha}{\alpha + \beta} \left( 1 - 2 F_{\alpha + 1, \beta}^{0, 1}(y) - \frac{2B(2 \alpha, 2 \beta)}{\alpha B(\alpha, \beta)^{2}} \right), \\ + \mathrm{CRPS}(F_{\alpha, \beta}^{l, u}, y) &= (u - l) \mathrm{CRPS} \left( F_{\alpha, \beta}^{0, 1}, \frac{y - l}{u - l} \right), +\end{align} +$$ + +where $B$ is the beta function, and $I$ is the regularised incomplete beta function. + + +### Continuous uniform distribution + +The continuous *uniform distribution* has lower and upper bound parameters $l,u \in \mathbb{R}$, $l < u$, +and point mass parameters $L, U \ge 0$, $L + U < 1$ that assign mass to the boundary points. +Its pdf+pmf, cdf, and CRPS are + +$$ +f_{l, u}^{L, U}(x) = +\begin{cases} + 0, & x < l \quad \text{or} \quad x > u, \\ + L, & x = l, \\ + U, & x = u, \\ + \frac{1 - L - U}{u - l}, & l < x < u, \\ +\end{cases} +$$ + +$$ +F_{l, u}^{L, U}(x) = +\begin{cases} + 0, & x < l, \\ + L + (1 - L - U) \frac{x - l}{u - l}, & l \le x \le u, \\ + 1, & x > u, \\ +\end{cases} +$$ + +$$ +\begin{align} + \mathrm{CRPS}(F_{0, 1}^{L, U}, y) &= | y - F(y) | + F(y)^{2}(1 - L - U) - F(y)(1 - 2L) + \frac{(1 - L - U)^{2}}{3} + (1 - L)U, \\ + \mathrm{CRPS}(F_{l, u}^{L, U}, y) &= (u - l) \mathrm{CRPS} \left( F_{0, 1}^{L, U}, \frac{y - l}{u - l} \right), +\end{align} +$$ + +where $F = F_{0, 1}^{0, 0}$ is the standard uniform distribution function. + + +### Exponential distribution with point mass + +The exponential distribution can be generalised to different supports, and to include a +point mass at the boundary. This *generalised exponential distribution* has one location parameter, $\mu \in \mathbb{R}$, +one scale parameter $\sigma > 0$, and one point mass parameter $M \in [0, 1]$. Its pdf+pmf, cdf, and CRPS are + +$$ +f_{\mu, \sigma, M}(x) = +\begin{cases} + 0, & x < \mu, \\ + M, & x = \mu, \\ + \frac{1 - M}{\sigma} \exp \left( - \frac{x - \mu}{\sigma} \right), & x > \mu, \\ +\end{cases} +$$ + +$$ +F_{\mu, \sigma, M}(x) = +\begin{cases} + 0, & x < \mu, \\ + M + (1 - M) \left(1 - \exp \left( - \frac{x - \mu}{\sigma} \right) \right), & x \ge \mu, \\ +\end{cases} +$$ + +$$ +\begin{align} + \mathrm{CRPS}(F_{0, 1, M}, y) &= |y| - 2(1 - M)F(y) + \frac{(1 - M)^{2}}{2}, \\ + \mathrm{CRPS}(F_{\mu, \sigma, M}, y) &= \sigma \mathrm{CRPS} \left( F_{0, 1, M}, \frac{y - \mu}{\sigma} \right), +\end{align} +$$ + +where $F = F_{0, 1, 0}$ is the standard exponential distribution function. + + +### Generalised extreme value (GEV) distribution + +The *generalised extreme value (GEV) distribution* has one location parameter, $\mu \in \mathbb{R}$, +one scale parameter, $\sigma > 0$, and one shape parameter, $\xi \in \mathbb{R}$. The GEV distribution can be divided +into three types, depending on the value of $\xi$. The pdf, cdf, and CRPS for each type are: + +If $\xi = 0$, + +$$ +f_{\mu, \sigma, \xi}(x) = \frac{1}{\sigma} \exp \left( - \frac{x - \mu}{\sigma} \right) \exp \left( - \exp \left( - \frac{x - \mu}{\sigma} \right) \right), +$$ + +$$ +F_{\mu, \sigma, \xi}(x) = \exp \left( - \exp \left( - \frac{x - \mu}{\sigma} \right) \right), +$$ + +$$ +\mathrm{CRPS}(F_{0, 1, \xi}, y) = - y - 2 \text{Ei}(\log F_{0, 1, \xi}(y)) + \gamma - \log 2, +$$ + +where $\text{Ei}$ is the exponential integral, and $\gamma$ is the Euler-Mascheroni constant. + +If $\xi > 0$, + +$$ +f_{\mu, \sigma, \xi}(x) = +\begin{cases} + 0, & x < \mu - \frac{\sigma}{\xi}, \\ + \frac{1}{\sigma} \left[ 1 + \xi \left( \frac{x - \mu}{\sigma} \right) \right]^{-(\xi + 1)/\xi} \exp \left( - \left[ 1 + \xi \left( \frac{x - \mu}{\sigma} \right) \right]^{-1/\xi} \right), & x \ge \mu - \frac{\sigma}{\xi}, +\end{cases} +$$ + +$$ +F_{\mu, \sigma, \xi}(x) = +\begin{cases} + 0, & x < \mu - \frac{\sigma}{\xi}, \\ + \exp \left( - \left[ 1 + \xi \left( \frac{x - \mu}{\sigma} \right) \right]^{-1/\xi} \right), & x \ge \mu - \frac{\sigma}{\xi}, +\end{cases} +$$ + +$$ +\mathrm{CRPS}(F_{0, 1, \xi}, y) = y (2 F_{0, 1, \xi}(y) - 1) - 2 G_{\xi} (y) - \frac{1 - (2 - 2^{\xi}) \Gamma(1 - \xi)}{\xi}, +$$ + +where + +$$ +G_{\xi}(x) = +\begin{cases} + 0, & x \le -\frac{1}{\xi}, \\ + -\frac{F_{0, 1, \xi}(x)}{\xi} + \frac{\Gamma_{u}(1 - \xi, - \log F_{0, 1, \xi}(x))}{\xi}, & x > -\frac{1}{\xi}, +\end{cases} +$$ + +with $\Gamma$ the gamma function, and $\Gamma_{u}$ the upper incomplete gamma function. + + +If $\xi < 0$, + +$$ +f_{\mu, \sigma, \xi}(x) = +\begin{cases} + \frac{1}{\sigma} \left[ 1 + \xi \left( \frac{x - \mu}{\sigma} \right) \right]^{-(\xi + 1)/\xi} \exp \left( - \left[ 1 + \xi \left( \frac{x - \mu}{\sigma} \right) \right]^{-1/\xi} \right), & x \le \mu - \frac{\sigma}{\xi}, \\ + 0, & x > \mu - \frac{\sigma}{\xi}, +\end{cases} +$$ + +$$ +F_{\mu, \sigma, \xi}(x) = +\begin{cases} + \exp \left( - \left[ 1 + \xi \left( \frac{x - \mu}{\sigma} \right) \right]^{-1/\xi} \right), & x \le \mu - \frac{\sigma}{\xi}, \\ + 1, & x > \mu - \frac{\sigma}{\xi}, +\end{cases} +$$ + +$$ +\mathrm{CRPS}(F_{0, 1, \xi}, y) = y (2 F_{0, 1, \xi}(y) - 1) - 2 G_{\xi} (y) - \frac{1 - (2 - 2^{\xi}) \Gamma(1 - \xi)}{\xi}, +$$ + +where + +$$ +G_{\xi}(x) = +\begin{cases} + -\frac{F_{0, 1, \xi}(x)}{\xi} + \frac{\Gamma_{u}(1 - \xi, - \log F_{0, 1, \xi}(x))}{\xi}, & x < -\frac{1}{\xi}, \\ + -\frac{1}{\xi} + \frac{\Gamma(1 - \xi)}{\xi}, & x \ge -\frac{1}{\xi}, +\end{cases} +$$ + +with $\Gamma$ the gamma function, and $\Gamma_{u}$ the upper incomplete gamma function. + +For all $\xi$, we have that + +$$ +\mathrm{CRPS}(F_{\mu, \sigma, \xi}, y) = \sigma \mathrm{CRPS} \left( F_{0, 1, \xi}, \frac{y - \mu}{\sigma} \right). +$$ + + +### Generalised Pareto distribution (GPD) with point mass + +The *generalised Pareto distribution (GPD)* has one location parameter, $\mu \in \mathbb{R}$, +one scale parameter, $\sigma > 0$, one shape parameter, $\xi \in \mathbb{R}$, and one point mass +parameter at the lower boundary, $M \in [0, 1]$. Its pdf+pmf, cdf, and CRPS are: + +If $\xi = 0$, + +$$ +f_{\mu, \sigma, \xi, M}(x) = +\begin{cases} + 0, & x < \mu, \\ + M, & x = \mu, \\ + \frac{1 - M}{\sigma} \exp \left( - \frac{x - \mu}{\sigma} \right), & x > \mu, +\end{cases} +$$ + +$$ +F_{\mu, \sigma, \xi, M}(x) = +\begin{cases} + 0, & x < \mu, \\ + M + (1 - M) \left( 1 - \exp \left( - \frac{x - \mu}{\sigma} \right) \right), & x \ge \mu. +\end{cases} +$$ + +If $\xi > 0$, + +$$ +f_{\mu, \sigma, \xi, M}(x) = +\begin{cases} + 0, & x < \mu, \\ + M, & x = \mu, \\ + \frac{1 - M}{\sigma} \left[ 1 + \xi \left( \frac{x - \mu}{\sigma} \right) \right]^{- (\xi + 1) / \xi}, & x > \mu, +\end{cases} +$$ + +$$ +F_{\mu, \sigma, \xi, M}(x) = +\begin{cases} + 0, & x < \mu, \\ + M + (1 - M) \left( 1 - \left[ 1 + \xi \left( \frac{x - \mu}{\sigma} \right) \right]^{- 1 / \xi} \right), & x \ge \mu. +\end{cases} +$$ + +If $\xi < 0$, + +$$ +f_{\mu, \sigma, \xi, M}(x) = +\begin{cases} + M, & x = \mu, \\ + \frac{1 - M}{\sigma} \left[ 1 + \xi \left( \frac{x - \mu}{\sigma} \right) \right]^{- (\xi + 1) / \xi}, & \mu < x \le \mu - \frac{\sigma}{\xi}, \\ + 0, & \text{otherwise}, +\end{cases} +$$ + +$$ +F_{\mu, \sigma, \xi, M}(x) = +\begin{cases} + 0, & x < \mu, \\ + M + (1 - M) \left( 1 - \left[ 1 + \xi \left( \frac{x - \mu}{\sigma} \right) \right]^{- 1 / \xi} \right), & \mu < x \le \mu - \frac{\sigma}{\xi}, \\ + 1, & x > \mu - \frac{\sigma}{\xi}. +\end{cases} +$$ + +For all $\xi$, + +$$ +\begin{align} + \mathrm{CRPS}(F_{0, 1, \xi, M}, y) &= |y| - \frac{2(1 - M)}{1 - \xi} \left( 1 - (1 - F_{0, 1, \xi, 0}(y))^{1 - \xi} \right) + \frac{(1 - M)^{2}}{2 - \xi}, \\ + \mathrm{CRPS}(F_{\mu, \sigma, \xi, M}, y) &= \sigma \mathrm{CRPS} \left( F_{0, 1, \xi, M}, \frac{y - \mu}{\sigma} \right). +\end{align} +$$ + + +### Generalised truncated or censored logistic distribution + +The logistic distribution can be generalised to account for truncation, censoring, and to +allow point masses at the boundary points of its support. This *generalised logistic distribution* +has one location parameter, $\mu \in \mathbb{R}$, one scale parameter $\sigma > 0$, +lower and upper bound parameters $l,u \in \mathbb{R}$, $l < u$, and point mass parameters +$L, U \ge 0$, $L + U < 1$, that assign mass to the boundary points $l$ and $u$ respectively. +Its pdf+pmf, cdf, and CRPS are + +$$ +f_{\mu, \sigma, l, L}^{u, U}(x) = +\begin{cases} + 0, & x < l, \\ + L, & x = l, \\ + (1 - L - U) f_{\mu, \sigma}(x), & l < x < u, \\ + U, & x = u, \\ + 0, & x > u, \\ +\end{cases} +$$ + +$$ +F_{\mu, \sigma, l, L}^{u, U}(x) = +\begin{cases} + 0, & x < l, \\ + L + (1 - L - U) \frac{F_{\mu, \sigma}(x) - F_{\mu, \sigma}(l)}{F_{\mu, \sigma}(u) - F_{\mu, \sigma}(l)}, & l \le x < u, \\ + 1, & x \ge u, \\ +\end{cases} +$$ + +$$ +\begin{align} + \mathrm{CRPS}(F_{0, 1, l, L}^{u, U}, y) &= |y - z| + uU^{2} - lL^{2} - \left( \frac{1 - L - U}{F(u) - F(l)} \right) z \left( \frac{(1 - 2L)F(u) + (1 - 2U)F(l)}{1 - L - U} \right) - \left( \frac{1 - L - U}{F(u) - F(l)} \right) (2 \log F(-z) - 2G(u)U - 2G(l)L) - \left( \frac{1 - L - U}{F(u) - F(l)} \right)^{2} (H(u) - H(l)) , \\ + \mathrm{CRPS}(F_{\mu, \sigma, l, L}^{u, U}, y) &= \sigma \mathrm{CRPS} \left( F_{0, 1, (l - \mu)/\sigma, L}^{(u - \mu)/\sigma, U}, \frac{y - \mu}{\sigma} \right), +\end{align} +$$ + +where $f_{\mu, \sigma}$ and $F_{\mu, \sigma}$ denote the pdf and cdf of the logistic distribution +with location parameter $\mu$ and scale parameter $\sigma$, $F = F_{0, 1}$ is the cdf of the +standard logistic distribution, and with + +$$ +\begin{align} + G(x) &= xF(x) + \log F(-x), \\ + H(x) &= F(x) - xF(x)^{2} + (1 - 2F(x)) \log F(-x), \\ + z &= \max \{l, \min \{y, u\}\}. +\end{align} +$$ + +### Generalised truncated or censored normal distribution + +The normal distribution can similarly be generalised to account for truncation, censoring, and to +allow point masses at the boundary points of its support. This *generalised normal distribution* +has one location parameter, $\mu \in \mathbb{R}$, one scale parameter $\sigma > 0$, +lower and upper bound parameters $l,u \in \mathbb{R}$, $l < u$, and point mass parameters +$L, U \ge 0$, $L + U < 1$, that assign mass to the boundary points $l$ and $u$ respectively. +Its pdf+pmf, cdf, and CRPS are + +$$ +f_{\mu, \sigma, l, L}^{u, U}(x) = +\begin{cases} + 0, & x < l, \\ + L, & x = l, \\ + (1 - L - U) f_{\mu, \sigma}(x), & l < x < u, \\ + U, & x = u, \\ + 0, & x > u, \\ +\end{cases} +$$ + +$$ +F_{\mu, \sigma, l, L}^{u, U}(x) = +\begin{cases} + 0, & x < l, \\ + L + (1 - L - U) \frac{F_{\mu, \sigma}(x) - F_{\mu, \sigma}(l)}{F_{\mu, \sigma}(u) - F_{\mu, \sigma}(l)}, & l \le x < u, \\ + 1, & x \ge u, \\ +\end{cases} +$$ + +$$ +\begin{align} + \mathrm{CRPS}(F_{0, 1, l, L}^{u, U}, y) &= |y - z| + uU^{2} - lL^{2} + \left( \frac{1 - L - U}{\Phi(u) - \Phi(l)} \right) z \left( 2 \Phi(z) - \frac{(1 - 2L)\Phi(u) + (1 - 2U)\Phi(l)}{1 - L - U} \right) + \left( \frac{1 - L - U}{\Phi(u) - \Phi(l)} \right) (2 \phi(z) - 2 \phi(u)U - 2 \phi(l)L) - \left( \frac{1 - L - U}{\Phi(u) - \Phi(l)} \right)^{2} \left( \frac{1}{\sqrt{\pi}} \right) \left(\Phi \left( u\sqrt{2} \right) - \Phi \left( l\sqrt{2} \right) \right), \\ + \mathrm{CRPS}(F_{\mu, \sigma, l, L}^{u, U}, y) &= \sigma \mathrm{CRPS} \left( F_{0, 1, (l - \mu)/\sigma, L}^{(u - \mu)/\sigma, U}, \frac{y - \mu}{\sigma} \right), +\end{align} +$$ + +where $f_{\mu, \sigma}$ and $F_{\mu, \sigma}$ denote the pdf and cdf of the normal distribution +with location parameter $\mu$ and scale parameter $\sigma$, and $\phi = f_{0, 1}$ and $\Phi = F_{0, 1}$ +are the pdf and cdf of the standard normal distribution. + + +### Generalised truncated or censored Student's t distribution + +The Student's t distribution can also be generalised to account for truncation, censoring, and to +allow point masses at the boundary points of its support. This *generalised t distribution* +has one location parameter, $\mu \in \mathbb{R}$, one scale parameter $\sigma > 0$, +one degrees of freedom parameter, $\nu > 0$, lower and upper bound parameters $l,u \in \mathbb{R}$, +$l < u$, and point mass parameters $L, U \ge 0$, $L + U < 1$, that assign mass to the boundary points $l$ and $u$ respectively. +Its pdf+pmf, cdf, and CRPS are + +$$ +f_{\mu, \sigma, \nu, l, L}^{u, U}(x) = +\begin{cases} + 0, & x < l, \\ + L, & x = l, \\ + (1 - L - U) f_{\mu, \sigma, \nu}(x), & l < x < u, \\ + U, & x = u, \\ + 0, & x > u, \\ +\end{cases} +$$ + +$$ +F_{\mu, \sigma, \nu, l, L}^{u, U}(x) = +\begin{cases} + 0, & x < l, \\ + L + (1 - L - U) \frac{F_{\mu, \sigma, \nu}(x) - F_{\mu, \sigma, \nu}(l)}{F_{\mu, \sigma, \nu}(u) - F_{\mu, \sigma, \nu}(l)}, & l \le x < u, \\ + 1, & x \ge u, \\ +\end{cases} +$$ + +$$ +\begin{align} + \mathrm{CRPS}(F_{0, 1, \nu, l, L}^{u, U}, y) &= |y - z| + uU^{2} - lL^{2} + \left( \frac{1 - L - U}{F_{\nu}(u) - F_{\nu}(l)} \right) z \left( 2 F_{\nu}(z) - \frac{(1 - 2L)F_{\nu}(u) + (1 - 2U)F_{\nu}(l)}{1 - L - U} \right) - \left( \frac{1 - L - U}{F_{\nu}(u) - F_{\nu}(l)} \right) (2 G_{\nu}(z) - 2 G_{\nu}(u)U - 2 G_{\nu}(l)L) - \left( \frac{1 - L - U}{F_{\nu}(u) - F_{\nu}(l)} \right)^{2} \bar{B}_{\nu} (H_{\nu}(u) - H_{\nu}(l)), \\ + \mathrm{CRPS}(F_{\mu, \sigma, \nu, l, L}^{u, U}, y) &= \sigma \mathrm{CRPS} \left( F_{0, 1, \nu, (l - \mu)/\sigma, L}^{(u - \mu)/\sigma, U}, \frac{y - \mu}{\sigma} \right), +\end{align} +$$ + +where $f_{\mu, \sigma, \nu}$ and $F_{\mu, \sigma, \nu}$ denote the pdf and cdf of the Student's t distribution +with location parameter $\mu$, scale parameter $\sigma$, and degrees of freedom parameter $\nu$; +$f_{\nu} = f_{0, 1, \nu}$ and $F_{\nu} = F_{0, 1, \nu}$ are the pdf and cdf of the standard t distribution with $\nu$ +degrees of freedom; and + +$$ +\begin{align} + G_{\nu}(x) &= - \left( \frac{\nu + x^{2}}{\nu - 1} \right) f_{\nu}(x), \\ + H_{\nu}(x) &= \frac{1}{2} + \frac{1}{2} \text{sgn}(x) I \left(\frac{1}{2}, \nu - \frac{1}{2}, \frac{x^{2}}{\nu + x^{2}} \right), \\ + \bar{B}_{\nu} &= \left( \frac{2 \sqrt{\nu}}{\nu - 1} \right) \frac{B \left(\frac{1}{2}, \nu - \frac{1}{2} \right)}{B(\frac{1}{2}, \frac{\nu}{2})}, \\ + z &= \max \{l, \min \{y, u\}\}, +\end{align} +$$ + +where $I(a, b, x)$ is the regularised incomplete beta function, and $B$ is the beta function. From cf5f95ec7c8c41376fd9bfd5c9ccaa88d25c4cfb Mon Sep 17 00:00:00 2001 From: sallen12 Date: Fri, 4 Apr 2025 18:00:41 +0200 Subject: [PATCH 24/79] add documentation page for weighted scoring rules --- docs/weighted_scores.md | 575 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 575 insertions(+) create mode 100644 docs/weighted_scores.md diff --git a/docs/weighted_scores.md b/docs/weighted_scores.md new file mode 100644 index 0000000..2ffda00 --- /dev/null +++ b/docs/weighted_scores.md @@ -0,0 +1,575 @@ +(weighted_scores)= +# Weighted scoring rules + +Most scoring rules used in practice evaluate the entire forecast distribution. However, +it is often the case that some outcomes are of more interest than others. For example, extreme +outcomes often have a large impact on forecast users, making forecasts for these outcomes +particularly valuable. One could therefore argue that these outcomes should be assigned a +higher weight when evaluating forecast performance. For example, Savage (1971) remarks +that "the scoring rule should encourage the respondent to work hardest at what +the questioner most wants to know." For this purpose, weighted scoring rules +have been introduced, which extend conventional scoring rules to incorporate a weight function +into the score. This weight function can then be chosen to emphasise the outcomes that +are of most interest to the forecast users, facilitating flexible, user-oriented evaluation. + +## The forecaster's dilemma + +Given a proper scoring rule $S$ on $\Omega$, one may try to emphasise particular outcomes by multiplying the +score by a weight depending on what outcome occurs. That is, to evaluate the forecast $F$ using + +$$ w(y)S(F, y), $$ + +for some non-negative weight function $w : \Omega \to [0, \infty)$. In this case, the score will +be scaled depending on what outcome occurs. + +However, Gneiting and Ranjan (2011) demonstrate that this is not a proper scoring rule. In particular, +if $Y \sim G$, then the expected score $\mathbb{E}[w(Y)S(F, Y)]$ is minimised by a weighted version of +$G$ rather than $G$ itself. This weighted version of $G$ assigns higher probability density to the +outcomes that are assigned higher weight by $w$. For example, if $G$ emits a density $g$, then +the weighted score is minimised by the distribution $G_{w}$ with density + +$$ g_{w}(x) = \frac{w(x)g(x)}{\int_{\Omega} w(z) g(z) dz}. $$ + +If the weight function is of the form $w(x) = \mathbb{1}\{x \in A\}$ for some subset of the outcomes +$A \subset \Omega$, then $G_{w}$ is the conditional distribution of $Y$ given that $Y \in A$. +Note that this easily extends to distributions that do not emit a density function. + +For more intuition, consider the case where the weight function restricts attention to extreme outcomes. +If we only evaluate forecasts made for extreme outcomes, then it becomes an attractive strategy to +always predict that an extreme event will occur, regardless of how likely such an event is. +However, such a forecast is useless in practice: if we always predict that an extreme +event will occur, then we cannot use this forecast to distinguish whether or not an extreme event will actually occur. +Lerch et al. (2017) term this the *forecaster's dilemma*. Instead, alternative, more +theoretically-desirable methods are required to emphasise particular outcomes when evaluating +forecast performance. + + +## Outcome-weighting (conditional scores) + +Holzmann and Klar (2017) remarked that if the expected scoring rule is minimised by a weighted +version of the true distribution of $Y$, then we can circumvent the forecaster's dilemma by +evaluating forecasts via their weighted version. That is, given a proper scoring rule $S$, +the weighted scoring rule + +$$ \mathrm{ow}S(F, y) = w(y)S(F_{w}, y) $$ + +is minimised when $F_{w} = G_{w}$, which is obviously the case (though generally not uniquely) when $F = G$. +This scoring rule is therefore generally proper but not strictly proper. + +This approach was initially applied to the Log score by Diks et al. (2011), but can be implemented +using any proper scoring rule, providing a very general framework +with which to weight particular outcomes during forecast evaluation whilst retaining the +propriety of the scoring rule. We refer to these scoring rules as outcome-weighted +scoring rules, since they weight the score depending on the outcome. They have also been +referred to as conditional scores, since, when the weight function is of the form +$w(x) = \mathbb{1}\{x \in A\}$, they evaluate the forecast via its conditional distribution +on $A$. + +In evaluating the conditional forecast distribution, these outcome-weighted scoring rules do not +take into account the forecast probability that an outcome of interest will occur. For example, +if $w(x) = \mathbb{1}\{x \in A\}$, then the score will not depend on the forecast probability +that $\mathbb{1}\{Y \notin A\}$. Two forecasts that have the same conditional distribution will +therefore receive the same score. + +Holzmann and Klar (2017) proposed to remedy this by adding a proper scoring rule for binary outcomes, +such as the Brier score, to the outcome-weighted scoring rule. However, there is no guarantee that +the scale of the two scores are compatible, and there is often no canonical choice for the scoring +rule for binary outcomes; the Log score is an exception, since this approach can recover the censored +likelihood score introduced by Diks et al. (2011), which is discussed later (Holzmann and Klar, 2017, Example 4). +Hence, these complemented outcome-weighted scores are not available in `scoringrules`. However, +`scoringrules` does offer outcome-weighted versions of some popular scoring rules, which +are listed in the following sections. + + +## Threshold-weighting (censored scores) + +When $\Omega \subseteq \mathbb{R}$, Matheson and Winkler (1976) propose evaluating probabilistic +forecasts by considering the probability forecast that a threshold will be exceeded, evaluating +this forecast using a proper scoring rules for binary outcomes, and then integrating over all thresholds. +For example, they introduce the Continuous Ranked Probability Score (CRPS) as + +$$ + \mathrm{CRPS}(F, y) = \int_{-\infty}^{\infty} (F(z) - \mathbb{1}\{y \le z\})^{2} dz, +$$ + +which is the integral of the Brier score. Matheson and Winkler (1976) note that the integral +over the binary scoring rule could be defined more generally with respect to any probability +distribution, essentially incorporating a non-negative weight $w(z)$ into the integrand. +They then remark that "if certain regions of values of the variable are of particular interest, +the experimenter might make [$w(z)$] higher in these regions than it is elsewhere." + +In the case of the CRPS, Gneiting and Ranjan (2011) refer to this weighted version as the +*threshold-weighted CRPS*, since the weight function emphasises particular thresholds at +which the binary scoring rules are evaluated. Todter and Ahrens (2012) replaced the Brier +score with the Log score and similarly introduced a *Continuous Ranked Logarithmic Score (CRLS)*, +as well as a *threshold-weighted CRLS*. + +Allen et al. (2024) demonstrate that the threshold-weighted CRPS corresponds to transforming +the forecast and observation, before evaluating the transformed forecast against the transformed +observation using the standard (unweighted) CRPS; the transformation is determined by the weight +function. For example, when $w(x) = \mathbb{1}\{x > t\}$, for $t \in \mathbb{R}$, one possible +transformation is $v(x) = \max\{x, t\}$. In this case, the forecast distribution and the observation +are essentially censored at the threshold $t$, which is akin to the censored likelihood score +proposed by Diks et al. (2011). + +Moreover, while the original idea of threshold-weighting was implicitly univariate and specific +to scoring rules constructed using the approach of Matheson and Winkler (1976) described above, +the idea of transforming forecasts and observations prior to evaluation can easily be implemented +on arbitrary outcome spaces $\Omega$, and with arbitrary scoring rules. This facilitates the +introduction of threshold-weighted versions of any proper scoring rule. Some examples constructed +using popular scoring rules are listed in the following sections. + + +## Vertical re-scaling + +Outcome-weighted and threshold-weighted scoring rules involve transforming the forecast distributions +and outcomes (e.g. via conditioning or censoring) before calculating the original unweighted scores. +Alternatively, since many scoring rules are defined in terms of (dis-)similarity metrics, we could +instead define weighted scoring rules by transforming the output of these metrics, rather than the +inputs. In particular, *kernel scores* are scoring rules of the form + +$$ + S_{k}(F, y) = \frac{1}{2} k(y, y) + \frac{1}{2} \mathbb{E} k(X, X^{\prime}) - \mathbb{E} k(X, y), +$$ + +where $k$ is a positive definite kernel on $\Omega$. A positive definite kernel can be interpreted +as a generalised inner product, so that $k(x, x^{\prime})$, $x, x^{\prime} \in \Omega$, loosely +represents a measure of similarity between $x$ and $x^{\prime}$. + +Rasmussen and Williams (2006) remark that if $k$ is a positive definite kernel on $\Omega$, then +so is the weighted kernel $\check{k}(x, x^{\prime}) = k(x, x^{\prime})w(x)w(x^{\prime})$, for +some non-negative weight function $w$ on $\Omega$. This can therefore be implemented within +the definition of kernel scores above to obtain weighted versions of any kernel score. +Allen et al. (2024) term these *vertically re-scaled kernel scores*, since they involve +re-scaling the output of the similarity measure. + +For an interpretation of these vertically re-scaled kernel scores, consider how $\check{k}$ adapts $k$. +The similarity between $x$ and $x^{\prime}$ (as measured using $k$) is weighted depending on their values. +For example, if $w(x) = \mathbb{1}\{x \in A\}$ for some subset of the outcomes $A \subset \Omega$, then + +$$ +\check{k}(x, x^{\prime}) = +\begin{cases} + k(x, x^{\prime}), & x \in A, x^{\prime} \in A, \\ + 0, & \text{otherwise.} +\end{cases} +$$ + +In this case, the measure of similarity is restricted to instances where both inputs are in the region +$A$ of interest. In the context of kernel scores, consider the term $\mathbb{E} \check{k}(X, y)$. +When the outcome $y \notin A$, this term is equal to zero, regardless of the forecast. +When $y \in A$, the term is maximised (resulting in a lower score) when $F$ assigns a higher probability +to the region $A$. Conversely, the second term $\mathbb{E} \check{k}(X, X^{\prime})$ does not depend on the +observation, and is minimised (resulting in a lower score) when $F$ assigns lower probability to $A$. +Hence, this vertically re-scaled scoring rule rewards forecasts that issue a high probability to $A$ +when $y \in A$, and a low probability to $A$ otherwise. + +Similarly to the threshold-weighted scores, the behaviour of the forecast outside of $A$ does not +contribute to the score beyond the probability that is assigned to $A$. In many relevant cases, +vertical re-scaling is equivalent to threshold-weighting (Allen et al., 2024, Proposition 4.10). + + +## Examples + +In the following, let $w : \Omega \to [0, \infty)$ be a weight function that +assigns a non-negative weight to each possible outcome, unless specified otherwise. + +### Log score + +Assuming the forecast distribution $F$ emits a density function $f$ on $\Omega$, the Log score is defined as + +$$ \mathrm{LS}(F, y) = -\log f(y). $$ + +The *conditional likelihood score* is defined as + +$$ +\begin{align} +\mathrm{coLS}(F, y; w) &= - w(y) \log \left( \frac{f(y)}{\int_{\Omega} w(z)f(z) dz} \right) \\ + &= - w(y) \log f(y) + w(y) \log \left[ \int_{\Omega} w(z)f(z) dz \right]. +\end{align} +$$ + +This can be interpreted as an outcome-weighted version of the Log score (Holzmann and Klar, 2017, Example 1). + +For a weight function $w : \Omega \to [0, 1]$, the *censored likelihood score* is defined as + +$$ +\mathrm{ceLS}(F, y; w) = - w(y) \log f(y) - (1 - w(y)) \log \left[ 1 - \int_{\Omega} w(z)f(z) dz \right]. +$$ + +The censored likelihood score evaluates the forecast via the censored density function, and can +therefore be interpreted as a threshold-weighted version of the Log score (see the discussion +in de Punder et al., 2023). + +Since the Log score is not a kernel score, no vertically re-scaled version of the score exists. + + + +### Kernel scores + +The kernel score corresponding to a positive definite kernel $k$ on $\Omega$ is defined as + +$$ + S_{k}(F, y) = \frac{1}{2} k(y, y) + \frac{1}{2} \mathbb{E} k(X, X^{\prime}) - \mathbb{E} k(X, y), +$$ + +where $X, X^{\prime} \sim F$ are independent. Allen et al. (2024) discuss how weighted versions +of kernel scores can be constructed. Since many popular scoring rules are kernel scores, +including the CRPS, energy score, and Variogram score, +this facilitates the introduction of weighted versions of these popular scores. These +examples are given in the following sections. + +The outcome-weighted version of a kernel score can be written as + +$$ + \mathrm{ow}S_{k}(F, y; w) = \frac{1}{2} k(y, y)w(y) + \frac{1}{2 \bar{w}_{F}^{2}} \mathbb{E} k(X, X^{\prime})w(X)w(X^{\prime})w(y) - \frac{1}{\bar{w}_{F}}\mathbb{E} k(X, y)w(X)w(y), +$$ + +where $\bar{w}_{F} = \mathbb{E}[w(X)]$. + +The threshold-weighted version of a kernel score can be written as + +$$ + \mathrm{tw}S_{k}(F, y; v) = \frac{1}{2} k(v(y), v(y)) + \frac{1}{2} \mathbb{E} k(v(X), v(X^{\prime})) - \mathbb{E} k(v(X), v(y)), +$$ + +where $v : \Omega \to \Omega$ is termed the *chaining function*. When $\Omega \subseteq \mathbb{R}$, +the chaining function can be defined as the anti-derivative of the weight function. That is, +$v(x) - v(x^{\prime}) = \int_{x^{\prime}}^{x} w(z) dz$ for all $x, x^{\prime} \in \mathbb{R}$. +However, more generally, there is no canonical way to map a weight function $w$ to a chaining function $v$. + +The vertically re-scaled version of a kernel score can be written as + +$$ + \mathrm{vr}S_{k}(F, y; w) = \frac{1}{2} k(y, y)w(y)^{2} + \frac{1}{2} \mathbb{E} k(X, X^{\prime})w(X)w(X^{\prime}) - \mathbb{E} k(X, y)w(X)w(y). +$$ + +For any positive definite kernel $k$ on $\Omega$, the functions $\tilde{k}(x, x^{\prime}) = k(v(x), v(x^{\prime}))$ +and $\check{k}(x, x^{\prime}) = k(x, x^{\prime})w(x)w(x^{\prime})$ are also positive definite +kernels on $\Omega$ (Rasmussen and Williams, 2006). Hence, the threshold-weighted and vertically +re-scaled versions of kernel scores are themselves kernel scores. + + +### CRPS + +The CRPS is defined as + +$$ +\begin{align} +\mathrm{CRPS}(F, y) &= \int_{-\infty}^{\infty} (F(z) - \mathbb{1}\{y \le z\})^{2} dz \\ +&= \mathbb{E}|X - y| - \frac{1}{2} \mathbb{E}|X - X^{\prime}|, +\end{align} +$$ +where $y \in \Omega \subseteq \mathbb{R}$, and $X, X^{\prime} \sim F$ are independent +(Matheson and Winkler, 1976; Gneiting and Rafery, 2007). + +Holzmann and Klar (2017) introduce the *outcome-weighted CRPS* as + +$$ +\begin{align} +\mathrm{owCRPS}(F, y; w) &= w(y) \int_{-\infty}^{\infty} (F_{w}(z) - \mathbb{1}\{y \le z\})^{2} dz \\ +&= \frac{1}{\bar{w}_{F}}\mathbb{E}|X - y|w(X)w(y) - \frac{1}{2\bar{w}_{F}^{2}} \mathbb{E}|X - X^{\prime}|w(X)w(X^{\prime})w(y), +\end{align} +$$ +where $\bar{w}_{F} = \mathbb{E}[w(X)]$. + +Gneiting and Ranjan (2011) introduce the *threshold-weighted CRPS* as + +$$ +\begin{align} +\mathrm{twCRPS}(F, y; w) &= \int_{-\infty}^{\infty} (F(x) - 1\{y \le x\})^{2} w(x) dx \\ +&= \mathbb{E} | v(X) - v(y) | - \frac{1}{2} \mathbb{E} |v(X) - v(X^{\prime}) |, +\end{align} +$$ + +where $v : \mathbb{R} \to \mathbb{R}$ is an anti-derivative of the weight function $w$ (Allen et al., 2024). + +Allen et al. (2023) introduce the *vertically re-scaled CRPS* as + +$$ +\mathrm{vrCRPS}(F, y; w, x_{0}) = \mathbb{E} | X - y |w(X)w(y) - \frac{1}{2} \mathbb{E} | X - X^{\prime} | w(X)w(X^{\prime}) + \left( \mathbb{E}|X - x_{0}|w(X) - |y - x_{0}|w(y) \right) \left( \mathbb{E}w(X) - w(y) \right), +$$ + +for some $x_{0} \in \mathbb{R}$. The canonical choice is $x_{0} = 0$, though if the weight function +is of the form $w(x) = \mathbb{1}\{x > t\}$ or $w(x) = \mathbb{1}\{x < t\}$, for some threshold +$t \in \mathbb{R}$, then setting $x_{0} = t$ recovers the threshold-weighted CRPS +(Allen et al., 2024, Proposition 4.10). + + +### Energy score + +The energy score is defined as + +$$ +\mathrm{ES}(F, \boldsymbol{y}) = \mathbb{E} \| \boldsymbol{X} - \boldsymbol{y} \| - \frac{1}{2} \mathbb{E} \| \boldsymbol{X} - \boldsymbol{X}^{\prime} \|, +$$ + +where $\| \cdot \|$ is the Euclidean distance on $\mathbb{R}^{d}$, +$\boldsymbol{y} \in \Omega \subseteq \mathbb{R}^{d}$, and $\boldsymbol{X}, \boldsymbol{X}^{\prime} \sim F$ are independent, +with $F$ a multivariate predictive distribution on $\Omega$ (Gneiting and Raftery, 2007). + +The *outcome-weighted energy score* is similarly defined as + +$$ +\mathrm{owES}(F, \boldsymbol{y}; w) = \frac{1}{\bar{w}_{F}} \mathbb{E} \| \boldsymbol{X} - \boldsymbol{y} \| w(\boldsymbol{X})w(\boldsymbol{y}) - \frac{1}{2\bar{w}_{F}^{2}} \mathbb{E} \| \boldsymbol{X} - \boldsymbol{X}^{\prime} \|w(\boldsymbol{X})w(\boldsymbol{X}^{\prime})w(\boldsymbol{y}), +$$ + +where $\boldsymbol{X}, \boldsymbol{X}^{\prime} \sim F$ are independent, and +$\bar{w}_{F} = \mathbb{E}[w(\boldsymbol{X})]$ for some weight function $w : \Omega \to [0, \infty)$ +(Holzmann and Klar, 2017). + +The *threshold-weighted energy score* constitutes a multivariate extension of the threshold-weighted CRPS, + +$$ +\mathrm{twES}(F, y\boldsymbol{y}; v) = \mathbb{E} | v(\boldsymbol{X}) - v(\boldsymbol{y}) | - \frac{1}{2} \mathbb{E} |v(\boldsymbol{X}) - v(\boldsymbol{X}^{\prime}) |, +$$ + +where $v : \mathbb{R} \to \mathbb{R}$, + +and the *vertically re-scaled energy score* constitutes a multivariate extension of the vertically re-scaled CRPS, + +$$ +\mathrm{vrES}(F, \boldsymbol{y}; w, \boldsymbol{x}_{0}) = \mathbb{E} | \boldsymbol{X} - \boldsymbol{y} |w(X)w(\boldsymbol{y}) - \frac{1}{2} \mathbb{E} | \boldsymbol{X} - \boldsymbol{X}^{\prime} | w(\boldsymbol{X})w(\boldsymbol{X}^{\prime}) + \left( \mathbb{E}|\boldsymbol{X} - \boldsymbol{x}_{0}|w(\boldsymbol{X}) - |\boldsymbol{y} - \boldsymbol{x}_{0}|w(\boldsymbol{y}) \right) \left( \mathbb{E}w(\boldsymbol{X}) - w(\boldsymbol{y}) \right) +$$ + +(Allen et al., 2024). As with the vertically re-scaled CRPS, the canonical choice is $\boldsymbol{x}_{0} = \boldsymbol{0}$, +though if the weight function is of the form $w(\boldsymbol{x}) = \mathbb{1}\{\boldsymbol{x} > \boldsymbol{x}\}$ or +$w(\boldsymbol{x}) = \mathbb{1}\{\boldsymbol{x} < t\}$, for some multivariate threshold +$\boldsymbol{x} \in \mathbb{R}$ (with $<$ and $>$ understood componentwise), then setting +$\boldsymbol{x}_{0} = \boldsymbol{t}$ recovers the threshold-weighted energy score (Allen et al., 2024, Proposition 4.10). + + +### Variogram score + +The Variogram score is defined as + +$$ + \mathrm{VS}_{p}(F, \boldsymbol{y}) = \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left( \mathbb{E} | X_{i} - X_{j} |^{p} - | y_{i} - y_{j} |^{p} \right)^{2}, +$$ + +where $p > 0$, $\boldsymbol{y} = (y_{1}, \dots, y_{d}) \in \Omega \subseteq \mathbb{R}^{d}$, +$\boldsymbol{X} = (X_{1}, \dots, X_{d}) \sim F$, with $F$ a multivariate predictive distribution on $\Omega$, +and $h_{i,j} \ge 0$ are weights assigned to different pairs of dimensions (Scheuerer and Hamill, 2015). + +Allen (2024) writes the *outcome-weighted Variogram score* as + +$$ + \mathrm{owVS}_{p}(F, \boldsymbol{y}; w) = w(\boldsymbol{y}) \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left( \frac{1}{\bar{w}_{F}} \mathbb{E} | X_{i} - X_{j} |^{p}w(\boldsymbol{X}) - | y_{i} - y_{j} |^{p} \right)^{2}. +$$ + +Since the Variogram score is a kernel score, the *threshold-weighted Variogram score* is defined as + +$$ + \mathrm{twVS}_{p}(F, \boldsymbol{y}; v) = \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left(\mathbb{E} | v(\boldsymbol{X})_{i} - v(\boldsymbol{X})_{j} |^{p} - | v(\boldsymbol{y})_{i} - v(\boldsymbol{y})_{j} |^{p} \right)^{2}, +$$ + +where $v(\boldsymbol{y}) = (v(\boldsymbol{y})_{1}, \dots, v(\boldsymbol{y})_{d}) \in \Omega \subseteq \mathbb{R}^{d}$ +for some chaining function $v : \Omega \to \Omega$ (Allen et al., 2024). + +The *vertically-rescaled Variogram score* is + +$$ + \mathrm{vrVS}_{p}(F, \boldsymbol{y}; w, x_{0}) = \mathbb{E} \left[ w(\boldsymbol{X})w(\boldsymbol{y}) \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left(| X_{i} - X_{j} |^{p} - | y_{i} - y_{j} |^{p} \right)^{2} \right] - \frac{1}{2} \mathbb{E} \left[ w(\boldsymbol{X})w(\boldsymbol{X^{\prime}}) \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left(| X_{i} - X_{j} |^{p} - | X_{i}^{\prime} - X_{j}^{\prime} |^{p} \right)^{2} \right] + \left( \mathbb{E} \left[ w(\boldsymbol{X}) \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left(| X_{i} - X_{j} |^{p} - | x_{0,i} - x_{0,j} |^{p} \right)^{2} \right] - w(\boldsymbol{y}) \sum_{i=1}^{d} \sum_{j=1}^{d} h_{i,j} \left(| y_{i} - y_{j} |^{p} - | x_{0,i} - x_{0,j} |^{p} \right)^{2} \right) \left( \mathbb{E}[w(\boldsymbol{X})] - w(\boldsymbol{y}) \right), +$$ + +for some $\boldsymbol{x}_{0} = (x_{0,1}, \dots, x_{0,d}) \in \mathbb{R}^{d}$. Analogously to the vertically +re-scaled energy score, the canonical choice is $\boldsymbol{x}_{0} = \boldsymbol{0}$, +unless the weight function is of the form $w(\boldsymbol{x}) = \mathbb{1}\{\boldsymbol{x} > \boldsymbol{t}\}$ or +$w(\boldsymbol{x}) = \mathbb{1}\{\boldsymbol{x} < t\}$, for some multivariate threshold +$\boldsymbol{t} \in \mathbb{R}$ (with $<$ and $>$ understood componentwise), in which case setting +$\boldsymbol{x}_{0} = \boldsymbol{t}$ recovers the threshold-weighted variogram score +(Allen et al., 2024, Proposition 4.10). + + +## Weight functions + +The weighted scoring rules introduced above are all recovered when the weight function is +constant, e.g. $w(z) = 1$ for all $z \in \Omega$. Hence, weighted scoring rules increase the +flexibility offered to practitioners when evaluating forecasts. +The weight function can be chosen to direct the scoring rules to particular outcomes, so that +poor forecasts for these outcomes are penalised more heavily than poor forecasts for other +outcomes. In practice, it is often not trivial to decide which outcomes are "of interest", +and how much they should be emphasised when calculating the score; that is, what weight function +should be used. This decision is very application-specific and it is therefore difficult to provide +any general guidance regarding which weight function should be employed in practical applications. +Nonetheless, we list some common weight functions below. + +### Univariate forecasts + +Suppose $\Omega \subseteq \mathbb{R}$. The most common weight function used in practice is +an indicator weight function of the form + +$$ w(z) = \mathbb{1}\{z > t\} \quad \text{or} \quad \mathbb{1}\{z < t\} $$ + +which restricts attention to outcomes above or below some relevant threshold $t \in \mathbb{R}$. +The threshold $t$ is typically chosen to be a fairly extreme quantile of previously observed +outcomes, though it could also correspond to a value used for policy making; a warning threshold, +for example. + +More generally, we can define indicator weights on arbitrary regions of $\Omega$. For example, +we may be interested in values within a certain range, + +$$ w(z) = \mathbb{1}\{a < z < b\} \quad \text{for some $-\infty \le a < b \le \infty$}, $$ + +which nests the above cases when $a = t$ and $b = \infty$ or when $a = -\infty$ and $b = t$; +or in values that either fall below a low threshold or exceed a high threshold, + +$$ w(z) = \mathbb{1}\{z < a\} + \mathbb{1}\{z > b\} \quad \text{for some $-\infty \le a < b \le \infty$}. $$ + +Alternatively, Gneiting and Ranjan (2011) suggested using normal density and distribution functions +to define continuous weight functions that change smoothly over $\mathbb{R}$. Let $\phi_{\mu,\sigma}$ +and $\Phi_{\mu, \sigma}$ denote the normal density and distribution functions respectively, with +location parameter $\mu$ and scale parameter $\sigma$. The weight function + +$$ w(z) = \Phi_{\mu, \sigma}(z) $$ + +assigns a higher weight to higher outcomes, with the weight increasing gradually from 0 to 1. +This can be interpreted as a smoothed step function, where $\mu$ defines a threshold of interest, +and $\sigma$ determines the speed at which the weight changes from 0 before the threshold to 1 afterwards; +as $\sigma \to 0$, the weight function tends towards the indicator weight function $w(z) = \mathbb{1}\{z > \mu\}$, +while a larger value of $\sigma$ increases the weight more slowly. Similarly, the weight function + +$$ w(z) = 1 - \Phi_{\mu, \sigma}(z) $$ + +assigns a higher weight to smaller outcomes, with analogous interpretation of $\mu$ and $\sigma$. + +Using the normal density function allows central values to be emphasised, + +$$ w(z) = \phi_{\mu, \sigma}(z). $$ + +In this case, $\sigma$ controls the concentration of the weight function around $\mu$, with the +weight function reducing to a point mass at $\mu$ as $\sigma \to 0$. Conversely, the lower and upper +tails can be targeted simultaneously using + +$$ w(z) = 1 - \frac{\phi_{\mu, \sigma}(z)}{\phi_{\mu, \sigma}(\mu)}. $$ + +While these all employ the normal density and distribution functions, the normal distribution +could readily be replaced by any other location-scale family distribution. Allen (2024), for example, +consider the same weights defined using the logistic distribution. + + +### Multivariate forecasts + +Multivariate weight functions (when $\Omega \subseteq \mathbb{R}^{d}$) can be defined similarly. +For example, it is common to employ an indicator weight function of the form + +$$ w(\boldsymbol{z}) = \mathbb{1}\{z_{1} > t_{1}, \dots, z_{d} > t_{d}\} $$ + +to emphasise values above a threshold $t_{i} \in \mathbb{R}$ in each dimension. Different regions +of the outcome space can again be targeted by interchanging $<$ and $>$ signs. More generally, +this can be expressed as + +$$ w(\boldsymbol{z}) = \mathbb{1}\{a_{1} < z_{1} < b_{1}, \dots, a_{d} < z_{d} < b_{d}\}, $$ + +for some $-\infty \le a_{i} < b_{i} \le \infty$ for $i = 1, \dots, d$. + +Smooth weight functions can again be constructed using multivariate probability distributions. +For example, let $\phi_{\boldsymbol{\mu}, \Sigma}$ and $\Phi_{\boldsymbol{\mu}, \Sigma}$ +denote the density and distribution functions of a multivariate normal distribution with +mean vector $\boldsymbol{\mu}$ and covariance matrix $\Sigma$. Then, high outcomes (in all dimensions) +could be targeted using a weight function of the form + +$$ w(\boldsymbol{z}) = \Phi_{\boldsymbol{\mu}, \Sigma}(\boldsymbol{z}). $$ + +The interpretation of $\boldsymbol{\mu}$ and $\Sigma$ is similar to in the univariate case: +$\boldsymbol{\mu}$ can be thought of as a vector of thresholds, with $\Sigma$ controlling +the rate at which the weight function increases from 0 to 1 along each dimension. Low outcomes +can similarly be emphasised using + +$$ w(\boldsymbol{z}) = 1 - \Phi_{\boldsymbol{\mu}, \Sigma}(\boldsymbol{z}), $$ + +while a high outcome in one dimension and a low outcome in another dimension could similarly +be targeted using appropriate manipulations of the multivariate normal distribution. For example, +when $d = 2$, the following weight function would target high values along the first dimension +and low values along the second, + +$$ w(\boldsymbol{z}) = \mathrm{P}(X_{1} \le z_{1}, X_{2} > z_{2}), $$ + +where $\boldsymbol{X} = (X_{1}, X_{2})$ follows a multivariate normal distribution. + +The multivariate normal density function can similarly be used to target central values, + +$$ w(\boldsymbol{z}) = \phi_{\boldsymbol{\mu}, \Sigma}(\boldsymbol{z}), $$ + +while tails (in all dimensions) can be emphasised using + +$$ w(\boldsymbol{z}) = 1 - \phi_{\boldsymbol{\mu}, \Sigma}(\boldsymbol{z})/\phi_{\boldsymbol{\mu}, \Sigma}(\boldsymbol{\mu}). $$ + +As in the univariate case, these are just particular examples, and by no means an exhaustive +list of the weight functions that should be employed in practice. + + +## Chaining functions + +While most weighted scoring rules depend directly on a weight function, threshold-weighted +scoring rules can be defined in terms of a transformation function, or chaining function +$v : \Omega \to \Omega$. In the univariate case, a chaining function can be derived from a +weight function. However, in the multivariate case, there is generally no canonical way to +obtain a chaining function from a weight function. In the following, we discuss the choice +of chaining function in more detail. + +### Univariate forecasts + +Due to the two representations of the threshold-weighted CRPS, it is straightforward to map +a weight function to a chaining function in the univariate case ($\Omega \subseteq \mathbb{R}$). +In particular, given a weight function $w : \Omega \to [0, \infty)$, the chaining function is +simply an anti-derivative of the weight function. That is, the chaining function $v$ satisfied +$v(x) - v(x^{\prime}) = \int_{x^{\prime}}^{x} w(z) dz$ for all $x, x^{\prime} \in \Omega$. + +When $w$ is constant, we recover the identity function $v(z) = z$, for all $z \in \Omega$, +and for all weight functions commonly used in practice, the chaining function is typically +straightforward to calculate. Several weight and chaining function combinations are listed +in Table 1 of Allen (2024). + +For example, when $w(z) = \mathbb{1}\{z > t\}$, $t \in \mathbb{R}$, we get (up to an unimportant constant) + +$$ v(z) = \max\{z, t\}, $$ + +and more generally for $w(z) = \mathbb{1}\{a < z < t\}$, $- \infty \le a < b \le \infty$ we have + +$$ v(z) = \min\{\max\{z, a\}, b\}. $$ + +For $w(z) = \Phi_{\mu, \sigma}(z)$, + +$$ v(z) = (z - \mu)\Phi_{\mu, \sigma}(z) + \sigma^{2} \phi_{\mu, \sigma}(z); $$ + +for $w(z) = 1 - \Phi_{\mu, \sigma}(z)$, + +$$ v(z) = z - (z - \mu)\Phi_{\mu, \sigma}(z) - \sigma^{2} \phi_{\mu, \sigma}(z); $$ + +and for $w(z) = \phi_{\mu, \sigma}(z)$, + +$$ v(z) = \Phi_{\mu, \sigma}(z). $$ + + +### Multivariate forecasts + +In the multivariate case $(\Omega \subseteq \mathbb{R}^{d})$, there is no default way to obtain +a chaining function given a multivariate weight function. One approach is to find the anti-derivative +of the weight function along each dimension separately. For example, for the weight function +$ w(\boldsymbol{z}) = \mathbb{1}\{a_{1} < z_{1} < b_{1}, \dots, a_{d} < z_{d} < b_{d}\}, $ +a possible chaining function is + +$$ v(\boldsymbol{z}) = (\min\{\max\{z_{1}, a_{1}\}, b_{1}\}, . . . , \min\{\max\{z_{d}, a_{d}\}, b_{d}\}). $$ + +In this case, the weight function represents a box in multivariate space, and $v$ projects +points not in the orthant onto its perimeter; the points inside the box, i.e., for which +the weight function is equal to one, remain unchanged. + +Similarly, for the smooth weight functions based on multivariate Gaussian distribution and +density functions, a chaining function can be derived from a component-wise extension of +the chaining functions corresponding to univariate Gaussian weight functions. For example, +for $w(\boldsymbol{z}) = \Phi_{\boldsymbol{\mu}, \Sigma}(\boldsymbol{z})$, the chaining +function would become + +$$ v(\boldsymbol{z}) = ((z_{1} - \mu_{1})\Phi_{\mu_{1}, \sigma_{1}}(z_{1}) + \sigma_{1}^{2} \phi_{\mu_{1}, \sigma_{1}}(z_{1}), \dots, (z_{d} - \mu_{d})\Phi_{\mu_{d}, \sigma_{d}}(z_{d}) + \sigma_{d}^{2} \phi_{\mu_{d}, \sigma_{d}}(z_{d})), $$ + +where $\sigma_{1}, \dots, \sigma_{d}$ are the standard deviations along each dimension. +This does not depend on the off-diagonal terms of the covariance matrix $\Sigma$, thereby +implicitly assuming that it is diagonal. However, when the weight function is +$w(\boldsymbol{z}) = \phi_{\boldsymbol{\mu}, \Sigma}(\boldsymbol{z})$, we can readily +use + +$$ v(\boldsymbol{z}) = \Phi_{\boldsymbol{\mu}, \Sigma}(\boldsymbol{z}). $$ + +A more detailed discussion on multivariate chaining functions is available in Allen et al. (2023). From 5eb6656582a4e7432782871191fd91c01a136d2f Mon Sep 17 00:00:00 2001 From: sallen12 Date: Fri, 4 Apr 2025 18:00:58 +0200 Subject: [PATCH 25/79] update docu index --- .gitignore | 1 + docs/index.md | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index be1ebf5..45599db 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,7 @@ instance/ # Sphinx documentation docs/_build/ docs/build/ +_build* # PyBuilder .pybuilder/ diff --git a/docs/index.md b/docs/index.md index 4d5c484..dea7f15 100644 --- a/docs/index.md +++ b/docs/index.md @@ -34,7 +34,9 @@ The scoring rules available in `scoringrules` include, but are not limited to, t :caption: Background theory.md +forecast_dists.md crps_estimators.md +weighted_scores.md ``` ```{toctree} From 7f4603f983db30d3378b3d313f501166d8cc1cca Mon Sep 17 00:00:00 2001 From: Nicholas Loveday <48701367+nicholasloveday@users.noreply.github.com> Date: Mon, 7 Apr 2025 22:18:07 +1000 Subject: [PATCH 26/79] update axis naming args (#95) * change ens member axis to m_axis * change categorical axis to k_axis * update mixture components axis name --- scoringrules/_brier.py | 16 ++++++----- scoringrules/_crps.py | 54 +++++++++++++++++------------------ scoringrules/_error_spread.py | 8 +++--- scoringrules/_kernels.py | 30 +++++++++---------- scoringrules/_logs.py | 46 ++++++++++++++--------------- tests/test_brier.py | 4 +-- tests/test_crps.py | 6 ++-- tests/test_error_spread.py | 2 +- tests/test_logs.py | 14 ++++----- tests/test_wcrps.py | 4 +-- 10 files changed, 93 insertions(+), 91 deletions(-) diff --git a/scoringrules/_brier.py b/scoringrules/_brier.py index 5230346..554f0f2 100644 --- a/scoringrules/_brier.py +++ b/scoringrules/_brier.py @@ -46,7 +46,7 @@ def rps_score( obs: "ArrayLike", fct: "ArrayLike", /, - axis: int = -1, + k_axis: int = -1, *, backend: "Backend" = None, ) -> "Array": @@ -70,7 +70,7 @@ def rps_score( Array of 0's and 1's corresponding to unobserved and observed categories forecasts : Array of forecast probabilities for each category. - axis: int + k_axis: int The axis corresponding to the categories. Default is the last axis. backend : str The name of the backend used for computations. Defaults to 'numpy'. @@ -84,8 +84,8 @@ def rps_score( B = backends.active if backend is None else backends[backend] fct = B.asarray(fct) - if axis != -1: - fct = B.moveaxis(fct, axis, -1) + if k_axis != -1: + fct = B.moveaxis(fct, k_axis, -1) return brier.rps_score(obs=obs, fct=fct, backend=backend) @@ -129,7 +129,7 @@ def rls_score( obs: "ArrayLike", fct: "ArrayLike", /, - axis: int = -1, + k_axis: int = -1, *, backend: "Backend" = None, ) -> "Array": @@ -153,6 +153,8 @@ def rls_score( Observed outcome, either 0 or 1. fct : array_like Forecasted probabilities between 0 and 1. + k_axis: int + The axis corresponding to the categories. Default is the last axis. backend : str The name of the backend used for computations. Defaults to 'numpy'. @@ -165,8 +167,8 @@ def rls_score( B = backends.active if backend is None else backends[backend] fct = B.asarray(fct) - if axis != -1: - fct = B.moveaxis(fct, axis, -1) + if k_axis != -1: + fct = B.moveaxis(fct, k_axis, -1) return brier.rls_score(obs=obs, fct=fct, backend=backend) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index 633ab53..33fb5ed 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -11,7 +11,7 @@ def crps_ensemble( obs: "ArrayLike", fct: "Array", /, - axis: int = -1, + m_axis: int = -1, *, sorted_ensemble: bool = False, estimator: str = "pwm", @@ -48,7 +48,7 @@ def crps_ensemble( fct : array_like, shape (..., m) The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - axis : int + m_axis : int The axis corresponding to the ensemble. Default is the last axis. sorted_ensemble : bool Boolean indicating whether the ensemble members are already in ascending order. @@ -99,8 +99,8 @@ def crps_ensemble( B = backends.active if backend is None else backends[backend] obs, fct = map(B.asarray, (obs, fct)) - if axis != -1: - fct = B.moveaxis(fct, axis, -1) + if m_axis != -1: + fct = B.moveaxis(fct, m_axis, -1) if not sorted_ensemble and estimator not in [ "nrg", @@ -108,7 +108,7 @@ def crps_ensemble( "akr_circperm", "fair", ]: - fct = B.sort(fct, axis=-1) + fct = B.sort(fct, m_axis=-1) if backend == "numba": if estimator not in crps.estimator_gufuncs: @@ -126,7 +126,7 @@ def twcrps_ensemble( fct: "Array", v_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, - axis: int = -1, + m_axis: int = -1, *, estimator: str = "pwm", sorted_ensemble: bool = False, @@ -156,7 +156,7 @@ def twcrps_ensemble( Chaining function used to emphasise particular outcomes. For example, a function that only considers values above a certain threshold :math:`t` by projecting forecasts and observations to :math:`[t, \inf)`. - axis : int + m_axis : int The axis corresponding to the ensemble. Default is the last axis. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. @@ -200,7 +200,7 @@ def twcrps_ensemble( return crps_ensemble( obs, fct, - axis=axis, + m_axis=m_axis, sorted_ensemble=sorted_ensemble, estimator=estimator, backend=backend, @@ -212,7 +212,7 @@ def owcrps_ensemble( fct: "Array", w_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, - axis: int = -1, + m_axis: int = -1, *, estimator: tp.Literal["nrg"] = "nrg", backend: "Backend" = None, @@ -245,7 +245,7 @@ def owcrps_ensemble( represented by the last axis. w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. - axis : int + m_axis : int The axis corresponding to the ensemble. Default is the last axis. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. @@ -288,8 +288,8 @@ def owcrps_ensemble( "Only the energy form of the estimator is available " "for the outcome-weighted CRPS." ) - if axis != -1: - fct = B.moveaxis(fct, axis, -1) + if m_axis != -1: + fct = B.moveaxis(fct, m_axis, -1) obs_weights, fct_weights = map(w_func, (obs, fct)) @@ -309,7 +309,7 @@ def vrcrps_ensemble( fct: "Array", w_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, - axis: int = -1, + m_axis: int = -1, *, estimator: tp.Literal["nrg"] = "nrg", backend: "Backend" = None, @@ -340,7 +340,7 @@ def vrcrps_ensemble( represented by the last axis. w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. - axis : int + m_axis : int The axis corresponding to the ensemble. Default is the last axis. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. @@ -383,8 +383,8 @@ def vrcrps_ensemble( "Only the energy form of the estimator is available " "for the outcome-weighted CRPS." ) - if axis != -1: - fct = B.moveaxis(fct, axis, -1) + if m_axis != -1: + fct = B.moveaxis(fct, m_axis, -1) obs_weights, fct_weights = map(w_func, (obs, fct)) @@ -404,7 +404,7 @@ def crps_quantile( fct: "Array", alpha: "Array", /, - axis: int = -1, + m_axis: int = -1, *, backend: "Backend" = None, ) -> "Array": @@ -435,7 +435,7 @@ def crps_quantile( represented by the last axis. alpha : array_like The percentile levels. We expect the quantile array to match the axis (see below) of the forecast array. - axis : int + m_axis : int The axis corresponding to the ensemble. Default is the last axis. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. @@ -456,8 +456,8 @@ def crps_quantile( B = backends.active if backend is None else backends[backend] obs, fct, alpha = map(B.asarray, (obs, fct, alpha)) - if axis != -1: - fct = B.moveaxis(fct, axis, -1) + if m_axis != -1: + fct = B.moveaxis(fct, m_axis, -1) if not fct.shape[-1] == alpha.shape[-1]: raise ValueError("Expected matching length of `fct` and `alpha` values.") @@ -1839,7 +1839,7 @@ def crps_mixnorm( s: "ArrayLike", /, w: "ArrayLike" = None, - axis: "ArrayLike" = -1, + m_axis: "ArrayLike" = -1, *, backend: "Backend" = None, ) -> "ArrayLike": @@ -1863,7 +1863,7 @@ def crps_mixnorm( Standard deviations of the component normal distributions. w: array_like Non-negative weights assigned to each component. - axis : int + m_axis : int The axis corresponding to the mixture components. Default is the last axis. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. @@ -1890,15 +1890,15 @@ def crps_mixnorm( obs, m, s = map(B.asarray, (obs, m, s)) if w is None: - M: int = m.shape[axis] + M: int = m.shape[m_axis] w = B.zeros(m.shape) + 1 / M else: w = B.asarray(w) - if axis != -1: - m = B.moveaxis(m, axis, -1) - s = B.moveaxis(s, axis, -1) - w = B.moveaxis(w, axis, -1) + if m_axis != -1: + m = B.moveaxis(m, m_axis, -1) + s = B.moveaxis(s, m_axis, -1) + w = B.moveaxis(w, m_axis, -1) return crps.mixnorm(obs, m, s, w, backend=backend) diff --git a/scoringrules/_error_spread.py b/scoringrules/_error_spread.py index 98cb1f7..e712c6b 100644 --- a/scoringrules/_error_spread.py +++ b/scoringrules/_error_spread.py @@ -11,7 +11,7 @@ def error_spread_score( observations: "ArrayLike", forecasts: "Array", /, - axis: int = -1, + m_axis: int = -1, *, backend: "Backend" = None, ) -> "Array": @@ -24,7 +24,7 @@ def error_spread_score( forecasts: Array The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - axis: int + m_axis: int The axis corresponding to the ensemble. Default is the last axis. backend: str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -37,8 +37,8 @@ def error_spread_score( B = backends.active if backend is None else backends[backend] observations, forecasts = map(B.asarray, (observations, forecasts)) - if axis != -1: - forecasts = B.moveaxis(forecasts, axis, -1) + if m_axis != -1: + forecasts = B.moveaxis(forecasts, m_axis, -1) if B.name == "numba": return error_spread._ess_gufunc(observations, forecasts) diff --git a/scoringrules/_kernels.py b/scoringrules/_kernels.py index b4da612..e21d192 100644 --- a/scoringrules/_kernels.py +++ b/scoringrules/_kernels.py @@ -12,7 +12,7 @@ def gksuv_ensemble( obs: "ArrayLike", fct: "Array", /, - axis: int = -1, + m_axis: int = -1, *, estimator: str = "nrg", backend: "Backend" = None, @@ -40,7 +40,7 @@ def gksuv_ensemble( fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - axis : int + m_axis : int The axis corresponding to the ensemble. Default is the last axis. estimator : str Indicates the estimator to be used. @@ -73,8 +73,8 @@ def gksuv_ensemble( f"Must be one of ['fair', 'nrg']" ) - if axis != -1: - fct = B.moveaxis(fct, axis, -1) + if m_axis != -1: + fct = B.moveaxis(fct, m_axis, -1) if backend == "numba": return kernels.estimator_gufuncs[estimator](obs, fct) @@ -87,7 +87,7 @@ def twgksuv_ensemble( fct: "Array", v_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, - axis: int = -1, + m_axis: int = -1, *, estimator: str = "nrg", backend: "Backend" = None, @@ -120,7 +120,7 @@ def twgksuv_ensemble( Chaining function used to emphasise particular outcomes. For example, a function that only considers values above a certain threshold :math:`t` by projecting forecasts and observations to :math:`[t, \inf)`. - axis : int + m_axis : int The axis corresponding to the ensemble. Default is the last axis. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -144,7 +144,7 @@ def twgksuv_ensemble( return gksuv_ensemble( obs, fct, - axis=axis, + m_axis=m_axis, estimator=estimator, backend=backend, ) @@ -155,7 +155,7 @@ def owgksuv_ensemble( fct: "Array", w_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, - axis: int = -1, + m_axis: int = -1, *, backend: "Backend" = None, ) -> "Array": @@ -186,7 +186,7 @@ def owgksuv_ensemble( represented by the last axis. w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. - axis : int + m_axis : int The axis corresponding to the ensemble. Default is the last axis. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -210,8 +210,8 @@ def owgksuv_ensemble( obs, fct = map(B.asarray, (obs, fct)) - if axis != -1: - fct = B.moveaxis(fct, axis, -1) + if m_axis != -1: + fct = B.moveaxis(fct, m_axis, -1) obs_weights, fct_weights = map(w_func, (obs, fct)) @@ -229,7 +229,7 @@ def vrgksuv_ensemble( fct: "Array", w_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, - axis: int = -1, + m_axis: int = -1, *, backend: "Backend" = None, ) -> "Array": @@ -259,7 +259,7 @@ def vrgksuv_ensemble( represented by the last axis. w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. - axis : int + m_axis : int The axis corresponding to the ensemble. Default is the last axis. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -283,8 +283,8 @@ def vrgksuv_ensemble( obs, fct = map(B.asarray, (obs, fct)) - if axis != -1: - fct = B.moveaxis(fct, axis, -1) + if m_axis != -1: + fct = B.moveaxis(fct, m_axis, -1) obs_weights, fct_weights = map(w_func, (obs, fct)) diff --git a/scoringrules/_logs.py b/scoringrules/_logs.py index deeb319..bccf5a6 100644 --- a/scoringrules/_logs.py +++ b/scoringrules/_logs.py @@ -11,7 +11,7 @@ def logs_ensemble( obs: "ArrayLike", fct: "Array", /, - axis: int = -1, + m_axis: int = -1, *, bw: "ArrayLike" = None, backend: "Backend" = None, @@ -32,7 +32,7 @@ def logs_ensemble( fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - axis : int + m_axis : int The axis corresponding to the ensemble. Default is the last axis. bw : array_like The bandwidth parameter for each forecast ensemble. If not given, estimated using @@ -53,19 +53,19 @@ def logs_ensemble( B = backends.active if backend is None else backends[backend] obs, fct = map(B.asarray, (obs, fct)) - if axis != -1: - fct = B.moveaxis(fct, axis, -1) + if m_axis != -1: + fct = B.moveaxis(fct, m_axis, -1) M = fct.shape[-1] # Silverman's rule of thumb for estimating the bandwidth parameter if bw is None: - sigmahat = B.std(fct, axis=-1) - q75 = B.quantile(fct, 0.75, axis=-1) - q25 = B.quantile(fct, 0.25, axis=-1) + sigmahat = B.std(fct, m_axis=-1) + q75 = B.quantile(fct, 0.75, m_axis=-1) + q25 = B.quantile(fct, 0.25, m_axis=-1) iqr = q75 - q25 bw = 1.06 * B.minimum(sigmahat, iqr / 1.34) * (M ** (-1 / 5)) - bw = B.stack([bw] * M, axis=-1) + bw = B.stack([bw] * M, m_axis=-1) w = B.zeros(fct.shape) + 1 / M @@ -78,7 +78,7 @@ def clogs_ensemble( /, a: "ArrayLike" = float("-inf"), b: "ArrayLike" = float("inf"), - axis: int = -1, + m_axis: int = -1, *, bw: "ArrayLike" = None, cens: bool = True, @@ -105,7 +105,7 @@ def clogs_ensemble( The lower bound in the weight function. b : array_like The upper bound in the weight function. - axis : int + m_axis : int The axis corresponding to the ensemble. Default is the last axis. bw : array_like The bandwidth parameter for each forecast ensemble. If not given, estimated using @@ -129,19 +129,19 @@ def clogs_ensemble( B = backends.active if backend is None else backends[backend] fct = B.asarray(fct) - if axis != -1: - fct = B.moveaxis(fct, axis, -1) + if m_axis != -1: + fct = B.moveaxis(fct, m_axis, -1) M = fct.shape[-1] # Silverman's rule of thumb for estimating the bandwidth parameter if bw is None: - sigmahat = B.std(fct, axis=-1) - q75 = B.quantile(fct, 0.75, axis=-1) - q25 = B.quantile(fct, 0.25, axis=-1) + sigmahat = B.std(fct, m_axis=-1) + q75 = B.quantile(fct, 0.75, m_axis=-1) + q25 = B.quantile(fct, 0.25, m_axis=-1) iqr = q75 - q25 bw = 1.06 * B.minimum(sigmahat, iqr / 1.34) * (M ** (-1 / 5)) - bw = B.stack([bw] * M, axis=-1) + bw = B.stack([bw] * M, m_axis=-1) return logarithmic.clogs_ensemble( obs, @@ -683,7 +683,7 @@ def logs_mixnorm( s: "ArrayLike", /, w: "ArrayLike" = None, - axis: "ArrayLike" = -1, + mc_axis: "ArrayLike" = -1, *, backend: "Backend" = None, ) -> "ArrayLike": @@ -701,7 +701,7 @@ def logs_mixnorm( Standard deviations of the component normal distributions. w : array_like Non-negative weights assigned to each component. - axis : int + mc_axis : int The axis corresponding to the mixture components. Default is the last axis. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -720,15 +720,15 @@ def logs_mixnorm( obs, m, s = map(B.asarray, (obs, m, s)) if w is None: - M: int = m.shape[axis] + M: int = m.shape[mc_axis] w = B.zeros(m.shape) + 1 / M else: w = B.asarray(w) - if axis != -1: - m = B.moveaxis(m, axis, -1) - s = B.moveaxis(s, axis, -1) - w = B.moveaxis(w, axis, -1) + if mc_axis != -1: + m = B.moveaxis(m, mc_axis, -1) + s = B.moveaxis(s, mc_axis, -1) + w = B.moveaxis(w, mc_axis, -1) return logarithmic.mixnorm(obs, m, s, w, backend=backend) diff --git a/tests/test_brier.py b/tests/test_brier.py index 55291db..f92d6e1 100644 --- a/tests/test_brier.py +++ b/tests/test_brier.py @@ -50,7 +50,7 @@ def test_rps(backend): ] res1 = sr.rps_score(obs, fct, backend=backend) fct = np.transpose(fct) - res2 = sr.rps_score(obs, fct, axis=0, backend=backend) + res2 = sr.rps_score(obs, fct, k_axis=0, backend=backend) assert np.allclose(res1, res2) @@ -100,6 +100,6 @@ def test_rls(backend): res1 = sr.rls_score(obs, fct, backend=backend) fct = np.transpose(fct) - res2 = sr.rls_score(obs, fct, axis=0, backend=backend) + res2 = sr.rls_score(obs, fct, k_axis=0, backend=backend) assert np.allclose(res1, res2) diff --git a/tests/test_crps.py b/tests/test_crps.py index 5fb7447..5989c8c 100644 --- a/tests/test_crps.py +++ b/tests/test_crps.py @@ -36,7 +36,7 @@ def test_crps_ensemble(estimator, backend): res = sr.crps_ensemble( obs, np.random.randn(ENSEMBLE_SIZE, N), - axis=0, + m_axis=0, estimator=estimator, backend=backend, ) @@ -64,7 +64,7 @@ def test_crps_quantile(backend): assert res.shape == (N,) fct = np.random.randn(ENSEMBLE_SIZE, N) res = sr.crps_quantile( - obs, np.random.randn(ENSEMBLE_SIZE, N), alpha, axis=0, backend=backend + obs, np.random.randn(ENSEMBLE_SIZE, N), alpha, m_axis=0, backend=backend ) assert res.shape == (N,) @@ -563,7 +563,7 @@ def test_crps_mixnorm(backend): obs = [-1.6, 0.3] m = [[0.0, -2.9], [0.6, 0.0], [-1.1, -2.3]] s = [[0.5, 1.7], [1.1, 0.7], [1.4, 1.5]] - res1 = sr.crps_mixnorm(obs, m, s, axis=0, backend=backend) + res1 = sr.crps_mixnorm(obs, m, s, m_axis=0, backend=backend) m = [[0.0, 0.6, -1.1], [-2.9, 0.0, -2.3]] s = [[0.5, 1.1, 1.4], [1.7, 0.7, 1.5]] diff --git a/tests/test_error_spread.py b/tests/test_error_spread.py index 5dd4f55..051ba1f 100644 --- a/tests/test_error_spread.py +++ b/tests/test_error_spread.py @@ -18,7 +18,7 @@ def test_error_spread_score(backend): obs = np.random.randn(N) fct = np.random.randn(ENSEMBLE_SIZE, N) - res = sr.error_spread_score(obs, fct, axis=0, backend=backend) + res = sr.error_spread_score(obs, fct, m_axis=0, backend=backend) assert res.shape == (N,) # test correctness diff --git a/tests/test_logs.py b/tests/test_logs.py index c30b1e7..af32aba 100644 --- a/tests/test_logs.py +++ b/tests/test_logs.py @@ -16,11 +16,11 @@ def test_ensemble(backend): sigma = abs(np.random.randn(N)) fct = np.random.randn(N, ENSEMBLE_SIZE) * sigma[..., None] + mu[..., None] - res = sr.logs_ensemble(obs, fct, axis=-1, backend=backend) + res = sr.logs_ensemble(obs, fct, m_axis=-1, backend=backend) assert res.shape == (N,) fct = fct.T - res0 = sr.logs_ensemble(obs, fct, axis=0, backend=backend) + res0 = sr.logs_ensemble(obs, fct, m_axis=0, backend=backend) assert np.allclose(res, res0) obs, fct = 6.2, [4.2, 5.1, 6.1, 7.6, 8.3, 9.5] @@ -41,16 +41,16 @@ def test_clogs(backend): sigma = abs(np.random.randn(N)) fct = np.random.randn(N, ENSEMBLE_SIZE) * sigma[..., None] + mu[..., None] - res0 = sr.logs_ensemble(obs, fct, axis=-1, backend=backend) - res = sr.clogs_ensemble(obs, fct, axis=-1, backend=backend) - res_co = sr.clogs_ensemble(obs, fct, axis=-1, cens=False, backend=backend) + res0 = sr.logs_ensemble(obs, fct, m_axis=-1, backend=backend) + res = sr.clogs_ensemble(obs, fct, m_axis=-1, backend=backend) + res_co = sr.clogs_ensemble(obs, fct, m_axis=-1, cens=False, backend=backend) assert res.shape == (N,) assert res_co.shape == (N,) assert np.allclose(res, res0, atol=1e-5) assert np.allclose(res_co, res0, atol=1e-5) fct = fct.T - res0 = sr.clogs_ensemble(obs, fct, axis=0, backend=backend) + res0 = sr.clogs_ensemble(obs, fct, m_axis=0, backend=backend) assert np.allclose(res, res0, atol=1e-5) obs, fct = 6.2, [4.2, 5.1, 6.1, 7.6, 8.3, 9.5] @@ -368,7 +368,7 @@ def test_mixnorm(backend): obs = [-1.6, 0.3] m = [[0.0, -2.9], [0.6, 0.0], [-1.1, -2.3]] s = [[0.5, 1.7], [1.1, 0.7], [1.4, 1.5]] - res1 = sr.logs_mixnorm(obs, m, s, axis=0, backend=backend) + res1 = sr.logs_mixnorm(obs, m, s, mc_axis=0, backend=backend) m = [[0.0, 0.6, -1.1], [-2.9, 0.0, -2.3]] s = [[0.5, 1.1, 1.4], [1.7, 0.7, 1.5]] diff --git a/tests/test_wcrps.py b/tests/test_wcrps.py index a29dc50..9f3d86a 100644 --- a/tests/test_wcrps.py +++ b/tests/test_wcrps.py @@ -20,7 +20,7 @@ def test_owcrps_ensemble(backend): res = sr.owcrps_ensemble(obs, np.random.randn(N, M), lambda x: x * 0.0 + 1.0) assert res.shape == (N,) res = sr.owcrps_ensemble( - obs, np.random.randn(M, N), lambda x: x * 0.0 + 1.0, axis=0 + obs, np.random.randn(M, N), lambda x: x * 0.0 + 1.0, m_axis=0 ) assert res.shape == (N,) @@ -37,7 +37,7 @@ def test_vrcrps_ensemble(backend): res = sr.vrcrps_ensemble(obs, np.random.randn(N, M), lambda x: x * 0.0 + 1.0) assert res.shape == (N,) res = sr.vrcrps_ensemble( - obs, np.random.randn(M, N), lambda x: x * 0.0 + 1.0, axis=0 + obs, np.random.randn(M, N), lambda x: x * 0.0 + 1.0, m_axis=0 ) assert res.shape == (N,) From 2e37351023f4ea9ec825159ed7a5c20619156db0 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Mon, 14 Apr 2025 15:46:11 +0200 Subject: [PATCH 27/79] fix m_axis bug --- scoringrules/_crps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index 33fb5ed..79ad571 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -108,7 +108,7 @@ def crps_ensemble( "akr_circperm", "fair", ]: - fct = B.sort(fct, m_axis=-1) + fct = B.sort(fct, axis=-1) if backend == "numba": if estimator not in crps.estimator_gufuncs: From 40fad39cc3fecbbf5f0a707a77a017ee86dcd746 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Mon, 14 Apr 2025 16:00:36 +0200 Subject: [PATCH 28/79] change test_kernels to use m_axis argument --- tests/test_kernels.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_kernels.py b/tests/test_kernels.py index 92bbb4a..e1c4629 100644 --- a/tests/test_kernels.py +++ b/tests/test_kernels.py @@ -26,11 +26,11 @@ def test_gksuv(estimator, backend): res = np.asarray(res) assert not np.any(res < 0.0) - # axis keyword + # m_axis keyword res = sr.gksuv_ensemble( obs, np.random.randn(ENSEMBLE_SIZE, N), - axis=0, + m_axis=0, estimator=estimator, backend=backend, ) @@ -241,12 +241,12 @@ def test_owgksuv(backend): sigma = abs(np.random.randn(N)) * 0.3 fct = np.random.randn(N, ENSEMBLE_SIZE) * sigma[..., None] + mu[..., None] - # axis keyword + # m_axis keyword res = sr.owgksuv_ensemble( obs, np.random.randn(ENSEMBLE_SIZE, N), lambda x: x * 0.0 + 1.0, - axis=0, + m_axis=0, backend=backend, ) res = np.asarray(res) @@ -340,12 +340,12 @@ def test_vrgksuv(backend): sigma = abs(np.random.randn(N)) * 0.3 fct = np.random.randn(N, ENSEMBLE_SIZE) * sigma[..., None] + mu[..., None] - # axis keyword + # m_axis keyword res = sr.vrgksuv_ensemble( obs, np.random.randn(ENSEMBLE_SIZE, N), lambda x: x * 0.0 + 1.0, - axis=0, + m_axis=0, backend=backend, ) res = np.asarray(res) From a79024fa83757ee045b7323030e05419c6f7aff1 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Mon, 14 Apr 2025 16:04:13 +0200 Subject: [PATCH 29/79] fix m_axis bug in logs_ensemble --- scoringrules/_logs.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scoringrules/_logs.py b/scoringrules/_logs.py index bccf5a6..6b5f96d 100644 --- a/scoringrules/_logs.py +++ b/scoringrules/_logs.py @@ -60,12 +60,12 @@ def logs_ensemble( # Silverman's rule of thumb for estimating the bandwidth parameter if bw is None: - sigmahat = B.std(fct, m_axis=-1) - q75 = B.quantile(fct, 0.75, m_axis=-1) - q25 = B.quantile(fct, 0.25, m_axis=-1) + sigmahat = B.std(fct, axis=-1) + q75 = B.quantile(fct, 0.75, axis=-1) + q25 = B.quantile(fct, 0.25, axis=-1) iqr = q75 - q25 bw = 1.06 * B.minimum(sigmahat, iqr / 1.34) * (M ** (-1 / 5)) - bw = B.stack([bw] * M, m_axis=-1) + bw = B.stack([bw] * M, axis=-1) w = B.zeros(fct.shape) + 1 / M @@ -136,12 +136,12 @@ def clogs_ensemble( # Silverman's rule of thumb for estimating the bandwidth parameter if bw is None: - sigmahat = B.std(fct, m_axis=-1) - q75 = B.quantile(fct, 0.75, m_axis=-1) - q25 = B.quantile(fct, 0.25, m_axis=-1) + sigmahat = B.std(fct, axis=-1) + q75 = B.quantile(fct, 0.75, axis=-1) + q25 = B.quantile(fct, 0.25, axis=-1) iqr = q75 - q25 bw = 1.06 * B.minimum(sigmahat, iqr / 1.34) * (M ** (-1 / 5)) - bw = B.stack([bw] * M, m_axis=-1) + bw = B.stack([bw] * M, axis=-1) return logarithmic.clogs_ensemble( obs, From c181cf3cbab8e80ddf2d665d3fd60a3de93a6d75 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Mon, 14 Apr 2025 16:11:20 +0200 Subject: [PATCH 30/79] increase N in crps ensemble tests --- tests/test_crps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_crps.py b/tests/test_crps.py index 5989c8c..390a314 100644 --- a/tests/test_crps.py +++ b/tests/test_crps.py @@ -6,7 +6,7 @@ from .conftest import BACKENDS ENSEMBLE_SIZE = 11 -N = 20 +N = 100 ESTIMATORS = ["nrg", "fair", "pwm", "int", "qd", "akr", "akr_circperm"] From ade193ea66d5d75e43f0b50a390c30019a1b1700 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Mon, 14 Apr 2025 16:14:44 +0200 Subject: [PATCH 31/79] add exception in crps ensemble tests for akr approximations --- tests/test_crps.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_crps.py b/tests/test_crps.py index 390a314..d61672b 100644 --- a/tests/test_crps.py +++ b/tests/test_crps.py @@ -6,7 +6,7 @@ from .conftest import BACKENDS ENSEMBLE_SIZE = 11 -N = 100 +N = 20 ESTIMATORS = ["nrg", "fair", "pwm", "int", "qd", "akr", "akr_circperm"] @@ -43,9 +43,10 @@ def test_crps_ensemble(estimator, backend): assert res.shape == (N,) # non-negative values - res = sr.crps_ensemble(obs, fct, estimator=estimator, backend=backend) - res = np.asarray(res) - assert not np.any(res < 0.0) + if estimator not in ["akr", "akr_circperm"]: + res = sr.crps_ensemble(obs, fct, estimator=estimator, backend=backend) + res = np.asarray(res) + assert not np.any(res < 0.0) # approx zero when perfect forecast perfect_fct = obs[..., None] + np.random.randn(N, ENSEMBLE_SIZE) * 0.00001 From 1db9052913325ab79d36c3be94930890de03554b Mon Sep 17 00:00:00 2001 From: sallen12 Date: Mon, 14 Apr 2025 18:12:50 +0200 Subject: [PATCH 32/79] change energy score function name --- scoringrules/__init__.py | 16 ++++++++-------- scoringrules/_energy.py | 20 ++++++++++---------- scoringrules/core/energy/__init__.py | 6 +++--- scoringrules/core/energy/_score.py | 6 +++--- tests/test_energy.py | 6 +++--- tests/test_wenergy.py | 20 ++++++++++---------- 6 files changed, 37 insertions(+), 37 deletions(-) diff --git a/scoringrules/__init__.py b/scoringrules/__init__.py index 4e2725b..1c8f62b 100644 --- a/scoringrules/__init__.py +++ b/scoringrules/__init__.py @@ -44,10 +44,10 @@ vrcrps_ensemble, ) from scoringrules._energy import ( - energy_score, - owenergy_score, - twenergy_score, - vrenergy_score, + es_ensemble, + owes_ensemble, + twes_ensemble, + vres_ensemble, ) from scoringrules._error_spread import error_spread_score from scoringrules._interval import interval_score, weighted_interval_score @@ -172,10 +172,10 @@ "log_score", "rls_score", "error_spread_score", - "energy_score", - "owenergy_score", - "twenergy_score", - "vrenergy_score", + "es_ensemble", + "owes_ensemble", + "twes_ensemble", + "vres_ensemble", "variogram_score", "owvariogram_score", "twvariogram_score", diff --git a/scoringrules/_energy.py b/scoringrules/_energy.py index f96f20b..b4fd990 100644 --- a/scoringrules/_energy.py +++ b/scoringrules/_energy.py @@ -8,7 +8,7 @@ from scoringrules.core.typing import Array, ArrayLike, Backend -def energy_score( +def es_ensemble( obs: "Array", fct: "Array", /, @@ -44,12 +44,12 @@ def energy_score( Returns ------- - energy_score : array_like + es_ensemble : array_like The computed Energy Score. See Also -------- - twenergy_score, owenergy_score, vrenergy_score + twes_ensemble, owes_ensemble, vres_ensemble Weighted variants of the Energy Score. crps_ensemble The univariate equivalent of the Energy Score. @@ -68,7 +68,7 @@ def energy_score( return energy.nrg(obs, fct, backend=backend) -def twenergy_score( +def twes_ensemble( obs: "Array", fct: "Array", v_func: tp.Callable[["ArrayLike"], "ArrayLike"], @@ -110,14 +110,14 @@ def twenergy_score( Returns ------- - twenergy_score : array_like + twes_ensemble : array_like The computed Threshold-Weighted Energy Score. """ obs, fct = map(v_func, (obs, fct)) - return energy_score(obs, fct, m_axis=m_axis, v_axis=v_axis, backend=backend) + return es_ensemble(obs, fct, m_axis=m_axis, v_axis=v_axis, backend=backend) -def owenergy_score( +def owes_ensemble( obs: "Array", fct: "Array", w_func: tp.Callable[["ArrayLike"], "ArrayLike"], @@ -162,7 +162,7 @@ def owenergy_score( Returns ------- - owenergy_score : array_like + owes_ensemble : array_like The computed Outcome-Weighted Energy Score. """ B = backends.active if backend is None else backends[backend] @@ -178,7 +178,7 @@ def owenergy_score( return energy.ownrg(obs, fct, obs_weights, fct_weights, backend=backend) -def vrenergy_score( +def vres_ensemble( obs: "Array", fct: "Array", w_func: tp.Callable[["ArrayLike"], "ArrayLike"], @@ -224,7 +224,7 @@ def vrenergy_score( Returns ------- - vrenergy_score : array_like + vres_ensemble : array_like The computed Vertically Re-scaled Energy Score. """ B = backends.active if backend is None else backends[backend] diff --git a/scoringrules/core/energy/__init__.py b/scoringrules/core/energy/__init__.py index 4fad1be..b9bc987 100644 --- a/scoringrules/core/energy/__init__.py +++ b/scoringrules/core/energy/__init__.py @@ -9,9 +9,9 @@ _owenergy_score_gufunc = None _vrenergy_score_gufunc = None -from ._score import energy_score as nrg -from ._score import owenergy_score as ownrg -from ._score import vrenergy_score as vrnrg +from ._score import es_ensemble as nrg +from ._score import owes_ensemble as ownrg +from ._score import vres_ensemble as vrnrg __all__ = [ "nrg", diff --git a/scoringrules/core/energy/_score.py b/scoringrules/core/energy/_score.py index 3a05419..e264c4b 100644 --- a/scoringrules/core/energy/_score.py +++ b/scoringrules/core/energy/_score.py @@ -6,7 +6,7 @@ from scoringrules.core.typing import Array, Backend -def energy_score(obs: "Array", fct: "Array", backend=None) -> "Array": +def es_ensemble(obs: "Array", fct: "Array", backend=None) -> "Array": """ Compute the energy score based on a finite ensemble. @@ -23,7 +23,7 @@ def energy_score(obs: "Array", fct: "Array", backend=None) -> "Array": return E_1 - 0.5 * E_2 -def owenergy_score( +def owes_ensemble( obs: "Array", # (... D) fct: "Array", # (... M D) ow: "Array", # (...) @@ -48,7 +48,7 @@ def owenergy_score( return E_1 - 0.5 * E_2 -def vrenergy_score( +def vres_ensemble( obs: "Array", fct: "Array", ow: "Array", diff --git a/tests/test_energy.py b/tests/test_energy.py index a4728a6..90c74d9 100644 --- a/tests/test_energy.py +++ b/tests/test_energy.py @@ -15,12 +15,12 @@ def test_energy_score(backend): with pytest.raises(ValueError): obs = np.random.randn(N, N_VARS) fct = np.random.randn(N, ENSEMBLE_SIZE, N_VARS - 1) - sr.energy_score(obs, fct, backend=backend) + sr.es_ensemble(obs, fct, backend=backend) # test shapes obs = np.random.randn(N, N_VARS) fct = np.expand_dims(obs, axis=-2) + np.random.randn(N, ENSEMBLE_SIZE, N_VARS) - res = sr.energy_score(obs, fct, backend=backend) + res = sr.es_ensemble(obs, fct, backend=backend) assert res.shape == (N,) # test correctness @@ -28,6 +28,6 @@ def test_energy_score(backend): [[0.79546742, 0.4777960, 0.2164079], [0.02461368, 0.7584595, 0.3181810]] ).T obs = np.array([0.2743836, 0.8146400]) - res = sr.energy_score(obs, fct, backend=backend) + res = sr.es_ensemble(obs, fct, backend=backend) expected = 0.334542 # TODO: test this against scoringRules np.testing.assert_allclose(res, expected, atol=1e-6) diff --git a/tests/test_wenergy.py b/tests/test_wenergy.py index ba4614d..5dca3fa 100644 --- a/tests/test_wenergy.py +++ b/tests/test_wenergy.py @@ -16,8 +16,8 @@ def test_owes_vs_es(backend): obs = np.random.randn(N, N_VARS) fct = np.expand_dims(obs, axis=-2) + np.random.randn(N, ENSEMBLE_SIZE, N_VARS) - res = sr.energy_score(obs, fct, backend=backend) - resw = sr.owenergy_score( + res = sr.es_ensemble(obs, fct, backend=backend) + resw = sr.owes_ensemble( obs, fct, lambda x: backends[backend].mean(x) * 0.0 + 1.0, @@ -31,8 +31,8 @@ def test_twes_vs_es(backend): obs = np.random.randn(N, N_VARS) fct = np.expand_dims(obs, axis=-2) + np.random.randn(N, ENSEMBLE_SIZE, N_VARS) - res = sr.energy_score(obs, fct, backend=backend) - resw = sr.twenergy_score(obs, fct, lambda x: x, backend=backend) + res = sr.es_ensemble(obs, fct, backend=backend) + resw = sr.twes_ensemble(obs, fct, lambda x: x, backend=backend) np.testing.assert_allclose(res, resw, rtol=1e-10) @@ -41,8 +41,8 @@ def test_vres_vs_es(backend): obs = np.random.randn(N, N_VARS) fct = np.expand_dims(obs, axis=-2) + np.random.randn(N, ENSEMBLE_SIZE, N_VARS) - res = sr.energy_score(obs, fct, backend=backend) - resw = sr.vrenergy_score( + res = sr.es_ensemble(obs, fct, backend=backend) + resw = sr.vres_ensemble( obs, fct, lambda x: backends[backend].mean(x) * 0.0 + 1.0, @@ -61,13 +61,13 @@ def test_owenergy_score_correctness(backend): def w_func(x): return backends[backend].all(x > 0.2) - res = sr.owenergy_score(obs, fct, w_func, backend=backend) + res = sr.owes_ensemble(obs, fct, w_func, backend=backend) np.testing.assert_allclose(res, 0.2274243, rtol=1e-6) def w_func(x): return backends[backend].all(x < 1.0) - res = sr.owenergy_score(obs, fct, w_func, backend=backend) + res = sr.owes_ensemble(obs, fct, w_func, backend=backend) np.testing.assert_allclose(res, 0.3345418, rtol=1e-6) @@ -81,11 +81,11 @@ def test_twenergy_score_correctness(backend): def v_func(x): return np.maximum(x, 0.2) - res = sr.twenergy_score(obs, fct, v_func, backend=backend) + res = sr.twes_ensemble(obs, fct, v_func, backend=backend) np.testing.assert_allclose(res, 0.3116075, rtol=1e-6) def v_func(x): return np.minimum(x, 1) - res = sr.twenergy_score(obs, fct, v_func, backend=backend) + res = sr.twes_ensemble(obs, fct, v_func, backend=backend) np.testing.assert_allclose(res, 0.3345418, rtol=1e-6) From e919a36611fc9f91009ff637fd8507b5e4567fbf Mon Sep 17 00:00:00 2001 From: sallen12 Date: Mon, 14 Apr 2025 18:19:04 +0200 Subject: [PATCH 33/79] change variogram score function name --- scoringrules/__init__.py | 16 +++++++-------- scoringrules/_variogram.py | 26 ++++++++++++------------- scoringrules/core/variogram/__init__.py | 6 +++--- scoringrules/core/variogram/_score.py | 6 +++--- tests/test_variogram.py | 8 ++++---- tests/test_wvariogram.py | 20 +++++++++---------- 6 files changed, 41 insertions(+), 41 deletions(-) diff --git a/scoringrules/__init__.py b/scoringrules/__init__.py index 1c8f62b..0213df4 100644 --- a/scoringrules/__init__.py +++ b/scoringrules/__init__.py @@ -80,10 +80,10 @@ clogs_ensemble, ) from scoringrules._variogram import ( - owvariogram_score, - twvariogram_score, - variogram_score, - vrvariogram_score, + owvs_ensemble, + twvs_ensemble, + vs_ensemble, + vrvs_ensemble, ) from scoringrules._kernels import ( gksuv_ensemble, @@ -176,10 +176,10 @@ "owes_ensemble", "twes_ensemble", "vres_ensemble", - "variogram_score", - "owvariogram_score", - "twvariogram_score", - "vrvariogram_score", + "vs_ensemble", + "owvs_ensemble", + "twvs_ensemble", + "vrvs_ensemble", "gksuv_ensemble", "twgksuv_ensemble", "owgksuv_ensemble", diff --git a/scoringrules/_variogram.py b/scoringrules/_variogram.py index 1ec34d3..b60db8e 100644 --- a/scoringrules/_variogram.py +++ b/scoringrules/_variogram.py @@ -8,7 +8,7 @@ from scoringrules.core.typing import Array, Backend -def variogram_score( +def vs_ensemble( obs: "Array", fct: "Array", /, @@ -47,7 +47,7 @@ def variogram_score( Returns ------- - variogram_score : array_like + vs_ensemble : array_like The computed Variogram Score. References @@ -63,7 +63,7 @@ def variogram_score( >>> rng = np.random.default_rng(123) >>> obs = rng.normal(size=(3, 5)) >>> fct = rng.normal(size=(3, 10, 5)) - >>> sr.variogram_score(obs, fct) + >>> sr.vs_ensemble(obs, fct) array([ 8.65630139, 6.84693866, 19.52993307]) """ obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) @@ -74,7 +74,7 @@ def variogram_score( return variogram.vs(obs, fct, p, backend=backend) -def twvariogram_score( +def twvs_ensemble( obs: "Array", fct: "Array", v_func: tp.Callable, @@ -116,7 +116,7 @@ def twvariogram_score( Returns ------- - twvariogram_score : array_like + twvs_ensemble : array_like The computed Threshold-Weighted Variogram Score. References @@ -133,14 +133,14 @@ def twvariogram_score( >>> rng = np.random.default_rng(123) >>> obs = rng.normal(size=(3, 5)) >>> fct = rng.normal(size=(3, 10, 5)) - >>> sr.twvariogram_score(obs, fct, lambda x: np.maximum(x, -0.2)) + >>> sr.twvs_ensemble(obs, fct, lambda x: np.maximum(x, -0.2)) array([5.94996894, 4.72029765, 6.08947229]) """ obs, fct = map(v_func, (obs, fct)) - return variogram_score(obs, fct, m_axis, v_axis, p=p, backend=backend) + return vs_ensemble(obs, fct, m_axis, v_axis, p=p, backend=backend) -def owvariogram_score( +def owvs_ensemble( obs: "Array", fct: "Array", w_func: tp.Callable, @@ -187,7 +187,7 @@ def owvariogram_score( Returns ------- - owvariogram_score : array_like + owvs_ensemble : array_like The computed Outcome-Weighted Variogram Score. Examples @@ -197,7 +197,7 @@ def owvariogram_score( >>> rng = np.random.default_rng(123) >>> obs = rng.normal(size=(3, 5)) >>> fct = rng.normal(size=(3, 10, 5)) - >>> sr.owvariogram_score(obs, fct, lambda x: x.mean() + 1.0) + >>> sr.owvs_ensemble(obs, fct, lambda x: x.mean() + 1.0) array([ 9.86816636, 6.75532522, 19.59353723]) """ B = backends.active if backend is None else backends[backend] @@ -215,7 +215,7 @@ def owvariogram_score( return variogram.owvs(obs, fct, obs_weights, fct_weights, p=p, backend=backend) -def vrvariogram_score( +def vrvs_ensemble( obs: "Array", fct: "Array", w_func: tp.Callable, @@ -264,7 +264,7 @@ def vrvariogram_score( Returns ------- - vrvariogram_score : array_like + vrvs_ensemble : array_like The computed Vertically Re-scaled Variogram Score. Examples @@ -274,7 +274,7 @@ def vrvariogram_score( >>> rng = np.random.default_rng(123) >>> obs = rng.normal(size=(3, 5)) >>> fct = rng.normal(size=(3, 10, 5)) - >>> sr.vrvariogram_score(obs, fct, lambda x: x.max() + 1.0) + >>> sr.vrvs_ensemble(obs, fct, lambda x: x.max() + 1.0) array([46.48256493, 57.90759816, 92.37153472]) """ B = backends.active if backend is None else backends[backend] diff --git a/scoringrules/core/variogram/__init__.py b/scoringrules/core/variogram/__init__.py index 2006d13..1e74527 100644 --- a/scoringrules/core/variogram/__init__.py +++ b/scoringrules/core/variogram/__init__.py @@ -9,9 +9,9 @@ _variogram_score_gufunc = None _vrvariogram_score_gufunc = None -from ._score import owvariogram_score as owvs -from ._score import variogram_score as vs -from ._score import vrvariogram_score as vrvs +from ._score import owvs_ensemble as owvs +from ._score import vs_ensemble as vs +from ._score import vrvs_ensemble as vrvs __all__ = [ "vs", diff --git a/scoringrules/core/variogram/_score.py b/scoringrules/core/variogram/_score.py index bafde56..8850cb5 100644 --- a/scoringrules/core/variogram/_score.py +++ b/scoringrules/core/variogram/_score.py @@ -6,7 +6,7 @@ from scoringrules.core.typing import Array, Backend -def variogram_score( +def vs_ensemble( obs: "Array", # (... D) fct: "Array", # (... M D) p: float = 1, @@ -22,7 +22,7 @@ def variogram_score( return B.sum((vobs - vfct) ** 2, axis=(-2, -1)) # (...) -def owvariogram_score( +def owvs_ensemble( obs: "Array", fct: "Array", ow: "Array", @@ -57,7 +57,7 @@ def owvariogram_score( return E_1 - 0.5 * E_2 -def vrvariogram_score( +def vrvs_ensemble( obs: "Array", fct: "Array", ow: "Array", diff --git a/tests/test_variogram.py b/tests/test_variogram.py index dcee9ae..9e471ec 100644 --- a/tests/test_variogram.py +++ b/tests/test_variogram.py @@ -16,7 +16,7 @@ def test_variogram_score(backend): obs = np.random.randn(N, N_VARS) fct = np.expand_dims(obs, axis=-2) + np.random.randn(N, ENSEMBLE_SIZE, N_VARS) - res = sr.variogram_score(obs, fct, backend=backend) + res = sr.vs_ensemble(obs, fct, backend=backend) if backend in ["numpy", "numba"]: assert isinstance(res, np.ndarray) @@ -29,7 +29,7 @@ def test_variogram_score_permuted_dims(backend): obs = np.random.randn(N, N_VARS) fct = np.expand_dims(obs, axis=-2) + np.random.randn(N, ENSEMBLE_SIZE, N_VARS) - res = sr.variogram_score(obs, fct, v_axis=-1, m_axis=-2, backend=backend) + res = sr.vs_ensemble(obs, fct, v_axis=-1, m_axis=-2, backend=backend) if backend in ["numpy", "numba"]: assert isinstance(res, np.ndarray) @@ -45,8 +45,8 @@ def test_variogram_score_correctness(backend): obs = np.array([0.2743836, 0.8146400]) - res = sr.variogram_score(obs, fct.T, p=0.5, backend=backend) + res = sr.vs_ensemble(obs, fct.T, p=0.5, backend=backend) np.testing.assert_allclose(res, 0.05083489, rtol=1e-5) - res = sr.variogram_score(obs, fct.T, p=1.0, backend=backend) + res = sr.vs_ensemble(obs, fct.T, p=1.0, backend=backend) np.testing.assert_allclose(res, 0.04856365, rtol=1e-5) diff --git a/tests/test_wvariogram.py b/tests/test_wvariogram.py index 0a3f47f..688db83 100644 --- a/tests/test_wvariogram.py +++ b/tests/test_wvariogram.py @@ -16,8 +16,8 @@ def test_owvs_vs_vs(backend): obs = np.random.randn(N, N_VARS) fct = np.expand_dims(obs, axis=-2) + np.random.randn(N, ENSEMBLE_SIZE, N_VARS) - res = sr.variogram_score(obs, fct, backend=backend) - resw = sr.owvariogram_score( + res = sr.vs_ensemble(obs, fct, backend=backend) + resw = sr.owvs_ensemble( obs, fct, lambda x: backends[backend].mean(x) * 0.0 + 1.0, @@ -33,8 +33,8 @@ def test_twvs_vs_vs(backend): obs = np.random.randn(N, N_VARS) fct = np.expand_dims(obs, axis=-2) + np.random.randn(N, ENSEMBLE_SIZE, N_VARS) - res = sr.variogram_score(obs, fct, backend=backend) - resw = sr.twvariogram_score(obs, fct, lambda x: x, backend=backend) + res = sr.vs_ensemble(obs, fct, backend=backend) + resw = sr.twvs_ensemble(obs, fct, lambda x: x, backend=backend) np.testing.assert_allclose(res, resw, rtol=5e-4) @@ -43,8 +43,8 @@ def test_vrvs_vs_vs(backend): obs = np.random.randn(N, N_VARS) fct = np.expand_dims(obs, axis=-2) + np.random.randn(N, ENSEMBLE_SIZE, N_VARS) - res = sr.variogram_score(obs, fct, backend=backend) - resw = sr.vrvariogram_score( + res = sr.vs_ensemble(obs, fct, backend=backend) + resw = sr.vrvs_ensemble( obs, fct, lambda x: backends[backend].mean(x) * 0.0 + 1.0, @@ -65,13 +65,13 @@ def w_func(x): backends[backend].all(x > 0.2) + 0.0 ) # + 0.0 works to convert to float in every backend - res = sr.owvariogram_score(obs, fct, w_func, p=0.5, backend=backend) + res = sr.owvs_ensemble(obs, fct, w_func, p=0.5, backend=backend) np.testing.assert_allclose(res, 0.1929739, rtol=1e-6) def w_func(x): return backends[backend].all(x < 1.0) + 0.0 - res = sr.owvariogram_score(obs, fct, w_func, p=1.0, backend=backend) + res = sr.owvs_ensemble(obs, fct, w_func, p=1.0, backend=backend) np.testing.assert_allclose(res, 0.04856366, rtol=1e-6) @@ -85,11 +85,11 @@ def test_twvariogram_score_correctness(backend): def v_func(x): return np.maximum(x, 0.2) - res = sr.twvariogram_score(obs, fct, v_func, p=0.5, backend=backend) + res = sr.twvs_ensemble(obs, fct, v_func, p=0.5, backend=backend) np.testing.assert_allclose(res, 0.07594679, rtol=1e-6) def v_func(x): return np.minimum(x, 1.0) - res = sr.twvariogram_score(obs, fct, v_func, p=1.0, backend=backend) + res = sr.twvs_ensemble(obs, fct, v_func, p=1.0, backend=backend) np.testing.assert_allclose(res, 0.04856366, rtol=1e-6) From fb905e92ac15b50f11e401d26cc1716e496fc5f4 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Mon, 14 Apr 2025 19:10:24 +0200 Subject: [PATCH 34/79] add default threshold arguments to twcrps_ensemble --- scoringrules/_crps.py | 23 +++++++++++++++++++---- tests/test_wcrps.py | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index 79ad571..e8a2b13 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -124,10 +124,12 @@ def crps_ensemble( def twcrps_ensemble( obs: "ArrayLike", fct: "Array", - v_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, + a: "ArrayLike" = float("-inf"), + b: "ArrayLike" = float("inf"), m_axis: int = -1, *, + v_func: tp.Callable[["ArrayLike"], "ArrayLike"] = None, estimator: str = "pwm", sorted_ensemble: bool = False, backend: "Backend" = None, @@ -152,12 +154,18 @@ def twcrps_ensemble( fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. + a : array_like + The lower bound(s) to be used in the default weight function that restricts attention + to values in the range [a, b]. + b : array_like + The upper bound(s) to be used in the default weight function that restricts attention + to values in the range [a, b]. + m_axis : int + The axis corresponding to the ensemble. Default is the last axis. v_func : callable, array_like -> array_like Chaining function used to emphasise particular outcomes. For example, a function that only considers values above a certain threshold :math:`t` by projecting forecasts and observations to :math:`[t, \inf)`. - m_axis : int - The axis corresponding to the ensemble. Default is the last axis. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. @@ -193,9 +201,16 @@ def twcrps_ensemble( ... >>> obs = rng.normal(size=3) >>> fct = rng.normal(size=(3, 10)) - >>> sr.twcrps_ensemble(obs, fct, v_func) + >>> sr.twcrps_ensemble(obs, fct, v_func=v_func) array([0.69605316, 0.32865417, 0.39048665]) """ + if v_func is None: + B = backends.active if backend is None else backends[backend] + a, b, obs, fct = map(B.asarray, (a, b, obs, fct)) + + def v_func(x): + return B.minimum(B.maximum(x, a), b) + obs, fct = map(v_func, (obs, fct)) return crps_ensemble( obs, diff --git a/tests/test_wcrps.py b/tests/test_wcrps.py index 9f3d86a..f790e53 100644 --- a/tests/test_wcrps.py +++ b/tests/test_wcrps.py @@ -50,7 +50,21 @@ def test_twcrps_vs_crps(backend): fct = np.random.randn(N, M) * sigma[..., None] + mu[..., None] res = sr.crps_ensemble(obs, fct, backend=backend, estimator="nrg") - resw = sr.twcrps_ensemble(obs, fct, lambda x: x, estimator="nrg", backend=backend) + + # no argument given + resw = sr.twcrps_ensemble(obs, fct, estimator="nrg", backend=backend) + np.testing.assert_allclose(res, resw, rtol=1e-10) + + # a and b + resw = sr.twcrps_ensemble( + obs, fct, a=float("-inf"), b=float("inf"), estimator="nrg", backend=backend + ) + np.testing.assert_allclose(res, resw, rtol=1e-10) + + # v_func as identity function + resw = sr.twcrps_ensemble( + obs, fct, v_func=lambda x: x, estimator="nrg", backend=backend + ) np.testing.assert_allclose(res, resw, rtol=1e-10) @@ -160,7 +174,16 @@ def v_func(x): res = np.mean( np.float64( - sr.twcrps_ensemble(obs, fct, v_func, estimator="nrg", backend=backend) + sr.twcrps_ensemble( + obs, fct, v_func=v_func, estimator="nrg", backend=backend + ) + ) + ) + np.testing.assert_allclose(res, 0.09489662, rtol=1e-6) + + res = np.mean( + np.float64( + sr.twcrps_ensemble(obs, fct, a=-1.0, estimator="nrg", backend=backend) ) ) np.testing.assert_allclose(res, 0.09489662, rtol=1e-6) @@ -170,7 +193,16 @@ def v_func(x): res = np.mean( np.float64( - sr.twcrps_ensemble(obs, fct, v_func, estimator="nrg", backend=backend) + sr.twcrps_ensemble( + obs, fct, v_func=v_func, estimator="nrg", backend=backend + ) + ) + ) + np.testing.assert_allclose(res, 0.0994809, rtol=1e-6) + + res = np.mean( + np.float64( + sr.twcrps_ensemble(obs, fct, b=1.85, estimator="nrg", backend=backend) ) ) np.testing.assert_allclose(res, 0.0994809, rtol=1e-6) From c05fe6b4a808bfff77abaa5d1c9adf3552aa8437 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Mon, 14 Apr 2025 19:32:26 +0200 Subject: [PATCH 35/79] add default threshold arguments to owcrps_ensemble --- scoringrules/_crps.py | 34 ++++++++++++++++++++++++---------- tests/test_wcrps.py | 34 +++++++++++++++++++++++++++++----- 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index e8a2b13..38a9087 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -125,8 +125,8 @@ def twcrps_ensemble( obs: "ArrayLike", fct: "Array", /, - a: "ArrayLike" = float("-inf"), - b: "ArrayLike" = float("inf"), + a: float = float("-inf"), + b: float = float("inf"), m_axis: int = -1, *, v_func: tp.Callable[["ArrayLike"], "ArrayLike"] = None, @@ -154,11 +154,11 @@ def twcrps_ensemble( fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - a : array_like - The lower bound(s) to be used in the default weight function that restricts attention + a : float + The lower bound to be used in the default weight function that restricts attention to values in the range [a, b]. - b : array_like - The upper bound(s) to be used in the default weight function that restricts attention + b : float + The upper bound to be used in the default weight function that restricts attention to values in the range [a, b]. m_axis : int The axis corresponding to the ensemble. Default is the last axis. @@ -225,10 +225,12 @@ def v_func(x): def owcrps_ensemble( obs: "ArrayLike", fct: "Array", - w_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, + a: float = float("-inf"), + b: float = float("inf"), m_axis: int = -1, *, + w_func: tp.Callable[["ArrayLike"], "ArrayLike"] = None, estimator: tp.Literal["nrg"] = "nrg", backend: "Backend" = None, ) -> "Array": @@ -258,10 +260,16 @@ def owcrps_ensemble( fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - w_func : callable, array_like -> array_like - Weight function used to emphasise particular outcomes. + a : float + The lower bound to be used in the default weight function that restricts attention + to values in the range [a, b]. + b : float + The upper bound to be used in the default weight function that restricts attention + to values in the range [a, b]. m_axis : int The axis corresponding to the ensemble. Default is the last axis. + w_func : callable, array_like -> array_like + Weight function used to emphasise particular outcomes. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. @@ -293,9 +301,10 @@ def owcrps_ensemble( ... >>> obs = rng.normal(size=3) >>> fct = rng.normal(size=(3, 10)) - >>> sr.owcrps_ensemble(obs, fct, w_func) + >>> sr.owcrps_ensemble(obs, fct, w_func=w_func) array([0.91103733, 0.45212402, 0.35686667]) """ + B = backends.active if backend is None else backends[backend] if estimator != "nrg": @@ -306,6 +315,11 @@ def owcrps_ensemble( if m_axis != -1: fct = B.moveaxis(fct, m_axis, -1) + if w_func is None: + + def w_func(x): + return ((a < x) & (x < b)).astype(float) + obs_weights, fct_weights = map(w_func, (obs, fct)) if backend == "numba": diff --git a/tests/test_wcrps.py b/tests/test_wcrps.py index f790e53..27e5add 100644 --- a/tests/test_wcrps.py +++ b/tests/test_wcrps.py @@ -17,10 +17,10 @@ def test_owcrps_ensemble(backend): # test shapes obs = np.random.randn(N) - res = sr.owcrps_ensemble(obs, np.random.randn(N, M), lambda x: x * 0.0 + 1.0) + res = sr.owcrps_ensemble(obs, np.random.randn(N, M), w_func=lambda x: x * 0.0 + 1.0) assert res.shape == (N,) res = sr.owcrps_ensemble( - obs, np.random.randn(M, N), lambda x: x * 0.0 + 1.0, m_axis=0 + obs, np.random.randn(M, N), w_func=lambda x: x * 0.0 + 1.0, m_axis=0 ) assert res.shape == (N,) @@ -76,7 +76,21 @@ def test_owcrps_vs_crps(backend): fct = np.random.randn(N, M) * sigma[..., None] + mu[..., None] res = sr.crps_ensemble(obs, fct, backend=backend, estimator="nrg") - resw = sr.owcrps_ensemble(obs, fct, lambda x: x * 0.0 + 1.0, backend=backend) + + # no argument given + resw = sr.owcrps_ensemble(obs, fct, estimator="nrg", backend=backend) + np.testing.assert_allclose(res, resw, rtol=1e-5) + + # a and b + resw = sr.owcrps_ensemble( + obs, fct, a=float("-inf"), b=float("inf"), estimator="nrg", backend=backend + ) + np.testing.assert_allclose(res, resw, rtol=1e-5) + + # w_func as identity function + resw = sr.owcrps_ensemble( + obs, fct, w_func=lambda x: x * 0.0 + 1.0, estimator="nrg", backend=backend + ) np.testing.assert_allclose(res, resw, rtol=1e-5) @@ -127,13 +141,23 @@ def test_owcrps_score_correctness(backend): def w_func(x): return (x > -1).astype(float) - res = np.mean(np.float64(sr.owcrps_ensemble(obs, fct, w_func, backend=backend))) + res = np.mean( + np.float64(sr.owcrps_ensemble(obs, fct, w_func=w_func, backend=backend)) + ) + np.testing.assert_allclose(res, 0.09320807, rtol=1e-6) + + res = np.mean(np.float64(sr.owcrps_ensemble(obs, fct, a=-1.0, backend=backend))) np.testing.assert_allclose(res, 0.09320807, rtol=1e-6) def w_func(x): return (x < 1.85).astype(float) - res = np.mean(np.float64(sr.owcrps_ensemble(obs, fct, w_func, backend=backend))) + res = np.mean( + np.float64(sr.owcrps_ensemble(obs, fct, w_func=w_func, backend=backend)) + ) + np.testing.assert_allclose(res, 0.09933139, rtol=1e-6) + + res = np.mean(np.float64(sr.owcrps_ensemble(obs, fct, b=1.85, backend=backend))) np.testing.assert_allclose(res, 0.09933139, rtol=1e-6) From 39f8eb0d89e53a8b991699b5501ad3c69ccb2754 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Mon, 14 Apr 2025 19:42:00 +0200 Subject: [PATCH 36/79] add default threshold arguments to vrcrps_ensemble --- scoringrules/_crps.py | 19 ++++++++++++++++--- tests/test_wcrps.py | 40 ++++++++++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index 38a9087..759b037 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -336,10 +336,12 @@ def w_func(x): def vrcrps_ensemble( obs: "ArrayLike", fct: "Array", - w_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, + a: float = float("-inf"), + b: float = float("inf"), m_axis: int = -1, *, + w_func: tp.Callable[["ArrayLike"], "ArrayLike"] = None, estimator: tp.Literal["nrg"] = "nrg", backend: "Backend" = None, ) -> "Array": @@ -367,10 +369,16 @@ def vrcrps_ensemble( fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - w_func : callable, array_like -> array_like - Weight function used to emphasise particular outcomes. + a : float + The lower bound to be used in the default weight function that restricts attention + to values in the range [a, b]. + b : float + The upper bound to be used in the default weight function that restricts attention + to values in the range [a, b]. m_axis : int The axis corresponding to the ensemble. Default is the last axis. + w_func : callable, array_like -> array_like + Weight function used to emphasise particular outcomes. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. @@ -415,6 +423,11 @@ def vrcrps_ensemble( if m_axis != -1: fct = B.moveaxis(fct, m_axis, -1) + if w_func is None: + + def w_func(x): + return ((a < x) & (x < b)).astype(float) + obs_weights, fct_weights = map(w_func, (obs, fct)) if backend == "numba": diff --git a/tests/test_wcrps.py b/tests/test_wcrps.py index 27e5add..d9be922 100644 --- a/tests/test_wcrps.py +++ b/tests/test_wcrps.py @@ -13,7 +13,7 @@ def test_owcrps_ensemble(backend): # test exceptions with pytest.raises(ValueError): est = "not_nrg" - sr.owcrps_ensemble(1, 1.1, lambda x: x, estimator=est, backend=backend) + sr.owcrps_ensemble(1, 1.1, w_func=lambda x: x, estimator=est, backend=backend) # test shapes obs = np.random.randn(N) @@ -30,14 +30,14 @@ def test_vrcrps_ensemble(backend): # test exceptions with pytest.raises(ValueError): est = "not_nrg" - sr.vrcrps_ensemble(1, 1.1, lambda x: x, estimator=est, backend=backend) + sr.vrcrps_ensemble(1, 1.1, w_func=lambda x: x, estimator=est, backend=backend) # test shapes obs = np.random.randn(N) - res = sr.vrcrps_ensemble(obs, np.random.randn(N, M), lambda x: x * 0.0 + 1.0) + res = sr.vrcrps_ensemble(obs, np.random.randn(N, M), w_func=lambda x: x * 0.0 + 1.0) assert res.shape == (N,) res = sr.vrcrps_ensemble( - obs, np.random.randn(M, N), lambda x: x * 0.0 + 1.0, m_axis=0 + obs, np.random.randn(M, N), w_func=lambda x: x * 0.0 + 1.0, m_axis=0 ) assert res.shape == (N,) @@ -102,8 +102,22 @@ def test_vrcrps_vs_crps(backend): fct = np.random.randn(N, M) * sigma[..., None] + mu[..., None] res = sr.crps_ensemble(obs, fct, backend=backend, estimator="nrg") - resw = sr.vrcrps_ensemble(obs, fct, lambda x: x * 0.0 + 1.0, backend=backend) - np.testing.assert_allclose(res, resw, atol=1e-6) + + # no argument given + resw = sr.vrcrps_ensemble(obs, fct, estimator="nrg", backend=backend) + np.testing.assert_allclose(res, resw, rtol=1e-5) + + # a and b + resw = sr.vrcrps_ensemble( + obs, fct, a=float("-inf"), b=float("inf"), estimator="nrg", backend=backend + ) + np.testing.assert_allclose(res, resw, rtol=1e-5) + + # w_func as identity function + resw = sr.vrcrps_ensemble( + obs, fct, w_func=lambda x: x * 0.0 + 1.0, estimator="nrg", backend=backend + ) + np.testing.assert_allclose(res, resw, rtol=1e-5) @pytest.mark.parametrize("backend", BACKENDS) @@ -267,11 +281,21 @@ def test_vrcrps_score_correctness(backend): def w_func(x): return (x > -1).astype(float) - res = np.mean(np.float64(sr.vrcrps_ensemble(obs, fct, w_func, backend=backend))) + res = np.mean( + np.float64(sr.vrcrps_ensemble(obs, fct, w_func=w_func, backend=backend)) + ) + np.testing.assert_allclose(res, 0.1003983, rtol=1e-6) + + res = np.mean(np.float64(sr.vrcrps_ensemble(obs, fct, a=-1.0, backend=backend))) np.testing.assert_allclose(res, 0.1003983, rtol=1e-6) def w_func(x): return (x < 1.85).astype(float) - res = np.mean(np.float64(sr.vrcrps_ensemble(obs, fct, w_func, backend=backend))) + res = np.mean( + np.float64(sr.vrcrps_ensemble(obs, fct, w_func=w_func, backend=backend)) + ) + np.testing.assert_allclose(res, 0.1950857, rtol=1e-6) + + res = np.mean(np.float64(sr.vrcrps_ensemble(obs, fct, b=1.85, backend=backend))) np.testing.assert_allclose(res, 0.1950857, rtol=1e-6) From 5b9b1cf0ca22ce3ef3699e81e33fac39b39bf757 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Mon, 14 Apr 2025 19:46:36 +0200 Subject: [PATCH 37/79] reduce tolerance on owgksuv_ensemble test --- tests/test_kernels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_kernels.py b/tests/test_kernels.py index e1c4629..efdec7e 100644 --- a/tests/test_kernels.py +++ b/tests/test_kernels.py @@ -254,7 +254,7 @@ def test_owgksuv(backend): res = sr.gksuv_ensemble(obs, fct, backend=backend) resw = sr.owgksuv_ensemble(obs, fct, lambda x: x * 0.0 + 1.0, backend=backend) - np.testing.assert_allclose(res, resw, rtol=1e-9) + np.testing.assert_allclose(res, resw, rtol=1e-6) # test correctness fct = np.array( From e7b161e377727f7797c484d03bed4d5fb69d1cef Mon Sep 17 00:00:00 2001 From: sallen12 Date: Mon, 14 Apr 2025 20:28:07 +0200 Subject: [PATCH 38/79] add default threshold argument to weighted gksuv scores --- scoringrules/_crps.py | 12 ++-- scoringrules/_kernels.py | 75 ++++++++++++++++------ tests/test_kernels.py | 132 +++++++++++++++++++++++++++++++++------ 3 files changed, 174 insertions(+), 45 deletions(-) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index 759b037..330e854 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -306,6 +306,7 @@ def owcrps_ensemble( """ B = backends.active if backend is None else backends[backend] + obs, fct = map(B.asarray, (obs, fct)) if estimator != "nrg": raise ValueError( @@ -318,18 +319,16 @@ def owcrps_ensemble( if w_func is None: def w_func(x): - return ((a < x) & (x < b)).astype(float) + return ((a < x) & (x < b)) * 1.0 obs_weights, fct_weights = map(w_func, (obs, fct)) + obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) if backend == "numba": return crps.estimator_gufuncs["ow" + estimator]( obs, fct, obs_weights, fct_weights ) - obs, fct, obs_weights, fct_weights = map( - B.asarray, (obs, fct, obs_weights, fct_weights) - ) return crps.ow_ensemble(obs, fct, obs_weights, fct_weights, backend=backend) @@ -414,6 +413,7 @@ def vrcrps_ensemble( array([0.90036433, 0.41515255, 0.41653833]) """ B = backends.active if backend is None else backends[backend] + obs, fct = map(B.asarray, (obs, fct)) if estimator != "nrg": raise ValueError( @@ -429,15 +429,13 @@ def w_func(x): return ((a < x) & (x < b)).astype(float) obs_weights, fct_weights = map(w_func, (obs, fct)) + obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) if backend == "numba": return crps.estimator_gufuncs["vr" + estimator]( obs, fct, obs_weights, fct_weights ) - obs, fct, obs_weights, fct_weights = map( - B.asarray, (obs, fct, obs_weights, fct_weights) - ) return crps.vr_ensemble(obs, fct, obs_weights, fct_weights, backend=backend) diff --git a/scoringrules/_kernels.py b/scoringrules/_kernels.py index e21d192..1d59da8 100644 --- a/scoringrules/_kernels.py +++ b/scoringrules/_kernels.py @@ -85,10 +85,12 @@ def gksuv_ensemble( def twgksuv_ensemble( obs: "ArrayLike", fct: "Array", - v_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, + a: float = float("-inf"), + b: float = float("inf"), m_axis: int = -1, *, + v_func: tp.Callable[["ArrayLike"], "ArrayLike"] = None, estimator: str = "nrg", backend: "Backend" = None, ) -> "Array": @@ -116,12 +118,18 @@ def twgksuv_ensemble( fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. + a : float + The lower bound to be used in the default weight function that restricts attention + to values in the range [a, b]. + b : float + The upper bound to be used in the default weight function that restricts attention + to values in the range [a, b]. + m_axis : int + The axis corresponding to the ensemble. Default is the last axis. v_func : callable, array_like -> array_like Chaining function used to emphasise particular outcomes. For example, a function that only considers values above a certain threshold :math:`t` by projecting forecasts and observations to :math:`[t, \inf)`. - m_axis : int - The axis corresponding to the ensemble. Default is the last axis. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -138,8 +146,15 @@ def twgksuv_ensemble( >>> def v_func(x): >>> return np.maximum(x, -1.0) >>> - >>> sr.twgksuv_ensemble(obs, pred, v_func) + >>> sr.twgksuv_ensemble(obs, pred, v_func=v_func) """ + if v_func is None: + B = backends.active if backend is None else backends[backend] + a, b, obs, fct = map(B.asarray, (a, b, obs, fct)) + + def v_func(x): + return B.minimum(B.maximum(x, a), b) + obs, fct = map(v_func, (obs, fct)) return gksuv_ensemble( obs, @@ -153,10 +168,12 @@ def twgksuv_ensemble( def owgksuv_ensemble( obs: "ArrayLike", fct: "Array", - w_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, + a: float = float("-inf"), + b: float = float("inf"), m_axis: int = -1, *, + w_func: tp.Callable[["ArrayLike"], "ArrayLike"] = None, backend: "Backend" = None, ) -> "Array": r"""Compute the univariate Outcome-Weighted Gaussian Kernel Score (owGKS) for a finite ensemble. @@ -184,10 +201,16 @@ def owgksuv_ensemble( fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - w_func : callable, array_like -> array_like - Weight function used to emphasise particular outcomes. + a : float + The lower bound to be used in the default weight function that restricts attention + to values in the range [a, b]. + b : float + The upper bound to be used in the default weight function that restricts attention + to values in the range [a, b]. m_axis : int The axis corresponding to the ensemble. Default is the last axis. + w_func : callable, array_like -> array_like + Weight function used to emphasise particular outcomes. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -204,33 +227,37 @@ def owgksuv_ensemble( >>> def w_func(x): >>> return (x > -1).astype(float) >>> - >>> sr.owgksuv_ensemble(obs, pred, w_func) + >>> sr.owgksuv_ensemble(obs, pred, w_func=w_func) """ B = backends.active if backend is None else backends[backend] - obs, fct = map(B.asarray, (obs, fct)) if m_axis != -1: fct = B.moveaxis(fct, m_axis, -1) + if w_func is None: + + def w_func(x): + return ((a < x) & (x < b)) * 1.0 + obs_weights, fct_weights = map(w_func, (obs, fct)) + obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) if backend == "numba": return kernels.estimator_gufuncs["ow"](obs, fct, obs_weights, fct_weights) - obs, fct, obs_weights, fct_weights = map( - B.asarray, (obs, fct, obs_weights, fct_weights) - ) return kernels.ow_ensemble_uv(obs, fct, obs_weights, fct_weights, backend=backend) def vrgksuv_ensemble( obs: "ArrayLike", fct: "Array", - w_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, + a: float = float("-inf"), + b: float = float("inf"), m_axis: int = -1, *, + w_func: tp.Callable[["ArrayLike"], "ArrayLike"] = None, backend: "Backend" = None, ) -> "Array": r"""Estimate the Vertically Re-scaled Gaussian Kernel Score (vrGKS) for a finite ensemble. @@ -257,10 +284,16 @@ def vrgksuv_ensemble( fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. - w_func : callable, array_like -> array_like - Weight function used to emphasise particular outcomes. + a : float + The lower bound to be used in the default weight function that restricts attention + to values in the range [a, b]. + b : float + The upper bound to be used in the default weight function that restricts attention + to values in the range [a, b]. m_axis : int The axis corresponding to the ensemble. Default is the last axis. + w_func : callable, array_like -> array_like + Weight function used to emphasise particular outcomes. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -277,23 +310,25 @@ def vrgksuv_ensemble( >>> def w_func(x): >>> return (x > -1).astype(float) >>> - >>> sr.vrgksuv_ensemble(obs, pred, w_func) + >>> sr.vrgksuv_ensemble(obs, pred, w_func=w_func) """ B = backends.active if backend is None else backends[backend] - obs, fct = map(B.asarray, (obs, fct)) if m_axis != -1: fct = B.moveaxis(fct, m_axis, -1) + if w_func is None: + + def w_func(x): + return ((a < x) & (x < b)) * 1.0 + obs_weights, fct_weights = map(w_func, (obs, fct)) + obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) if backend == "numba": return kernels.estimator_gufuncs["vr"](obs, fct, obs_weights, fct_weights) - obs, fct, obs_weights, fct_weights = map( - B.asarray, (obs, fct, obs_weights, fct_weights) - ) return kernels.vr_ensemble_uv(obs, fct, obs_weights, fct_weights, backend=backend) diff --git a/tests/test_kernels.py b/tests/test_kernels.py index efdec7e..f8e1a3e 100644 --- a/tests/test_kernels.py +++ b/tests/test_kernels.py @@ -121,9 +121,21 @@ def test_twgksuv(estimator, backend): sigma = abs(np.random.randn(N)) * 0.3 fct = np.random.randn(N, ENSEMBLE_SIZE) * sigma[..., None] + mu[..., None] - res = sr.gksuv_ensemble(obs, fct, estimator=estimator, backend=backend) + res = sr.gksuv_ensemble(obs, fct, backend=backend, estimator=estimator) + + # no argument given + resw = sr.twgksuv_ensemble(obs, fct, estimator=estimator, backend=backend) + np.testing.assert_allclose(res, resw, rtol=1e-10) + + # a and b + resw = sr.twgksuv_ensemble( + obs, fct, a=float("-inf"), b=float("inf"), estimator=estimator, backend=backend + ) + np.testing.assert_allclose(res, resw, rtol=1e-10) + + # v_func as identity function resw = sr.twgksuv_ensemble( - obs, fct, lambda x: x, estimator=estimator, backend=backend + obs, fct, v_func=lambda x: x, estimator=estimator, backend=backend ) np.testing.assert_allclose(res, resw, rtol=1e-10) @@ -168,7 +180,16 @@ def v_func2(x): res = np.mean( np.float64( sr.twgksuv_ensemble( - obs, fct, v_func1, estimator=estimator, backend=backend + obs, fct, v_func=v_func1, estimator=estimator, backend=backend + ) + ) + ) + np.testing.assert_allclose(res, 0.01018774, rtol=1e-6) + + res = np.mean( + np.float64( + sr.twgksuv_ensemble( + obs, fct, a=-1.0, estimator=estimator, backend=backend ) ) ) @@ -177,7 +198,16 @@ def v_func2(x): res = np.mean( np.float64( sr.twgksuv_ensemble( - obs, fct, v_func2, estimator=estimator, backend=backend + obs, fct, v_func=v_func2, estimator=estimator, backend=backend + ) + ) + ) + np.testing.assert_allclose(res, 0.0089314, rtol=1e-6) + + res = np.mean( + np.float64( + sr.twgksuv_ensemble( + obs, fct, b=1.85, estimator=estimator, backend=backend ) ) ) @@ -187,7 +217,16 @@ def v_func2(x): res = np.mean( np.float64( sr.twgksuv_ensemble( - obs, fct, v_func1, estimator=estimator, backend=backend + obs, fct, v_func=v_func1, estimator=estimator, backend=backend + ) + ) + ) + np.testing.assert_allclose(res, 0.130842, rtol=1e-6) + + res = np.mean( + np.float64( + sr.twgksuv_ensemble( + obs, fct, a=-1.0, estimator=estimator, backend=backend ) ) ) @@ -196,7 +235,16 @@ def v_func2(x): res = np.mean( np.float64( sr.twgksuv_ensemble( - obs, fct, v_func2, estimator=estimator, backend=backend + obs, fct, v_func=v_func2, estimator=estimator, backend=backend + ) + ) + ) + np.testing.assert_allclose(res, 0.1283745, rtol=1e-6) + + res = np.mean( + np.float64( + sr.twgksuv_ensemble( + obs, fct, b=1.85, estimator=estimator, backend=backend ) ) ) @@ -245,16 +293,30 @@ def test_owgksuv(backend): res = sr.owgksuv_ensemble( obs, np.random.randn(ENSEMBLE_SIZE, N), - lambda x: x * 0.0 + 1.0, + w_func=lambda x: x * 0.0 + 1.0, m_axis=0, backend=backend, ) res = np.asarray(res) assert not np.any(res < 0.0) - res = sr.gksuv_ensemble(obs, fct, backend=backend) - resw = sr.owgksuv_ensemble(obs, fct, lambda x: x * 0.0 + 1.0, backend=backend) - np.testing.assert_allclose(res, resw, rtol=1e-6) + res = sr.gksuv_ensemble(obs, fct, backend=backend, estimator="nrg") + + # no argument given + resw = sr.owgksuv_ensemble(obs, fct, backend=backend) + np.testing.assert_allclose(res, resw, rtol=1e-4) + + # a and b + resw = sr.owgksuv_ensemble( + obs, fct, a=float("-inf"), b=float("inf"), backend=backend + ) + np.testing.assert_allclose(res, resw, rtol=1e-4) + + # w_func as identity function + resw = sr.owgksuv_ensemble( + obs, fct, w_func=lambda x: x * 0.0 + 1.0, backend=backend + ) + np.testing.assert_allclose(res, resw, rtol=1e-4) # test correctness fct = np.array( @@ -290,13 +352,23 @@ def test_owgksuv(backend): def w_func(x): return (x > -1) * 1.0 - res = np.mean(np.float64(sr.owgksuv_ensemble(obs, fct, w_func, backend=backend))) + res = np.mean( + np.float64(sr.owgksuv_ensemble(obs, fct, w_func=w_func, backend=backend)) + ) + np.testing.assert_allclose(res, 0.01036335, rtol=1e-5) + + res = np.mean(np.float64(sr.owgksuv_ensemble(obs, fct, a=-1.0, backend=backend))) np.testing.assert_allclose(res, 0.01036335, rtol=1e-5) def w_func(x): return (x < 1.85) * 1.0 - res = np.mean(np.float64(sr.owgksuv_ensemble(obs, fct, w_func, backend=backend))) + res = np.mean( + np.float64(sr.owgksuv_ensemble(obs, fct, w_func=w_func, backend=backend)) + ) + np.testing.assert_allclose(res, 0.008905213, rtol=1e-5) + + res = np.mean(np.float64(sr.owgksuv_ensemble(obs, fct, b=1.85, backend=backend))) np.testing.assert_allclose(res, 0.008905213, rtol=1e-5) @@ -344,16 +416,30 @@ def test_vrgksuv(backend): res = sr.vrgksuv_ensemble( obs, np.random.randn(ENSEMBLE_SIZE, N), - lambda x: x * 0.0 + 1.0, + w_func=lambda x: x * 0.0 + 1.0, m_axis=0, backend=backend, ) res = np.asarray(res) assert not np.any(res < 0.0) - res = sr.gksuv_ensemble(obs, fct, backend=backend) - resw = sr.vrgksuv_ensemble(obs, fct, lambda x: x * 0.0 + 1.0, backend=backend) - np.testing.assert_allclose(res, resw, rtol=1e-10) + res = sr.gksuv_ensemble(obs, fct, backend=backend, estimator="nrg") + + # no argument given + resw = sr.vrgksuv_ensemble(obs, fct, backend=backend) + np.testing.assert_allclose(res, resw, rtol=1e-5) + + # a and b + resw = sr.vrgksuv_ensemble( + obs, fct, a=float("-inf"), b=float("inf"), backend=backend + ) + np.testing.assert_allclose(res, resw, rtol=1e-5) + + # w_func as identity function + resw = sr.vrgksuv_ensemble( + obs, fct, w_func=lambda x: x * 0.0 + 1.0, backend=backend + ) + np.testing.assert_allclose(res, resw, rtol=1e-5) # test correctness fct = np.array( @@ -389,13 +475,23 @@ def test_vrgksuv(backend): def w_func(x): return (x > -1) * 1.0 - res = np.mean(np.float64(sr.vrgksuv_ensemble(obs, fct, w_func, backend=backend))) + res = np.mean( + np.float64(sr.vrgksuv_ensemble(obs, fct, w_func=w_func, backend=backend)) + ) + np.testing.assert_allclose(res, 0.01476682, rtol=1e-6) + + res = np.mean(np.float64(sr.vrgksuv_ensemble(obs, fct, a=-1.0, backend=backend))) np.testing.assert_allclose(res, 0.01476682, rtol=1e-6) def w_func(x): return (x < 1.85) * 1.0 - res = np.mean(np.float64(sr.vrgksuv_ensemble(obs, fct, w_func, backend=backend))) + res = np.mean( + np.float64(sr.vrgksuv_ensemble(obs, fct, w_func=w_func, backend=backend)) + ) + np.testing.assert_allclose(res, 0.04011836, rtol=1e-6) + + res = np.mean(np.float64(sr.vrgksuv_ensemble(obs, fct, b=1.85, backend=backend))) np.testing.assert_allclose(res, 0.04011836, rtol=1e-6) From 1ed244a23b8d9a2364b0c0c391b230dee0b85d93 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Mon, 14 Apr 2025 20:30:31 +0200 Subject: [PATCH 39/79] update default w_func in vrcrps_ensemble --- scoringrules/_crps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index 330e854..a6f51b3 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -426,7 +426,7 @@ def vrcrps_ensemble( if w_func is None: def w_func(x): - return ((a < x) & (x < b)).astype(float) + return ((a < x) & (x < b)) * 1.0 obs_weights, fct_weights = map(w_func, (obs, fct)) obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) From 8442fc8cc33fd52a91c84a88e0ed689adfa14766 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Mon, 14 Apr 2025 20:33:13 +0200 Subject: [PATCH 40/79] update test_wcrps --- tests/test_wcrps.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_wcrps.py b/tests/test_wcrps.py index d9be922..e39d02d 100644 --- a/tests/test_wcrps.py +++ b/tests/test_wcrps.py @@ -153,7 +153,7 @@ def test_owcrps_score_correctness(backend): ) def w_func(x): - return (x > -1).astype(float) + return (x > -1) * 1.0 res = np.mean( np.float64(sr.owcrps_ensemble(obs, fct, w_func=w_func, backend=backend)) @@ -164,7 +164,7 @@ def w_func(x): np.testing.assert_allclose(res, 0.09320807, rtol=1e-6) def w_func(x): - return (x < 1.85).astype(float) + return (x < 1.85) * 1.0 res = np.mean( np.float64(sr.owcrps_ensemble(obs, fct, w_func=w_func, backend=backend)) @@ -279,7 +279,7 @@ def test_vrcrps_score_correctness(backend): ) def w_func(x): - return (x > -1).astype(float) + return (x > -1) * 1.0 res = np.mean( np.float64(sr.vrcrps_ensemble(obs, fct, w_func=w_func, backend=backend)) @@ -290,7 +290,7 @@ def w_func(x): np.testing.assert_allclose(res, 0.1003983, rtol=1e-6) def w_func(x): - return (x < 1.85).astype(float) + return (x < 1.85) * 1.0 res = np.mean( np.float64(sr.vrcrps_ensemble(obs, fct, w_func=w_func, backend=backend)) From 9d8efefe1857303d5b30af86cf60ea9633af246b Mon Sep 17 00:00:00 2001 From: sallen12 Date: Tue, 15 Apr 2025 09:26:11 +0200 Subject: [PATCH 41/79] reduce tolerance of owgks tests --- scoringrules/_crps.py | 4 ++-- scoringrules/_kernels.py | 4 ++-- tests/test_kernels.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index a6f51b3..281a826 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -319,7 +319,7 @@ def owcrps_ensemble( if w_func is None: def w_func(x): - return ((a < x) & (x < b)) * 1.0 + return ((a <= x) & (x <= b)) * 1.0 obs_weights, fct_weights = map(w_func, (obs, fct)) obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) @@ -426,7 +426,7 @@ def vrcrps_ensemble( if w_func is None: def w_func(x): - return ((a < x) & (x < b)) * 1.0 + return ((a <= x) & (x <= b)) * 1.0 obs_weights, fct_weights = map(w_func, (obs, fct)) obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) diff --git a/scoringrules/_kernels.py b/scoringrules/_kernels.py index 1d59da8..3ea14c2 100644 --- a/scoringrules/_kernels.py +++ b/scoringrules/_kernels.py @@ -238,7 +238,7 @@ def owgksuv_ensemble( if w_func is None: def w_func(x): - return ((a < x) & (x < b)) * 1.0 + return ((a <= x) & (x <= b)) * 1.0 obs_weights, fct_weights = map(w_func, (obs, fct)) obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) @@ -321,7 +321,7 @@ def vrgksuv_ensemble( if w_func is None: def w_func(x): - return ((a < x) & (x < b)) * 1.0 + return ((a <= x) & (x <= b)) * 1.0 obs_weights, fct_weights = map(w_func, (obs, fct)) obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) diff --git a/tests/test_kernels.py b/tests/test_kernels.py index f8e1a3e..0e9fe88 100644 --- a/tests/test_kernels.py +++ b/tests/test_kernels.py @@ -304,19 +304,19 @@ def test_owgksuv(backend): # no argument given resw = sr.owgksuv_ensemble(obs, fct, backend=backend) - np.testing.assert_allclose(res, resw, rtol=1e-4) + np.testing.assert_allclose(res, resw, rtol=1e-3) # a and b resw = sr.owgksuv_ensemble( obs, fct, a=float("-inf"), b=float("inf"), backend=backend ) - np.testing.assert_allclose(res, resw, rtol=1e-4) + np.testing.assert_allclose(res, resw, rtol=1e-3) # w_func as identity function resw = sr.owgksuv_ensemble( obs, fct, w_func=lambda x: x * 0.0 + 1.0, backend=backend ) - np.testing.assert_allclose(res, resw, rtol=1e-4) + np.testing.assert_allclose(res, resw, rtol=1e-3) # test correctness fct = np.array( From 0c209f970c9a48ea1d35de50acbdb005704e57d2 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Tue, 15 Apr 2025 09:33:37 +0200 Subject: [PATCH 42/79] reduce tolerance of owgks tests --- tests/test_kernels.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_kernels.py b/tests/test_kernels.py index 0e9fe88..579c3c2 100644 --- a/tests/test_kernels.py +++ b/tests/test_kernels.py @@ -304,19 +304,19 @@ def test_owgksuv(backend): # no argument given resw = sr.owgksuv_ensemble(obs, fct, backend=backend) - np.testing.assert_allclose(res, resw, rtol=1e-3) + np.testing.assert_allclose(res, resw, rtol=1e-2) # a and b resw = sr.owgksuv_ensemble( obs, fct, a=float("-inf"), b=float("inf"), backend=backend ) - np.testing.assert_allclose(res, resw, rtol=1e-3) + np.testing.assert_allclose(res, resw, rtol=1e-2) # w_func as identity function resw = sr.owgksuv_ensemble( obs, fct, w_func=lambda x: x * 0.0 + 1.0, backend=backend ) - np.testing.assert_allclose(res, resw, rtol=1e-3) + np.testing.assert_allclose(res, resw, rtol=1e-2) # test correctness fct = np.array( From fdf78ea01d07bf8746cf4787096e0767661a7b1b Mon Sep 17 00:00:00 2001 From: sallen12 Date: Tue, 15 Apr 2025 11:14:59 +0200 Subject: [PATCH 43/79] add argsort method to backends --- scoringrules/backend/base.py | 11 +++++++++++ scoringrules/backend/jax.py | 10 ++++++++++ scoringrules/backend/numpy.py | 11 +++++++++++ scoringrules/backend/tensorflow.py | 11 +++++++++++ scoringrules/backend/torch.py | 11 +++++++++++ 5 files changed, 54 insertions(+) diff --git a/scoringrules/backend/base.py b/scoringrules/backend/base.py index 10d6200..669d4f7 100644 --- a/scoringrules/backend/base.py +++ b/scoringrules/backend/base.py @@ -201,6 +201,17 @@ def any( ) -> "Array": """Test whether any input array element evaluates to ``True`` along a specified axis.""" + @abc.abstractmethod + def argsort( + self, + x: "Array", + /, + *, + axis: int = -1, + descending: bool = False, + ) -> "Array": + """Return the indices of a sorted copy of an input array ``x``.""" + @abc.abstractmethod def sort( self, diff --git a/scoringrules/backend/jax.py b/scoringrules/backend/jax.py index 80332cb..bf6c259 100644 --- a/scoringrules/backend/jax.py +++ b/scoringrules/backend/jax.py @@ -194,6 +194,16 @@ def sort( out = jnp.sort(x, axis=axis) # TODO: this is slow! why? return -out if descending else out + def argsort( + self, + x: "Array", + /, + *, + axis: int = -1, + descending: bool = False, + ) -> "Array": + return jnp.argsort(x, axis=axis, descending=descending) + def norm(self, x: "Array", axis: int | tuple[int, ...] | None = None) -> "Array": return jnp.linalg.norm(x, axis=axis) diff --git a/scoringrules/backend/numpy.py b/scoringrules/backend/numpy.py index 1fce50f..922d242 100644 --- a/scoringrules/backend/numpy.py +++ b/scoringrules/backend/numpy.py @@ -198,6 +198,17 @@ def sort( out = np.sort(x, axis=axis) return -out if descending else out + def argsort( + self, + x: "NDArray", + /, + *, + axis: int = -1, + descending: bool = False, + ) -> "NDArray": + x = -x if descending else x + return np.argsort(x, axis=axis) + def norm( self, x: "NDArray", axis: int | tuple[int, ...] | None = None ) -> "NDArray": diff --git a/scoringrules/backend/tensorflow.py b/scoringrules/backend/tensorflow.py index 9143607..fde7949 100644 --- a/scoringrules/backend/tensorflow.py +++ b/scoringrules/backend/tensorflow.py @@ -225,6 +225,17 @@ def sort( direction = "DESCENDING" if descending else "ASCENDING" return tf.sort(x, axis=axis, direction=direction) + def argsort( + self, + x: "Tensor", + /, + *, + axis: int = -1, + descending: bool = False, + ) -> "Tensor": + direction = "DESCENDING" if descending else "ASCENDING" + return tf.argsort(x, axis=axis, direction=direction) + def norm(self, x: "Tensor", axis: int | tuple[int, ...] | None = None) -> "Tensor": return tf.norm(x, axis=axis) diff --git a/scoringrules/backend/torch.py b/scoringrules/backend/torch.py index 3c7528b..f007366 100644 --- a/scoringrules/backend/torch.py +++ b/scoringrules/backend/torch.py @@ -204,6 +204,17 @@ def sort( ) -> "Tensor": return torch.sort(x, stable=stable, dim=axis, descending=descending)[0] + def argsort( + self, + x: "Tensor", + /, + *, + axis: int = -1, + descending: bool = False, + stable: bool = True, + ) -> "Tensor": + return torch.argsort(x, stable=stable, dim=axis, descending=descending) + def norm(self, x: "Tensor", axis: int | tuple[int, ...] | None = None) -> "Tensor": return torch.norm(x, dim=axis) From 9312992a800e94aa37dda3b291341a384a118d1c Mon Sep 17 00:00:00 2001 From: sallen12 Date: Wed, 16 Apr 2025 09:45:18 +0200 Subject: [PATCH 44/79] add weight function in ensemble crps for non-numba backends --- scoringrules/_crps.py | 38 ++++++++++++++++++++----------- scoringrules/core/crps/_approx.py | 36 +++++++++++++++-------------- 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index 281a826..910c95f 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -13,6 +13,7 @@ def crps_ensemble( /, m_axis: int = -1, *, + w: "Array" = None, sorted_ensemble: bool = False, estimator: str = "pwm", backend: "Backend" = None, @@ -45,11 +46,14 @@ def crps_ensemble( ---------- obs : array_like The observed values. - fct : array_like, shape (..., m) + fct : array, shape (..., m) The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. m_axis : int The axis corresponding to the ensemble. Default is the last axis. + w : array, shape (..., m) + Weights assigned to the ensemble members. Array with the same shape as fct. + Default is equal weighting. sorted_ensemble : bool Boolean indicating whether the ensemble members are already in ascending order. Default is False. @@ -102,12 +106,20 @@ def crps_ensemble( if m_axis != -1: fct = B.moveaxis(fct, m_axis, -1) - if not sorted_ensemble and estimator not in [ - "nrg", - "akr", - "akr_circperm", - "fair", - ]: + sort_ensemble = not sorted_ensemble and estimator in ["qd", "pwm"] + + if w is None: + M = fct.shape[-1] + w = B.zeros(fct.shape) + 1.0 / M + else: + w = map(B.asarray, w) + w = B.moveaxis(w, m_axis, -1) + w = w / B.sum(w, axis=-1, keepdims=True) + if sort_ensemble: + ind = B.argsort(fct, axis=-1) + w = w[ind] + + if sort_ensemble: fct = B.sort(fct, axis=-1) if backend == "numba": @@ -116,9 +128,9 @@ def crps_ensemble( f"{estimator} is not a valid estimator. " f"Must be one of {crps.estimator_gufuncs.keys()}" ) - return crps.estimator_gufuncs[estimator](obs, fct) + return crps.estimator_gufuncs[estimator](obs, fct, w) - return crps.ensemble(obs, fct, estimator, backend=backend) + return crps.ensemble(obs, fct, w, estimator, backend=backend) def twcrps_ensemble( @@ -151,7 +163,7 @@ def twcrps_ensemble( ---------- obs : array_like The observed values. - fct : array_like + fct : array The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. a : float @@ -257,7 +269,7 @@ def owcrps_ensemble( ---------- obs : array_like The observed values. - fct : array_like + fct : array The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. a : float @@ -365,7 +377,7 @@ def vrcrps_ensemble( ---------- obs : array_like The observed values. - fct : array_like + fct : array The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. a : float @@ -470,7 +482,7 @@ def crps_quantile( ---------- obs : array_like The observed values. - fct : array_like + fct : array The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. alpha : array_like diff --git a/scoringrules/core/crps/_approx.py b/scoringrules/core/crps/_approx.py index 8a40abd..bea70b2 100644 --- a/scoringrules/core/crps/_approx.py +++ b/scoringrules/core/crps/_approx.py @@ -9,16 +9,17 @@ def ensemble( obs: "ArrayLike", fct: "Array", + w: "Array", estimator: str = "pwm", backend: "Backend" = None, ) -> "Array": """Compute the CRPS for a finite ensemble.""" if estimator == "nrg": - out = _crps_ensemble_nrg(obs, fct, backend=backend) + out = _crps_ensemble_nrg(obs, fct, w, backend=backend) elif estimator == "pwm": - out = _crps_ensemble_pwm(obs, fct, backend=backend) + out = _crps_ensemble_pwm(obs, fct, w, backend=backend) elif estimator == "fair": - out = _crps_ensemble_fair(obs, fct, backend=backend) + out = _crps_ensemble_fair(obs, fct, w, backend=backend) else: raise ValueError( f"{estimator} can only be used with `numpy` " @@ -29,39 +30,40 @@ def ensemble( def _crps_ensemble_fair( - obs: "Array", fct: "Array", backend: "Backend" = None + obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None ) -> "Array": """Fair version of the CRPS estimator based on the energy form.""" B = backends.active if backend is None else backends[backend] - M: int = fct.shape[-1] - e_1 = B.sum(B.abs(obs[..., None] - fct), axis=-1) / M + e_1 = B.sum(B.abs(obs[..., None] - fct) * w, axis=-1) e_2 = B.sum( - B.abs(fct[..., None] - fct[..., None, :]), + B.abs(fct[..., None] - fct[..., None, :]) * w[..., None] * w[..., None, :], axis=(-1, -2), - ) / (M * (M - 1)) + ) / (1 - B.sum(w * w, axis=-1)) return e_1 - 0.5 * e_2 def _crps_ensemble_nrg( - obs: "Array", fct: "Array", backend: "Backend" = None + obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None ) -> "Array": """CRPS estimator based on the energy form.""" B = backends.active if backend is None else backends[backend] - M: int = fct.shape[-1] - e_1 = B.sum(B.abs(obs[..., None] - fct), axis=-1) / M - e_2 = B.sum(B.abs(fct[..., None] - fct[..., None, :]), (-1, -2)) / (M**2) + e_1 = B.sum(B.abs(obs[..., None] - fct) * w, axis=-1) + e_2 = B.sum( + B.abs(fct[..., None] - fct[..., None, :]) * w[..., None] * w[..., None, :], + (-1, -2), + ) return e_1 - 0.5 * e_2 def _crps_ensemble_pwm( - obs: "Array", fct: "Array", backend: "Backend" = None + obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None ) -> "Array": """CRPS estimator based on the probability weighted moment (PWM) form.""" B = backends.active if backend is None else backends[backend] - M: int = fct.shape[-1] - expected_diff = B.sum(B.abs(obs[..., None] - fct), axis=-1) / M - β_0 = B.sum(fct, axis=-1) / M - β_1 = B.sum(fct * B.arange(0, M), axis=-1) / (M * (M - 1.0)) + w_sum = B.cumsum(w, axis=-1) + expected_diff = B.sum(B.abs(obs[..., None] - fct) * w, axis=-1) + β_0 = B.sum(fct * w * (1.0 - w), axis=-1) + β_1 = B.sum(fct * w * (w_sum - w), axis=-1) return expected_diff + β_0 - 2.0 * β_1 From 8ba346ff115ba71147a1b9048e4975b5f378853e Mon Sep 17 00:00:00 2001 From: sallen12 Date: Wed, 16 Apr 2025 13:30:03 +0200 Subject: [PATCH 45/79] add weight function in ensemble crps for non-numba backends --- scoringrules/core/crps/_gufuncs.py | 85 +++++++++++++++++------------- 1 file changed, 49 insertions(+), 36 deletions(-) diff --git a/scoringrules/core/crps/_gufuncs.py b/scoringrules/core/crps/_gufuncs.py index b28745e..7996cee 100644 --- a/scoringrules/core/crps/_gufuncs.py +++ b/scoringrules/core/crps/_gufuncs.py @@ -73,15 +73,16 @@ def _crps_ensemble_int_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray) @guvectorize( [ - "void(float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", ], "(),(n)->()", ) -def _crps_ensemble_qd_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): +def _crps_ensemble_qd_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): """CRPS estimator based on the quantile decomposition form.""" obs = obs[0] - M = fct.shape[-1] if np.isnan(obs): out[0] = np.nan @@ -89,24 +90,27 @@ def _crps_ensemble_qd_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): obs_cdf = 0.0 integral = 0.0 + a = np.cumsum(w) - w / 2 for i, forecast in enumerate(fct): if obs < forecast: obs_cdf = 1.0 - integral += (forecast - obs) * (M * obs_cdf - (i + 1) + 0.5) + integral += w[i] * (obs_cdf - a[i]) * (forecast - obs) - out[0] = (2 / M**2) * integral + out[0] = 2 * integral @guvectorize( [ - "void(float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", ], "(),(n)->()", ) -def _crps_ensemble_nrg_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): +def _crps_ensemble_nrg_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): """CRPS estimator based on the energy form.""" obs = obs[0] M = fct.shape[-1] @@ -119,21 +123,23 @@ def _crps_ensemble_nrg_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray) e_2 = 0 for i in range(M): - e_1 += abs(fct[i] - obs) + e_1 += abs(fct[i] - obs) * w[i] for j in range(i + 1, M): - e_2 += 2 * abs(fct[j] - fct[i]) + e_2 += abs(fct[j] - fct[i]) * w[j] * w[i] - out[0] = e_1 / M - 0.5 * e_2 / (M**2) + out[0] = e_1 - e_2 @guvectorize( [ - "void(float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", ], "(),(n)->()", ) -def _crps_ensemble_fair_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): +def _crps_ensemble_fair_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): """Fair version of the CRPS estimator based on the energy form.""" obs = obs[0] M = fct.shape[-1] @@ -146,11 +152,13 @@ def _crps_ensemble_fair_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray e_2 = 0 for i in range(M): - e_1 += abs(fct[i] - obs) + e_1 += abs(fct[i] - obs) * w[i] for j in range(i + 1, M): - e_2 += 2 * abs(fct[j] - fct[i]) + e_2 += abs(fct[j] - fct[i]) * w[j] * w[i] + + fair_c = 1 - np.sum(w**2) - out[0] = e_1 / M - 0.5 * e_2 / (M * (M - 1)) + out[0] = e_1 - e_2 / fair_c @guvectorize( @@ -160,10 +168,11 @@ def _crps_ensemble_fair_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray ], "(),(n)->()", ) -def _crps_ensemble_pwm_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): +def _crps_ensemble_pwm_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): """CRPS estimator based on the probability weighted moment (PWM) form.""" obs = obs[0] - M = fct.shape[-1] if np.isnan(obs): out[0] = np.nan @@ -173,22 +182,26 @@ def _crps_ensemble_pwm_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray) β_0 = 0.0 β_1 = 0.0 + w_sum = np.cumsum(w) + for i, forecast in enumerate(fct): - expected_diff += np.abs(forecast - obs) - β_0 += forecast - β_1 += forecast * i + expected_diff += np.abs(forecast - obs) * w[i] + β_0 += forecast * w[i] * (1.0 - w[i]) + β_1 += forecast * w[i] * (w_sum[i] - w[i]) - out[0] = expected_diff / M + β_0 / M - 2 * β_1 / (M * (M - 1)) + out[0] = expected_diff + β_0 - 2 * β_1 @guvectorize( [ - "void(float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", ], "(),(n)->()", ) -def _crps_ensemble_akr_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): +def _crps_ensemble_akr_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): """CRPS estimaton based on the approximate kernel representation.""" M = fct.shape[-1] obs = obs[0] @@ -197,20 +210,20 @@ def _crps_ensemble_akr_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray) for i, forecast in enumerate(fct): if i == 0: i = M - 1 - e_1 += abs(forecast - obs) - e_2 += abs(forecast - fct[i - 1]) - out[0] = e_1 / M - 0.5 * 1 / M * e_2 + e_1 += abs(forecast - obs) * w[i] + e_2 += abs(forecast - fct[i - 1]) * w[i] + out[0] = e_1 - 0.5 * e_2 @guvectorize( [ - "void(float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", ], "(),(n)->()", ) def _crps_ensemble_akr_circperm_gufunc( - obs: np.ndarray, fct: np.ndarray, out: np.ndarray + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray ): """CRPS estimaton based on the AKR with cyclic permutation.""" M = fct.shape[-1] @@ -219,9 +232,9 @@ def _crps_ensemble_akr_circperm_gufunc( e_2 = 0.0 for i, forecast in enumerate(fct): sigma_i = int((i + 1 + ((M - 1) / 2)) % M) - e_1 += abs(forecast - obs) - e_2 += abs(forecast - fct[sigma_i]) - out[0] = e_1 / M - 0.5 * 1 / M * e_2 + e_1 += abs(forecast - obs) * w[i] + e_2 += abs(forecast - fct[sigma_i]) * w[i] + out[0] = e_1 - 0.5 * e_2 @guvectorize( From 7003da9ea0103643ea1cb49869ac537488c01f11 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Wed, 16 Apr 2025 14:02:06 +0200 Subject: [PATCH 46/79] add weight function in ensemble crps for non-numba backends --- scoringrules/core/crps/_approx.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scoringrules/core/crps/_approx.py b/scoringrules/core/crps/_approx.py index bea70b2..4f1c8d5 100644 --- a/scoringrules/core/crps/_approx.py +++ b/scoringrules/core/crps/_approx.py @@ -20,6 +20,8 @@ def ensemble( out = _crps_ensemble_pwm(obs, fct, w, backend=backend) elif estimator == "fair": out = _crps_ensemble_fair(obs, fct, w, backend=backend) + elif estimator == "fair": + out = _crps_ensemble_qd(obs, fct, w, backend=backend) else: raise ValueError( f"{estimator} can only be used with `numpy` " @@ -67,6 +69,19 @@ def _crps_ensemble_pwm( return expected_diff + β_0 - 2.0 * β_1 +def _crps_ensemble_qd( + obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None +) -> "Array": + """CRPS estimator based on the quantile score decomposition.""" + B = backends.active if backend is None else backends[backend] + w_sum = B.cumsum(w, axis=-1) + a = w_sum - 0.5 * w + dif = fct - obs[..., None] + c = B.where(dif > 0, 1 - a, -a) + s = B.sum(w * c * dif, axis=-1) + return 2 * s + + def quantile_pinball( obs: "Array", fct: "Array", alpha: "Array", backend: "Backend" = None ) -> "Array": From 9e49e10ab1d9e97961ad2f7f3b40e9f95f07089c Mon Sep 17 00:00:00 2001 From: sallen12 Date: Wed, 16 Apr 2025 14:23:47 +0200 Subject: [PATCH 47/79] add akr estimators for crps_ensemble for non-numba backends --- scoringrules/core/crps/_approx.py | 36 ++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/scoringrules/core/crps/_approx.py b/scoringrules/core/crps/_approx.py index 4f1c8d5..7cdfef7 100644 --- a/scoringrules/core/crps/_approx.py +++ b/scoringrules/core/crps/_approx.py @@ -20,13 +20,14 @@ def ensemble( out = _crps_ensemble_pwm(obs, fct, w, backend=backend) elif estimator == "fair": out = _crps_ensemble_fair(obs, fct, w, backend=backend) - elif estimator == "fair": + elif estimator == "qd": out = _crps_ensemble_qd(obs, fct, w, backend=backend) + elif estimator == "akr": + out = _crps_ensemble_akr(obs, fct, w, backend=backend) + elif estimator == "akr_circperm": + out = _crps_ensemble_akr_circperm(obs, fct, w, backend=backend) else: - raise ValueError( - f"{estimator} can only be used with `numpy` " - "backend and needs `numba` to be installed" - ) + raise ValueError(f"{estimator} is not an available estimator") return out @@ -82,6 +83,31 @@ def _crps_ensemble_qd( return 2 * s +def _crps_ensemble_akr( + obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None +) -> "Array": + """CRPS estimator based on the approximate kernel representation.""" + B = backends.active if backend is None else backends[backend] + M = fct.shape[-1] + e_1 = B.sum(B.abs(obs[..., None] - fct) * w, axis=-1) + ind = [(i + 1) % M for i in range(M)] + e_2 = B.sum(B.abs(fct[..., ind] - fct) * w[..., ind], axis=-1) + return e_1 - 0.5 * e_2 + + +def _crps_ensemble_akr_circperm( + obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None +) -> "Array": + """CRPS estimator based on the AKR with cyclic permutation.""" + B = backends.active if backend is None else backends[backend] + M = fct.shape[-1] + e_1 = B.sum(B.abs(obs[..., None] - fct) * w, axis=-1) + shift = int((M - 1) / 2) + ind = [(i + shift) % M for i in range(M)] + e_2 = B.sum(B.abs(fct[..., ind] - fct) * w[..., ind], axis=-1) + return e_1 - 0.5 * e_2 + + def quantile_pinball( obs: "Array", fct: "Array", alpha: "Array", backend: "Backend" = None ) -> "Array": From f50211a6f6112d7f5d91c3b4fdb9bfb2bbd250e9 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Wed, 16 Apr 2025 15:11:17 +0200 Subject: [PATCH 48/79] add ensemble weights to weighted ensemble_crps functions --- scoringrules/_crps.py | 74 +++++++++++++++++++----------- scoringrules/core/crps/_approx.py | 45 ++++++++++-------- scoringrules/core/crps/_gufuncs.py | 52 +++++++++++---------- 3 files changed, 100 insertions(+), 71 deletions(-) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index 910c95f..bdeffb3 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -13,7 +13,7 @@ def crps_ensemble( /, m_axis: int = -1, *, - w: "Array" = None, + ens_w: "Array" = None, sorted_ensemble: bool = False, estimator: str = "pwm", backend: "Backend" = None, @@ -51,7 +51,7 @@ def crps_ensemble( represented by the last axis. m_axis : int The axis corresponding to the ensemble. Default is the last axis. - w : array, shape (..., m) + ens_w : array, shape (..., m) Weights assigned to the ensemble members. Array with the same shape as fct. Default is equal weighting. sorted_ensemble : bool @@ -108,16 +108,17 @@ def crps_ensemble( sort_ensemble = not sorted_ensemble and estimator in ["qd", "pwm"] - if w is None: + if ens_w is None: M = fct.shape[-1] - w = B.zeros(fct.shape) + 1.0 / M + ens_w = B.zeros(fct.shape) + 1.0 / M else: - w = map(B.asarray, w) - w = B.moveaxis(w, m_axis, -1) - w = w / B.sum(w, axis=-1, keepdims=True) + ens_w = map(B.asarray, ens_w) + ens_w = ens_w / B.sum(ens_w, axis=-1, keepdims=True) + if m_axis != -1: + ens_w = B.moveaxis(ens_w, -1, -1) if sort_ensemble: ind = B.argsort(fct, axis=-1) - w = w[ind] + ens_w = ens_w[ind] if sort_ensemble: fct = B.sort(fct, axis=-1) @@ -128,9 +129,9 @@ def crps_ensemble( f"{estimator} is not a valid estimator. " f"Must be one of {crps.estimator_gufuncs.keys()}" ) - return crps.estimator_gufuncs[estimator](obs, fct, w) + return crps.estimator_gufuncs[estimator](obs, fct, ens_w) - return crps.ensemble(obs, fct, w, estimator, backend=backend) + return crps.ensemble(obs, fct, ens_w, estimator, backend=backend) def twcrps_ensemble( @@ -141,6 +142,7 @@ def twcrps_ensemble( b: float = float("inf"), m_axis: int = -1, *, + ens_w: "Array" = None, v_func: tp.Callable[["ArrayLike"], "ArrayLike"] = None, estimator: str = "pwm", sorted_ensemble: bool = False, @@ -174,6 +176,9 @@ def twcrps_ensemble( to values in the range [a, b]. m_axis : int The axis corresponding to the ensemble. Default is the last axis. + ens_w : array + Weights assigned to the ensemble members. Array with the same shape as fct. + Default is equal weighting. v_func : callable, array_like -> array_like Chaining function used to emphasise particular outcomes. For example, a function that only considers values above a certain threshold :math:`t` by projecting forecasts and observations @@ -228,6 +233,7 @@ def v_func(x): obs, fct, m_axis=m_axis, + ens_w=ens_w, sorted_ensemble=sorted_ensemble, estimator=estimator, backend=backend, @@ -242,8 +248,8 @@ def owcrps_ensemble( b: float = float("inf"), m_axis: int = -1, *, + ens_w: "Array" = None, w_func: tp.Callable[["ArrayLike"], "ArrayLike"] = None, - estimator: tp.Literal["nrg"] = "nrg", backend: "Backend" = None, ) -> "Array": r"""Estimate the outcome-weighted CRPS (owCRPS) for a finite ensemble. @@ -280,6 +286,9 @@ def owcrps_ensemble( to values in the range [a, b]. m_axis : int The axis corresponding to the ensemble. Default is the last axis. + ens_w : array + Weights assigned to the ensemble members. Array with the same shape as fct. + Default is equal weighting. w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. backend : str, optional @@ -320,13 +329,16 @@ def owcrps_ensemble( B = backends.active if backend is None else backends[backend] obs, fct = map(B.asarray, (obs, fct)) - if estimator != "nrg": - raise ValueError( - "Only the energy form of the estimator is available " - "for the outcome-weighted CRPS." - ) + if ens_w is None: + M = fct.shape[m_axis] + ens_w = B.zeros(fct.shape) + 1.0 / M + else: + ens_w = map(B.asarray, ens_w) + ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) + if m_axis != -1: fct = B.moveaxis(fct, m_axis, -1) + ens_w = B.moveaxis(ens_w, m_axis, -1) if w_func is None: @@ -337,11 +349,11 @@ def w_func(x): obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) if backend == "numba": - return crps.estimator_gufuncs["ow" + estimator]( - obs, fct, obs_weights, fct_weights + return crps.estimator_gufuncs["ownrg"]( + obs, fct, obs_weights, fct_weights, ens_w ) - return crps.ow_ensemble(obs, fct, obs_weights, fct_weights, backend=backend) + return crps.ow_ensemble(obs, fct, obs_weights, fct_weights, ens_w, backend=backend) def vrcrps_ensemble( @@ -352,8 +364,8 @@ def vrcrps_ensemble( b: float = float("inf"), m_axis: int = -1, *, + ens_w: "Array" = None, w_func: tp.Callable[["ArrayLike"], "ArrayLike"] = None, - estimator: tp.Literal["nrg"] = "nrg", backend: "Backend" = None, ) -> "Array": r"""Estimate the vertically re-scaled CRPS (vrCRPS) for a finite ensemble. @@ -388,6 +400,9 @@ def vrcrps_ensemble( to values in the range [a, b]. m_axis : int The axis corresponding to the ensemble. Default is the last axis. + ens_w : array + Weights assigned to the ensemble members. Array with the same shape as fct. + Default is equal weighting. w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. backend : str, optional @@ -427,13 +442,16 @@ def vrcrps_ensemble( B = backends.active if backend is None else backends[backend] obs, fct = map(B.asarray, (obs, fct)) - if estimator != "nrg": - raise ValueError( - "Only the energy form of the estimator is available " - "for the outcome-weighted CRPS." - ) + if ens_w is None: + M = fct.shape[m_axis] + ens_w = B.zeros(fct.shape) + 1.0 / M + else: + ens_w = map(B.asarray, ens_w) + ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) + if m_axis != -1: fct = B.moveaxis(fct, m_axis, -1) + ens_w = B.moveaxis(ens_w, m_axis, -1) if w_func is None: @@ -444,11 +462,11 @@ def w_func(x): obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) if backend == "numba": - return crps.estimator_gufuncs["vr" + estimator]( - obs, fct, obs_weights, fct_weights + return crps.estimator_gufuncs["vrnrg"]( + obs, fct, obs_weights, fct_weights, ens_w ) - return crps.vr_ensemble(obs, fct, obs_weights, fct_weights, backend=backend) + return crps.vr_ensemble(obs, fct, obs_weights, fct_weights, ens_w, backend=backend) def crps_quantile( diff --git a/scoringrules/core/crps/_approx.py b/scoringrules/core/crps/_approx.py index 7cdfef7..50d0731 100644 --- a/scoringrules/core/crps/_approx.py +++ b/scoringrules/core/crps/_approx.py @@ -9,23 +9,23 @@ def ensemble( obs: "ArrayLike", fct: "Array", - w: "Array", + ens_w: "Array", estimator: str = "pwm", backend: "Backend" = None, ) -> "Array": """Compute the CRPS for a finite ensemble.""" if estimator == "nrg": - out = _crps_ensemble_nrg(obs, fct, w, backend=backend) + out = _crps_ensemble_nrg(obs, fct, ens_w, backend=backend) elif estimator == "pwm": - out = _crps_ensemble_pwm(obs, fct, w, backend=backend) + out = _crps_ensemble_pwm(obs, fct, ens_w, backend=backend) elif estimator == "fair": - out = _crps_ensemble_fair(obs, fct, w, backend=backend) + out = _crps_ensemble_fair(obs, fct, ens_w, backend=backend) elif estimator == "qd": - out = _crps_ensemble_qd(obs, fct, w, backend=backend) + out = _crps_ensemble_qd(obs, fct, ens_w, backend=backend) elif estimator == "akr": - out = _crps_ensemble_akr(obs, fct, w, backend=backend) + out = _crps_ensemble_akr(obs, fct, ens_w, backend=backend) elif estimator == "akr_circperm": - out = _crps_ensemble_akr_circperm(obs, fct, w, backend=backend) + out = _crps_ensemble_akr_circperm(obs, fct, ens_w, backend=backend) else: raise ValueError(f"{estimator} is not an available estimator") @@ -123,18 +123,22 @@ def ow_ensemble( fct: "Array", ow: "Array", fw: "Array", + ens_w: "Array", backend: "Backend" = None, ) -> "Array": """Outcome-Weighted CRPS estimator based on the energy form.""" B = backends.active if backend is None else backends[backend] - M: int = fct.shape[-1] - wbar = B.mean(fw, axis=-1) - e_1 = B.sum(B.abs(obs[..., None] - fct) * fw, axis=-1) * ow / (M * wbar) + wbar = B.mean(ens_w * fw, axis=-1) + e_1 = B.sum(ens_w * B.abs(obs[..., None] - fct) * fw, axis=-1) * ow / wbar e_2 = B.sum( - B.abs(fct[..., None] - fct[..., None, :]) * fw[..., None] * fw[..., None, :], + ens_w[..., None] + * ens_w[..., None, :] + * B.abs(fct[..., None] - fct[..., None, :]) + * fw[..., None] + * fw[..., None, :], axis=(-1, -2), ) - e_2 *= ow / (M**2 * wbar**2) + e_2 *= ow / (wbar**2) return e_1 - 0.5 * e_2 @@ -143,17 +147,20 @@ def vr_ensemble( fct: "Array", ow: "Array", fw: "Array", + ens_w: "Array", backend: "Backend" = None, ) -> "Array": """Vertically Re-scaled CRPS estimator based on the energy form.""" B = backends.active if backend is None else backends[backend] - M: int = fct.shape[-1] - e_1 = B.sum(B.abs(obs[..., None] - fct) * fw, axis=-1) * ow / M + e_1 = B.sum(ens_w * B.abs(obs[..., None] - fct) * fw, axis=-1) * ow e_2 = B.sum( - B.abs(B.expand_dims(fct, axis=-1) - B.expand_dims(fct, axis=-2)) - * (B.expand_dims(fw, axis=-1) * B.expand_dims(fw, axis=-2)), + ens_w[..., None] + * ens_w[..., None, :] + * B.abs(fct[..., None] - fct[..., None, :]) + * fw[..., None] + * fw[..., None, :], axis=(-1, -2), - ) / (M**2) - e_3 = B.mean(B.abs(fct) * fw, axis=-1) - B.abs(obs) * ow - e_3 *= B.mean(fw, axis=1) - ow + ) + e_3 = B.mean(ens_w * B.abs(fct) * fw, axis=-1) - B.abs(obs) * ow + e_3 *= B.mean(ens_w * fw, axis=1) - ow return e_1 - 0.5 * e_2 + e_3 diff --git a/scoringrules/core/crps/_gufuncs.py b/scoringrules/core/crps/_gufuncs.py index 7996cee..d530872 100644 --- a/scoringrules/core/crps/_gufuncs.py +++ b/scoringrules/core/crps/_gufuncs.py @@ -28,13 +28,15 @@ def quantile_pinball_gufunc( @guvectorize( [ - "void(float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", ], - "(),(n)->()", + "(),(n),(n)->()", ) -def _crps_ensemble_int_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): - """CRPS estimator based on the integral form.""" +def _crps_ensemble_int_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): + """CRPS estimator based on the integral form.""" # TODO: currently does not use weight argument obs = obs[0] M = fct.shape[0] @@ -76,7 +78,7 @@ def _crps_ensemble_int_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray) "void(float32[:], float32[:], float32[:], float32[:])", "void(float64[:], float64[:], float64[:], float64[:])", ], - "(),(n)->()", + "(),(n),(n)->()", ) def _crps_ensemble_qd_gufunc( obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray @@ -106,7 +108,7 @@ def _crps_ensemble_qd_gufunc( "void(float32[:], float32[:], float32[:], float32[:])", "void(float64[:], float64[:], float64[:], float64[:])", ], - "(),(n)->()", + "(),(n),(n)->()", ) def _crps_ensemble_nrg_gufunc( obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray @@ -135,7 +137,7 @@ def _crps_ensemble_nrg_gufunc( "void(float32[:], float32[:], float32[:], float32[:])", "void(float64[:], float64[:], float64[:], float64[:])", ], - "(),(n)->()", + "(),(n),(n)->()", ) def _crps_ensemble_fair_gufunc( obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray @@ -166,7 +168,7 @@ def _crps_ensemble_fair_gufunc( "void(float32[:], float32[:], float32[:])", "void(float64[:], float64[:], float64[:])", ], - "(),(n)->()", + "(),(n),(n)->()", ) def _crps_ensemble_pwm_gufunc( obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray @@ -197,7 +199,7 @@ def _crps_ensemble_pwm_gufunc( "void(float32[:], float32[:], float32[:], float32[:])", "void(float64[:], float64[:], float64[:], float64[:])", ], - "(),(n)->()", + "(),(n),(n)->()", ) def _crps_ensemble_akr_gufunc( obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray @@ -220,7 +222,7 @@ def _crps_ensemble_akr_gufunc( "void(float32[:], float32[:], float32[:], float32[:])", "void(float64[:], float64[:], float64[:], float64[:])", ], - "(),(n)->()", + "(),(n),(n)->()", ) def _crps_ensemble_akr_circperm_gufunc( obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray @@ -239,16 +241,17 @@ def _crps_ensemble_akr_circperm_gufunc( @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:], float64[:], float64[:])", ], - "(),(n),(),(n)->()", + "(),(n),(),(n),(n)->()", ) def _owcrps_ensemble_nrg_gufunc( obs: np.ndarray, fct: np.ndarray, ow: np.ndarray, fw: np.ndarray, + ens_w: np.ndarray, out: np.ndarray, ): """Outcome-weighted CRPS estimator based on the energy form.""" @@ -264,13 +267,13 @@ def _owcrps_ensemble_nrg_gufunc( e_2 = 0.0 for i in range(M): - e_1 += abs(fct[i] - obs) * fw[i] * ow + e_1 += ens_w[i] * abs(fct[i] - obs) * fw[i] * ow for j in range(i + 1, M): - e_2 += 2 * abs(fct[i] - fct[j]) * fw[i] * fw[j] * ow + e_2 += 2 * ens_w[i] * ens_w[j] * abs(fct[i] - fct[j]) * fw[i] * fw[j] * ow - wbar = np.mean(fw) + wbar = np.mean(ens_w * fw) - out[0] = e_1 / (M * wbar) - 0.5 * e_2 / ((M * wbar) ** 2) + out[0] = e_1 / wbar - 0.5 * e_2 / (wbar**2) @guvectorize( @@ -278,13 +281,14 @@ def _owcrps_ensemble_nrg_gufunc( "void(float32[:], float32[:], float32[:], float32[:], float32[:])", "void(float64[:], float64[:], float64[:], float64[:], float64[:])", ], - "(),(n),(),(n)->()", + "(),(n),(),(n),(n)->()", ) def _vrcrps_ensemble_nrg_gufunc( obs: np.ndarray, fct: np.ndarray, ow: np.ndarray, fw: np.ndarray, + ens_w: np.ndarray, out: np.ndarray, ): """Vertically re-scaled CRPS estimator based on the energy form.""" @@ -300,15 +304,15 @@ def _vrcrps_ensemble_nrg_gufunc( e_2 = 0.0 for i in range(M): - e_1 += abs(fct[i] - obs) * fw[i] * ow + e_1 += ens_w[i] * abs(fct[i] - obs) * fw[i] * ow for j in range(i + 1, M): - e_2 += 2 * abs(fct[i] - fct[j]) * fw[i] * fw[j] + e_2 += 2 * ens_w[i] * ens_w[j] * abs(fct[i] - fct[j]) * fw[i] * fw[j] - wbar = np.mean(fw) - wabs_x = np.mean(np.abs(fct) * fw) + wbar = np.mean(ens_w * fw) + wabs_x = np.mean(ens_w * np.abs(fct) * fw) wabs_y = abs(obs) * ow - out[0] = e_1 / M - 0.5 * e_2 / (M**2) + (wabs_x - wabs_y) * (wbar - ow) + out[0] = e_1 - 0.5 * e_2 + (wabs_x - wabs_y) * (wbar - ow) @njit(["float32(float32)", "float64(float64)"]) From 4be8135cd0e151cc4270124baabcccc1b95c07a6 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Wed, 16 Apr 2025 17:27:00 +0200 Subject: [PATCH 49/79] fix bugs when using ensemble weights in crps_ensemble --- scoringrules/_crps.py | 10 +++--- scoringrules/backend/base.py | 4 +++ scoringrules/backend/jax.py | 3 ++ scoringrules/backend/numpy.py | 3 ++ scoringrules/backend/tensorflow.py | 4 +++ scoringrules/backend/torch.py | 3 ++ scoringrules/core/crps/_approx.py | 11 +++--- scoringrules/core/crps/_gufuncs.py | 14 ++++---- tests/test_crps.py | 56 ++++++++++++++++++++++++++++-- tests/test_wcrps.py | 28 ++++----------- 10 files changed, 97 insertions(+), 39 deletions(-) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index bdeffb3..bee64f1 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -112,13 +112,13 @@ def crps_ensemble( M = fct.shape[-1] ens_w = B.zeros(fct.shape) + 1.0 / M else: - ens_w = map(B.asarray, ens_w) + ens_w = B.asarray(ens_w) ens_w = ens_w / B.sum(ens_w, axis=-1, keepdims=True) if m_axis != -1: - ens_w = B.moveaxis(ens_w, -1, -1) + ens_w = B.moveaxis(ens_w, m_axis, -1) if sort_ensemble: ind = B.argsort(fct, axis=-1) - ens_w = ens_w[ind] + ens_w = B.gather(ens_w, ind, axis=-1) if sort_ensemble: fct = B.sort(fct, axis=-1) @@ -333,7 +333,7 @@ def owcrps_ensemble( M = fct.shape[m_axis] ens_w = B.zeros(fct.shape) + 1.0 / M else: - ens_w = map(B.asarray, ens_w) + ens_w = B.asarray(ens_w) ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) if m_axis != -1: @@ -446,7 +446,7 @@ def vrcrps_ensemble( M = fct.shape[m_axis] ens_w = B.zeros(fct.shape) + 1.0 / M else: - ens_w = map(B.asarray, ens_w) + ens_w = B.asarray(ens_w) ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) if m_axis != -1: diff --git a/scoringrules/backend/base.py b/scoringrules/backend/base.py index 669d4f7..64f0828 100644 --- a/scoringrules/backend/base.py +++ b/scoringrules/backend/base.py @@ -308,3 +308,7 @@ def size(self, x: "Array") -> int: @abc.abstractmethod def indices(self, x: "Array") -> int: """Return an array representing the indices of a grid.""" + + @abc.abstractmethod + def gather(self, x: "Array", ind: "Array", axis: int) -> "Array": + """Reorder an array ``x`` depending on a template ``ind`` across an axis ``axis``.""" diff --git a/scoringrules/backend/jax.py b/scoringrules/backend/jax.py index bf6c259..2e5f872 100644 --- a/scoringrules/backend/jax.py +++ b/scoringrules/backend/jax.py @@ -279,6 +279,9 @@ def size(self, x: "Array") -> int: def indices(self, dimensions: tuple) -> "Array": return jnp.indices(dimensions) + def gather(self, x: "Array", ind: "Array", axis: int) -> "Array": + return jnp.take_along_axis(x, ind, axis=axis) + if __name__ == "__main__": B = JaxBackend() diff --git a/scoringrules/backend/numpy.py b/scoringrules/backend/numpy.py index 922d242..c1f9b08 100644 --- a/scoringrules/backend/numpy.py +++ b/scoringrules/backend/numpy.py @@ -276,6 +276,9 @@ def size(self, x: "NDArray") -> int: def indices(self, dimensions: tuple) -> "NDArray": return np.indices(dimensions) + def gather(self, x: "NDArray", ind: "NDArray", axis: int) -> "NDArray": + return np.take_along_axis(x, ind, axis=axis) + class NumbaBackend(NumpyBackend): """Numba backend.""" diff --git a/scoringrules/backend/tensorflow.py b/scoringrules/backend/tensorflow.py index fde7949..453bfd6 100644 --- a/scoringrules/backend/tensorflow.py +++ b/scoringrules/backend/tensorflow.py @@ -315,6 +315,10 @@ def indices(self, dimensions: tuple) -> "Tensor": indices = tf.stack(index_grids) return indices + def gather(self, x: "Tensor", ind: "Tensor", axis: int) -> "Tensor": + d = len(x.shape) + return tf.gather(x, ind, axis=axis, batch_dims=d) + if __name__ == "__main__": B = TensorflowBackend() diff --git a/scoringrules/backend/torch.py b/scoringrules/backend/torch.py index f007366..1ff0e59 100644 --- a/scoringrules/backend/torch.py +++ b/scoringrules/backend/torch.py @@ -297,3 +297,6 @@ def indices(self, dimensions: tuple) -> "Tensor": index_grids = torch.meshgrid(*ranges, indexing="ij") indices = torch.stack(index_grids) return indices + + def gather(self, x: "Tensor", ind: "Tensor", axis: int) -> "Tensor": + return torch.gather(x, index=ind, dim=axis) diff --git a/scoringrules/core/crps/_approx.py b/scoringrules/core/crps/_approx.py index 50d0731..863c329 100644 --- a/scoringrules/core/crps/_approx.py +++ b/scoringrules/core/crps/_approx.py @@ -27,7 +27,10 @@ def ensemble( elif estimator == "akr_circperm": out = _crps_ensemble_akr_circperm(obs, fct, ens_w, backend=backend) else: - raise ValueError(f"{estimator} is not an available estimator") + raise ValueError( + f"{estimator} can only be used with `numpy` " + "backend and needs `numba` to be installed" + ) return out @@ -128,7 +131,7 @@ def ow_ensemble( ) -> "Array": """Outcome-Weighted CRPS estimator based on the energy form.""" B = backends.active if backend is None else backends[backend] - wbar = B.mean(ens_w * fw, axis=-1) + wbar = B.sum(ens_w * fw, axis=-1) e_1 = B.sum(ens_w * B.abs(obs[..., None] - fct) * fw, axis=-1) * ow / wbar e_2 = B.sum( ens_w[..., None] @@ -161,6 +164,6 @@ def vr_ensemble( * fw[..., None, :], axis=(-1, -2), ) - e_3 = B.mean(ens_w * B.abs(fct) * fw, axis=-1) - B.abs(obs) * ow - e_3 *= B.mean(ens_w * fw, axis=1) - ow + e_3 = B.sum(ens_w * B.abs(fct) * fw, axis=-1) - B.abs(obs) * ow + e_3 *= B.sum(ens_w * fw, axis=1) - ow return e_1 - 0.5 * e_2 + e_3 diff --git a/scoringrules/core/crps/_gufuncs.py b/scoringrules/core/crps/_gufuncs.py index d530872..0eb7908 100644 --- a/scoringrules/core/crps/_gufuncs.py +++ b/scoringrules/core/crps/_gufuncs.py @@ -165,8 +165,8 @@ def _crps_ensemble_fair_gufunc( @guvectorize( [ - "void(float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", ], "(),(n),(n)->()", ) @@ -271,15 +271,15 @@ def _owcrps_ensemble_nrg_gufunc( for j in range(i + 1, M): e_2 += 2 * ens_w[i] * ens_w[j] * abs(fct[i] - fct[j]) * fw[i] * fw[j] * ow - wbar = np.mean(ens_w * fw) + wbar = np.sum(ens_w * fw) out[0] = e_1 / wbar - 0.5 * e_2 / (wbar**2) @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:], float64[:], float64[:])", ], "(),(n),(),(n),(n)->()", ) @@ -308,8 +308,8 @@ def _vrcrps_ensemble_nrg_gufunc( for j in range(i + 1, M): e_2 += 2 * ens_w[i] * ens_w[j] * abs(fct[i] - fct[j]) * fw[i] * fw[j] - wbar = np.mean(ens_w * fw) - wabs_x = np.mean(ens_w * np.abs(fct) * fw) + wbar = np.sum(ens_w * fw) + wabs_x = np.sum(ens_w * np.abs(fct) * fw) wabs_y = abs(obs) * ow out[0] = e_1 - 0.5 * e_2 + (wabs_x - wabs_y) * (wbar - ow) diff --git a/tests/test_crps.py b/tests/test_crps.py index d61672b..3f0c62c 100644 --- a/tests/test_crps.py +++ b/tests/test_crps.py @@ -21,7 +21,7 @@ def test_crps_ensemble(estimator, backend): # test exceptions if backend in ["numpy", "jax", "torch", "tensorflow"]: - if estimator not in ["nrg", "fair", "pwm"]: + if estimator == "int": with pytest.raises(ValueError): sr.crps_ensemble(obs, fct, estimator=estimator, backend=backend) return @@ -43,7 +43,7 @@ def test_crps_ensemble(estimator, backend): assert res.shape == (N,) # non-negative values - if estimator not in ["akr", "akr_circperm"]: + if estimator not in ["akr", "akr_circperm", "int"]: res = sr.crps_ensemble(obs, fct, estimator=estimator, backend=backend) res = np.asarray(res) assert not np.any(res < 0.0) @@ -54,6 +54,58 @@ def test_crps_ensemble(estimator, backend): res = np.asarray(res) assert not np.any(res - 0.0 > 0.0001) + # test equivalence of different estimators + res_nrg = sr.crps_ensemble(obs, fct, estimator="nrg", backend=backend) + res_pwm = sr.crps_ensemble(obs, fct, estimator="pwm", backend=backend) + res_qd = sr.crps_ensemble(obs, fct, estimator="qd", backend=backend) + assert np.allclose(res_nrg, res_pwm) + assert np.allclose(res_nrg, res_qd) + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_crps_ensemble_corr(backend): + obs = np.random.randn(N) + mu = obs + np.random.randn(N) * 0.3 + sigma = abs(np.random.randn(N)) * 0.5 + fct = np.random.randn(N, ENSEMBLE_SIZE) * sigma[..., None] + mu[..., None] + + # test equivalence of different estimators + res_nrg = sr.crps_ensemble(obs, fct, estimator="nrg", backend=backend) + res_pwm = sr.crps_ensemble(obs, fct, estimator="pwm", backend=backend) + res_qd = sr.crps_ensemble(obs, fct, estimator="qd", backend=backend) + assert np.allclose(res_nrg, res_pwm) + assert np.allclose(res_nrg, res_qd) + + w = np.abs(np.random.randn(N, ENSEMBLE_SIZE) * sigma[..., None]) + res_nrg = sr.crps_ensemble(obs, fct, ens_w=w, estimator="nrg", backend=backend) + res_pwm = sr.crps_ensemble(obs, fct, ens_w=w, estimator="pwm", backend=backend) + res_qd = sr.crps_ensemble(obs, fct, ens_w=w, estimator="qd", backend=backend) + assert np.allclose(res_nrg, res_pwm) + assert np.allclose(res_nrg, res_qd) + + # test correctness + obs = -0.6042506 + fct = np.array( + [ + 1.7812118, + 0.5863797, + 0.7038174, + -0.7743998, + -0.2751647, + 1.1863249, + 1.2990966, + -0.3242982, + -0.5968781, + 0.9064937, + ] + ) + res = sr.crps_ensemble(obs, fct) + assert np.isclose(res, 0.6126602) + + w = np.arange(10) + res = sr.crps_ensemble(obs, fct, ens_w=w) + assert np.isclose(res, 0.4923673) + @pytest.mark.parametrize("backend", BACKENDS) def test_crps_quantile(backend): diff --git a/tests/test_wcrps.py b/tests/test_wcrps.py index e39d02d..601d316 100644 --- a/tests/test_wcrps.py +++ b/tests/test_wcrps.py @@ -10,11 +10,6 @@ @pytest.mark.parametrize("backend", BACKENDS) def test_owcrps_ensemble(backend): - # test exceptions - with pytest.raises(ValueError): - est = "not_nrg" - sr.owcrps_ensemble(1, 1.1, w_func=lambda x: x, estimator=est, backend=backend) - # test shapes obs = np.random.randn(N) res = sr.owcrps_ensemble(obs, np.random.randn(N, M), w_func=lambda x: x * 0.0 + 1.0) @@ -27,11 +22,6 @@ def test_owcrps_ensemble(backend): @pytest.mark.parametrize("backend", BACKENDS) def test_vrcrps_ensemble(backend): - # test exceptions - with pytest.raises(ValueError): - est = "not_nrg" - sr.vrcrps_ensemble(1, 1.1, w_func=lambda x: x, estimator=est, backend=backend) - # test shapes obs = np.random.randn(N) res = sr.vrcrps_ensemble(obs, np.random.randn(N, M), w_func=lambda x: x * 0.0 + 1.0) @@ -75,22 +65,20 @@ def test_owcrps_vs_crps(backend): sigma = abs(np.random.randn(N)) * 0.5 fct = np.random.randn(N, M) * sigma[..., None] + mu[..., None] - res = sr.crps_ensemble(obs, fct, backend=backend, estimator="nrg") + res = sr.crps_ensemble(obs, fct, backend=backend) # no argument given - resw = sr.owcrps_ensemble(obs, fct, estimator="nrg", backend=backend) + resw = sr.owcrps_ensemble(obs, fct, backend=backend) np.testing.assert_allclose(res, resw, rtol=1e-5) # a and b resw = sr.owcrps_ensemble( - obs, fct, a=float("-inf"), b=float("inf"), estimator="nrg", backend=backend + obs, fct, a=float("-inf"), b=float("inf"), backend=backend ) np.testing.assert_allclose(res, resw, rtol=1e-5) # w_func as identity function - resw = sr.owcrps_ensemble( - obs, fct, w_func=lambda x: x * 0.0 + 1.0, estimator="nrg", backend=backend - ) + resw = sr.owcrps_ensemble(obs, fct, w_func=lambda x: x * 0.0 + 1.0, backend=backend) np.testing.assert_allclose(res, resw, rtol=1e-5) @@ -104,19 +92,17 @@ def test_vrcrps_vs_crps(backend): res = sr.crps_ensemble(obs, fct, backend=backend, estimator="nrg") # no argument given - resw = sr.vrcrps_ensemble(obs, fct, estimator="nrg", backend=backend) + resw = sr.vrcrps_ensemble(obs, fct, backend=backend) np.testing.assert_allclose(res, resw, rtol=1e-5) # a and b resw = sr.vrcrps_ensemble( - obs, fct, a=float("-inf"), b=float("inf"), estimator="nrg", backend=backend + obs, fct, a=float("-inf"), b=float("inf"), backend=backend ) np.testing.assert_allclose(res, resw, rtol=1e-5) # w_func as identity function - resw = sr.vrcrps_ensemble( - obs, fct, w_func=lambda x: x * 0.0 + 1.0, estimator="nrg", backend=backend - ) + resw = sr.vrcrps_ensemble(obs, fct, w_func=lambda x: x * 0.0 + 1.0, backend=backend) np.testing.assert_allclose(res, resw, rtol=1e-5) From e44ce58cd7d5be757fe80a2afbf8dbaa0b67df5f Mon Sep 17 00:00:00 2001 From: sallen12 Date: Thu, 17 Apr 2025 15:15:29 +0200 Subject: [PATCH 50/79] add check_pars argument and parameter checks for crps distributions --- scoringrules/_crps.py | 444 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 437 insertions(+), 7 deletions(-) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index bee64f1..5c106e2 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -113,7 +113,9 @@ def crps_ensemble( ens_w = B.zeros(fct.shape) + 1.0 / M else: ens_w = B.asarray(ens_w) - ens_w = ens_w / B.sum(ens_w, axis=-1, keepdims=True) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) if m_axis != -1: ens_w = B.moveaxis(ens_w, m_axis, -1) if sort_ensemble: @@ -334,6 +336,8 @@ def owcrps_ensemble( ens_w = B.zeros(fct.shape) + 1.0 / M else: ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) if m_axis != -1: @@ -346,7 +350,8 @@ def w_func(x): return ((a <= x) & (x <= b)) * 1.0 obs_weights, fct_weights = map(w_func, (obs, fct)) - obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) + if B.any(obs_weights < 0) or B.any(fct_weights < 0): + raise ValueError("`w_func` returns negative values") if backend == "numba": return crps.estimator_gufuncs["ownrg"]( @@ -447,6 +452,8 @@ def vrcrps_ensemble( ens_w = B.zeros(fct.shape) + 1.0 / M else: ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) if m_axis != -1: @@ -459,7 +466,8 @@ def w_func(x): return ((a <= x) & (x <= b)) * 1.0 obs_weights, fct_weights = map(w_func, (obs, fct)) - obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) + if B.any(obs_weights < 0) or B.any(fct_weights < 0): + raise ValueError("`w_func` returns negative values") if backend == "numba": return crps.estimator_gufuncs["vrnrg"]( @@ -521,11 +529,14 @@ def crps_quantile( Journal of Econometrics, 237(2), 105221. Available at https://arxiv.org/abs/2102.00968. - # TODO: add example + # TODO: add example, change reference """ B = backends.active if backend is None else backends[backend] obs, fct, alpha = map(B.asarray, (obs, fct, alpha)) + if B.any(alpha <= 0) or B.any(alpha >= 1): + raise ValueError("`alpha` contains entries that are not between 0 and 1.") + if m_axis != -1: fct = B.moveaxis(fct, m_axis, -1) @@ -547,6 +558,7 @@ def crps_beta( upper: "ArrayLike" = 1.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the beta distribution. @@ -580,6 +592,9 @@ def crps_beta( Upper bound of the forecast beta distribution. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -599,6 +614,19 @@ def crps_beta( >>> sr.crps_beta(0.3, 0.7, 1.1) 0.08501024366637236 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + a, b, lower, upper = map(B.asarray, (a, b, lower, upper)) + if B.any(a <= 0): + raise ValueError( + "`a` contains non-positive entries. The shape parameters of the Beta distribution must be positive." + ) + if B.any(b <= 0): + raise ValueError( + "`b` contains non-positive entries. The shape parameters of the Beta distribution must be positive." + ) + if B.any(lower >= upper): + raise ValueError("`lower` is not always smaller than `upper`.") return crps.beta(obs, a, b, lower, upper, backend=backend) @@ -609,6 +637,7 @@ def crps_binomial( /, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the binomial distribution. @@ -631,6 +660,9 @@ def crps_binomial( Probability parameter of the forecast binomial distribution as a float or array of floats. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -650,6 +682,11 @@ def crps_binomial( >>> sr.crps_binomial(4, 10, 0.5) 0.5955772399902344 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + prob = B.asarray(prob) + if B.any(prob < 0) or B.any(prob > 1): + raise ValueError("`prob` contains values outside the range [0, 1].") return crps.binomial(obs, n, prob, backend=backend) @@ -659,6 +696,7 @@ def crps_exponential( /, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the exponential distribution. @@ -677,6 +715,9 @@ def crps_exponential( Rate parameter of the forecast exponential distribution. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -699,6 +740,13 @@ def crps_exponential( >>> sr.crps_exponential(np.array([0.8, 0.9]), np.array([3.0, 2.0])) array([0.36047864, 0.31529889]) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + rate = B.asarray(rate) + if B.any(rate <= 0): + raise ValueError( + "`rate` contains non-positive entries. The rate parameter of the exponential distribution must be positive." + ) return crps.exponential(obs, rate, backend=backend) @@ -710,6 +758,7 @@ def crps_exponentialM( scale: "ArrayLike" = 1.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the standard exponential distribution with a point mass at the boundary. @@ -743,6 +792,9 @@ def crps_exponentialM( Scale parameter of the forecast exponential distribution. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -762,6 +814,15 @@ def crps_exponentialM( >>> sr.crps_exponentialM(0.4, 0.2, 0.0, 1.0) 0.19251207365702294 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, mass = map(B.asarray, (scale, mass)) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter must be positive." + ) + if B.any(mass < 0) or B.any(mass > 1): + raise ValueError("`mass` contains entries outside the range [0, 1].") return crps.exponentialM(obs, mass, location, scale, backend=backend) @@ -773,6 +834,7 @@ def crps_2pexponential( /, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the two-piece exponential distribution. @@ -799,6 +861,9 @@ def crps_2pexponential( Location parameter of the forecast two-piece exponential distribution. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -818,6 +883,17 @@ def crps_2pexponential( >>> sr.crps_2pexponential(0.8, 3.0, 1.4, 0.0) array(1.18038524) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale1, scale2 = map(B.asarray, (scale1, scale2)) + if B.any(scale1 <= 0): + raise ValueError( + "`scale1` contains non-positive entries. The scale parameters of the two-piece exponential distribution must be positive." + ) + if B.any(scale2 <= 0): + raise ValueError( + "`scale2` contains non-positive entries. The scale parameters of the two-piece exponential distribution must be positive." + ) return crps.twopexponential(obs, scale1, scale2, location, backend=backend) @@ -829,6 +905,7 @@ def crps_gamma( *, scale: "ArrayLike | None" = None, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the gamma distribution. @@ -857,6 +934,9 @@ def crps_gamma( Either ``rate`` or ``scale`` must be provided. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -889,6 +969,18 @@ def crps_gamma( if rate is None: rate = 1.0 / scale + if check_pars: + B = backends.active if backend is None else backends[backend] + shape, rate = map(B.asarray, (shape, rate)) + if B.any(shape <= 0): + raise ValueError( + "`shape` contains non-positive entries. The shape parameter of the gamma distribution must be positive." + ) + if B.any(rate <= 0): + raise ValueError( + "`rate` or `scale` contains non-positive entries. The rate and scale parameters of the gamma distribution must be positive." + ) + return crps.gamma(obs, shape, rate, backend=backend) @@ -900,6 +992,7 @@ def crps_gev( scale: "ArrayLike" = 1.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the generalised extreme value (GEV) distribution. @@ -923,6 +1016,9 @@ def crps_gev( Scale parameter of the forecast GEV distribution. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -984,6 +1080,17 @@ def crps_gev( >>> sr.crps_gev(0.3, 0.1) 0.2924712413052034 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, shape = map(B.asarray, (scale, shape)) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the GEV distribution must be positive." + ) + if B.any(shape >= 1): + raise ValueError( + "`shape` contains entries larger than 1. The CRPS for the GEV distribution is only valid for shape values less than 1." + ) return crps.gev(obs, shape, location, scale, backend=backend) @@ -996,6 +1103,7 @@ def crps_gpd( mass: "ArrayLike" = 0.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the generalised pareto distribution (GPD). @@ -1028,6 +1136,9 @@ def crps_gpd( Mass parameter at the lower boundary of the forecast GPD distribution. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1047,6 +1158,21 @@ def crps_gpd( >>> sr.crps_gpd(0.3, 0.9) 0.6849331901197213 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, shape, mass = map(B.asarray, (scale, shape, mass)) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the GPD distribution must be positive. `nan` is returned in these places." + ) + if B.any(shape >= 1): + raise ValueError( + "`shape` contains entries larger than 1. The CRPS for the GPD distribution is only valid for shape values less than 1. `nan` is returned in these places." + ) + if B.any(mass < 0) or B.any(mass > 1): + raise ValueError( + "`mass` contains entries outside the range [0, 1]. `nan` is returned in these places." + ) return crps.gpd(obs, shape, location, scale, mass, backend=backend) @@ -1061,6 +1187,7 @@ def crps_gtclogistic( umass: "ArrayLike" = 0.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the generalised truncated and censored logistic distribution. @@ -1107,6 +1234,9 @@ def crps_gtclogistic( Point mass assigned to the lower boundary of the forecast distribution. umass : array_like Point mass assigned to the upper boundary of the forecast distribution. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1119,6 +1249,23 @@ def crps_gtclogistic( >>> sr.crps_gtclogistic(0.0, 0.1, 0.4, -1.0, 1.0, 0.1, 0.1) 0.1658713056903939 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, lmass, umass, lower, upper = map( + B.asarray, (scale, lmass, umass, lower, upper) + ) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the generalised logistic distribution must be positive." + ) + if B.any(lmass < 0) or B.any(lmass > 1): + raise ValueError("`lmass` contains entries outside the range [0, 1].") + if B.any(umass < 0) or B.any(umass > 1): + raise ValueError("`umass` contains entries outside the range [0, 1].") + if B.any(umass + lmass >= 1): + raise ValueError("The sum of `umass` and `lmass` should be smaller than 1.") + if B.any(lower >= upper): + raise ValueError("`lower` is not always smaller than `upper`.") return crps.gtclogistic( obs, location, @@ -1140,6 +1287,7 @@ def crps_tlogistic( upper: "ArrayLike" = float("inf"), *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the truncated logistic distribution. @@ -1158,6 +1306,9 @@ def crps_tlogistic( Lower boundary of the truncated forecast distribution. upper : array_like Upper boundary of the truncated forecast distribution. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1170,8 +1321,24 @@ def crps_tlogistic( >>> sr.crps_tlogistic(0.0, 0.1, 0.4, -1.0, 1.0) 0.12714830546327846 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, lower, upper = map(B.asarray, (scale, lower, upper)) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the truncated logistic distribution must be positive." + ) + if B.any(lower >= upper): + raise ValueError("`lower` is not always smaller than `upper`.") return crps.gtclogistic( - obs, location, scale, lower, upper, 0.0, 0.0, backend=backend + obs, + location, + scale, + lower, + upper, + 0.0, + 0.0, + backend=backend, ) @@ -1184,6 +1351,7 @@ def crps_clogistic( upper: "ArrayLike" = float("inf"), *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the censored logistic distribution. @@ -1202,6 +1370,9 @@ def crps_clogistic( Lower boundary of the truncated forecast distribution. upper : array_like Upper boundary of the truncated forecast distribution. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1214,6 +1385,15 @@ def crps_clogistic( >>> sr.crps_clogistic(0.0, 0.1, 0.4, -1.0, 1.0) 0.15805632276434345 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, lower, upper = map(B.asarray, (scale, lower, upper)) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the censored logistic distribution must be positive." + ) + if B.any(lower >= upper): + raise ValueError("`lower` is not always smaller than `upper`.") lmass = stats._logis_cdf((lower - location) / scale) umass = 1 - stats._logis_cdf((upper - location) / scale) return crps.gtclogistic( @@ -1239,6 +1419,7 @@ def crps_gtcnormal( umass: "ArrayLike" = 0.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the generalised truncated and censored normal distribution. @@ -1272,6 +1453,23 @@ def crps_gtcnormal( >>> sr.crps_gtcnormal(0.0, 0.1, 0.4, -1.0, 1.0, 0.1, 0.1) 0.1351100832878575 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, lmass, umass, lower, upper = map( + B.asarray, (scale, lmass, umass, lower, upper) + ) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the generalised normal distribution must be positive." + ) + if B.any(lmass < 0) or B.any(lmass > 1): + raise ValueError("`lmass` contains entries outside the range [0, 1].") + if B.any(umass < 0) or B.any(umass > 1): + raise ValueError("`umass` contains entries outside the range [0, 1].") + if B.any(umass + lmass >= 1): + raise ValueError("The sum of `umass` and `lmass` should be smaller than 1.") + if B.any(lower >= upper): + raise ValueError("`lower` is not always smaller than `upper`.") return crps.gtcnormal( obs, location, @@ -1293,6 +1491,7 @@ def crps_tnormal( upper: "ArrayLike" = float("inf"), *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the truncated normal distribution. @@ -1311,6 +1510,9 @@ def crps_tnormal( Lower boundary of the truncated forecast distribution. upper : array_like Upper boundary of the truncated forecast distribution. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1323,6 +1525,15 @@ def crps_tnormal( >>> sr.crps_tnormal(0.0, 0.1, 0.4, -1.0, 1.0) 0.10070146718008832 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, lower, upper = map(B.asarray, (scale, lower, upper)) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the truncated normal distribution must be positive." + ) + if B.any(lower >= upper): + raise ValueError("`lower` is not always smaller than `upper`.") return crps.gtcnormal(obs, location, scale, lower, upper, 0.0, 0.0, backend=backend) @@ -1335,6 +1546,7 @@ def crps_cnormal( upper: "ArrayLike" = float("inf"), *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the censored normal distribution. @@ -1353,6 +1565,9 @@ def crps_cnormal( Lower boundary of the truncated forecast distribution. upper : array_like Upper boundary of the truncated forecast distribution. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1365,6 +1580,15 @@ def crps_cnormal( >>> sr.crps_cnormal(0.0, 0.1, 0.4, -1.0, 1.0) 0.10338851213123085 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, lower, upper = map(B.asarray, (scale, lower, upper)) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the censored normal distribution must be positive." + ) + if B.any(lower >= upper): + raise ValueError("`lower` is not always smaller than `upper`.") lmass = stats._norm_cdf((lower - location) / scale) umass = 1 - stats._norm_cdf((upper - location) / scale) return crps.gtcnormal( @@ -1391,6 +1615,7 @@ def crps_gtct( umass: "ArrayLike" = 0.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the generalised truncated and censored t distribution. @@ -1435,6 +1660,9 @@ def crps_gtct( Point mass assigned to the lower boundary of the forecast distribution. umass : array_like Point mass assigned to the upper boundary of the forecast distribution. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1454,6 +1682,27 @@ def crps_gtct( >>> sr.crps_gtct(0.0, 2.0, 0.1, 0.4, -1.0, 1.0, 0.1, 0.1) 0.13997789333289662 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, lmass, umass, lower, upper = map( + B.asarray, (scale, lmass, umass, lower, upper) + ) + if B.any(df <= 0): + raise ValueError( + "`df` contains non-positive entries. The degrees of freedom parameter of the generalised t distribution must be positive." + ) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the generalised t distribution must be positive." + ) + if B.any(lmass < 0) or B.any(lmass > 1): + raise ValueError("`lmass` contains entries outside the range [0, 1].") + if B.any(umass < 0) or B.any(umass > 1): + raise ValueError("`umass` contains entries outside the range [0, 1].") + if B.any(umass + lmass >= 1): + raise ValueError("The sum of `umass` and `lmass` should be smaller than 1.") + if B.any(lower >= upper): + raise ValueError("`lower` is not always smaller than `upper`.") return crps.gtct( obs, df, @@ -1477,6 +1726,7 @@ def crps_tt( upper: "ArrayLike" = float("inf"), *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the truncated t distribution. @@ -1497,6 +1747,9 @@ def crps_tt( Lower boundary of the truncated forecast distribution. upper : array_like Upper boundary of the truncated forecast distribution. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1509,6 +1762,19 @@ def crps_tt( >>> sr.crps_tt(0.0, 2.0, 0.1, 0.4, -1.0, 1.0) 0.10323007471747117 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, lower, upper = map(B.asarray, (scale, lower, upper)) + if B.any(df <= 0): + raise ValueError( + "`df` contains non-positive entries. The degrees of freedom parameter of the truncated t distribution must be positive." + ) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the truncated t distribution must be positive." + ) + if B.any(lower >= upper): + raise ValueError("`lower` is not always smaller than `upper`.") return crps.gtct( obs, df, @@ -1532,6 +1798,7 @@ def crps_ct( upper: "ArrayLike" = float("inf"), *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the censored t distribution. @@ -1552,6 +1819,9 @@ def crps_ct( Lower boundary of the truncated forecast distribution. upper : array_like Upper boundary of the truncated forecast distribution. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1564,6 +1834,19 @@ def crps_ct( >>> sr.crps_ct(0.0, 2.0, 0.1, 0.4, -1.0, 1.0) 0.12672580744453948 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, lower, upper = map(B.asarray, (scale, lower, upper)) + if B.any(df <= 0): + raise ValueError( + "`df` contains non-positive entries. The degrees of freedom parameter of the censored t distribution must be positive." + ) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the censored t distribution must be positive." + ) + if B.any(lower >= upper): + raise ValueError("`lower` is not always smaller than `upper`.") lmass = stats._t_cdf((lower - location) / scale, df) umass = 1 - stats._t_cdf((upper - location) / scale, df) return crps.gtct( @@ -1587,6 +1870,7 @@ def crps_hypergeometric( /, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the hypergeometric distribution. @@ -1612,6 +1896,9 @@ def crps_hypergeometric( Number of draws, without replacement. Must be in 0, 1, ..., m + n. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1631,6 +1918,7 @@ def crps_hypergeometric( >>> sr.crps_hypergeometric(5, 7, 13, 12) 0.44697415547610597 """ + # TODO: add check that m,n,k are integers return crps.hypergeometric(obs, m, n, k, backend=backend) @@ -1641,6 +1929,7 @@ def crps_laplace( scale: "ArrayLike" = 1.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the laplace distribution. @@ -1663,6 +1952,9 @@ def crps_laplace( Scale parameter of the forecast laplace distribution. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1682,6 +1974,13 @@ def crps_laplace( >>> sr.crps_laplace(0.3, 0.1, 0.2) 0.12357588823428847 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale = B.asarray(scale) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the laplace must be positive." + ) return crps.laplace(obs, location, scale, backend=backend) @@ -1692,6 +1991,7 @@ def crps_logistic( /, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the logistic distribution. @@ -1711,6 +2011,9 @@ def crps_logistic( Location parameter of the forecast logistic distribution. sigma: array_like Scale parameter of the forecast logistic distribution. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1730,6 +2033,13 @@ def crps_logistic( >>> sr.crps_logistic(0.0, 0.4, 0.1) 0.3036299855835619 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + sigma = B.asarray(sigma) + if B.any(sigma <= 0): + raise ValueError( + "`sigma` contains non-positive entries. The scale parameter of the logistic distribution must be positive." + ) return crps.logistic(obs, mu, sigma, backend=backend) @@ -1739,6 +2049,7 @@ def crps_loglaplace( scalelog: "ArrayLike", *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the log-Laplace distribution. @@ -1771,6 +2082,9 @@ def crps_loglaplace( Scale parameter of the forecast log-laplace distribution. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1790,6 +2104,13 @@ def crps_loglaplace( >>> sr.crps_loglaplace(3.0, 0.1, 0.9) 1.162020513653791 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scalelog = B.asarray(scalelog) + if B.any(scalelog <= 0) or B.any(scalelog >= 1): + raise ValueError( + "`scalelog` contains entries outside of the range (0, 1). The scale parameter of the log-laplace distribution must be between 0 and 1." + ) return crps.loglaplace(obs, locationlog, scalelog, backend=backend) @@ -1798,6 +2119,7 @@ def crps_loglogistic( mulog: "ArrayLike", sigmalog: "ArrayLike", backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the log-logistic distribution. @@ -1830,7 +2152,9 @@ def crps_loglogistic( Scale parameter of the log-logistic distribution. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. - + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1850,6 +2174,13 @@ def crps_loglogistic( >>> sr.crps_loglogistic(3.0, 0.1, 0.9) 1.1329527730161177 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + sigmalog = B.asarray(sigmalog) + if B.any(sigmalog <= 0) or B.any(sigmalog >= 1): + raise ValueError( + "`sigmalog` contains entries outside of the range (0, 1). The scale parameter of the log-logistic distribution must be between 0 and 1." + ) return crps.loglogistic(obs, mulog, sigmalog, backend=backend) @@ -1858,6 +2189,7 @@ def crps_lognormal( mulog: "ArrayLike", sigmalog: "ArrayLike", backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the lognormal distribution. @@ -1882,6 +2214,9 @@ def crps_lognormal( Mean of the normal underlying distribution. sigmalog : array_like Standard deviation of the underlying normal distribution. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1900,6 +2235,13 @@ def crps_lognormal( >>> sr.crps_lognormal(0.1, 0.4, 0.0) 1.3918246976412703 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + sigmalog = B.asarray(sigmalog) + if B.any(sigmalog <= 0): + raise ValueError( + "`sigmalog` contains non-positive entries. The scale parameter of the log-normal distribution must be positive." + ) return crps.lognormal(obs, mulog, sigmalog, backend=backend) @@ -1912,6 +2254,7 @@ def crps_mixnorm( m_axis: "ArrayLike" = -1, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for a mixture of normal distributions. @@ -1937,6 +2280,9 @@ def crps_mixnorm( The axis corresponding to the mixture components. Default is the last axis. backend : str, optional The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1964,6 +2310,15 @@ def crps_mixnorm( w = B.zeros(m.shape) + 1 / M else: w = B.asarray(w) + w = w / B.sum(w, axis=m_axis, keepdims=True) + + if check_pars: + if B.any(s <= 0): + raise ValueError( + "`s` contains non-positive entries. The scale parameters of the normal distributions should be positive." + ) + if B.any(w < 0): + raise ValueError("`w` contains negative entries") if m_axis != -1: m = B.moveaxis(m, m_axis, -1) @@ -1981,6 +2336,7 @@ def crps_negbinom( *, mu: "ArrayLike | None" = None, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the negative binomial distribution. @@ -2003,6 +2359,9 @@ def crps_negbinom( Probability parameter of the forecast negative binomial distribution. mu: array_like Mean of the forecast negative binomial distribution. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -2034,6 +2393,12 @@ def crps_negbinom( if prob is None: prob = n / (n + mu) + if check_pars: + B = backends.active if backend is None else backends[backend] + prob = B.asarray(prob) + if B.any(prob < 0) or B.any(prob > 1): + raise ValueError("`prob` contains values outside the range [0, 1].") + return crps.negbinom(obs, n, prob, backend=backend) @@ -2044,6 +2409,7 @@ def crps_normal( /, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the normal distribution. @@ -2063,6 +2429,9 @@ def crps_normal( Mean of the forecast normal distribution. sigma: array_like Standard deviation of the forecast normal distribution. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -2081,6 +2450,13 @@ def crps_normal( >>> sr.crps_normal(0.0, 0.1, 0.4) 0.10339992515976162 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + sigma = B.asarray(sigma) + if B.any(sigma <= 0): + raise ValueError( + "`sigma` contains non-positive entries. The standard deviation of the normal distribution must be positive." + ) return crps.normal(obs, mu, sigma, backend=backend) @@ -2092,6 +2468,7 @@ def crps_2pnormal( /, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the two-piece normal distribution. @@ -2116,6 +2493,9 @@ def crps_2pnormal( Scale parameter of the upper half of the forecast two-piece normal distribution. mu: array_like Location parameter of the forecast two-piece normal distribution. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -2137,6 +2517,15 @@ def crps_2pnormal( """ B = backends.active if backend is None else backends[backend] obs, scale1, scale2, location = map(B.asarray, (obs, scale1, scale2, location)) + if check_pars: + if B.any(scale1 <= 0): + raise ValueError( + "`scale1` contains non-positive entries. The scale parameters of the two-piece normal distribution must be positive." + ) + if B.any(scale2 <= 0): + raise ValueError( + "`scale2` contains non-positive entries. The scale parameters of the two-piece normal distribution must be positive." + ) lower = float("-inf") upper = 0.0 lmass = 0.0 @@ -2162,6 +2551,7 @@ def crps_poisson( /, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the Poisson distribution. @@ -2181,6 +2571,9 @@ def crps_poisson( The observed values. mean : array_like Mean parameter of the forecast poisson distribution. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -2199,6 +2592,13 @@ def crps_poisson( >>> sr.crps_poisson(1, 2) 0.4991650450203817 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + mean = B.asarray(mean) + if B.any(mean <= 0): + raise ValueError( + "`mean` contains non-positive entries. The mean parameter of the Poisson distribution must be positive." + ) return crps.poisson(obs, mean, backend=backend) @@ -2210,6 +2610,7 @@ def crps_t( scale: "ArrayLike" = 1.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the student's t distribution. @@ -2234,8 +2635,11 @@ def crps_t( Degrees of freedom parameter of the forecast t distribution. location : array_like Location parameter of the forecast t distribution. - sigma : array_like + scale : array_like Scale parameter of the forecast t distribution. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -2255,6 +2659,17 @@ def crps_t( >>> sr.crps_t(0.0, 0.1, 0.4, 0.1) 0.07687151141732129 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + df, scale = map(B.asarray, (df, scale)) + if B.any(df <= 0): + raise ValueError( + "`df` contains non-positive entries. The degrees of freedom parameter of the t distribution must be positive." + ) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the t distribution must be positive." + ) return crps.t(obs, df, location, scale, backend=backend) @@ -2267,6 +2682,7 @@ def crps_uniform( umass: "ArrayLike" = 0.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the closed form of the CRPS for the uniform distribution. @@ -2294,6 +2710,9 @@ def crps_uniform( Point mass on the lower bound of the forecast uniform distribution. umass : array_like Point mass on the upper bound of the forecast uniform distribution. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -2313,6 +2732,17 @@ def crps_uniform( >>> sr.crps_uniform(0.4, 0.0, 1.0, 0.0, 0.0) 0.09333333333333332 """ + if check_pars: + B = backends.active if backend is None else backends[backend] + lmass, umass, min, max = map(B.asarray, (lmass, umass, min, max)) + if B.any(lmass < 0) or B.any(lmass > 1): + raise ValueError("`lmass` contains entries outside the range [0, 1].") + if B.any(umass < 0) or B.any(umass > 1): + raise ValueError("`umass` contains entries outside the range [0, 1].") + if B.any(umass + lmass >= 1): + raise ValueError("The sum of `umass` and `lmass` should be smaller than 1.") + if B.any(min >= max): + raise ValueError("`min` is not always smaller than `max`.") return crps.uniform(obs, min, max, lmass, umass, backend=backend) From b00c5f16873e4277fb9293d9f54be8046af0f57f Mon Sep 17 00:00:00 2001 From: sallen12 Date: Thu, 17 Apr 2025 16:30:52 +0200 Subject: [PATCH 51/79] add default parameter arguments to crps and log scores for parametric distributions --- scoringrules/_crps.py | 111 ++++++----- scoringrules/_logs.py | 455 +++++++++++++++++++++++++++++++++++------- tests/test_crps.py | 18 +- tests/test_logs.py | 2 +- 4 files changed, 458 insertions(+), 128 deletions(-) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index 5c106e2..453eab0 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -480,8 +480,8 @@ def w_func(x): def crps_quantile( obs: "ArrayLike", fct: "Array", - alpha: "Array", /, + alpha: "Array", m_axis: int = -1, *, backend: "Backend" = None, @@ -551,9 +551,9 @@ def crps_quantile( def crps_beta( obs: "ArrayLike", + /, a: "ArrayLike", b: "ArrayLike", - /, lower: "ArrayLike" = 0.0, upper: "ArrayLike" = 1.0, *, @@ -632,9 +632,9 @@ def crps_beta( def crps_binomial( obs: "ArrayLike", + /, n: "ArrayLike", prob: "ArrayLike", - /, *, backend: "Backend" = None, check_pars: bool = False, @@ -692,8 +692,8 @@ def crps_binomial( def crps_exponential( obs: "ArrayLike", - rate: "ArrayLike", /, + rate: "ArrayLike", *, backend: "Backend" = None, check_pars: bool = False, @@ -753,9 +753,9 @@ def crps_exponential( def crps_exponentialM( obs: "ArrayLike", /, - mass: "ArrayLike" = 0.0, location: "ArrayLike" = 0.0, scale: "ArrayLike" = 1.0, + mass: "ArrayLike" = 0.0, *, backend: "Backend" = None, check_pars: bool = False, @@ -823,15 +823,15 @@ def crps_exponentialM( ) if B.any(mass < 0) or B.any(mass > 1): raise ValueError("`mass` contains entries outside the range [0, 1].") - return crps.exponentialM(obs, mass, location, scale, backend=backend) + return crps.exponentialM(obs, location, scale, mass, backend=backend) def crps_2pexponential( obs: "ArrayLike", + /, scale1: "ArrayLike", scale2: "ArrayLike", - location: "ArrayLike", - /, + location: "ArrayLike" = 0.0, *, backend: "Backend" = None, check_pars: bool = False, @@ -899,8 +899,8 @@ def crps_2pexponential( def crps_gamma( obs: "ArrayLike", - shape: "ArrayLike", /, + shape: "ArrayLike", rate: "ArrayLike | None" = None, *, scale: "ArrayLike | None" = None, @@ -986,8 +986,8 @@ def crps_gamma( def crps_gev( obs: "ArrayLike", - shape: "ArrayLike", /, + shape: "ArrayLike", location: "ArrayLike" = 0.0, scale: "ArrayLike" = 1.0, *, @@ -1096,8 +1096,8 @@ def crps_gev( def crps_gpd( obs: "ArrayLike", - shape: "ArrayLike", /, + shape: "ArrayLike", location: "ArrayLike" = 0.0, scale: "ArrayLike" = 1.0, mass: "ArrayLike" = 0.0, @@ -1178,9 +1178,9 @@ def crps_gpd( def crps_gtclogistic( obs: "ArrayLike", - location: "ArrayLike", - scale: "ArrayLike", /, + location: "ArrayLike" = 0.0, + scale: "ArrayLike" = 1.0, lower: "ArrayLike" = float("-inf"), upper: "ArrayLike" = float("inf"), lmass: "ArrayLike" = 0.0, @@ -1280,9 +1280,9 @@ def crps_gtclogistic( def crps_tlogistic( obs: "ArrayLike", - location: "ArrayLike", - scale: "ArrayLike", /, + location: "ArrayLike" = 0.0, + scale: "ArrayLike" = 1.0, lower: "ArrayLike" = float("-inf"), upper: "ArrayLike" = float("inf"), *, @@ -1344,9 +1344,9 @@ def crps_tlogistic( def crps_clogistic( obs: "ArrayLike", - location: "ArrayLike", - scale: "ArrayLike", /, + location: "ArrayLike" = 0.0, + scale: "ArrayLike" = 1.0, lower: "ArrayLike" = float("-inf"), upper: "ArrayLike" = float("inf"), *, @@ -1410,9 +1410,9 @@ def crps_clogistic( def crps_gtcnormal( obs: "ArrayLike", - location: "ArrayLike", - scale: "ArrayLike", /, + location: "ArrayLike" = 0.0, + scale: "ArrayLike" = 1.0, lower: "ArrayLike" = float("-inf"), upper: "ArrayLike" = float("inf"), lmass: "ArrayLike" = 0.0, @@ -1484,9 +1484,9 @@ def crps_gtcnormal( def crps_tnormal( obs: "ArrayLike", - location: "ArrayLike", - scale: "ArrayLike", /, + location: "ArrayLike" = 0.0, + scale: "ArrayLike" = 1.0, lower: "ArrayLike" = float("-inf"), upper: "ArrayLike" = float("inf"), *, @@ -1539,9 +1539,9 @@ def crps_tnormal( def crps_cnormal( obs: "ArrayLike", - location: "ArrayLike", - scale: "ArrayLike", /, + location: "ArrayLike" = 0.0, + scale: "ArrayLike" = 1.0, lower: "ArrayLike" = float("-inf"), upper: "ArrayLike" = float("inf"), *, @@ -1605,8 +1605,8 @@ def crps_cnormal( def crps_gtct( obs: "ArrayLike", - df: "ArrayLike", /, + df: "ArrayLike", location: "ArrayLike" = 0.0, scale: "ArrayLike" = 1.0, lower: "ArrayLike" = float("-inf"), @@ -1718,8 +1718,8 @@ def crps_gtct( def crps_tt( obs: "ArrayLike", - df: "ArrayLike", /, + df: "ArrayLike", location: "ArrayLike" = 0.0, scale: "ArrayLike" = 1.0, lower: "ArrayLike" = float("-inf"), @@ -1790,8 +1790,8 @@ def crps_tt( def crps_ct( obs: "ArrayLike", - df: "ArrayLike", /, + df: "ArrayLike", location: "ArrayLike" = 0.0, scale: "ArrayLike" = 1.0, lower: "ArrayLike" = float("-inf"), @@ -1864,10 +1864,10 @@ def crps_ct( def crps_hypergeometric( obs: "ArrayLike", + /, m: "ArrayLike", n: "ArrayLike", k: "ArrayLike", - /, *, backend: "Backend" = None, check_pars: bool = False, @@ -1986,9 +1986,9 @@ def crps_laplace( def crps_logistic( obs: "ArrayLike", - mu: "ArrayLike", - sigma: "ArrayLike", /, + location: "ArrayLike" = 0.0, + scale: "ArrayLike" = 1.0, *, backend: "Backend" = None, check_pars: bool = False, @@ -2007,9 +2007,9 @@ def crps_logistic( ---------- obs : array_like Observed values. - mu: array_like + location: array_like Location parameter of the forecast logistic distribution. - sigma: array_like + scale: array_like Scale parameter of the forecast logistic distribution. check_pars: bool Boolean indicating whether distribution parameter checks should be carried out prior to implementation. @@ -2018,7 +2018,7 @@ def crps_logistic( Returns ------- crps : array_like - The CRPS for the Logistic(mu, sigma) forecasts given the observations. + The CRPS for the Logistic(location, scale) forecasts given the observations. References ---------- @@ -2035,18 +2035,19 @@ def crps_logistic( """ if check_pars: B = backends.active if backend is None else backends[backend] - sigma = B.asarray(sigma) - if B.any(sigma <= 0): + scale = B.asarray(scale) + if B.any(scale <= 0): raise ValueError( "`sigma` contains non-positive entries. The scale parameter of the logistic distribution must be positive." ) - return crps.logistic(obs, mu, sigma, backend=backend) + return crps.logistic(obs, location, scale, backend=backend) def crps_loglaplace( obs: "ArrayLike", - locationlog: "ArrayLike", - scalelog: "ArrayLike", + /, + locationlog: "ArrayLike" = 0.0, + scalelog: "ArrayLike" = 1.0, *, backend: "Backend" = None, check_pars: bool = False, @@ -2116,8 +2117,10 @@ def crps_loglaplace( def crps_loglogistic( obs: "ArrayLike", - mulog: "ArrayLike", - sigmalog: "ArrayLike", + /, + mulog: "ArrayLike" = 0.0, + sigmalog: "ArrayLike" = 1.0, + *, backend: "Backend" = None, check_pars: bool = False, ) -> "ArrayLike": @@ -2186,8 +2189,10 @@ def crps_loglogistic( def crps_lognormal( obs: "ArrayLike", - mulog: "ArrayLike", - sigmalog: "ArrayLike", + /, + mulog: "ArrayLike" = 0.0, + sigmalog: "ArrayLike" = 1.0, + *, backend: "Backend" = None, check_pars: bool = False, ) -> "ArrayLike": @@ -2247,9 +2252,9 @@ def crps_lognormal( def crps_mixnorm( obs: "ArrayLike", - m: "ArrayLike", - s: "ArrayLike", /, + m: "ArrayLike" = 0.0, + s: "ArrayLike" = 1.0, w: "ArrayLike" = None, m_axis: "ArrayLike" = -1, *, @@ -2330,8 +2335,8 @@ def crps_mixnorm( def crps_negbinom( obs: "ArrayLike", - n: "ArrayLike", /, + n: "ArrayLike", prob: "ArrayLike | None" = None, *, mu: "ArrayLike | None" = None, @@ -2404,9 +2409,9 @@ def crps_negbinom( def crps_normal( obs: "ArrayLike", - mu: "ArrayLike", - sigma: "ArrayLike", /, + mu: "ArrayLike" = 0.0, + sigma: "ArrayLike" = 1.0, *, backend: "Backend" = None, check_pars: bool = False, @@ -2462,10 +2467,10 @@ def crps_normal( def crps_2pnormal( obs: "ArrayLike", - scale1: "ArrayLike", - scale2: "ArrayLike", - location: "ArrayLike", /, + scale1: "ArrayLike" = 1.0, + scale2: "ArrayLike" = 1.0, + location: "ArrayLike" = 0.0, *, backend: "Backend" = None, check_pars: bool = False, @@ -2547,8 +2552,8 @@ def crps_2pnormal( def crps_poisson( obs: "ArrayLike", - mean: "ArrayLike", /, + mean: "ArrayLike", *, backend: "Backend" = None, check_pars: bool = False, @@ -2604,8 +2609,8 @@ def crps_poisson( def crps_t( obs: "ArrayLike", - df: "ArrayLike", /, + df: "ArrayLike", location: "ArrayLike" = 0.0, scale: "ArrayLike" = 1.0, *, @@ -2675,9 +2680,9 @@ def crps_t( def crps_uniform( obs: "ArrayLike", - min: "ArrayLike", - max: "ArrayLike", /, + min: "ArrayLike" = 0.0, + max: "ArrayLike" = 1.0, lmass: "ArrayLike" = 0.0, umass: "ArrayLike" = 0.0, *, diff --git a/scoringrules/_logs.py b/scoringrules/_logs.py index 6b5f96d..81ec8bc 100644 --- a/scoringrules/_logs.py +++ b/scoringrules/_logs.py @@ -156,13 +156,14 @@ def clogs_ensemble( def logs_beta( obs: "ArrayLike", + /, a: "ArrayLike", b: "ArrayLike", - /, lower: "ArrayLike" = 0.0, upper: "ArrayLike" = 1.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the beta distribution. @@ -180,8 +181,11 @@ def logs_beta( Lower bound of the forecast beta distribution. upper : array_like Upper bound of the forecast beta distribution. - backend : str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -193,16 +197,30 @@ def logs_beta( >>> import scoringrules as sr >>> sr.logs_beta(0.3, 0.7, 1.1) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + a, b, lower, upper = map(B.asarray, (a, b, lower, upper)) + if B.any(a <= 0): + raise ValueError( + "`a` contains non-positive entries. The shape parameters of the Beta distribution must be positive." + ) + if B.any(b <= 0): + raise ValueError( + "`b` contains non-positive entries. The shape parameters of the Beta distribution must be positive." + ) + if B.any(lower >= upper): + raise ValueError("`lower` is not always smaller than `upper`.") return logarithmic.beta(obs, a, b, lower, upper, backend=backend) def logs_binomial( obs: "ArrayLike", + /, n: "ArrayLike", prob: "ArrayLike", - /, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the binomial distribution. @@ -216,8 +234,11 @@ def logs_binomial( Size parameter of the forecast binomial distribution as an integer or array of integers. prob : array_like Probability parameter of the forecast binomial distribution as a float or array of floats. - backend : str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -229,15 +250,21 @@ def logs_binomial( >>> import scoringrules as sr >>> sr.logs_binomial(4, 10, 0.5) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + prob = B.asarray(prob) + if B.any(prob < 0) or B.any(prob > 1): + raise ValueError("`prob` contains values outside the range [0, 1].") return logarithmic.binomial(obs, n, prob, backend=backend) def logs_exponential( obs: "ArrayLike", - rate: "ArrayLike", /, + rate: "ArrayLike", *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the exponential distribution. @@ -249,8 +276,11 @@ def logs_exponential( The observed values. rate : array_like Rate parameter of the forecast exponential distribution. - backend : str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -262,6 +292,13 @@ def logs_exponential( >>> import scoringrules as sr >>> sr.logs_exponential(0.8, 3.0) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + rate = B.asarray(rate) + if B.any(rate <= 0): + raise ValueError( + "`rate` contains non-positive entries. The rate parameter of the exponential distribution must be positive." + ) return logarithmic.exponential(obs, rate, backend=backend) @@ -272,6 +309,7 @@ def logs_exponential2( scale: "ArrayLike" = 1.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the exponential distribution with location and scale parameters. @@ -285,8 +323,11 @@ def logs_exponential2( Location parameter of the forecast exponential distribution. scale : array_like Scale parameter of the forecast exponential distribution. - backend : str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -298,17 +339,25 @@ def logs_exponential2( >>> import scoringrules as sr >>> sr.logs_exponential2(0.2, 0.0, 1.0) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale = B.asarray(scale) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the exponential distribution must be positive." + ) return logarithmic.exponential2(obs, location, scale, backend=backend) def logs_2pexponential( obs: "ArrayLike", + /, scale1: "ArrayLike", scale2: "ArrayLike", - location: "ArrayLike", - /, + location: "ArrayLike" = 0.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the two-piece exponential distribution. @@ -324,8 +373,11 @@ def logs_2pexponential( Second scale parameter of the forecast two-piece exponential distribution. location : array_like Location parameter of the forecast two-piece exponential distribution. - backend : str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -337,17 +389,29 @@ def logs_2pexponential( >>> import scoringrules as sr >>> sr.logs_2pexponential(0.8, 3.0, 1.4, 0.0) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale1, scale2 = map(B.asarray, (scale1, scale2)) + if B.any(scale1 <= 0): + raise ValueError( + "`scale1` contains non-positive entries. The scale parameters of the two-piece exponential distribution must be positive." + ) + if B.any(scale2 <= 0): + raise ValueError( + "`scale2` contains non-positive entries. The scale parameters of the two-piece exponential distribution must be positive." + ) return logarithmic.twopexponential(obs, scale1, scale2, location, backend=backend) def logs_gamma( obs: "ArrayLike", - shape: "ArrayLike", /, + shape: "ArrayLike", rate: "ArrayLike | None" = None, *, scale: "ArrayLike | None" = None, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the gamma distribution. @@ -363,6 +427,11 @@ def logs_gamma( Rate parameter of the forecast gamma distribution. scale : array_like Scale parameter of the forecast gamma distribution, where `scale = 1 / rate`. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -387,17 +456,30 @@ def logs_gamma( if rate is None: rate = 1.0 / scale + if check_pars: + B = backends.active if backend is None else backends[backend] + shape, rate = map(B.asarray, (shape, rate)) + if B.any(shape <= 0): + raise ValueError( + "`shape` contains non-positive entries. The shape parameter of the gamma distribution must be positive." + ) + if B.any(rate <= 0): + raise ValueError( + "`rate` or `scale` contains non-positive entries. The rate and scale parameters of the gamma distribution must be positive." + ) + return logarithmic.gamma(obs, shape, rate, backend=backend) def logs_gev( obs: "ArrayLike", - shape: "ArrayLike", /, + shape: "ArrayLike", location: "ArrayLike" = 0.0, scale: "ArrayLike" = 1.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the generalised extreme value (GEV) distribution. @@ -413,8 +495,11 @@ def logs_gev( Location parameter of the forecast GEV distribution. scale : array_like Scale parameter of the forecast GEV distribution. - backend : str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -426,17 +511,25 @@ def logs_gev( >>> import scoringrules as sr >>> sr.logs_gev(0.3, 0.1) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, shape = map(B.asarray, (scale, shape)) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the GEV distribution must be positive." + ) return logarithmic.gev(obs, shape, location, scale, backend=backend) def logs_gpd( obs: "ArrayLike", - shape: "ArrayLike", /, + shape: "ArrayLike", location: "ArrayLike" = 0.0, scale: "ArrayLike" = 1.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the generalised Pareto distribution (GPD). @@ -453,8 +546,11 @@ def logs_gpd( Location parameter of the forecast GPD distribution. scale : array_like Scale parameter of the forecast GPD distribution. - backend : str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -466,17 +562,25 @@ def logs_gpd( >>> import scoringrules as sr >>> sr.logs_gpd(0.3, 0.9) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, shape = map(B.asarray, (scale, shape)) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the GPD distribution must be positive. `nan` is returned in these places." + ) return logarithmic.gpd(obs, shape, location, scale, backend=backend) def logs_hypergeometric( obs: "ArrayLike", + /, m: "ArrayLike", n: "ArrayLike", k: "ArrayLike", - /, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the hypergeometric distribution. @@ -492,8 +596,11 @@ def logs_hypergeometric( Number of failure states in the population. k : array_like Number of draws, without replacement. Must be in 0, 1, ..., m + n. - backend : str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -505,16 +612,18 @@ def logs_hypergeometric( >>> import scoringrules as sr >>> sr.logs_hypergeometric(5, 7, 13, 12) """ + # TODO: add check that m,n,k are integers return logarithmic.hypergeometric(obs, m, n, k, backend=backend) def logs_laplace( obs: "ArrayLike", + /, location: "ArrayLike" = 0.0, scale: "ArrayLike" = 1.0, - /, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the Laplace distribution. @@ -530,6 +639,11 @@ def logs_laplace( scale : array_like Scale parameter of the forecast laplace distribution. The LS between obs and Laplace(location, scale). + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -541,15 +655,24 @@ def logs_laplace( >>> import scoringrules as sr >>> sr.logs_laplace(0.3, 0.1, 0.2) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale = B.asarray(scale) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the laplace must be positive." + ) return logarithmic.laplace(obs, location, scale, backend=backend) def logs_loglaplace( obs: "ArrayLike", - locationlog: "ArrayLike", - scalelog: "ArrayLike", + /, + locationlog: "ArrayLike" = 0.0, + scalelog: "ArrayLike" = 1.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the log-Laplace distribution. @@ -563,6 +686,11 @@ def logs_loglaplace( Location parameter of the forecast log-laplace distribution. scalelog : array_like Scale parameter of the forecast log-laplace distribution. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -574,16 +702,24 @@ def logs_loglaplace( >>> import scoringrules as sr >>> sr.logs_loglaplace(3.0, 0.1, 0.9) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scalelog = B.asarray(scalelog) + if B.any(scalelog <= 0) or B.any(scalelog >= 1): + raise ValueError( + "`scalelog` contains entries outside of the range (0, 1). The scale parameter of the log-laplace distribution must be between 0 and 1." + ) return logarithmic.loglaplace(obs, locationlog, scalelog, backend=backend) def logs_logistic( obs: "ArrayLike", - mu: "ArrayLike", - sigma: "ArrayLike", /, + location: "ArrayLike" = 0.0, + scale: "ArrayLike" = 1.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the logistic distribution. @@ -597,6 +733,11 @@ def logs_logistic( Location parameter of the forecast logistic distribution. sigma : array_like Scale parameter of the forecast logistic distribution. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -608,14 +749,24 @@ def logs_logistic( >>> import scoringrules as sr >>> sr.logs_logistic(0.0, 0.4, 0.1) """ - return logarithmic.logistic(obs, mu, sigma, backend=backend) + if check_pars: + B = backends.active if backend is None else backends[backend] + scale = B.asarray(scale) + if B.any(scale <= 0): + raise ValueError( + "`sigma` contains non-positive entries. The scale parameter of the logistic distribution must be positive." + ) + return logarithmic.logistic(obs, location, scale, backend=backend) def logs_loglogistic( obs: "ArrayLike", - mulog: "ArrayLike", - sigmalog: "ArrayLike", + /, + mulog: "ArrayLike" = 0.0, + sigmalog: "ArrayLike" = 1.0, + *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the log-logistic distribution. @@ -629,8 +780,11 @@ def logs_loglogistic( Location parameter of the log-logistic distribution. sigmalog : array_like Scale parameter of the log-logistic distribution. - backend : str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -642,14 +796,24 @@ def logs_loglogistic( >>> import scoringrules as sr >>> sr.logs_loglogistic(3.0, 0.1, 0.9) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + sigmalog = B.asarray(sigmalog) + if B.any(sigmalog <= 0) or B.any(sigmalog >= 1): + raise ValueError( + "`sigmalog` contains entries outside of the range (0, 1). The scale parameter of the log-logistic distribution must be between 0 and 1." + ) return logarithmic.loglogistic(obs, mulog, sigmalog, backend=backend) def logs_lognormal( obs: "ArrayLike", - mulog: "ArrayLike", - sigmalog: "ArrayLike", + /, + mulog: "ArrayLike" = 0.0, + sigmalog: "ArrayLike" = 1.0, + *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the log-normal distribution. @@ -663,6 +827,11 @@ def logs_lognormal( Mean of the normal underlying distribution. sigmalog : array_like Standard deviation of the underlying normal distribution. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -674,18 +843,26 @@ def logs_lognormal( >>> import scoringrules as sr >>> sr.logs_lognormal(0.0, 0.4, 0.1) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + sigmalog = B.asarray(sigmalog) + if B.any(sigmalog <= 0): + raise ValueError( + "`sigmalog` contains non-positive entries. The scale parameter of the log-normal distribution must be positive." + ) return logarithmic.lognormal(obs, mulog, sigmalog, backend=backend) def logs_mixnorm( obs: "ArrayLike", - m: "ArrayLike", - s: "ArrayLike", /, + m: "ArrayLike" = 0.0, + s: "ArrayLike" = 1.0, w: "ArrayLike" = None, - mc_axis: "ArrayLike" = -1, + m_axis: "ArrayLike" = -1, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score for a mixture of normal distributions. @@ -701,10 +878,13 @@ def logs_mixnorm( Standard deviations of the component normal distributions. w : array_like Non-negative weights assigned to each component. - mc_axis : int + m_axis : int The axis corresponding to the mixture components. Default is the last axis. - backend : str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -720,27 +900,37 @@ def logs_mixnorm( obs, m, s = map(B.asarray, (obs, m, s)) if w is None: - M: int = m.shape[mc_axis] + M: int = m.shape[m_axis] w = B.zeros(m.shape) + 1 / M else: w = B.asarray(w) + w = w / B.sum(w, axis=m_axis, keepdims=True) + + if check_pars: + if B.any(s <= 0): + raise ValueError( + "`s` contains non-positive entries. The scale parameters of the normal distributions should be positive." + ) + if B.any(w < 0): + raise ValueError("`w` contains negative entries") - if mc_axis != -1: - m = B.moveaxis(m, mc_axis, -1) - s = B.moveaxis(s, mc_axis, -1) - w = B.moveaxis(w, mc_axis, -1) + if m_axis != -1: + m = B.moveaxis(m, m_axis, -1) + s = B.moveaxis(s, m_axis, -1) + w = B.moveaxis(w, m_axis, -1) return logarithmic.mixnorm(obs, m, s, w, backend=backend) def logs_negbinom( obs: "ArrayLike", - n: "ArrayLike", /, + n: "ArrayLike", prob: "ArrayLike | None" = None, *, mu: "ArrayLike | None" = None, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the negative binomial distribution. @@ -756,6 +946,11 @@ def logs_negbinom( Probability parameter of the forecast negative binomial distribution. mu : array_like Mean of the forecast negative binomial distribution. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -780,17 +975,24 @@ def logs_negbinom( if prob is None: prob = n / (n + mu) + if check_pars: + B = backends.active if backend is None else backends[backend] + prob = B.asarray(prob) + if B.any(prob < 0) or B.any(prob > 1): + raise ValueError("`prob` contains values outside the range [0, 1].") + return logarithmic.negbinom(obs, n, prob, backend=backend) def logs_normal( obs: "ArrayLike", - mu: "ArrayLike", - sigma: "ArrayLike", /, + mu: "ArrayLike" = 0.0, + sigma: "ArrayLike" = 1.0, *, negative: bool = True, backend: "Backend" = None, + check_pars: bool = False, ) -> "Array": r"""Compute the logarithmic score (LS) for the normal distribution. @@ -805,7 +1007,10 @@ def logs_normal( sigma : array_like Standard deviation of the forecast normal distribution. backend : str, optional - The backend used for computations. + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -817,17 +1022,25 @@ def logs_normal( >>> import scoringrules as sr >>> sr.logs_normal(0.0, 0.4, 0.1) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + sigma = B.asarray(sigma) + if B.any(sigma <= 0): + raise ValueError( + "`sigma` contains non-positive entries. The standard deviation of the normal distribution must be positive." + ) return logarithmic.normal(obs, mu, sigma, negative=negative, backend=backend) def logs_2pnormal( obs: "ArrayLike", - scale1: "ArrayLike", - scale2: "ArrayLike", - location: "ArrayLike", /, + scale1: "ArrayLike" = 1.0, + scale2: "ArrayLike" = 1.0, + location: "ArrayLike" = 0.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the two-piece normal distribution. @@ -843,8 +1056,11 @@ def logs_2pnormal( Scale parameter of the upper half of the forecast two-piece normal distribution. location : array_like Location parameter of the forecast two-piece normal distribution. - backend : str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -855,15 +1071,27 @@ def logs_2pnormal( >>> import scoringrules as sr >>> sr.logs_2pnormal(0.0, 0.4, 2.0, 0.1) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale1, scale2 = map(B.asarray, (scale1, scale2)) + if B.any(scale1 <= 0): + raise ValueError( + "`scale1` contains non-positive entries. The scale parameters of the two-piece normal distribution must be positive." + ) + if B.any(scale2 <= 0): + raise ValueError( + "`scale2` contains non-positive entries. The scale parameters of the two-piece normal distribution must be positive." + ) return logarithmic.twopnormal(obs, scale1, scale2, location, backend=backend) def logs_poisson( obs: "ArrayLike", - mean: "ArrayLike", /, + mean: "ArrayLike", *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the Poisson distribution. @@ -875,8 +1103,11 @@ def logs_poisson( The observed values. mean : array_like Mean parameter of the forecast poisson distribution. - backend : str - The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -888,17 +1119,25 @@ def logs_poisson( >>> import scoringrules as sr >>> sr.logs_poisson(1, 2) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + mean = B.asarray(mean) + if B.any(mean <= 0): + raise ValueError( + "`mean` contains non-positive entries. The mean parameter of the Poisson distribution must be positive." + ) return logarithmic.poisson(obs, mean, backend=backend) def logs_t( obs: "ArrayLike", - df: "ArrayLike", /, + df: "ArrayLike", location: "ArrayLike" = 0.0, scale: "ArrayLike" = 1.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the Student's t distribution. @@ -912,8 +1151,13 @@ def logs_t( Degrees of freedom parameter of the forecast t distribution. location : array_like Location parameter of the forecast t distribution. - sigma : array_like + scale : array_like Scale parameter of the forecast t distribution. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -925,18 +1169,30 @@ def logs_t( >>> import scoringrules as sr >>> sr.logs_t(0.0, 0.1, 0.4, 0.1) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + df, scale = map(B.asarray, (df, scale)) + if B.any(df <= 0): + raise ValueError( + "`df` contains non-positive entries. The degrees of freedom parameter of the t distribution must be positive." + ) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the t distribution must be positive." + ) return logarithmic.t(obs, df, location, scale, backend=backend) def logs_tlogistic( obs: "ArrayLike", - location: "ArrayLike", - scale: "ArrayLike", /, + location: "ArrayLike" = 0.0, + scale: "ArrayLike" = 1.0, lower: "ArrayLike" = float("-inf"), upper: "ArrayLike" = float("inf"), *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the truncated logistic distribution. @@ -954,6 +1210,11 @@ def logs_tlogistic( Lower boundary of the truncated forecast distribution. upper : array_like Upper boundary of the truncated forecast distribution. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -965,18 +1226,28 @@ def logs_tlogistic( >>> import scoringrules as sr >>> sr.logs_tlogistic(0.0, 0.1, 0.4, -1.0, 1.0) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, lower, upper = map(B.asarray, (scale, lower, upper)) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the truncated logistic distribution must be positive." + ) + if B.any(lower >= upper): + raise ValueError("`lower` is not always smaller than `upper`.") return logarithmic.tlogistic(obs, location, scale, lower, upper, backend=backend) def logs_tnormal( obs: "ArrayLike", - location: "ArrayLike", - scale: "ArrayLike", /, + location: "ArrayLike" = 0.0, + scale: "ArrayLike" = 1.0, lower: "ArrayLike" = float("-inf"), upper: "ArrayLike" = float("inf"), *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the truncated normal distribution. @@ -994,6 +1265,11 @@ def logs_tnormal( Lower boundary of the truncated forecast distribution. upper : array_like Upper boundary of the truncated forecast distribution. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1005,19 +1281,29 @@ def logs_tnormal( >>> import scoringrules as sr >>> sr.logs_tnormal(0.0, 0.1, 0.4, -1.0, 1.0) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, lower, upper = map(B.asarray, (scale, lower, upper)) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the truncated normal distribution must be positive." + ) + if B.any(lower >= upper): + raise ValueError("`lower` is not always smaller than `upper`.") return logarithmic.tnormal(obs, location, scale, lower, upper, backend=backend) def logs_tt( obs: "ArrayLike", - df: "ArrayLike", /, + df: "ArrayLike", location: "ArrayLike" = 0.0, scale: "ArrayLike" = 1.0, lower: "ArrayLike" = float("-inf"), upper: "ArrayLike" = float("inf"), *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the truncated Student's t distribution. @@ -1037,6 +1323,11 @@ def logs_tt( Lower boundary of the truncated forecast distribution. upper : array_like Upper boundary of the truncated forecast distribution. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- @@ -1048,16 +1339,30 @@ def logs_tt( >>> import scoringrules as sr >>> sr.logs_tt(0.0, 2.0, 0.1, 0.4, -1.0, 1.0) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + scale, lower, upper = map(B.asarray, (scale, lower, upper)) + if B.any(df <= 0): + raise ValueError( + "`df` contains non-positive entries. The degrees of freedom parameter of the truncated t distribution must be positive." + ) + if B.any(scale <= 0): + raise ValueError( + "`scale` contains non-positive entries. The scale parameter of the truncated t distribution must be positive." + ) + if B.any(lower >= upper): + raise ValueError("`lower` is not always smaller than `upper`.") return logarithmic.tt(obs, df, location, scale, lower, upper, backend=backend) def logs_uniform( obs: "ArrayLike", - min: "ArrayLike", - max: "ArrayLike", /, + min: "ArrayLike" = 0.0, + max: "ArrayLike" = 1.0, *, backend: "Backend" = None, + check_pars: bool = False, ) -> "ArrayLike": r"""Compute the logarithmic score (LS) for the uniform distribution. @@ -1071,17 +1376,27 @@ def logs_uniform( Lower bound of the forecast uniform distribution. max : array_like Upper bound of the forecast uniform distribution. + backend : str, optional + The name of the backend used for computations. Defaults to ``numba`` if available, else ``numpy``. + check_pars: bool + Boolean indicating whether distribution parameter checks should be carried out prior to implementation. + Default is False. Returns ------- score : array_like - The LS between U(min, max, lmass, umass) and obs. + The LS between U(min, max) and obs. Examples -------- >>> import scoringrules as sr >>> sr.logs_uniform(0.4, 0.0, 1.0) """ + if check_pars: + B = backends.active if backend is None else backends[backend] + min, max = map(B.asarray, (min, max)) + if B.any(min >= max): + raise ValueError("`min` is not always smaller than `max`.") return logarithmic.uniform(obs, min, max, backend=backend) diff --git a/tests/test_crps.py b/tests/test_crps.py index 3f0c62c..d663fd2 100644 --- a/tests/test_crps.py +++ b/tests/test_crps.py @@ -163,7 +163,17 @@ def test_crps_beta(backend): # test exceptions with pytest.raises(ValueError): - sr.crps_beta(0.3, 0.7, 1.1, lower=1.0, upper=0.0, backend=backend) + sr.crps_beta( + 0.3, 0.7, 1.1, lower=1.0, upper=0.0, backend=backend, check_pars=True + ) + return + + with pytest.raises(ValueError): + sr.crps_beta(0.3, -0.7, 1.1, backend=backend, check_pars=True) + return + + with pytest.raises(ValueError): + sr.crps_beta(0.3, 0.7, -1.1, backend=backend, check_pars=True) return # correctness tests @@ -234,17 +244,17 @@ def test_crps_exponential(backend): @pytest.mark.parametrize("backend", BACKENDS) def test_crps_exponentialM(backend): obs, mass, location, scale = 0.3, 0.1, 0.0, 1.0 - res = sr.crps_exponentialM(obs, mass, location, scale, backend=backend) + res = sr.crps_exponentialM(obs, location, scale, mass, backend=backend) expected = 0.2384728 assert np.isclose(res, expected) obs, mass, location, scale = 0.3, 0.1, -2.0, 3.0 - res = sr.crps_exponentialM(obs, mass, location, scale, backend=backend) + res = sr.crps_exponentialM(obs, location, scale, mass, backend=backend) expected = 0.6236187 assert np.isclose(res, expected) obs, mass, location, scale = -1.2, 0.1, -2.0, 3.0 - res = sr.crps_exponentialM(obs, mass, location, scale, backend=backend) + res = sr.crps_exponentialM(obs, location, scale, mass, backend=backend) expected = 0.751013 assert np.isclose(res, expected) diff --git a/tests/test_logs.py b/tests/test_logs.py index af32aba..ca3a579 100644 --- a/tests/test_logs.py +++ b/tests/test_logs.py @@ -368,7 +368,7 @@ def test_mixnorm(backend): obs = [-1.6, 0.3] m = [[0.0, -2.9], [0.6, 0.0], [-1.1, -2.3]] s = [[0.5, 1.7], [1.1, 0.7], [1.4, 1.5]] - res1 = sr.logs_mixnorm(obs, m, s, mc_axis=0, backend=backend) + res1 = sr.logs_mixnorm(obs, m, s, m_axis=0, backend=backend) m = [[0.0, 0.6, -1.1], [-2.9, 0.0, -2.3]] s = [[0.5, 1.1, 1.4], [1.7, 0.7, 1.5]] From fbfa09073211779fcb159f3030ceee2501145108 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Thu, 17 Apr 2025 16:59:03 +0200 Subject: [PATCH 52/79] fix bug in crps for beta distribution --- scoringrules/core/crps/_closed.py | 42 ++++++++++--------------------- 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/scoringrules/core/crps/_closed.py b/scoringrules/core/crps/_closed.py index 7edc13e..3ccaec9 100644 --- a/scoringrules/core/crps/_closed.py +++ b/scoringrules/core/crps/_closed.py @@ -35,28 +35,19 @@ def beta( """Compute the CRPS for the beta distribution.""" B = backends.active if backend is None else backends[backend] obs, a, b, lower, upper = map(B.asarray, (obs, a, b, lower, upper)) + a = B.where(a > 0.0, a, B.nan) + b = B.where(b > 0.0, b, B.nan) + lower = B.where(lower < upper, lower, B.nan) - if _is_scalar_value(lower, 0.0) and _is_scalar_value(upper, 1.0): - special_limits = False - else: - if B.any(lower >= upper): - raise ValueError("lower must be less than upper") - special_limits = True - - if special_limits: - obs = (obs - lower) / (upper - lower) + obs = (obs - lower) / (upper - lower) + obs_std = B.minimum(B.maximum(obs, 0.0), 1.0) - I_ab = B.betainc(a, b, obs) - I_a1b = B.betainc(a + 1, b, obs) - F_ab = B.minimum(B.maximum(I_ab, 0), 1) - F_a1b = B.minimum(B.maximum(I_a1b, 0), 1) + F_ab = B.betainc(a, b, obs_std) + F_a1b = B.betainc(a + 1, b, obs_std) bet_rat = 2 * B.beta(2 * a, 2 * b) / (a * B.beta(a, b) ** 2) s = obs * (2 * F_ab - 1) + (a / (a + b)) * (1 - 2 * F_a1b - bet_rat) - if special_limits: - s = s * (upper - lower) - - return s + return s * (upper - lower) def binomial( @@ -133,26 +124,18 @@ def exponential( def exponentialM( obs: "ArrayLike", - mass: "ArrayLike", location: "ArrayLike", scale: "ArrayLike", + mass: "ArrayLike", backend: "Backend" = None, ) -> "Array": """Compute the CRPS for the standard exponential distribution with a point mass at the boundary.""" B = backends.active if backend is None else backends[backend] obs, location, scale, mass = map(B.asarray, (obs, location, scale, mass)) - - if not _is_scalar_value(location, 0.0): - obs -= location - - a = 1.0 if _is_scalar_value(mass, 0.0) else 1 - mass + obs -= location + a = 1.0 - mass s = B.abs(obs) - - if _is_scalar_value(scale, 1.0): - s -= a * (2 * _exp_cdf(obs, 1.0, backend=backend) - 0.5 * a) - else: - s -= scale * a * (2 * _exp_cdf(obs, 1 / scale, backend=backend) - 0.5 * a) - + s -= scale * a * (2 * _exp_cdf(obs, 1 / scale, backend=backend) - 0.5 * a) return s @@ -266,6 +249,7 @@ def gpd( ) shape = B.where(shape < 1.0, shape, B.nan) mass = B.where((mass >= 0.0) & (mass <= 1.0), mass, B.nan) + scale = B.where(scale > 0.0, scale, B.nan) ω = (obs - location) / scale F_xi = _gpd_cdf(ω, shape, backend=backend) s = ( From ca483b56605e3263e6c5fa6c5f89afd2a3475f42 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Thu, 29 May 2025 13:35:32 +0200 Subject: [PATCH 53/79] update documentation link and prepare release tag --- README.md | 4 +--- docs/index.md | 4 +--- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e2c658a..62f51a6 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ pip install scoringrules ## Documentation -Learn more about `scoringrules` in its official documentation at https://frazane.github.io/scoringrules/. +Learn more about `scoringrules` in its official documentation at https://scoringrules.readthedocs.io/en/latest/. ## Quick example @@ -77,5 +77,3 @@ grateful for fruitful discussions with the authors. - The quality of this library has also benefited a lot from discussions with (and contributions from) Nick Loveday and Tennessee Leeuwenburg, whose python library [`scores`](https://github.com/nci/scores) similarly provides a comprehensive collection of forecast evaluation methods. -- The implementation of the ensemble-based metrics as jit-compiled numpy generalized `ufuncs` -was first proposed in [`properscoring`](https://github.com/properscoring/properscoring), released under Apache License, Version 2.0. diff --git a/docs/index.md b/docs/index.md index dea7f15..739f9c0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -65,7 +65,7 @@ pip install scoringrules ## Documentation -Learn more about `scoringrules` in its official documentation at https://frazane.github.io/scoringrules/. +Learn more about `scoringrules` in its official documentation at https://scoringrules.readthedocs.io/en/latest/. ## Quick example @@ -97,5 +97,3 @@ grateful for fruitful discussions with the authors. - The quality of this library has also benefited a lot from discussions with (and contributions from) Nick Loveday and Tennessee Leeuwenburg, whose python library [`scores`](https://github.com/nci/scores) similarly provides a comprehensive collection of forecast evaluation methods. -- The implementation of the ensemble-based metrics as jit-compiled numpy generalized `ufuncs` -was first proposed in [`properscoring`](https://github.com/properscoring/properscoring), released under Apache License, Version 2.0. diff --git a/pyproject.toml b/pyproject.toml index 6b3144e..4ec95cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "scoringrules" -version = "0.7.1" +version = "0.8.0" description = "Scoring rules for probabilistic forecast evaluation." readme = "README.md" requires-python = ">=3.10" From 4d91ec2cc58792d50fa2089a0435ec4941662b77 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Thu, 29 May 2025 14:38:46 +0200 Subject: [PATCH 54/79] add deprecated aliases --- scoringrules/__init__.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/scoringrules/__init__.py b/scoringrules/__init__.py index 0213df4..9d04e9d 100644 --- a/scoringrules/__init__.py +++ b/scoringrules/__init__.py @@ -1,4 +1,6 @@ from importlib.metadata import version +import functools +import warnings from scoringrules._brier import ( brier_score, @@ -95,13 +97,36 @@ owgksmv_ensemble, vrgksmv_ensemble, ) - from scoringrules._quantile import quantile_score - from scoringrules.backend import backends, register_backend -__version__ = version("scoringrules") +def _deprecated_alias(new_func, old_func_name, remove_version): + """Return a deprecated alias for a renamed function.""" + + @functools.wraps(new_func) + def wrapper(*args, **kwargs): + warnings.warn( + f"{old_func_name} is deprecated and will be removed in {remove_version}. " + f"Use {new_func.__name__} instead.", + DeprecationWarning, + ) + return new_func(*args, **kwargs) + + return wrapper + + +energy_score = _deprecated_alias(es_ensemble, "energy_score", "v1.0.0") +owenergy_score = _deprecated_alias(owes_ensemble, "owenergy_score", "v1.0.0") +twenergy_score = _deprecated_alias(twes_ensemble, "twenergy_score", "v1.0.0") +vrenergy_score = _deprecated_alias(vres_ensemble, "vrenergy_score", "v1.0.0") + +variogram_score = _deprecated_alias(vs_ensemble, "variogram_score", "v1.0.0") +owvariogram_score = _deprecated_alias(owvs_ensemble, "owvariogram_score", "v1.0.0") +twvariogram_score = _deprecated_alias(twvs_ensemble, "twvariogram_score", "v1.0.0") +vrvariogram_score = _deprecated_alias(vrvs_ensemble, "vrvariogram_score", "v1.0.0") + +__version__ = version("scoringrules") __all__ = [ "register_backend", From 8db42c4428501b577531e6d72142f20a5f65e2f5 Mon Sep 17 00:00:00 2001 From: Francesco Zanetta Date: Fri, 30 May 2025 10:35:09 +0200 Subject: [PATCH 55/79] more minor fixes --- docs/conf.py | 4 ++-- docs/contributing.md | 7 +++---- docs/reference.md | 22 ++++++++++++---------- scoringrules/_crps.py | 2 +- uv.lock | 30 +++++++++++++++--------------- 5 files changed, 33 insertions(+), 32 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 7b08fb8..b5d98c9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,8 +8,8 @@ project = "scoringrules" author = "scoringrules contributors" -copyright = "2024" -release = "0.7.0" +copyright = "2024-2025" +release = "0.8.0" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/docs/contributing.md b/docs/contributing.md index 756fccf..3294e06 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -39,11 +39,10 @@ uv run pytest tests/ ### Contributing to the documentation -You can work on the documentation by modifying `mkdocs.yaml` and files in `docs/`. The most convenient way to do it is to run +You can work on the documentation by modifying files in `docs/`. The most convenient way to do it is to run ``` -uvx --with-requirements docs/requirements.txt mkdocs serve +uvx --with-requirements docs/requirements.txt sphinx-autobuild docs/ docs/_build/ ``` -and open the locally hosted documentation on your browser. It will be updated automatically every time you make changes and save. If you edit or add pieces of LaTex math, please make sure they -are rendered correctly. +and open the locally hosted documentation on your browser. It will be updated automatically every time you make changes and save. If you edit or add pieces of LaTex math, please make sure they are rendered correctly. diff --git a/docs/reference.md b/docs/reference.md index 00edc03..b3274fd 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -32,14 +32,16 @@ Multivariate .. autosummary:: :toctree: generated - energy_score - twenergy_score - owenergy_score - vrenergy_score - variogram_score - owvariogram_score - twvariogram_score - vrvariogram_score + es_ensemble + owes_ensemble + twes_ensemble + vres_ensemble + + vs_ensemble + owvs_ensemble + twvs_ensemble + vrvs_ensemble + gksmv_ensemble twgksmv_ensemble owgksmv_ensemble @@ -108,7 +110,7 @@ Parametric distributions forecasts logs_uniform Consistent scoring functions -========================== +============================ .. autosummary:: :toctree: generated @@ -117,7 +119,7 @@ Consistent scoring functions Categorical forecasts -================================= +===================== .. autosummary:: :toctree: generated diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index 281a826..135ffe5 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -69,7 +69,7 @@ def crps_ensemble( Weighted variants of the CRPS. crps_quantile CRPS for quantile forecasts. - energy_score + es_ensemble The multivariate equivalent of the CRPS. Notes diff --git a/uv.lock b/uv.lock index d519ab3..1cca528 100644 --- a/uv.lock +++ b/uv.lock @@ -1,4 +1,5 @@ version = 1 +revision = 1 requires-python = ">=3.10" resolution-markers = [ "python_full_version < '3.11'", @@ -942,7 +943,6 @@ name = "nvidia-nccl-cu12" version = "2.20.5" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/bb/d09dda47c881f9ff504afd6f9ca4f502ded6d8fc2f572cacc5e39da91c28/nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_aarch64.whl", hash = "sha256:1fc150d5c3250b170b29410ba682384b14581db722b2531b0d8d33c595f33d01", size = 176238458 }, { url = "https://files.pythonhosted.org/packages/4b/2a/0a131f572aa09f741c30ccd45a8e56316e8be8dfc7bc19bf0ab7cfef7b19/nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl", hash = "sha256:057f6bf9685f75215d0c53bf3ac4a10b3e6578351de307abad9e18a99182af56", size = 176249402 }, ] @@ -951,7 +951,6 @@ name = "nvidia-nvjitlink-cu12" version = "12.6.68" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/58/8c/69c9e39cd6bfa813852a94e9bd3c075045e2707d163e9dc2326c82d2c330/nvidia_nvjitlink_cu12-12.6.68-py3-none-manylinux2014_aarch64.whl", hash = "sha256:b3fd0779845f68b92063ab1393abab1ed0a23412fc520df79a8190d098b5cd6b", size = 19253287 }, { url = "https://files.pythonhosted.org/packages/a8/48/a9775d377cb95585fb188b469387f58ba6738e268de22eae2ad4cedb2c41/nvidia_nvjitlink_cu12-12.6.68-py3-none-manylinux2014_x86_64.whl", hash = "sha256:125a6c2a44e96386dda634e13d944e60b07a0402d391a070e8fb4104b34ea1ab", size = 19725597 }, ] @@ -1314,7 +1313,7 @@ wheels = [ [[package]] name = "scoringrules" -version = "0.7.1" +version = "0.8.0" source = { editable = "." } dependencies = [ { name = "numpy" }, @@ -1352,6 +1351,7 @@ requires-dist = [ { name = "tensorflow", marker = "extra == 'tensorflow'", specifier = ">=2.17.0" }, { name = "torch", marker = "extra == 'torch'", specifier = ">=2.4.1" }, ] +provides-extras = ["numba", "jax", "torch", "tensorflow"] [package.metadata.requires-dev] dev = [ @@ -1510,19 +1510,19 @@ dependencies = [ { name = "fsspec" }, { name = "jinja2" }, { name = "networkx" }, - { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, - { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, - { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, - { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, - { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, - { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, - { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, - { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, - { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, - { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, - { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "sympy" }, - { name = "triton", marker = "python_full_version < '3.13' and platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "triton", marker = "python_full_version < '3.13' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "typing-extensions" }, ] wheels = [ From fa4edcede7579427fbbe247eb735a913333d984e Mon Sep 17 00:00:00 2001 From: sallen12 Date: Wed, 9 Jul 2025 12:19:34 +0200 Subject: [PATCH 56/79] minor bug fixes --- scoringrules/_crps.py | 1 + tests/test_wcrps.py | 25 ++++++++++++++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index c92b3dd..65731a2 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -1335,6 +1335,7 @@ def crps_tlogistic( location, scale, lower, + upper, 0.0, 0.0, backend=backend, diff --git a/tests/test_wcrps.py b/tests/test_wcrps.py index 0a58545..79b94e9 100644 --- a/tests/test_wcrps.py +++ b/tests/test_wcrps.py @@ -12,27 +12,34 @@ def test_owcrps_ensemble(backend): # test shapes obs = np.random.randn(N) - res = sr.owcrps_ensemble(obs, np.random.randn(N, M), w_func=lambda x: x * 0.0 + 1.0) + res = sr.owcrps_ensemble( + obs, np.random.randn(N, M), w_func=lambda x: x * 0.0 + 1.0, backend=backend + ) assert res.shape == (N,) res = sr.owcrps_ensemble( - obs, np.random.randn(M, N), w_func=lambda x: x * 0.0 + 1.0, m_axis=0 + obs, + np.random.randn(M, N), + w_func=lambda x: x * 0.0 + 1.0, + m_axis=0, + backend=backend, ) assert res.shape == (N,) @pytest.mark.parametrize("backend", BACKENDS) def test_vrcrps_ensemble(backend): - # test exceptions - with pytest.raises(ValueError): - est = "not_nrg" - sr.vrcrps_ensemble(1, 1.1, w_func=lambda x: x, estimator=est, backend=backend) - # test shapes obs = np.random.randn(N) - res = sr.vrcrps_ensemble(obs, np.random.randn(N, M), w_func=lambda x: x * 0.0 + 1.0) + res = sr.vrcrps_ensemble( + obs, np.random.randn(N, M), w_func=lambda x: x * 0.0 + 1.0, backend=backend + ) assert res.shape == (N,) res = sr.vrcrps_ensemble( - obs, np.random.randn(M, N), w_func=lambda x: x * 0.0 + 1.0, m_axis=0 + obs, + np.random.randn(M, N), + w_func=lambda x: x * 0.0 + 1.0, + m_axis=0, + backend=backend, ) assert res.shape == (N,) From 0f9802596612cdb9851d3ca8d0f6d78af95f6578 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Wed, 9 Jul 2025 13:20:25 +0200 Subject: [PATCH 57/79] minor jax tolerance bug fix --- tests/test_crps.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/test_crps.py b/tests/test_crps.py index d663fd2..a237c26 100644 --- a/tests/test_crps.py +++ b/tests/test_crps.py @@ -54,13 +54,6 @@ def test_crps_ensemble(estimator, backend): res = np.asarray(res) assert not np.any(res - 0.0 > 0.0001) - # test equivalence of different estimators - res_nrg = sr.crps_ensemble(obs, fct, estimator="nrg", backend=backend) - res_pwm = sr.crps_ensemble(obs, fct, estimator="pwm", backend=backend) - res_qd = sr.crps_ensemble(obs, fct, estimator="qd", backend=backend) - assert np.allclose(res_nrg, res_pwm) - assert np.allclose(res_nrg, res_qd) - @pytest.mark.parametrize("backend", BACKENDS) def test_crps_ensemble_corr(backend): @@ -73,15 +66,23 @@ def test_crps_ensemble_corr(backend): res_nrg = sr.crps_ensemble(obs, fct, estimator="nrg", backend=backend) res_pwm = sr.crps_ensemble(obs, fct, estimator="pwm", backend=backend) res_qd = sr.crps_ensemble(obs, fct, estimator="qd", backend=backend) - assert np.allclose(res_nrg, res_pwm) - assert np.allclose(res_nrg, res_qd) + if backend == "torch": + assert np.allclose(res_nrg, res_pwm, rtol=1e-04) + assert np.allclose(res_nrg, res_qd, rtol=1e-04) + else: + assert np.allclose(res_nrg, res_pwm) + assert np.allclose(res_nrg, res_qd) w = np.abs(np.random.randn(N, ENSEMBLE_SIZE) * sigma[..., None]) res_nrg = sr.crps_ensemble(obs, fct, ens_w=w, estimator="nrg", backend=backend) res_pwm = sr.crps_ensemble(obs, fct, ens_w=w, estimator="pwm", backend=backend) res_qd = sr.crps_ensemble(obs, fct, ens_w=w, estimator="qd", backend=backend) - assert np.allclose(res_nrg, res_pwm) - assert np.allclose(res_nrg, res_qd) + if backend == "torch": + assert np.allclose(res_nrg, res_pwm, rtol=1e-04) + assert np.allclose(res_nrg, res_qd, rtol=1e-04) + else: + assert np.allclose(res_nrg, res_pwm) + assert np.allclose(res_nrg, res_qd) # test correctness obs = -0.6042506 From 6dfbd8b78ff5f757bf47545103b731e6abf4a55c Mon Sep 17 00:00:00 2001 From: sallen12 Date: Wed, 9 Jul 2025 13:24:58 +0200 Subject: [PATCH 58/79] minor jax tolerance bug fix --- tests/test_crps.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_crps.py b/tests/test_crps.py index a237c26..dce17a6 100644 --- a/tests/test_crps.py +++ b/tests/test_crps.py @@ -67,8 +67,8 @@ def test_crps_ensemble_corr(backend): res_pwm = sr.crps_ensemble(obs, fct, estimator="pwm", backend=backend) res_qd = sr.crps_ensemble(obs, fct, estimator="qd", backend=backend) if backend == "torch": - assert np.allclose(res_nrg, res_pwm, rtol=1e-04) - assert np.allclose(res_nrg, res_qd, rtol=1e-04) + assert np.allclose(res_nrg, res_pwm, rtol=1e-03) + assert np.allclose(res_nrg, res_qd, rtol=1e-03) else: assert np.allclose(res_nrg, res_pwm) assert np.allclose(res_nrg, res_qd) @@ -78,8 +78,8 @@ def test_crps_ensemble_corr(backend): res_pwm = sr.crps_ensemble(obs, fct, ens_w=w, estimator="pwm", backend=backend) res_qd = sr.crps_ensemble(obs, fct, ens_w=w, estimator="qd", backend=backend) if backend == "torch": - assert np.allclose(res_nrg, res_pwm, rtol=1e-04) - assert np.allclose(res_nrg, res_qd, rtol=1e-04) + assert np.allclose(res_nrg, res_pwm, rtol=1e-03) + assert np.allclose(res_nrg, res_qd, rtol=1e-03) else: assert np.allclose(res_nrg, res_pwm) assert np.allclose(res_nrg, res_qd) From 41d71b98e618097e7ad33f069880d61601891a1a Mon Sep 17 00:00:00 2001 From: sallen12 Date: Wed, 9 Jul 2025 13:29:28 +0200 Subject: [PATCH 59/79] minor jax tolerance bug fix --- tests/test_wcrps.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_wcrps.py b/tests/test_wcrps.py index 79b94e9..da3c779 100644 --- a/tests/test_wcrps.py +++ b/tests/test_wcrps.py @@ -81,17 +81,17 @@ def test_owcrps_vs_crps(backend): # no argument given resw = sr.owcrps_ensemble(obs, fct, backend=backend) - np.testing.assert_allclose(res, resw, rtol=1e-5) + np.testing.assert_allclose(res, resw, rtol=1e-4) # a and b resw = sr.owcrps_ensemble( obs, fct, a=float("-inf"), b=float("inf"), backend=backend ) - np.testing.assert_allclose(res, resw, rtol=1e-5) + np.testing.assert_allclose(res, resw, rtol=1e-4) # w_func as identity function resw = sr.owcrps_ensemble(obs, fct, w_func=lambda x: x * 0.0 + 1.0, backend=backend) - np.testing.assert_allclose(res, resw, rtol=1e-5) + np.testing.assert_allclose(res, resw, rtol=1e-4) @pytest.mark.parametrize("backend", BACKENDS) From 49da2f3463b4fb07e62ea831124ab3a8b43c2820 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Wed, 9 Jul 2025 16:59:20 +0200 Subject: [PATCH 60/79] add weight function argument to energy score --- scoringrules/_energy.py | 16 +++++++++++++--- scoringrules/core/energy/_gufuncs.py | 13 +++++++------ scoringrules/core/energy/_score.py | 10 ++++++---- scoringrules/core/utils.py | 4 ++-- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/scoringrules/_energy.py b/scoringrules/_energy.py index b4fd990..eba3990 100644 --- a/scoringrules/_energy.py +++ b/scoringrules/_energy.py @@ -15,6 +15,7 @@ def es_ensemble( m_axis: int = -2, v_axis: int = -1, *, + ens_w: "Array" = None, backend: "Backend" = None, ) -> "Array": r"""Compute the Energy Score for a finite multivariate ensemble. @@ -39,6 +40,9 @@ def es_ensemble( v_axis : int The axis corresponding to the variables dimension on the forecasts array (or the observations array with an extra dimension on `m_axis`). Defaults to -1. + ens_w : array_like + Weights assigned to the ensemble members. Array with one less dimension than fct (without the v_axis dimension). + Default is equal weighting. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -59,13 +63,19 @@ def es_ensemble( :ref:`theory.multivariate` Some theoretical background on scoring rules for multivariate forecasts. """ - backend = backend if backend is not None else backends._active + B = backends.active if backend is None else backends[backend] obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) + if ens_w is None: + M = fct.shape[m_axis] + ens_w = B.zeros(fct.shape[:v_axis] + fct.shape[(v_axis + 1) :]) + 1.0 / M + else: + ens_w = B.moveaxis(ens_w, m_axis, -2) + if backend == "numba": - return energy._energy_score_gufunc(obs, fct) + return energy._energy_score_gufunc(obs, fct, ens_w) - return energy.nrg(obs, fct, backend=backend) + return energy.nrg(obs, fct, ens_w, backend=backend) def twes_ensemble( diff --git a/scoringrules/core/energy/_gufuncs.py b/scoringrules/core/energy/_gufuncs.py index e371d76..c63f338 100644 --- a/scoringrules/core/energy/_gufuncs.py +++ b/scoringrules/core/energy/_gufuncs.py @@ -4,14 +4,15 @@ @guvectorize( [ - "void(float32[:], float32[:,:], float32[:])", - "void(float64[:], float64[:,:], float64[:])", + "void(float32[:], float32[:,:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:])", ], - "(d),(m,d)->()", + "(d),(m,d),(m)->()", ) def _energy_score_gufunc( obs: np.ndarray, fct: np.ndarray, + ens_w: np.ndarray, out: np.ndarray, ): """Compute the Energy Score for a finite ensemble.""" @@ -20,11 +21,11 @@ def _energy_score_gufunc( e_1 = 0.0 e_2 = 0.0 for i in range(M): - e_1 += float(np.linalg.norm(fct[i] - obs)) + e_1 += float(np.linalg.norm(fct[i] - obs)) * ens_w[i] for j in range(i + 1, M): - e_2 += 2 * float(np.linalg.norm(fct[i] - fct[j])) + e_2 += 2 * float(np.linalg.norm(fct[i] - fct[j])) * ens_w[i] * ens_w[j] - out[0] = e_1 / M - 0.5 / (M**2) * e_2 + out[0] = e_1 - 0.5 * e_2 @guvectorize( diff --git a/scoringrules/core/energy/_score.py b/scoringrules/core/energy/_score.py index e264c4b..52bd26a 100644 --- a/scoringrules/core/energy/_score.py +++ b/scoringrules/core/energy/_score.py @@ -6,20 +6,22 @@ from scoringrules.core.typing import Array, Backend -def es_ensemble(obs: "Array", fct: "Array", backend=None) -> "Array": +def es_ensemble(obs: "Array", fct: "Array", ens_w: "Array", backend=None) -> "Array": """ Compute the energy score based on a finite ensemble. The ensemble and variables axes are on the second last and last dimensions respectively. """ B = backends.active if backend is None else backends[backend] - M: int = fct.shape[-2] err_norm = B.norm(fct - B.expand_dims(obs, -2), -1) - E_1 = B.sum(err_norm, -1) / M + E_1 = B.sum(err_norm * ens_w, -1) spread_norm = B.norm(B.expand_dims(fct, -3) - B.expand_dims(fct, -2), -1) - E_2 = B.sum(spread_norm, (-2, -1)) / (M**2) + E_2 = B.sum( + spread_norm * B.expand_dims(ens_w, -1) * B.expand_dims(ens_w, -2), (-2, -1) + ) + return E_1 - 0.5 * E_2 diff --git a/scoringrules/core/utils.py b/scoringrules/core/utils.py index 07ec329..8901dbf 100644 --- a/scoringrules/core/utils.py +++ b/scoringrules/core/utils.py @@ -4,7 +4,7 @@ _M_AXIS = -2 -def _multivariate_shape_compatibility(obs, fct, m_axis) -> None: +def _multivariate_shape_compatibility(obs, fct, ens_w, m_axis) -> None: f_shape = fct.shape o_shape = obs.shape o_shape_broadcast = o_shape[:m_axis] + (f_shape[m_axis],) + o_shape[m_axis:] @@ -28,5 +28,5 @@ def multivariate_array_check(obs, fct, m_axis, v_axis, backend=None): obs, fct = map(B.asarray, (obs, fct)) m_axis = m_axis if m_axis >= 0 else fct.ndim + m_axis v_axis = v_axis if v_axis >= 0 else fct.ndim + v_axis - _multivariate_shape_compatibility(obs, fct, m_axis) + _multivariate_shape_compatibility(obs, fct, m_axis, v_axis) return _multivariate_shape_permute(obs, fct, m_axis, v_axis, backend=backend) From 517c4fa8e1bf8a700753a7d83ae52993e061caec Mon Sep 17 00:00:00 2001 From: sallen12 Date: Thu, 10 Jul 2025 09:44:49 +0200 Subject: [PATCH 61/79] add ensemble weight argument to weighted energy scores --- scoringrules/_energy.py | 46 ++++++++++++++++++++++------ scoringrules/core/energy/_gufuncs.py | 41 ++++++++++++++++--------- scoringrules/core/energy/_score.py | 23 ++++++++------ 3 files changed, 77 insertions(+), 33 deletions(-) diff --git a/scoringrules/_energy.py b/scoringrules/_energy.py index eba3990..1c0f5c9 100644 --- a/scoringrules/_energy.py +++ b/scoringrules/_energy.py @@ -68,7 +68,7 @@ def es_ensemble( if ens_w is None: M = fct.shape[m_axis] - ens_w = B.zeros(fct.shape[:v_axis] + fct.shape[(v_axis + 1) :]) + 1.0 / M + ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M else: ens_w = B.moveaxis(ens_w, m_axis, -2) @@ -86,6 +86,7 @@ def twes_ensemble( m_axis: int = -2, v_axis: int = -1, *, + ens_w: "Array" = None, backend: "Backend" = None, ) -> "Array": r"""Compute the Threshold-Weighted Energy Score (twES) for a finite multivariate ensemble. @@ -115,6 +116,9 @@ def twes_ensemble( The axis corresponding to the ensemble dimension. Defaults to -2. v_axis : int or tuple of int The axis corresponding to the variables dimension. Defaults to -1. + ens_w : array_like + Weights assigned to the ensemble members. Array with one less dimension than fct (without the v_axis dimension). + Default is equal weighting. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -124,7 +128,9 @@ def twes_ensemble( The computed Threshold-Weighted Energy Score. """ obs, fct = map(v_func, (obs, fct)) - return es_ensemble(obs, fct, m_axis=m_axis, v_axis=v_axis, backend=backend) + return es_ensemble( + obs, fct, m_axis=m_axis, v_axis=v_axis, ens_w=ens_w, backend=backend + ) def owes_ensemble( @@ -135,6 +141,7 @@ def owes_ensemble( m_axis: int = -2, v_axis: int = -1, *, + ens_w: "Array" = None, backend: "Backend" = None, ) -> "Array": r"""Compute the Outcome-Weighted Energy Score (owES) for a finite multivariate ensemble. @@ -167,6 +174,9 @@ def owes_ensemble( The axis corresponding to the ensemble dimension. Defaults to -2. v_axis : int or tuple of ints The axis corresponding to the variables dimension. Defaults to -1. + ens_w : array_like + Weights assigned to the ensemble members. Array with one less dimension than fct (without the v_axis dimension). + Default is equal weighting. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -176,16 +186,23 @@ def owes_ensemble( The computed Outcome-Weighted Energy Score. """ B = backends.active if backend is None else backends[backend] - obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) fct_weights = B.apply_along_axis(w_func, fct, -1) obs_weights = B.apply_along_axis(w_func, obs, -1) + if ens_w is None: + M = fct.shape[m_axis] + ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M + else: + ens_w = B.moveaxis(ens_w, m_axis, -2) + if B.name == "numba": - return energy._owenergy_score_gufunc(obs, fct, obs_weights, fct_weights) + return energy._owenergy_score_gufunc(obs, fct, obs_weights, fct_weights, ens_w) - return energy.ownrg(obs, fct, obs_weights, fct_weights, backend=backend) + return energy.ownrg( + obs, fct, obs_weights, fct_weights, ens_w=ens_w, backend=backend + ) def vres_ensemble( @@ -193,9 +210,10 @@ def vres_ensemble( fct: "Array", w_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, - *, m_axis: int = -2, v_axis: int = -1, + *, + ens_w: "Array" = None, backend: "Backend" = None, ) -> "Array": r"""Compute the Vertically Re-scaled Energy Score (vrES) for a finite multivariate ensemble. @@ -229,6 +247,9 @@ def vres_ensemble( The axis corresponding to the ensemble dimension. Defaults to -2. v_axis : int or tuple of int The axis corresponding to the variables dimension. Defaults to -1. + ens_w : array_like + Weights assigned to the ensemble members. Array with one less dimension than fct (without the v_axis dimension). + Default is equal weighting. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -238,13 +259,20 @@ def vres_ensemble( The computed Vertically Re-scaled Energy Score. """ B = backends.active if backend is None else backends[backend] - obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) fct_weights = B.apply_along_axis(w_func, fct, -1) obs_weights = B.apply_along_axis(w_func, obs, -1) + if ens_w is None: + M = fct.shape[m_axis] + ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M + else: + ens_w = B.moveaxis(ens_w, m_axis, -2) + if backend == "numba": - return energy._vrenergy_score_gufunc(obs, fct, obs_weights, fct_weights) + return energy._vrenergy_score_gufunc(obs, fct, obs_weights, fct_weights, ens_w) - return energy.vrnrg(obs, fct, obs_weights, fct_weights, backend=backend) + return energy.vrnrg( + obs, fct, obs_weights, fct_weights, ens_w=ens_w, backend=backend + ) diff --git a/scoringrules/core/energy/_gufuncs.py b/scoringrules/core/energy/_gufuncs.py index c63f338..fc95a39 100644 --- a/scoringrules/core/energy/_gufuncs.py +++ b/scoringrules/core/energy/_gufuncs.py @@ -30,16 +30,17 @@ def _energy_score_gufunc( @guvectorize( [ - "void(float32[:], float32[:,:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:,:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:,:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:], float64[:], float64[:])", ], - "(d),(m,d),(),(m)->()", + "(d),(m,d),(),(m),(m)->()", ) def _owenergy_score_gufunc( obs: np.ndarray, fct: np.ndarray, ow: np.ndarray, fw: np.ndarray, + ens_w: np.ndarray, out: np.ndarray, ): """Compute the Outcome-Weighted Energy Score for a finite ensemble.""" @@ -49,27 +50,33 @@ def _owenergy_score_gufunc( e_1 = 0.0 e_2 = 0.0 for i in range(M): - e_1 += float(np.linalg.norm(fct[i] - obs) * fw[i] * ow) + e_1 += float(np.linalg.norm(fct[i] - obs) * fw[i] * ow) * ens_w[i] for j in range(i + 1, M): - e_2 += 2 * float(np.linalg.norm(fct[i] - fct[j]) * fw[i] * fw[j] * ow) + e_2 += ( + 2 + * float(np.linalg.norm(fct[i] - fct[j]) * fw[i] * fw[j] * ow) + * ens_w[i] + * ens_w[j] + ) wbar = np.mean(fw) - out[0] = e_1 / (M * wbar) - 0.5 * e_2 / (M**2 * wbar**2) + out[0] = e_1 / (wbar) - 0.5 * e_2 / (wbar**2) @guvectorize( [ - "void(float32[:], float32[:,:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:,:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:,:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:], float64[:], float64[:])", ], - "(d),(m,d),(),(m)->()", + "(d),(m,d),(),(m),(m)->()", ) def _vrenergy_score_gufunc( obs: np.ndarray, fct: np.ndarray, ow: np.ndarray, fw: np.ndarray, + ens_w: np.ndarray, out: np.ndarray, ): """Compute the Vertically Re-scaled Energy Score for a finite ensemble.""" @@ -80,13 +87,17 @@ def _vrenergy_score_gufunc( e_2 = 0.0 wabs_x = 0.0 for i in range(M): - e_1 += float(np.linalg.norm(fct[i] - obs) * fw[i] * ow) - wabs_x += np.linalg.norm(fct[i]) * fw[i] + e_1 += float(np.linalg.norm(fct[i] - obs) * fw[i] * ow) * ens_w[i] + wabs_x += np.linalg.norm(fct[i]) * fw[i] * ens_w[i] for j in range(i + 1, M): - e_2 += 2 * float(np.linalg.norm(fct[i] - fct[j]) * fw[i] * fw[j]) + e_2 += ( + 2 + * float(np.linalg.norm(fct[i] - fct[j]) * fw[i] * fw[j]) + * ens_w[i] + * ens_w[j] + ) - wabs_x = wabs_x / M - wbar = np.mean(fw) + wbar = np.sum(fw * ens_w) wabs_y = np.linalg.norm(obs) * ow - out[0] = e_1 / M - 0.5 * e_2 / (M**2) + (wabs_x - wabs_y) * (wbar - ow) + out[0] = e_1 - 0.5 * e_2 + (wabs_x - wabs_y) * (wbar - ow) diff --git a/scoringrules/core/energy/_score.py b/scoringrules/core/energy/_score.py index 52bd26a..ff5b615 100644 --- a/scoringrules/core/energy/_score.py +++ b/scoringrules/core/energy/_score.py @@ -30,22 +30,24 @@ def owes_ensemble( fct: "Array", # (... M D) ow: "Array", # (...) fw: "Array", # (... M) + ens_w: "Array", # (... M) backend: "Backend" = None, ) -> "Array": """Compute the outcome-weighted energy score based on a finite ensemble.""" B = backends.active if backend is None else backends[backend] - M = fct.shape[-2] - wbar = B.sum(fw, -1) / M + wbar = B.sum(fw * ens_w, -1) err_norm = B.norm(fct - B.expand_dims(obs, -2), -1) # (... M) - E_1 = B.sum(err_norm * fw * B.expand_dims(ow, -1), -1) / (M * wbar) # (...) + E_1 = B.sum(err_norm * fw * B.expand_dims(ow, -1) * ens_w, -1) / wbar # (...) spread_norm = B.norm( B.expand_dims(fct, -2) - B.expand_dims(fct, -3), -1 ) # (... M M) fw_prod = B.expand_dims(fw, -1) * B.expand_dims(fw, -2) # (... M M) spread_norm *= fw_prod * B.expand_dims(ow, (-2, -1)) # (... M M) - E_2 = B.sum(spread_norm, (-2, -1)) / (M**2 * wbar**2) # (...) + E_2 = B.sum( + spread_norm * B.expand_dims(ens_w, -1) * B.expand_dims(ens_w, -2), (-2, -1) + ) / (wbar**2) # (...) return E_1 - 0.5 * E_2 @@ -55,23 +57,26 @@ def vres_ensemble( fct: "Array", ow: "Array", fw: "Array", + ens_w: "Array", backend: "Backend" = None, ) -> "Array": """Compute the vertically re-scaled energy score based on a finite ensemble.""" B = backends.active if backend is None else backends[backend] - M, D = fct.shape[-2:] - wbar = B.sum(fw, -1) / M + wbar = B.sum(fw * ens_w, -1) err_norm = B.norm(fct - B.expand_dims(obs, -2), -1) # (... M) err_norm *= fw * B.expand_dims(ow, -1) # (... M) - E_1 = B.sum(err_norm, -1) / M # (...) + E_1 = B.sum(err_norm * ens_w, -1) # (...) spread_norm = B.norm( B.expand_dims(fct, -2) - B.expand_dims(fct, -3), -1 ) # (... M M) fw_prod = B.expand_dims(fw, -2) * B.expand_dims(fw, -1) # (... M M) - E_2 = B.sum(spread_norm * fw_prod, (-2, -1)) / (M**2) # (...) + E_2 = B.sum( + spread_norm * fw_prod * B.expand_dims(ens_w, -1) * B.expand_dims(ens_w, -2), + (-2, -1), + ) # (...) - rhobar = B.sum(B.norm(fct, -1) * fw, -1) / M # (...) + rhobar = B.sum(B.norm(fct, -1) * fw * ens_w, -1) # (...) E_3 = (rhobar - B.norm(obs, -1) * ow) * (wbar - ow) # (...) return E_1 - 0.5 * E_2 + E_3 From 73dcb6dc7173cc5d493c079d34bbabef463486ca Mon Sep 17 00:00:00 2001 From: sallen12 Date: Thu, 10 Jul 2025 11:35:03 +0200 Subject: [PATCH 62/79] add ensemble weights for the variogram score --- scoringrules/_variogram.py | 19 +++++++++++++++---- scoringrules/core/utils.py | 4 ++-- scoringrules/core/variogram/_gufuncs.py | 11 +++++------ scoringrules/core/variogram/_score.py | 6 ++++-- tests/test_wenergy.py | 2 +- 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/scoringrules/_variogram.py b/scoringrules/_variogram.py index b60db8e..9b77d4f 100644 --- a/scoringrules/_variogram.py +++ b/scoringrules/_variogram.py @@ -15,6 +15,7 @@ def vs_ensemble( m_axis: int = -2, v_axis: int = -1, *, + ens_w: "Array" = None, p: float = 1.0, backend: "Backend" = None, ) -> "Array": @@ -36,12 +37,15 @@ def vs_ensemble( fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. - p : float - The order of the Variogram Score. Typical values are 0.5, 1.0 or 2.0. Defaults to 1.0. m_axis : int The axis corresponding to the ensemble dimension. Defaults to -2. v_axis : int The axis corresponding to the variables dimension. Defaults to -1. + ens_w : array_like + Weights assigned to the ensemble members. Array with one less dimension than fct (without the v_axis dimension). + Default is equal weighting. + p : float + The order of the Variogram Score. Typical values are 0.5, 1.0 or 2.0. Defaults to 1.0. backend: str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -66,12 +70,19 @@ def vs_ensemble( >>> sr.vs_ensemble(obs, fct) array([ 8.65630139, 6.84693866, 19.52993307]) """ + B = backends.active if backend is None else backends[backend] obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) + if ens_w is None: + M = fct.shape[m_axis] + ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M + else: + ens_w = B.moveaxis(ens_w, m_axis, -2) + if backend == "numba": - return variogram._variogram_score_gufunc(obs, fct, p) + return variogram._variogram_score_gufunc(obs, fct, ens_w, p) - return variogram.vs(obs, fct, p, backend=backend) + return variogram.vs(obs, fct, ens_w, p, backend=backend) def twvs_ensemble( diff --git a/scoringrules/core/utils.py b/scoringrules/core/utils.py index 8901dbf..07ec329 100644 --- a/scoringrules/core/utils.py +++ b/scoringrules/core/utils.py @@ -4,7 +4,7 @@ _M_AXIS = -2 -def _multivariate_shape_compatibility(obs, fct, ens_w, m_axis) -> None: +def _multivariate_shape_compatibility(obs, fct, m_axis) -> None: f_shape = fct.shape o_shape = obs.shape o_shape_broadcast = o_shape[:m_axis] + (f_shape[m_axis],) + o_shape[m_axis:] @@ -28,5 +28,5 @@ def multivariate_array_check(obs, fct, m_axis, v_axis, backend=None): obs, fct = map(B.asarray, (obs, fct)) m_axis = m_axis if m_axis >= 0 else fct.ndim + m_axis v_axis = v_axis if v_axis >= 0 else fct.ndim + v_axis - _multivariate_shape_compatibility(obs, fct, m_axis, v_axis) + _multivariate_shape_compatibility(obs, fct, m_axis) return _multivariate_shape_permute(obs, fct, m_axis, v_axis, backend=backend) diff --git a/scoringrules/core/variogram/_gufuncs.py b/scoringrules/core/variogram/_gufuncs.py index d3c298f..8dd04aa 100644 --- a/scoringrules/core/variogram/_gufuncs.py +++ b/scoringrules/core/variogram/_gufuncs.py @@ -4,12 +4,12 @@ @guvectorize( [ - "void(float32[:], float32[:,:], float32, float32[:])", - "void(float64[:], float64[:,:], float64, float64[:])", + "void(float32[:], float32[:,:], float32[:], float32, float32[:])", + "void(float64[:], float64[:,:], float64[:], float64, float64[:])", ], - "(d),(m,d),()->()", + "(d),(m,d),(m),()->()", ) -def _variogram_score_gufunc(obs, fct, p, out): +def _variogram_score_gufunc(obs, fct, ens_w, p, out): M = fct.shape[-2] D = fct.shape[-1] out[0] = 0.0 @@ -17,8 +17,7 @@ def _variogram_score_gufunc(obs, fct, p, out): for j in range(D): vfct = 0.0 for m in range(M): - vfct += abs(fct[m, i] - fct[m, j]) ** p - vfct = vfct / M + vfct += ens_w[m] * abs(fct[m, i] - fct[m, j]) ** p vobs = abs(obs[i] - obs[j]) ** p out[0] += (vobs - vfct) ** 2 diff --git a/scoringrules/core/variogram/_score.py b/scoringrules/core/variogram/_score.py index 8850cb5..464396e 100644 --- a/scoringrules/core/variogram/_score.py +++ b/scoringrules/core/variogram/_score.py @@ -9,14 +9,16 @@ def vs_ensemble( obs: "Array", # (... D) fct: "Array", # (... M D) + ens_w: "Array", # (... M) p: float = 1, backend: "Backend" = None, ) -> "Array": """Compute the Variogram Score for a multivariate finite ensemble.""" B = backends.active if backend is None else backends[backend] - M: int = fct.shape[-2] fct_diff = B.expand_dims(fct, -2) - B.expand_dims(fct, -1) # (... M D D) - vfct = B.sum(B.abs(fct_diff) ** p, axis=-3) / M # (... D D) + vfct = B.sum( + B.expand_dims(ens_w, (-1, -2)) * B.abs(fct_diff) ** p, axis=-3 + ) # (... D D) obs_diff = B.expand_dims(obs, -2) - B.expand_dims(obs, -1) # (... D D) vobs = B.abs(obs_diff) ** p # (... D D) return B.sum((vobs - vfct) ** 2, axis=(-2, -1)) # (...) diff --git a/tests/test_wenergy.py b/tests/test_wenergy.py index 5dca3fa..f717763 100644 --- a/tests/test_wenergy.py +++ b/tests/test_wenergy.py @@ -48,7 +48,7 @@ def test_vres_vs_es(backend): lambda x: backends[backend].mean(x) * 0.0 + 1.0, backend=backend, ) - np.testing.assert_allclose(res, resw, rtol=1e-10) + np.testing.assert_allclose(res, resw, rtol=1e-7) @pytest.mark.parametrize("backend", BACKENDS) From f034542522735346d0452afd23b1444d6d78305c Mon Sep 17 00:00:00 2001 From: sallen12 Date: Thu, 10 Jul 2025 14:13:04 +0200 Subject: [PATCH 63/79] add ensemble weights to weighted variogram scores --- scoringrules/_variogram.py | 27 +++++++++++++++++++------ scoringrules/core/variogram/_gufuncs.py | 16 +++++++-------- scoringrules/core/variogram/_score.py | 22 +++++++++++--------- 3 files changed, 41 insertions(+), 24 deletions(-) diff --git a/scoringrules/_variogram.py b/scoringrules/_variogram.py index 9b77d4f..75579d3 100644 --- a/scoringrules/_variogram.py +++ b/scoringrules/_variogram.py @@ -93,6 +93,7 @@ def twvs_ensemble( m_axis: int = -2, v_axis: int = -1, *, + ens_w: "Array" = None, p: float = 1.0, backend: "Backend" = None, ) -> "Array": @@ -122,6 +123,9 @@ def twvs_ensemble( The axis corresponding to the ensemble dimension. Defaults to -2. v_axis : int The axis corresponding to the variables dimension. Defaults to -1. + ens_w : array_like + Weights assigned to the ensemble members. Array with one less dimension than fct (without the v_axis dimension). + Default is equal weighting. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -148,7 +152,7 @@ def twvs_ensemble( array([5.94996894, 4.72029765, 6.08947229]) """ obs, fct = map(v_func, (obs, fct)) - return vs_ensemble(obs, fct, m_axis, v_axis, p=p, backend=backend) + return vs_ensemble(obs, fct, m_axis, v_axis, ens_w=ens_w, p=p, backend=backend) def owvs_ensemble( @@ -159,6 +163,7 @@ def owvs_ensemble( m_axis: int = -2, v_axis: int = -1, *, + ens_w: "Array" = None, p: float = 1.0, backend: "Backend" = None, ) -> "Array": @@ -193,6 +198,9 @@ def owvs_ensemble( The axis corresponding to the ensemble dimension. Defaults to -2. v_axis : int The axis corresponding to the variables dimension. Defaults to -1. + ens_w : array_like + Weights assigned to the ensemble members. Array with one less dimension than fct (without the v_axis dimension). + Default is equal weighting. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -212,7 +220,6 @@ def owvs_ensemble( array([ 9.86816636, 6.75532522, 19.59353723]) """ B = backends.active if backend is None else backends[backend] - obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) obs_weights = B.apply_along_axis(w_func, obs, -1) @@ -220,10 +227,12 @@ def owvs_ensemble( if backend == "numba": return variogram._owvariogram_score_gufunc( - obs, fct, p, obs_weights, fct_weights + obs, fct, obs_weights, fct_weights, ens_w, p ) - return variogram.owvs(obs, fct, obs_weights, fct_weights, p=p, backend=backend) + return variogram.owvs( + obs, fct, obs_weights, fct_weights, ens_w=ens_w, p=p, backend=backend + ) def vrvs_ensemble( @@ -234,6 +243,7 @@ def vrvs_ensemble( m_axis: int = -2, v_axis: int = -1, *, + ens_w: "Array" = None, p: float = 1.0, backend: "Backend" = None, ) -> "Array": @@ -270,6 +280,9 @@ def vrvs_ensemble( The axis corresponding to the ensemble dimension. Defaults to -2. v_axis : int The axis corresponding to the variables dimension. Defaults to -1. + ens_w : array_like + Weights assigned to the ensemble members. Array with one less dimension than fct (without the v_axis dimension). + Default is equal weighting. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -297,7 +310,9 @@ def vrvs_ensemble( if backend == "numba": return variogram._vrvariogram_score_gufunc( - obs, fct, p, obs_weights, fct_weights + obs, fct, obs_weights, fct_weights, ens_w, p ) - return variogram.vrvs(obs, fct, obs_weights, fct_weights, p=p, backend=backend) + return variogram.vrvs( + obs, fct, obs_weights, fct_weights, ensw=ens_w, p=p, backend=backend + ) diff --git a/scoringrules/core/variogram/_gufuncs.py b/scoringrules/core/variogram/_gufuncs.py index 8dd04aa..90e564c 100644 --- a/scoringrules/core/variogram/_gufuncs.py +++ b/scoringrules/core/variogram/_gufuncs.py @@ -24,12 +24,12 @@ def _variogram_score_gufunc(obs, fct, ens_w, p, out): @guvectorize( [ - "void(float32[:], float32[:,:], float32, float32, float32[:], float32[:])", - "void(float64[:], float64[:,:], float64, float64, float64[:], float64[:])", + "void(float32[:], float32[:,:], float32, float32[:], float32[:], float32, float32[:])", + "void(float64[:], float64[:,:], float64, float64[:], float64[:], float64, float64[:])", ], - "(d),(m,d),(),(),(m)->()", + "(d),(m,d),(),(m),(m),()->()", ) -def _owvariogram_score_gufunc(obs, fct, p, ow, fw, out): +def _owvariogram_score_gufunc(obs, fct, ow, fw, ens_w, p, out): M = fct.shape[-2] D = fct.shape[-1] @@ -55,12 +55,12 @@ def _owvariogram_score_gufunc(obs, fct, p, ow, fw, out): @guvectorize( [ - "void(float32[:], float32[:,:], float32, float32, float32[:], float32[:])", - "void(float64[:], float64[:,:], float64, float64, float64[:], float64[:])", + "void(float32[:], float32[:,:], float32, float32[:], float32[:], float32, float32[:])", + "void(float64[:], float64[:,:], float64, float64[:], float64[:], float64, float64[:])", ], - "(d),(m,d),(),(),(m)->()", + "(d),(m,d),(),(m),(m),()->()", ) -def _vrvariogram_score_gufunc(obs, fct, p, ow, fw, out): +def _vrvariogram_score_gufunc(obs, fct, ow, fw, ens_w, p, out): M = fct.shape[-2] D = fct.shape[-1] diff --git a/scoringrules/core/variogram/_score.py b/scoringrules/core/variogram/_score.py index 464396e..01b70e2 100644 --- a/scoringrules/core/variogram/_score.py +++ b/scoringrules/core/variogram/_score.py @@ -29,13 +29,13 @@ def owvs_ensemble( fct: "Array", ow: "Array", fw: "Array", + ens_w: "Array", p: float = 1, backend: "Backend" = None, ) -> "Array": """Compute the Outcome-Weighted Variogram Score for a multivariate finite ensemble.""" B = backends.active if backend is None else backends[backend] - M: int = fct.shape[-2] - wbar = B.mean(fw, axis=-1) + wbar = B.mean(fw * ens_w, axis=-1) fct_diff = B.expand_dims(fct, -2) - B.expand_dims(fct, -1) # (... M D D) fct_diff = B.abs(fct_diff) ** p # (... M D D) @@ -46,15 +46,16 @@ def owvs_ensemble( E_1 = (fct_diff - B.expand_dims(obs_diff, -3)) ** 2 # (... M D D) E_1 = B.sum(E_1, axis=(-2, -1)) # (... M) - E_1 = B.sum(E_1 * fw * B.expand_dims(ow, -1), axis=-1) / (M * wbar) # (...) + E_1 = B.sum(E_1 * fw * B.expand_dims(ow, -1) * ens_w, axis=-1) / wbar # (...) fct_diff_spread = B.expand_dims(fct_diff, -3) - B.expand_dims( fct_diff, -4 ) # (... M M D D) fw_prod = B.expand_dims(fw, -2) * B.expand_dims(fw, -1) # (... M M) + ew_prod = B.expand_dims(ens_w, -2) * B.expand_dims(ens_w, -1) # (... M M) E_2 = B.sum(fct_diff_spread**2, axis=(-2, -1)) # (... M M) - E_2 *= fw_prod * B.expand_dims(ow, (-2, -1)) # (... M M) - E_2 = B.sum(E_2, axis=(-2, -1)) / (M**2 * wbar**2) # (...) + E_2 *= fw_prod * B.expand_dims(ow, (-2, -1)) * ew_prod # (... M M) + E_2 = B.sum(E_2, axis=(-2, -1)) / (wbar**2) # (...) return E_1 - 0.5 * E_2 @@ -64,13 +65,13 @@ def vrvs_ensemble( fct: "Array", ow: "Array", fw: "Array", + ens_w: "Array", p: float = 1, backend: "Backend" = None, ) -> "Array": """Compute the Vertically Re-scaled Variogram Score for a multivariate finite ensemble.""" B = backends.active if backend is None else backends[backend] - M: int = fct.shape[-2] - wbar = B.mean(fw, axis=-1) + wbar = B.mean(fw * ens_w, axis=-1) fct_diff = ( B.abs(B.expand_dims(fct, -2) - B.expand_dims(fct, -1)) ** p @@ -79,7 +80,7 @@ def vrvs_ensemble( E_1 = (fct_diff - B.expand_dims(obs_diff, axis=-3)) ** 2 # (... M D D) E_1 = B.sum(E_1, axis=(-2, -1)) # (... M) - E_1 = B.sum(E_1 * fw * B.expand_dims(ow, axis=-1), axis=-1) / M # (...) + E_1 = B.sum(E_1 * fw * B.expand_dims(ow, axis=-1) * ens_w, axis=-1) # (...) E_2 = ( B.expand_dims(fct_diff, -3) - B.expand_dims(fct_diff, -4) @@ -87,10 +88,11 @@ def vrvs_ensemble( E_2 = B.sum(E_2, axis=(-2, -1)) # (... M M) fw_prod = B.expand_dims(fw, axis=-2) * B.expand_dims(fw, axis=-1) # (... M M) - E_2 = B.sum(E_2 * fw_prod, axis=(-2, -1)) / (M**2) # (...) + ew_prod = B.expand_dims(ens_w, -2) * B.expand_dims(ens_w, -1) # (... M M) + E_2 = B.sum(E_2 * fw_prod * ew_prod, axis=(-2, -1)) # (...) E_3 = B.sum(fct_diff**2, axis=(-2, -1)) # (... M) - E_3 = B.sum(E_3 * fw, axis=-1) / M # (...) + E_3 = B.sum(E_3 * fw * ens_w, axis=-1) # (...) E_3 -= B.sum(obs_diff**2, axis=(-2, -1)) * ow # (...) E_3 *= wbar - ow # (...) From f164f62b925144c42cfa02b95e5ec8b92fdbb050 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Thu, 10 Jul 2025 14:23:14 +0200 Subject: [PATCH 64/79] add ensemble weights to weighted variogram scores --- scoringrules/_variogram.py | 10 ++++++--- scoringrules/core/variogram/_gufuncs.py | 29 ++++++++++++++++--------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/scoringrules/_variogram.py b/scoringrules/_variogram.py index 75579d3..1f80ce8 100644 --- a/scoringrules/_variogram.py +++ b/scoringrules/_variogram.py @@ -12,6 +12,7 @@ def vs_ensemble( obs: "Array", fct: "Array", /, + w: "Array", m_axis: int = -2, v_axis: int = -1, *, @@ -24,7 +25,7 @@ def vs_ensemble( For a :math:`D`-variate ensemble the Variogram Score [1]_ is defined as: .. math:: - \text{VS}_{p}(F_{ens}, \mathbf{y})= \sum_{i=1}^{d} \sum_{j=1}^{d} + \text{VS}_{p}(F_{ens}, \mathbf{y})= \sum_{i=1}^{d} \sum_{j=1}^{d} w_{i,j} \left( \frac{1}{M} \sum_{m=1}^{M} | x_{m,i} - x_{m,j} |^{p} - | y_{i} - y_{j} |^{p} \right)^{2}, where :math:`\mathbf{X}` and :math:`\mathbf{X'}` are independently sampled ensembles from from :math:`F`. @@ -37,6 +38,9 @@ def vs_ensemble( fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. + w : array_like + The weights assigned to pairs of dimensions. Must be of shape (..., D, D), where + D is the dimension, so that the weights are in the last two axes. m_axis : int The axis corresponding to the ensemble dimension. Defaults to -2. v_axis : int @@ -80,9 +84,9 @@ def vs_ensemble( ens_w = B.moveaxis(ens_w, m_axis, -2) if backend == "numba": - return variogram._variogram_score_gufunc(obs, fct, ens_w, p) + return variogram._variogram_score_gufunc(obs, fct, w, ens_w, p) - return variogram.vs(obs, fct, ens_w, p, backend=backend) + return variogram.vs(obs, fct, w, ens_w, p, backend=backend) def twvs_ensemble( diff --git a/scoringrules/core/variogram/_gufuncs.py b/scoringrules/core/variogram/_gufuncs.py index 90e564c..f431c2e 100644 --- a/scoringrules/core/variogram/_gufuncs.py +++ b/scoringrules/core/variogram/_gufuncs.py @@ -40,17 +40,25 @@ def _owvariogram_score_gufunc(obs, fct, ow, fw, ens_w, p, out): for j in range(D): rho1 = abs(fct[k, i] - fct[k, j]) ** p rho2 = abs(obs[i] - obs[j]) ** p - e_1 += (rho1 - rho2) ** 2 * fw[k] * ow + e_1 += (rho1 - rho2) ** 2 * fw[k] * ow * ens_w[k] for m in range(k + 1, M): for i in range(D): for j in range(D): rho1 = abs(fct[k, i] - fct[k, j]) ** p rho2 = abs(fct[m, i] - fct[m, j]) ** p - e_2 += 2 * ((rho1 - rho2) ** 2) * fw[k] * fw[m] * ow + e_2 += ( + 2 + * ((rho1 - rho2) ** 2) + * fw[k] + * fw[m] + * ow + * ens_w[k] + * ens_w[m] + ) - wbar = np.mean(fw) + wbar = np.sum(fw * ens_w) - out[0] = e_1 / (M * wbar) - 0.5 * e_2 / (M**2 * wbar**2) + out[0] = e_1 / wbar - 0.5 * e_2 / (wbar**2) @guvectorize( @@ -72,21 +80,22 @@ def _vrvariogram_score_gufunc(obs, fct, ow, fw, ens_w, p, out): for j in range(D): rho1 = abs(fct[k, i] - fct[k, j]) ** p rho2 = abs(obs[i] - obs[j]) ** p - e_1 += (rho1 - rho2) ** 2 * fw[k] * ow - e_3_x += (rho1) ** 2 * fw[k] + e_1 += (rho1 - rho2) ** 2 * fw[k] * ow * ens_w[k] + e_3_x += (rho1) ** 2 * fw[k] * ens_w[k] for m in range(k + 1, M): for i in range(D): for j in range(D): rho1 = abs(fct[k, i] - fct[k, j]) ** p rho2 = abs(fct[m, i] - fct[m, j]) ** p - e_2 += 2 * ((rho1 - rho2) ** 2) * fw[k] * fw[m] + e_2 += ( + 2 * ((rho1 - rho2) ** 2) * fw[k] * fw[m] * ens_w[k] * ens_w[m] + ) - e_3_x *= 1 / M - wbar = np.mean(fw) + wbar = np.sum(fw * ens_w) e_3_y = 0.0 for i in range(D): for j in range(D): rho1 = abs(obs[i] - obs[j]) ** p e_3_y += (rho1) ** 2 * ow - out[0] = e_1 / M - 0.5 * e_2 / (M**2) + (e_3_x - e_3_y) * (wbar - ow) + out[0] = e_1 - 0.5 * e_2 + (e_3_x - e_3_y) * (wbar - ow) From d226627c69b30913acffd37cbaafd8bbd82ca109 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Thu, 10 Jul 2025 15:08:43 +0200 Subject: [PATCH 65/79] add dimension weights to variogram scores --- scoringrules/_energy.py | 6 +-- scoringrules/_variogram.py | 62 +++++++++++++++++++------ scoringrules/core/variogram/_gufuncs.py | 38 +++++++-------- scoringrules/core/variogram/_score.py | 23 +++++---- 4 files changed, 85 insertions(+), 44 deletions(-) diff --git a/scoringrules/_energy.py b/scoringrules/_energy.py index 1c0f5c9..72fcbe9 100644 --- a/scoringrules/_energy.py +++ b/scoringrules/_energy.py @@ -67,7 +67,7 @@ def es_ensemble( obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) if ens_w is None: - M = fct.shape[m_axis] + M = fct.shape[-2] ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M else: ens_w = B.moveaxis(ens_w, m_axis, -2) @@ -192,7 +192,7 @@ def owes_ensemble( obs_weights = B.apply_along_axis(w_func, obs, -1) if ens_w is None: - M = fct.shape[m_axis] + M = fct.shape[-2] ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M else: ens_w = B.moveaxis(ens_w, m_axis, -2) @@ -265,7 +265,7 @@ def vres_ensemble( obs_weights = B.apply_along_axis(w_func, obs, -1) if ens_w is None: - M = fct.shape[m_axis] + M = fct.shape[-2] ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M else: ens_w = B.moveaxis(ens_w, m_axis, -2) diff --git a/scoringrules/_variogram.py b/scoringrules/_variogram.py index 1f80ce8..cdf6963 100644 --- a/scoringrules/_variogram.py +++ b/scoringrules/_variogram.py @@ -12,7 +12,7 @@ def vs_ensemble( obs: "Array", fct: "Array", /, - w: "Array", + w: "Array" = None, m_axis: int = -2, v_axis: int = -1, *, @@ -78,11 +78,15 @@ def vs_ensemble( obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) if ens_w is None: - M = fct.shape[m_axis] + M = fct.shape[-2] ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M else: ens_w = B.moveaxis(ens_w, m_axis, -2) + if w is None: + D = fct.shape[-1] + w = B.zeros(obs.shape + (D,)) + 1.0 + if backend == "numba": return variogram._variogram_score_gufunc(obs, fct, w, ens_w, p) @@ -94,6 +98,7 @@ def twvs_ensemble( fct: "Array", v_func: tp.Callable, /, + w: "Array" = None, m_axis: int = -2, v_axis: int = -1, *, @@ -119,8 +124,9 @@ def twvs_ensemble( fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. - p : float - The order of the Variogram Score. Typical values are 0.5, 1.0 or 2.0. Defaults to 1.0. + w : array_like + The weights assigned to pairs of dimensions. Must be of shape (..., D, D), where + D is the dimension, so that the weights are in the last two axes. v_func : callable, array_like -> array_like Chaining function used to emphasise particular outcomes. m_axis : int @@ -130,6 +136,8 @@ def twvs_ensemble( ens_w : array_like Weights assigned to the ensemble members. Array with one less dimension than fct (without the v_axis dimension). Default is equal weighting. + p : float + The order of the Variogram Score. Typical values are 0.5, 1.0 or 2.0. Defaults to 1.0. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -156,7 +164,7 @@ def twvs_ensemble( array([5.94996894, 4.72029765, 6.08947229]) """ obs, fct = map(v_func, (obs, fct)) - return vs_ensemble(obs, fct, m_axis, v_axis, ens_w=ens_w, p=p, backend=backend) + return vs_ensemble(obs, fct, w, m_axis, v_axis, ens_w=ens_w, p=p, backend=backend) def owvs_ensemble( @@ -164,6 +172,7 @@ def owvs_ensemble( fct: "Array", w_func: tp.Callable, /, + w: "Array" = None, m_axis: int = -2, v_axis: int = -1, *, @@ -194,10 +203,11 @@ def owvs_ensemble( fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. - p : float - The order of the Variogram Score. Typical values are 0.5, 1.0 or 2.0. Defaults to 1.0. w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. + w : array_like + The weights assigned to pairs of dimensions. Must be of shape (..., D, D), where + D is the dimension, so that the weights are in the last two axes. m_axis : int The axis corresponding to the ensemble dimension. Defaults to -2. v_axis : int @@ -205,6 +215,8 @@ def owvs_ensemble( ens_w : array_like Weights assigned to the ensemble members. Array with one less dimension than fct (without the v_axis dimension). Default is equal weighting. + p : float + The order of the Variogram Score. Typical values are 0.5, 1.0 or 2.0. Defaults to 1.0. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -229,13 +241,23 @@ def owvs_ensemble( obs_weights = B.apply_along_axis(w_func, obs, -1) fct_weights = B.apply_along_axis(w_func, fct, -1) + if ens_w is None: + M = fct.shape[-2] + ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M + else: + ens_w = B.moveaxis(ens_w, m_axis, -2) + + if w is None: + D = fct.shape[-1] + w = B.zeros(obs.shape + (D,)) + 1.0 + if backend == "numba": return variogram._owvariogram_score_gufunc( - obs, fct, obs_weights, fct_weights, ens_w, p + obs, fct, w, obs_weights, fct_weights, ens_w, p ) return variogram.owvs( - obs, fct, obs_weights, fct_weights, ens_w=ens_w, p=p, backend=backend + obs, fct, w, obs_weights, fct_weights, ens_w=ens_w, p=p, backend=backend ) @@ -244,6 +266,7 @@ def vrvs_ensemble( fct: "Array", w_func: tp.Callable, /, + w: "Array" = None, m_axis: int = -2, v_axis: int = -1, *, @@ -276,10 +299,11 @@ def vrvs_ensemble( fct : array_like The predicted forecast ensemble, where the ensemble dimension is by default represented by the second last axis and the variables dimension by the last axis. - p : float - The order of the Variogram Score. Typical values are 0.5, 1.0 or 2.0. Defaults to 1.0. w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. + w : array_like + The weights assigned to pairs of dimensions. Must be of shape (..., D, D), where + D is the dimension, so that the weights are in the last two axes. m_axis : int The axis corresponding to the ensemble dimension. Defaults to -2. v_axis : int @@ -287,6 +311,8 @@ def vrvs_ensemble( ens_w : array_like Weights assigned to the ensemble members. Array with one less dimension than fct (without the v_axis dimension). Default is equal weighting. + p : float + The order of the Variogram Score. Typical values are 0.5, 1.0 or 2.0. Defaults to 1.0. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -312,11 +338,21 @@ def vrvs_ensemble( obs_weights = B.apply_along_axis(w_func, obs, -1) fct_weights = B.apply_along_axis(w_func, fct, -1) + if ens_w is None: + M = fct.shape[-2] + ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M + else: + ens_w = B.moveaxis(ens_w, m_axis, -2) + + if w is None: + D = fct.shape[-1] + w = B.zeros(obs.shape + (D,)) + 1.0 + if backend == "numba": return variogram._vrvariogram_score_gufunc( - obs, fct, obs_weights, fct_weights, ens_w, p + obs, fct, w, obs_weights, fct_weights, ens_w, p ) return variogram.vrvs( - obs, fct, obs_weights, fct_weights, ensw=ens_w, p=p, backend=backend + obs, fct, w, obs_weights, fct_weights, ens_w=ens_w, p=p, backend=backend ) diff --git a/scoringrules/core/variogram/_gufuncs.py b/scoringrules/core/variogram/_gufuncs.py index f431c2e..46b8a96 100644 --- a/scoringrules/core/variogram/_gufuncs.py +++ b/scoringrules/core/variogram/_gufuncs.py @@ -4,12 +4,12 @@ @guvectorize( [ - "void(float32[:], float32[:,:], float32[:], float32, float32[:])", - "void(float64[:], float64[:,:], float64[:], float64, float64[:])", + "void(float32[:], float32[:,:], float32[:,:], float32[:], float32, float32[:])", + "void(float64[:], float64[:,:], float64[:,:], float64[:], float64, float64[:])", ], - "(d),(m,d),(m),()->()", + "(d),(m,d),(d,d),(m),()->()", ) -def _variogram_score_gufunc(obs, fct, ens_w, p, out): +def _variogram_score_gufunc(obs, fct, w, ens_w, p, out): M = fct.shape[-2] D = fct.shape[-1] out[0] = 0.0 @@ -19,17 +19,17 @@ def _variogram_score_gufunc(obs, fct, ens_w, p, out): for m in range(M): vfct += ens_w[m] * abs(fct[m, i] - fct[m, j]) ** p vobs = abs(obs[i] - obs[j]) ** p - out[0] += (vobs - vfct) ** 2 + out[0] += w[i, j] * (vobs - vfct) ** 2 @guvectorize( [ - "void(float32[:], float32[:,:], float32, float32[:], float32[:], float32, float32[:])", - "void(float64[:], float64[:,:], float64, float64[:], float64[:], float64, float64[:])", + "void(float32[:], float32[:,:], float32[:,:], float32, float32[:], float32[:], float32, float32[:])", + "void(float64[:], float64[:,:], float64[:,:], float64, float64[:], float64[:], float64, float64[:])", ], - "(d),(m,d),(),(m),(m),()->()", + "(d),(m,d),(d,d),(),(m),(m),()->()", ) -def _owvariogram_score_gufunc(obs, fct, ow, fw, ens_w, p, out): +def _owvariogram_score_gufunc(obs, fct, w, ow, fw, ens_w, p, out): M = fct.shape[-2] D = fct.shape[-1] @@ -40,13 +40,13 @@ def _owvariogram_score_gufunc(obs, fct, ow, fw, ens_w, p, out): for j in range(D): rho1 = abs(fct[k, i] - fct[k, j]) ** p rho2 = abs(obs[i] - obs[j]) ** p - e_1 += (rho1 - rho2) ** 2 * fw[k] * ow * ens_w[k] + e_1 += w[i, j] * (rho1 - rho2) ** 2 * fw[k] * ow * ens_w[k] for m in range(k + 1, M): for i in range(D): for j in range(D): rho1 = abs(fct[k, i] - fct[k, j]) ** p rho2 = abs(fct[m, i] - fct[m, j]) ** p - e_2 += ( + e_2 += w[i, j] * ( 2 * ((rho1 - rho2) ** 2) * fw[k] @@ -63,12 +63,12 @@ def _owvariogram_score_gufunc(obs, fct, ow, fw, ens_w, p, out): @guvectorize( [ - "void(float32[:], float32[:,:], float32, float32[:], float32[:], float32, float32[:])", - "void(float64[:], float64[:,:], float64, float64[:], float64[:], float64, float64[:])", + "void(float32[:], float32[:,:], float32[:,:], float32, float32[:], float32[:], float32, float32[:])", + "void(float64[:], float64[:,:], float64[:,:], float64, float64[:], float64[:], float64, float64[:])", ], - "(d),(m,d),(),(m),(m),()->()", + "(d),(m,d),(d,d),(),(m),(m),()->()", ) -def _vrvariogram_score_gufunc(obs, fct, ow, fw, ens_w, p, out): +def _vrvariogram_score_gufunc(obs, fct, w, ow, fw, ens_w, p, out): M = fct.shape[-2] D = fct.shape[-1] @@ -80,14 +80,14 @@ def _vrvariogram_score_gufunc(obs, fct, ow, fw, ens_w, p, out): for j in range(D): rho1 = abs(fct[k, i] - fct[k, j]) ** p rho2 = abs(obs[i] - obs[j]) ** p - e_1 += (rho1 - rho2) ** 2 * fw[k] * ow * ens_w[k] - e_3_x += (rho1) ** 2 * fw[k] * ens_w[k] + e_1 += w[i, j] * (rho1 - rho2) ** 2 * fw[k] * ow * ens_w[k] + e_3_x += w[i, j] * (rho1) ** 2 * fw[k] * ens_w[k] for m in range(k + 1, M): for i in range(D): for j in range(D): rho1 = abs(fct[k, i] - fct[k, j]) ** p rho2 = abs(fct[m, i] - fct[m, j]) ** p - e_2 += ( + e_2 += w[i, j] * ( 2 * ((rho1 - rho2) ** 2) * fw[k] * fw[m] * ens_w[k] * ens_w[m] ) @@ -96,6 +96,6 @@ def _vrvariogram_score_gufunc(obs, fct, ow, fw, ens_w, p, out): for i in range(D): for j in range(D): rho1 = abs(obs[i] - obs[j]) ** p - e_3_y += (rho1) ** 2 * ow + e_3_y += w[i, j] * (rho1) ** 2 * ow out[0] = e_1 - 0.5 * e_2 + (e_3_x - e_3_y) * (wbar - ow) diff --git a/scoringrules/core/variogram/_score.py b/scoringrules/core/variogram/_score.py index 01b70e2..46a7205 100644 --- a/scoringrules/core/variogram/_score.py +++ b/scoringrules/core/variogram/_score.py @@ -9,6 +9,7 @@ def vs_ensemble( obs: "Array", # (... D) fct: "Array", # (... M D) + w: "Array", # (..., D D) ens_w: "Array", # (... M) p: float = 1, backend: "Backend" = None, @@ -21,12 +22,13 @@ def vs_ensemble( ) # (... D D) obs_diff = B.expand_dims(obs, -2) - B.expand_dims(obs, -1) # (... D D) vobs = B.abs(obs_diff) ** p # (... D D) - return B.sum((vobs - vfct) ** 2, axis=(-2, -1)) # (...) + return B.sum(w * (vobs - vfct) ** 2, axis=(-2, -1)) # (...) def owvs_ensemble( obs: "Array", fct: "Array", + w: "Array", ow: "Array", fw: "Array", ens_w: "Array", @@ -35,7 +37,7 @@ def owvs_ensemble( ) -> "Array": """Compute the Outcome-Weighted Variogram Score for a multivariate finite ensemble.""" B = backends.active if backend is None else backends[backend] - wbar = B.mean(fw * ens_w, axis=-1) + wbar = B.sum(fw * ens_w, axis=-1) fct_diff = B.expand_dims(fct, -2) - B.expand_dims(fct, -1) # (... M D D) fct_diff = B.abs(fct_diff) ** p # (... M D D) @@ -45,7 +47,7 @@ def owvs_ensemble( del obs, fct E_1 = (fct_diff - B.expand_dims(obs_diff, -3)) ** 2 # (... M D D) - E_1 = B.sum(E_1, axis=(-2, -1)) # (... M) + E_1 = B.sum(B.expand_dims(w, -3) * E_1, axis=(-2, -1)) # (... M) E_1 = B.sum(E_1 * fw * B.expand_dims(ow, -1) * ens_w, axis=-1) / wbar # (...) fct_diff_spread = B.expand_dims(fct_diff, -3) - B.expand_dims( @@ -53,7 +55,9 @@ def owvs_ensemble( ) # (... M M D D) fw_prod = B.expand_dims(fw, -2) * B.expand_dims(fw, -1) # (... M M) ew_prod = B.expand_dims(ens_w, -2) * B.expand_dims(ens_w, -1) # (... M M) - E_2 = B.sum(fct_diff_spread**2, axis=(-2, -1)) # (... M M) + E_2 = B.sum( + B.expand_dims(w, (-3, -4)) * fct_diff_spread**2, axis=(-2, -1) + ) # (... M M) E_2 *= fw_prod * B.expand_dims(ow, (-2, -1)) * ew_prod # (... M M) E_2 = B.sum(E_2, axis=(-2, -1)) / (wbar**2) # (...) @@ -63,6 +67,7 @@ def owvs_ensemble( def vrvs_ensemble( obs: "Array", fct: "Array", + w: "Array", ow: "Array", fw: "Array", ens_w: "Array", @@ -71,7 +76,7 @@ def vrvs_ensemble( ) -> "Array": """Compute the Vertically Re-scaled Variogram Score for a multivariate finite ensemble.""" B = backends.active if backend is None else backends[backend] - wbar = B.mean(fw * ens_w, axis=-1) + wbar = B.sum(fw * ens_w, axis=-1) fct_diff = ( B.abs(B.expand_dims(fct, -2) - B.expand_dims(fct, -1)) ** p @@ -79,21 +84,21 @@ def vrvs_ensemble( obs_diff = B.abs(B.expand_dims(obs, -2) - B.expand_dims(obs, -1)) ** p # (... D D) E_1 = (fct_diff - B.expand_dims(obs_diff, axis=-3)) ** 2 # (... M D D) - E_1 = B.sum(E_1, axis=(-2, -1)) # (... M) + E_1 = B.sum(B.expand_dims(w, -3) * E_1, axis=(-2, -1)) # (... M) E_1 = B.sum(E_1 * fw * B.expand_dims(ow, axis=-1) * ens_w, axis=-1) # (...) E_2 = ( B.expand_dims(fct_diff, -3) - B.expand_dims(fct_diff, -4) ) ** 2 # (... M M D D) - E_2 = B.sum(E_2, axis=(-2, -1)) # (... M M) + E_2 = B.sum(B.expand_dims(w, (-3, -4)) * E_2, axis=(-2, -1)) # (... M M) fw_prod = B.expand_dims(fw, axis=-2) * B.expand_dims(fw, axis=-1) # (... M M) ew_prod = B.expand_dims(ens_w, -2) * B.expand_dims(ens_w, -1) # (... M M) E_2 = B.sum(E_2 * fw_prod * ew_prod, axis=(-2, -1)) # (...) - E_3 = B.sum(fct_diff**2, axis=(-2, -1)) # (... M) + E_3 = B.sum(B.expand_dims(w, -3) * fct_diff**2, axis=(-2, -1)) # (... M) E_3 = B.sum(E_3 * fw * ens_w, axis=-1) # (...) - E_3 -= B.sum(obs_diff**2, axis=(-2, -1)) * ow # (...) + E_3 -= B.sum(w * obs_diff**2, axis=(-2, -1)) * ow # (...) E_3 *= wbar - ow # (...) return E_1 - 0.5 * E_2 + E_3 From b7fa375ed0879f1af96462f59b441b3f7fa6d38f Mon Sep 17 00:00:00 2001 From: sallen12 Date: Thu, 10 Jul 2025 15:12:13 +0200 Subject: [PATCH 66/79] fix tolerance bug in vres tests --- tests/test_wenergy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_wenergy.py b/tests/test_wenergy.py index f717763..ed1a100 100644 --- a/tests/test_wenergy.py +++ b/tests/test_wenergy.py @@ -23,7 +23,7 @@ def test_owes_vs_es(backend): lambda x: backends[backend].mean(x) * 0.0 + 1.0, backend=backend, ) - np.testing.assert_allclose(res, resw, atol=1e-7) + np.testing.assert_allclose(res, resw, atol=1e-6) @pytest.mark.parametrize("backend", BACKENDS) @@ -48,7 +48,7 @@ def test_vres_vs_es(backend): lambda x: backends[backend].mean(x) * 0.0 + 1.0, backend=backend, ) - np.testing.assert_allclose(res, resw, rtol=1e-7) + np.testing.assert_allclose(res, resw, rtol=1e-6) @pytest.mark.parametrize("backend", BACKENDS) From 5cdd2e44bbbe333c16e57216103a8dc14732edd2 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Thu, 10 Jul 2025 15:18:32 +0200 Subject: [PATCH 67/79] fix tolerance in crps jax tests --- tests/test_crps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_crps.py b/tests/test_crps.py index dce17a6..a2340b1 100644 --- a/tests/test_crps.py +++ b/tests/test_crps.py @@ -66,7 +66,7 @@ def test_crps_ensemble_corr(backend): res_nrg = sr.crps_ensemble(obs, fct, estimator="nrg", backend=backend) res_pwm = sr.crps_ensemble(obs, fct, estimator="pwm", backend=backend) res_qd = sr.crps_ensemble(obs, fct, estimator="qd", backend=backend) - if backend == "torch": + if backend in ["torch", "jax"]: assert np.allclose(res_nrg, res_pwm, rtol=1e-03) assert np.allclose(res_nrg, res_qd, rtol=1e-03) else: @@ -77,7 +77,7 @@ def test_crps_ensemble_corr(backend): res_nrg = sr.crps_ensemble(obs, fct, ens_w=w, estimator="nrg", backend=backend) res_pwm = sr.crps_ensemble(obs, fct, ens_w=w, estimator="pwm", backend=backend) res_qd = sr.crps_ensemble(obs, fct, ens_w=w, estimator="qd", backend=backend) - if backend == "torch": + if backend in ["torch", "jax"]: assert np.allclose(res_nrg, res_pwm, rtol=1e-03) assert np.allclose(res_nrg, res_qd, rtol=1e-03) else: From c7f4b57a26cf83a4819741b4968501ef13cefbe0 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Thu, 24 Jul 2025 16:12:45 +0200 Subject: [PATCH 68/79] add ensemble weight arguments to gaussian kernel scores --- scoringrules/_kernels.py | 144 ++++++++++++++++++++++---- scoringrules/core/kernels/_approx.py | 87 +++++++++------- scoringrules/core/kernels/_gufuncs.py | 134 +++++++++++++----------- tests/test_kernels.py | 100 ++++-------------- 4 files changed, 269 insertions(+), 196 deletions(-) diff --git a/scoringrules/_kernels.py b/scoringrules/_kernels.py index 3ea14c2..65482ec 100644 --- a/scoringrules/_kernels.py +++ b/scoringrules/_kernels.py @@ -14,6 +14,7 @@ def gksuv_ensemble( /, m_axis: int = -1, *, + ens_w: "Array" = None, estimator: str = "nrg", backend: "Backend" = None, ) -> "Array": @@ -37,11 +38,14 @@ def gksuv_ensemble( ---------- obs : array_like The observed values. - fct : array_like + fct : array The predicted forecast ensemble, where the ensemble dimension is by default represented by the last axis. m_axis : int The axis corresponding to the ensemble. Default is the last axis. + ens_w : array + Weights assigned to the ensemble members. Array with the same shape as fct. + Default is equal weighting. estimator : str Indicates the estimator to be used. backend : str @@ -60,6 +64,15 @@ def gksuv_ensemble( B = backends.active if backend is None else backends[backend] obs, fct = map(B.asarray, (obs, fct)) + if ens_w is None: + M = fct.shape[m_axis] + ens_w = B.zeros(fct.shape) + 1.0 / M + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) + if backend == "numba": if estimator not in kernels.estimator_gufuncs: raise ValueError( @@ -75,11 +88,14 @@ def gksuv_ensemble( if m_axis != -1: fct = B.moveaxis(fct, m_axis, -1) + ens_w = B.moveaxis(ens_w, m_axis, -1) if backend == "numba": - return kernels.estimator_gufuncs[estimator](obs, fct) + return kernels.estimator_gufuncs[estimator](obs, fct, ens_w) - return kernels.ensemble_uv(obs, fct, estimator, backend=backend) + return kernels.ensemble_uv( + obs, fct, ens_w=ens_w, estimator=estimator, backend=backend + ) def twgksuv_ensemble( @@ -90,6 +106,7 @@ def twgksuv_ensemble( b: float = float("inf"), m_axis: int = -1, *, + ens_w: "Array" = None, v_func: tp.Callable[["ArrayLike"], "ArrayLike"] = None, estimator: str = "nrg", backend: "Backend" = None, @@ -126,6 +143,9 @@ def twgksuv_ensemble( to values in the range [a, b]. m_axis : int The axis corresponding to the ensemble. Default is the last axis. + ens_w : array + Weights assigned to the ensemble members. Array with the same shape as fct. + Default is equal weighting. v_func : callable, array_like -> array_like Chaining function used to emphasise particular outcomes. For example, a function that only considers values above a certain threshold :math:`t` by projecting forecasts and observations @@ -160,6 +180,7 @@ def v_func(x): obs, fct, m_axis=m_axis, + ens_w=ens_w, estimator=estimator, backend=backend, ) @@ -173,6 +194,7 @@ def owgksuv_ensemble( b: float = float("inf"), m_axis: int = -1, *, + ens_w: "Array" = None, w_func: tp.Callable[["ArrayLike"], "ArrayLike"] = None, backend: "Backend" = None, ) -> "Array": @@ -209,6 +231,9 @@ def owgksuv_ensemble( to values in the range [a, b]. m_axis : int The axis corresponding to the ensemble. Default is the last axis. + ens_w : array + Weights assigned to the ensemble members. Array with the same shape as fct. + Default is equal weighting. w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. backend : str @@ -232,8 +257,18 @@ def owgksuv_ensemble( B = backends.active if backend is None else backends[backend] obs, fct = map(B.asarray, (obs, fct)) + if ens_w is None: + M = fct.shape[m_axis] + ens_w = B.zeros(fct.shape) + 1.0 / M + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) + if m_axis != -1: fct = B.moveaxis(fct, m_axis, -1) + ens_w = B.moveaxis(ens_w, m_axis, -1) if w_func is None: @@ -244,9 +279,13 @@ def w_func(x): obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) if backend == "numba": - return kernels.estimator_gufuncs["ow"](obs, fct, obs_weights, fct_weights) + return kernels.estimator_gufuncs["ow"]( + obs, fct, obs_weights, fct_weights, ens_w + ) - return kernels.ow_ensemble_uv(obs, fct, obs_weights, fct_weights, backend=backend) + return kernels.ow_ensemble_uv( + obs, fct, obs_weights, fct_weights, ens_w=ens_w, backend=backend + ) def vrgksuv_ensemble( @@ -257,6 +296,7 @@ def vrgksuv_ensemble( b: float = float("inf"), m_axis: int = -1, *, + ens_w: "Array" = None, w_func: tp.Callable[["ArrayLike"], "ArrayLike"] = None, backend: "Backend" = None, ) -> "Array": @@ -292,6 +332,9 @@ def vrgksuv_ensemble( to values in the range [a, b]. m_axis : int The axis corresponding to the ensemble. Default is the last axis. + ens_w : array + Weights assigned to the ensemble members. Array with the same shape as fct. + Default is equal weighting. w_func : callable, array_like -> array_like Weight function used to emphasise particular outcomes. backend : str @@ -315,8 +358,18 @@ def vrgksuv_ensemble( B = backends.active if backend is None else backends[backend] obs, fct = map(B.asarray, (obs, fct)) + if ens_w is None: + M = fct.shape[m_axis] + ens_w = B.zeros(fct.shape) + 1.0 / M + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) + if m_axis != -1: fct = B.moveaxis(fct, m_axis, -1) + ens_w = B.moveaxis(ens_w, m_axis, -1) if w_func is None: @@ -327,9 +380,13 @@ def w_func(x): obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) if backend == "numba": - return kernels.estimator_gufuncs["vr"](obs, fct, obs_weights, fct_weights) + return kernels.estimator_gufuncs["vr"]( + obs, fct, obs_weights, fct_weights, ens_w + ) - return kernels.vr_ensemble_uv(obs, fct, obs_weights, fct_weights, backend=backend) + return kernels.vr_ensemble_uv( + obs, fct, obs_weights, fct_weights, ens_w, backend=backend + ) def gksmv_ensemble( @@ -339,6 +396,7 @@ def gksmv_ensemble( m_axis: int = -2, v_axis: int = -1, *, + ens_w: "Array" = None, estimator: str = "nrg", backend: "Backend" = None, ) -> "Array": @@ -373,6 +431,9 @@ def gksmv_ensemble( v_axis : int The axis corresponding to the variables dimension on the forecasts array (or the observations array with an extra dimension on `m_axis`). Defaults to -1. + ens_w : array_like + Weights assigned to the ensemble members. Array with one less dimension than fct (without the v_axis dimension). + Default is equal weighting. estimator : str Indicates the estimator to be used. backend : str @@ -383,9 +444,15 @@ def gksmv_ensemble( score : array_like The GKS between the forecast ensemble and obs. """ - backend = backend if backend is not None else backends._active + B = backends.active if backend is None else backends[backend] obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) + if ens_w is None: + M = fct.shape[-2] + ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M + else: + ens_w = B.moveaxis(ens_w, m_axis, -2) + if estimator not in kernels.estimator_gufuncs_mv: raise ValueError( f"{estimator} is not a valid estimator. " @@ -393,9 +460,11 @@ def gksmv_ensemble( ) if backend == "numba": - return kernels.estimator_gufuncs_mv[estimator](obs, fct) + return kernels.estimator_gufuncs_mv[estimator](obs, fct, ens_w) - return kernels.ensemble_mv(obs, fct, estimator, backend=backend) + return kernels.ensemble_mv( + obs, fct, ens_w=ens_w, estimator=estimator, backend=backend + ) def twgksmv_ensemble( @@ -406,6 +475,8 @@ def twgksmv_ensemble( m_axis: int = -2, v_axis: int = -1, *, + ens_w: "Array" = None, + estimator: str = "nrg", backend: "Backend" = None, ) -> "Array": r"""Compute the Threshold-Weighted Gaussian Kernel Score (twGKS) for a finite multivariate ensemble. @@ -438,6 +509,9 @@ def twgksmv_ensemble( The axis corresponding to the ensemble dimension. Defaults to -2. v_axis : int or tuple of ints The axis corresponding to the variables dimension. Defaults to -1. + ens_w : array_like + Weights assigned to the ensemble members. Array with one less dimension than fct (without the v_axis dimension). + Default is equal weighting. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -447,7 +521,15 @@ def twgksmv_ensemble( The computed Threshold-Weighted Gaussian Kernel Score. """ obs, fct = map(v_func, (obs, fct)) - return gksmv_ensemble(obs, fct, m_axis=m_axis, v_axis=v_axis, backend=backend) + return gksmv_ensemble( + obs, + fct, + m_axis=m_axis, + v_axis=v_axis, + ens_w=ens_w, + estimator=estimator, + backend=backend, + ) def owgksmv_ensemble( @@ -458,6 +540,7 @@ def owgksmv_ensemble( m_axis: int = -2, v_axis: int = -1, *, + ens_w: "Array" = None, backend: "Backend" = None, ) -> "Array": r"""Compute the multivariate Outcome-Weighted Gaussian Kernel Score (owGKS) for a finite ensemble. @@ -505,6 +588,9 @@ def owgksmv_ensemble( The axis corresponding to the ensemble dimension. Defaults to -2. v_axis : int or tuple of ints The axis corresponding to the variables dimension. Defaults to -1. + ens_w : array_like + Weights assigned to the ensemble members. Array with one less dimension than fct (without the v_axis dimension). + Default is equal weighting. backend : str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -514,16 +600,25 @@ def owgksmv_ensemble( The computed Outcome-Weighted GKS. """ B = backends.active if backend is None else backends[backend] - obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) + if ens_w is None: + M = fct.shape[-2] + ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M + else: + ens_w = B.moveaxis(ens_w, m_axis, -2) + fct_weights = B.apply_along_axis(w_func, fct, -1) obs_weights = B.apply_along_axis(w_func, obs, -1) if B.name == "numba": - return kernels.estimator_gufuncs_mv["ow"](obs, fct, obs_weights, fct_weights) + return kernels.estimator_gufuncs_mv["ow"]( + obs, fct, obs_weights, fct_weights, ens_w + ) - return kernels.ow_ensemble_mv(obs, fct, obs_weights, fct_weights, backend=backend) + return kernels.ow_ensemble_mv( + obs, fct, obs_weights, fct_weights, ens_w=ens_w, backend=backend + ) def vrgksmv_ensemble( @@ -531,9 +626,10 @@ def vrgksmv_ensemble( fct: "Array", w_func: tp.Callable[["ArrayLike"], "ArrayLike"], /, - *, m_axis: int = -2, v_axis: int = -1, + *, + ens_w: "Array" = None, backend: "Backend" = None, ) -> "Array": r"""Compute the Vertically Re-scaled Gaussian Kernel Score (vrGKS) for a finite multivariate ensemble. @@ -567,6 +663,9 @@ def vrgksmv_ensemble( The axis corresponding to the ensemble dimension. Defaults to -2. v_axis : int or tuple of ints The axis corresponding to the variables dimension. Defaults to -1. + ens_w : array_like + Weights assigned to the ensemble members. Array with one less dimension than fct (without the v_axis dimension). + Default is equal weighting. backend: str The name of the backend used for computations. Defaults to 'numba' if available, else 'numpy'. @@ -576,16 +675,25 @@ def vrgksmv_ensemble( The computed Vertically Re-scaled Gaussian Kernel Score. """ B = backends.active if backend is None else backends[backend] - obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) + if ens_w is None: + M = fct.shape[-2] + ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M + else: + ens_w = B.moveaxis(ens_w, m_axis, -2) + fct_weights = B.apply_along_axis(w_func, fct, -1) obs_weights = B.apply_along_axis(w_func, obs, -1) if B.name == "numba": - return kernels.estimator_gufuncs_mv["vr"](obs, fct, obs_weights, fct_weights) + return kernels.estimator_gufuncs_mv["vr"]( + obs, fct, obs_weights, fct_weights, ens_w + ) - return kernels.vr_ensemble_mv(obs, fct, obs_weights, fct_weights, backend=backend) + return kernels.vr_ensemble_mv( + obs, fct, obs_weights, fct_weights, ens_w=ens_w, backend=backend + ) __all__ = [ diff --git a/scoringrules/core/kernels/_approx.py b/scoringrules/core/kernels/_approx.py index 1d581bf..42d5f5f 100644 --- a/scoringrules/core/kernels/_approx.py +++ b/scoringrules/core/kernels/_approx.py @@ -25,30 +25,33 @@ def gauss_kern_mv( def ensemble_uv( obs: "ArrayLike", fct: "Array", + ens_w: "Array", estimator: str = "nrg", backend: "Backend" = None, ) -> "Array": """Compute a kernel score for a finite ensemble.""" B = backends.active if backend is None else backends[backend] - M: int = fct.shape[-1] - e_1 = B.sum(gauss_kern_uv(obs[..., None], fct, backend=backend), axis=-1) / M + + e_1 = B.sum(gauss_kern_uv(obs[..., None], fct, backend=backend) * ens_w, axis=-1) e_2 = B.sum( - gauss_kern_uv(fct[..., None], fct[..., None, :], backend=backend), + gauss_kern_uv(fct[..., None], fct[..., None, :], backend=backend) + * ens_w[..., None] + * ens_w[..., None, :], axis=(-1, -2), - ) / (M**2) + ) e_3 = gauss_kern_uv(obs, obs) - if estimator == "nrg": - out = e_1 - 0.5 * e_2 - 0.5 * e_3 - elif estimator == "fair": - out = e_1 - 0.5 * e_2 * (M / (M - 1)) - 0.5 * e_3 + if estimator == "fair": + e_2 = e_2 / (1 - B.sum(ens_w * ens_w, axis=-1)) + out = e_1 - 0.5 * e_2 - 0.5 * e_3 return -out def ensemble_mv( obs: "ArrayLike", fct: "Array", + ens_w: "Array", estimator: str = "nrg", backend: "Backend" = None, ) -> "Array": @@ -57,22 +60,22 @@ def ensemble_mv( The ensemble and variables axes are on the second last and last dimensions respectively. """ B = backends.active if backend is None else backends[backend] - M: int = fct.shape[-2] - e_1 = ( - B.sum(gauss_kern_mv(B.expand_dims(obs, -2), fct, backend=backend), axis=-1) / M + e_1 = B.sum( + ens_w * gauss_kern_mv(B.expand_dims(obs, -2), fct, backend=backend), axis=-1 ) e_2 = B.sum( - gauss_kern_mv(B.expand_dims(fct, -3), B.expand_dims(fct, -2), backend=backend), + gauss_kern_mv(B.expand_dims(fct, -3), B.expand_dims(fct, -2), backend=backend) + * B.expand_dims(ens_w, -1) + * B.expand_dims(ens_w, -2), axis=(-2, -1), - ) / (M**2) + ) e_3 = gauss_kern_mv(obs, obs) - if estimator == "nrg": - out = e_1 - 0.5 * e_2 - 0.5 * e_3 - elif estimator == "fair": - out = e_1 - 0.5 * e_2 * (M / (M - 1)) - 0.5 * e_3 + if estimator == "fair": + e_2 = e_2 / (1 - B.sum(ens_w * ens_w, axis=-1)) + out = e_1 - 0.5 * e_2 - 0.5 * e_3 return -out @@ -81,24 +84,26 @@ def ow_ensemble_uv( fct: "Array", ow: "Array", fw: "Array", + ens_w: "Array", backend: "Backend" = None, ) -> "Array": """Compute an outcome-weighted kernel score for a finite univariate ensemble.""" B = backends.active if backend is None else backends[backend] - M: int = fct.shape[-1] - wbar = B.mean(fw, axis=-1) + wbar = B.sum(ens_w * fw, axis=-1) e_1 = ( - B.sum(gauss_kern_uv(obs[..., None], fct, backend=backend) * fw, axis=-1) + B.sum(ens_w * gauss_kern_uv(obs[..., None], fct, backend=backend) * fw, axis=-1) * ow - / (M * wbar) + / wbar ) e_2 = B.sum( - gauss_kern_uv(fct[..., None], fct[..., None, :], backend=backend) + ens_w[..., None] + * ens_w[..., None, :] + * gauss_kern_uv(fct[..., None], fct[..., None, :], backend=backend) * fw[..., None] * fw[..., None, :], axis=(-1, -2), ) - e_2 *= ow / (M**2 * wbar**2) + e_2 *= ow / (wbar**2) e_3 = gauss_kern_uv(obs, obs, backend=backend) * ow out = e_1 - 0.5 * e_2 - 0.5 * e_3 @@ -110,6 +115,7 @@ def ow_ensemble_mv( fct: "Array", ow: "Array", fw: "Array", + ens_w: "Array", backend: "Backend" = None, ) -> "Array": """Compute an outcome-weighted kernel score for a finite multivariate ensemble. @@ -117,18 +123,19 @@ def ow_ensemble_mv( The ensemble and variables axes are on the second last and last dimensions respectively. """ B = backends.active if backend is None else backends[backend] - M: int = fct.shape[-2] - wbar = B.sum(fw, -1) / M + wbar = B.sum(fw * ens_w, -1) err_kern = gauss_kern_mv(B.expand_dims(obs, -2), fct, backend=backend) - E_1 = B.sum(err_kern * fw * B.expand_dims(ow, -1), axis=-1) / (M * wbar) + E_1 = B.sum(err_kern * fw * B.expand_dims(ow, -1) * ens_w, axis=-1) / wbar spread_kern = gauss_kern_mv( B.expand_dims(fct, -3), B.expand_dims(fct, -2), backend=backend ) fw_prod = B.expand_dims(fw, -1) * B.expand_dims(fw, -2) spread_kern *= fw_prod * B.expand_dims(ow, (-2, -1)) - E_2 = B.sum(spread_kern, (-2, -1)) / (M**2 * wbar**2) + E_2 = B.sum( + spread_kern * B.expand_dims(ens_w, -1) * B.expand_dims(ens_w, -2), (-2, -1) + ) / (wbar**2) E_3 = gauss_kern_mv(obs, obs, backend=backend) * ow @@ -142,26 +149,24 @@ def vr_ensemble_uv( fct: "Array", ow: "Array", fw: "Array", + ens_w: "Array", backend: "Backend" = None, ) -> "Array": """Compute a vertically re-scaled kernel score for a finite univariate ensemble.""" B = backends.active if backend is None else backends[backend] - M: int = fct.shape[-1] e_1 = ( - B.sum(gauss_kern_uv(obs[..., None], fct, backend=backend) * fw, axis=-1) + B.sum(ens_w * gauss_kern_uv(obs[..., None], fct, backend=backend) * fw, axis=-1) * ow - / M ) e_2 = B.sum( - gauss_kern_uv( - B.expand_dims(fct, axis=-1), - B.expand_dims(fct, axis=-2), - backend=backend, - ) - * (B.expand_dims(fw, axis=-1) * B.expand_dims(fw, axis=-2)), + ens_w[..., None] + * ens_w[..., None, :] + * gauss_kern_uv(fct[..., None], fct[..., None, :], backend=backend) + * fw[..., None] + * fw[..., None, :], axis=(-1, -2), - ) / (M**2) + ) e_3 = gauss_kern_uv(obs, obs, backend=backend) * ow * ow out = e_1 - 0.5 * e_2 - 0.5 * e_3 @@ -174,6 +179,7 @@ def vr_ensemble_mv( fct: "Array", ow: "Array", fw: "Array", + ens_w: "Array", backend: "Backend" = None, ) -> "Array": """Compute a vertically re-scaled kernel score for a finite multivariate ensemble. @@ -181,17 +187,18 @@ def vr_ensemble_mv( The ensemble and variables axes are on the second last and last dimensions respectively. """ B = backends.active if backend is None else backends[backend] - M: int = fct.shape[-2] err_kern = gauss_kern_mv(B.expand_dims(obs, -2), fct, backend=backend) - E_1 = B.sum(err_kern * fw * B.expand_dims(ow, -1), axis=-1) / M + E_1 = B.sum(err_kern * fw * B.expand_dims(ow, -1) * ens_w, axis=-1) spread_kern = gauss_kern_mv( B.expand_dims(fct, -3), B.expand_dims(fct, -2), backend=backend ) fw_prod = B.expand_dims(fw, -1) * B.expand_dims(fw, -2) spread_kern *= fw_prod - E_2 = B.sum(spread_kern, (-2, -1)) / (M**2) + E_2 = B.sum( + spread_kern * B.expand_dims(ens_w, -1) * B.expand_dims(ens_w, -2), (-2, -1) + ) E_3 = gauss_kern_mv(obs, obs, backend=backend) * ow * ow diff --git a/scoringrules/core/kernels/_gufuncs.py b/scoringrules/core/kernels/_gufuncs.py index 2aa7a3e..24a3666 100644 --- a/scoringrules/core/kernels/_gufuncs.py +++ b/scoringrules/core/kernels/_gufuncs.py @@ -20,12 +20,14 @@ def _gauss_kern_mv(x1: float, x2: float) -> float: @guvectorize( [ - "void(float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", ], - "(),(n)->()", + "(),(n),(n)->()", ) -def _ks_ensemble_uv_nrg_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): +def _ks_ensemble_uv_nrg_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): """Standard version of the kernel score.""" obs = obs[0] M = fct.shape[-1] @@ -37,23 +39,25 @@ def _ks_ensemble_uv_nrg_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray e_1 = 0 e_2 = 0 - for x_i in fct: - e_1 += _gauss_kern_uv(x_i, obs) - for x_j in fct: - e_2 += _gauss_kern_uv(x_i, x_j) + for i in range(M): + e_1 += _gauss_kern_uv(fct[i], obs) * w[i] + for j in range(M): + e_2 += _gauss_kern_uv(fct[i], fct[j]) * w[i] * w[j] e_3 = _gauss_kern_uv(obs, obs) - out[0] = -(e_1 / M - 0.5 * e_2 / (M**2) - 0.5 * e_3) + out[0] = -(e_1 - 0.5 * e_2 - 0.5 * e_3) @guvectorize( [ - "void(float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", ], - "(),(n)->()", + "(),(n),(n)->()", ) -def _ks_ensemble_uv_fair_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): +def _ks_ensemble_uv_fair_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): """Fair version of the kernel score.""" obs = obs[0] M = fct.shape[-1] @@ -65,33 +69,33 @@ def _ks_ensemble_uv_fair_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarra e_1 = 0 e_2 = 0 - for x_i in fct: - e_1 += _gauss_kern_uv(x_i, obs) - for x_j in fct: - e_2 += _gauss_kern_uv(x_i, x_j) + for i in range(M): + e_1 += _gauss_kern_uv(fct[i], obs) * w[i] + for j in range(i + 1, M): + e_2 += _gauss_kern_uv(fct[i], fct[j]) * w[i] * w[j] e_3 = _gauss_kern_uv(obs, obs) - out[0] = -(e_1 / M - 0.5 * e_2 / (M * (M - 1)) - 0.5 * e_3) + out[0] = -(e_1 - e_2 / (1 - np.sum(w * w)) - 0.5 * e_3) @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:], float64[:], float64[:])", ], - "(),(n),(),(n)->()", + "(),(n),(),(n),(n)->()", ) def _owks_ensemble_uv_gufunc( obs: np.ndarray, fct: np.ndarray, ow: np.ndarray, fw: np.ndarray, + w: np.ndarray, out: np.ndarray, ): """Outcome-weighted kernel score for univariate ensembles.""" obs = obs[0] ow = ow[0] - M = fct.shape[-1] if np.isnan(obs): out[0] = np.nan @@ -101,34 +105,34 @@ def _owks_ensemble_uv_gufunc( e_2 = 0.0 for i, x_i in enumerate(fct): - e_1 += _gauss_kern_uv(x_i, obs) * fw[i] * ow + e_1 += _gauss_kern_uv(x_i, obs) * fw[i] * ow * w[i] for j, x_j in enumerate(fct): - e_2 += _gauss_kern_uv(x_i, x_j) * fw[i] * fw[j] * ow + e_2 += _gauss_kern_uv(x_i, x_j) * fw[i] * fw[j] * ow * w[i] * w[j] e_3 = _gauss_kern_uv(obs, obs) * ow - wbar = np.mean(fw) + wbar = np.sum(fw * w) - out[0] = -(e_1 / (M * wbar) - 0.5 * e_2 / ((M * wbar) ** 2) - 0.5 * e_3) + out[0] = -(e_1 / wbar - 0.5 * e_2 / (wbar**2) - 0.5 * e_3) @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:], float64[:], float64[:])", ], - "(),(n),(),(n)->()", + "(),(n),(),(n),(n)->()", ) def _vrks_ensemble_uv_gufunc( obs: np.ndarray, fct: np.ndarray, ow: np.ndarray, fw: np.ndarray, + w: np.ndarray, out: np.ndarray, ): """Vertically re-scaled kernel score for univariate ensembles.""" obs = obs[0] ow = ow[0] - M = fct.shape[-1] if np.isnan(obs): out[0] = np.nan @@ -138,70 +142,75 @@ def _vrks_ensemble_uv_gufunc( e_2 = 0.0 for i, x_i in enumerate(fct): - e_1 += _gauss_kern_uv(x_i, obs) * fw[i] * ow + e_1 += _gauss_kern_uv(x_i, obs) * fw[i] * ow * w[i] for j, x_j in enumerate(fct): - e_2 += _gauss_kern_uv(x_i, x_j) * fw[i] * fw[j] + e_2 += _gauss_kern_uv(x_i, x_j) * fw[i] * fw[j] * w[i] * w[j] e_3 = _gauss_kern_uv(obs, obs) * ow * ow - out[0] = -(e_1 / M - 0.5 * e_2 / (M**2) - 0.5 * e_3) + out[0] = -(e_1 - 0.5 * e_2 - 0.5 * e_3) @guvectorize( [ - "void(float32[:], float32[:,:], float32[:])", - "void(float64[:], float64[:,:], float64[:])", + "void(float32[:], float32[:,:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:])", ], - "(d),(m,d)->()", + "(d),(m,d),(m)->()", ) -def _ks_ensemble_mv_nrg_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): +def _ks_ensemble_mv_nrg_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): """Standard version of the multivariate kernel score.""" M = fct.shape[0] e_1 = 0.0 e_2 = 0.0 for i in range(M): - e_1 += float(_gauss_kern_mv(fct[i], obs)) + e_1 += float(_gauss_kern_mv(fct[i], obs)) * w[i] for j in range(M): - e_2 += float(_gauss_kern_mv(fct[i], fct[j])) + e_2 += float(_gauss_kern_mv(fct[i], fct[j])) * w[i] * w[j] e_3 = float(_gauss_kern_mv(obs, obs)) - out[0] = -(e_1 / M - 0.5 * e_2 / (M**2) - 0.5 * e_3) + out[0] = -(e_1 - 0.5 * e_2 - 0.5 * e_3) @guvectorize( [ - "void(float32[:], float32[:,:], float32[:])", - "void(float64[:], float64[:,:], float64[:])", + "void(float32[:], float32[:,:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:])", ], - "(d),(m,d)->()", + "(d),(m,d),(m)->()", ) -def _ks_ensemble_mv_fair_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): +def _ks_ensemble_mv_fair_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): """Fair version of the multivariate kernel score.""" M = fct.shape[0] e_1 = 0.0 e_2 = 0.0 for i in range(M): - e_1 += float(_gauss_kern_mv(fct[i], obs)) - for j in range(M): - e_2 += float(_gauss_kern_mv(fct[i], fct[j])) + e_1 += float(_gauss_kern_mv(fct[i], obs) * w[i]) + for j in range(i + 1, M): + e_2 += float(_gauss_kern_mv(fct[i], fct[j]) * w[i] * w[j]) e_3 = float(_gauss_kern_mv(obs, obs)) - out[0] = -(e_1 / M - 0.5 * e_2 / (M * (M - 1)) - 0.5 * e_3) + out[0] = -(e_1 - e_2 / (1 - np.sum(w * w)) - 0.5 * e_3) @guvectorize( [ - "void(float32[:], float32[:,:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:,:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:,:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:], float64[:], float64[:])", ], - "(d),(m,d),(),(m)->()", + "(d),(m,d),(),(m),(m)->()", ) def _owks_ensemble_mv_gufunc( obs: np.ndarray, fct: np.ndarray, ow: np.ndarray, fw: np.ndarray, + w: np.ndarray, out: np.ndarray, ): """Outcome-weighted kernel score for multivariate ensembles.""" @@ -211,28 +220,31 @@ def _owks_ensemble_mv_gufunc( e_1 = 0.0 e_2 = 0.0 for i in range(M): - e_1 += float(_gauss_kern_mv(fct[i], obs) * fw[i] * ow) + e_1 += float(_gauss_kern_mv(fct[i], obs) * fw[i] * ow * w[i]) for j in range(M): - e_2 += float(_gauss_kern_mv(fct[i], fct[j]) * fw[i] * fw[j] * ow) + e_2 += float( + _gauss_kern_mv(fct[i], fct[j]) * fw[i] * fw[j] * ow * w[i] * w[j] + ) e_3 = float(_gauss_kern_mv(obs, obs)) * ow - wbar = np.mean(fw) + wbar = np.sum(fw * w) - out[0] = -(e_1 / (M * wbar) - 0.5 * e_2 / (M**2 * wbar**2) - 0.5 * e_3) + out[0] = -(e_1 / wbar - 0.5 * e_2 / (wbar**2) - 0.5 * e_3) @guvectorize( [ - "void(float32[:], float32[:,:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:,:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:,:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:], float64[:], float64[:])", ], - "(d),(m,d),(),(m)->()", + "(d),(m,d),(),(m),(m)->()", ) def _vrks_ensemble_mv_gufunc( obs: np.ndarray, fct: np.ndarray, ow: np.ndarray, fw: np.ndarray, + w: np.ndarray, out: np.ndarray, ): """Vertically re-scaled kernel score for multivariate ensembles.""" @@ -242,12 +254,12 @@ def _vrks_ensemble_mv_gufunc( e_1 = 0.0 e_2 = 0.0 for i in range(M): - e_1 += float(_gauss_kern_mv(fct[i], obs) * fw[i] * ow) + e_1 += float(_gauss_kern_mv(fct[i], obs) * fw[i] * ow * w[i]) for j in range(M): - e_2 += float(_gauss_kern_mv(fct[i], fct[j]) * fw[i] * fw[j]) + e_2 += float(_gauss_kern_mv(fct[i], fct[j]) * fw[i] * fw[j] * w[i] * w[j]) e_3 = float(_gauss_kern_mv(obs, obs)) * ow * ow - out[0] = -(e_1 / M - 0.5 * e_2 / (M**2) - 0.5 * e_3) + out[0] = -(e_1 - 0.5 * e_2 - 0.5 * e_3) estimator_gufuncs = { diff --git a/tests/test_kernels.py b/tests/test_kernels.py index 579c3c2..a2e1f54 100644 --- a/tests/test_kernels.py +++ b/tests/test_kernels.py @@ -21,23 +21,23 @@ def test_gksuv(estimator, backend): sigma = abs(np.random.randn(N)) * 0.3 fct = np.random.randn(N, ENSEMBLE_SIZE) * sigma[..., None] + mu[..., None] - # non-negative values - res = sr.gksuv_ensemble(obs, fct, estimator=estimator, backend=backend) - res = np.asarray(res) - assert not np.any(res < 0.0) - - # m_axis keyword - res = sr.gksuv_ensemble( - obs, - np.random.randn(ENSEMBLE_SIZE, N), - m_axis=0, - estimator=estimator, - backend=backend, - ) - res = np.asarray(res) - assert not np.any(res < 0.0) - if estimator == "nrg": + # non-negative values + res = sr.gksuv_ensemble(obs, fct, estimator=estimator, backend=backend) + res = np.asarray(res) + assert not np.any(res < 0.0) + + # m_axis keyword + res = sr.gksuv_ensemble( + obs, + np.random.randn(ENSEMBLE_SIZE, N), + m_axis=0, + estimator=estimator, + backend=backend, + ) + res = np.asarray(res) + assert not np.any(res < 0.0) + # approx zero when perfect forecast perfect_fct = obs[..., None] + np.random.randn(N, ENSEMBLE_SIZE) * 0.00001 res = sr.gksuv_ensemble(obs, perfect_fct, estimator=estimator, backend=backend) @@ -50,13 +50,6 @@ def test_gksuv(estimator, backend): expected = 0.2490516 assert np.isclose(res, expected) - elif estimator == "fair": - # test correctness - obs, fct = 11.6, np.array([9.8, 8.7, 11.9, 12.1, 13.4]) - res = sr.gksuv_ensemble(obs, fct, estimator=estimator, backend=backend) - expected = 0.2987752 - assert np.isclose(res, expected) - # test exceptions with pytest.raises(ValueError): est = "undefined_estimator" @@ -95,16 +88,6 @@ def test_gksmv(estimator, backend): expected = 0.5868737 assert np.isclose(res, expected) - elif estimator == "fair": - # test correctness - obs = np.array([11.6, -23.1]) - fct = np.array( - [[9.8, 8.7, 11.9, 12.1, 13.4], [-24.8, -18.5, -29.9, -18.3, -21.0]] - ).transpose() - res = sr.gksmv_ensemble(obs, fct, estimator=estimator, backend=backend) - expected = 0.6120162 - assert np.isclose(res, expected) - # test exceptions with pytest.raises(ValueError): est = "undefined_estimator" @@ -213,43 +196,6 @@ def v_func2(x): ) np.testing.assert_allclose(res, 0.0089314, rtol=1e-6) - elif estimator == "fair": - res = np.mean( - np.float64( - sr.twgksuv_ensemble( - obs, fct, v_func=v_func1, estimator=estimator, backend=backend - ) - ) - ) - np.testing.assert_allclose(res, 0.130842, rtol=1e-6) - - res = np.mean( - np.float64( - sr.twgksuv_ensemble( - obs, fct, a=-1.0, estimator=estimator, backend=backend - ) - ) - ) - np.testing.assert_allclose(res, 0.130842, rtol=1e-6) - - res = np.mean( - np.float64( - sr.twgksuv_ensemble( - obs, fct, v_func=v_func2, estimator=estimator, backend=backend - ) - ) - ) - np.testing.assert_allclose(res, 0.1283745, rtol=1e-6) - - res = np.mean( - np.float64( - sr.twgksuv_ensemble( - obs, fct, b=1.85, estimator=estimator, backend=backend - ) - ) - ) - np.testing.assert_allclose(res, 0.1283745, rtol=1e-6) - @pytest.mark.parametrize("backend", BACKENDS) def test_twgksmv(backend): @@ -427,19 +373,19 @@ def test_vrgksuv(backend): # no argument given resw = sr.vrgksuv_ensemble(obs, fct, backend=backend) - np.testing.assert_allclose(res, resw, rtol=1e-5) + np.testing.assert_allclose(res, resw, rtol=1e-4) # a and b resw = sr.vrgksuv_ensemble( obs, fct, a=float("-inf"), b=float("inf"), backend=backend ) - np.testing.assert_allclose(res, resw, rtol=1e-5) + np.testing.assert_allclose(res, resw, rtol=1e-4) # w_func as identity function resw = sr.vrgksuv_ensemble( obs, fct, w_func=lambda x: x * 0.0 + 1.0, backend=backend ) - np.testing.assert_allclose(res, resw, rtol=1e-5) + np.testing.assert_allclose(res, resw, rtol=1e-4) # test correctness fct = np.array( @@ -478,10 +424,10 @@ def w_func(x): res = np.mean( np.float64(sr.vrgksuv_ensemble(obs, fct, w_func=w_func, backend=backend)) ) - np.testing.assert_allclose(res, 0.01476682, rtol=1e-6) + np.testing.assert_allclose(res, 0.01476682, rtol=1e-4) res = np.mean(np.float64(sr.vrgksuv_ensemble(obs, fct, a=-1.0, backend=backend))) - np.testing.assert_allclose(res, 0.01476682, rtol=1e-6) + np.testing.assert_allclose(res, 0.01476682, rtol=1e-4) def w_func(x): return (x < 1.85) * 1.0 @@ -489,10 +435,10 @@ def w_func(x): res = np.mean( np.float64(sr.vrgksuv_ensemble(obs, fct, w_func=w_func, backend=backend)) ) - np.testing.assert_allclose(res, 0.04011836, rtol=1e-6) + np.testing.assert_allclose(res, 0.04011836, rtol=1e-4) res = np.mean(np.float64(sr.vrgksuv_ensemble(obs, fct, b=1.85, backend=backend))) - np.testing.assert_allclose(res, 0.04011836, rtol=1e-6) + np.testing.assert_allclose(res, 0.04011836, rtol=1e-4) @pytest.mark.parametrize("backend", BACKENDS) From 792a8b637abc297349c2152175bf2f814d6a7d45 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Thu, 24 Jul 2025 16:16:37 +0200 Subject: [PATCH 69/79] reduce tol of vrgksuv tests --- tests/test_kernels.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_kernels.py b/tests/test_kernels.py index a2e1f54..33a4326 100644 --- a/tests/test_kernels.py +++ b/tests/test_kernels.py @@ -373,19 +373,19 @@ def test_vrgksuv(backend): # no argument given resw = sr.vrgksuv_ensemble(obs, fct, backend=backend) - np.testing.assert_allclose(res, resw, rtol=1e-4) + np.testing.assert_allclose(res, resw, rtol=1e-3) # a and b resw = sr.vrgksuv_ensemble( obs, fct, a=float("-inf"), b=float("inf"), backend=backend ) - np.testing.assert_allclose(res, resw, rtol=1e-4) + np.testing.assert_allclose(res, resw, rtol=1e-3) # w_func as identity function resw = sr.vrgksuv_ensemble( obs, fct, w_func=lambda x: x * 0.0 + 1.0, backend=backend ) - np.testing.assert_allclose(res, resw, rtol=1e-4) + np.testing.assert_allclose(res, resw, rtol=1e-3) # test correctness fct = np.array( @@ -424,10 +424,10 @@ def w_func(x): res = np.mean( np.float64(sr.vrgksuv_ensemble(obs, fct, w_func=w_func, backend=backend)) ) - np.testing.assert_allclose(res, 0.01476682, rtol=1e-4) + np.testing.assert_allclose(res, 0.01476682, rtol=1e-3) res = np.mean(np.float64(sr.vrgksuv_ensemble(obs, fct, a=-1.0, backend=backend))) - np.testing.assert_allclose(res, 0.01476682, rtol=1e-4) + np.testing.assert_allclose(res, 0.01476682, rtol=1e-3) def w_func(x): return (x < 1.85) * 1.0 @@ -435,10 +435,10 @@ def w_func(x): res = np.mean( np.float64(sr.vrgksuv_ensemble(obs, fct, w_func=w_func, backend=backend)) ) - np.testing.assert_allclose(res, 0.04011836, rtol=1e-4) + np.testing.assert_allclose(res, 0.04011836, rtol=1e-3) res = np.mean(np.float64(sr.vrgksuv_ensemble(obs, fct, b=1.85, backend=backend))) - np.testing.assert_allclose(res, 0.04011836, rtol=1e-4) + np.testing.assert_allclose(res, 0.04011836, rtol=1e-3) @pytest.mark.parametrize("backend", BACKENDS) From a50a5bafd8fa43c1fb0a7a1ce88b593d5f8a99ba Mon Sep 17 00:00:00 2001 From: sallen12 Date: Thu, 24 Jul 2025 16:22:23 +0200 Subject: [PATCH 70/79] skip torch tests for vrgksuv" --- tests/test_kernels.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_kernels.py b/tests/test_kernels.py index 33a4326..f14d34f 100644 --- a/tests/test_kernels.py +++ b/tests/test_kernels.py @@ -351,8 +351,8 @@ def w_func(x): @pytest.mark.parametrize("backend", BACKENDS) def test_vrgksuv(backend): - if backend == "jax": - pytest.skip("Not implemented in jax backend") + if backend in ["jax", "torch"]: + pytest.skip("Not implemented in torch and jax backends") obs = np.random.randn(N) mu = obs + np.random.randn(N) * 0.1 sigma = abs(np.random.randn(N)) * 0.3 @@ -373,19 +373,19 @@ def test_vrgksuv(backend): # no argument given resw = sr.vrgksuv_ensemble(obs, fct, backend=backend) - np.testing.assert_allclose(res, resw, rtol=1e-3) + np.testing.assert_allclose(res, resw, rtol=1e-6) # a and b resw = sr.vrgksuv_ensemble( obs, fct, a=float("-inf"), b=float("inf"), backend=backend ) - np.testing.assert_allclose(res, resw, rtol=1e-3) + np.testing.assert_allclose(res, resw, rtol=1e-6) # w_func as identity function resw = sr.vrgksuv_ensemble( obs, fct, w_func=lambda x: x * 0.0 + 1.0, backend=backend ) - np.testing.assert_allclose(res, resw, rtol=1e-3) + np.testing.assert_allclose(res, resw, rtol=1e-6) # test correctness fct = np.array( @@ -424,10 +424,10 @@ def w_func(x): res = np.mean( np.float64(sr.vrgksuv_ensemble(obs, fct, w_func=w_func, backend=backend)) ) - np.testing.assert_allclose(res, 0.01476682, rtol=1e-3) + np.testing.assert_allclose(res, 0.01476682, rtol=1e-6) res = np.mean(np.float64(sr.vrgksuv_ensemble(obs, fct, a=-1.0, backend=backend))) - np.testing.assert_allclose(res, 0.01476682, rtol=1e-3) + np.testing.assert_allclose(res, 0.01476682, rtol=1e-6) def w_func(x): return (x < 1.85) * 1.0 @@ -435,10 +435,10 @@ def w_func(x): res = np.mean( np.float64(sr.vrgksuv_ensemble(obs, fct, w_func=w_func, backend=backend)) ) - np.testing.assert_allclose(res, 0.04011836, rtol=1e-3) + np.testing.assert_allclose(res, 0.04011836, rtol=1e-6) res = np.mean(np.float64(sr.vrgksuv_ensemble(obs, fct, b=1.85, backend=backend))) - np.testing.assert_allclose(res, 0.04011836, rtol=1e-3) + np.testing.assert_allclose(res, 0.04011836, rtol=1e-6) @pytest.mark.parametrize("backend", BACKENDS) From ccb5784b31a6f4f898374722b1e577e4df1cd8da Mon Sep 17 00:00:00 2001 From: sallen12 Date: Mon, 4 Aug 2025 10:52:58 +0200 Subject: [PATCH 71/79] update brier score doc strings --- docs/{ => background}/crps_estimators.md | 0 docs/{ => background}/forecast_dists.md | 0 docs/{ => background}/theory.md | 0 docs/{ => background}/weighted_scores.md | 0 docs/contributing.md | 48 ---------- docs/index.md | 19 ++-- docs/library/contributing.md | 49 ++++++++++ .../generated/scoringrules.brier_score.rst | 6 ++ .../generated/scoringrules.clogs_ensemble.rst | 6 ++ .../scoringrules.crps_2pexponential.rst | 6 ++ .../generated/scoringrules.crps_2pnormal.rst | 6 ++ .../generated/scoringrules.crps_beta.rst | 6 ++ .../generated/scoringrules.crps_binomial.rst | 6 ++ .../generated/scoringrules.crps_clogistic.rst | 6 ++ .../generated/scoringrules.crps_cnormal.rst | 6 ++ .../generated/scoringrules.crps_ct.rst | 6 ++ .../generated/scoringrules.crps_ensemble.rst | 6 ++ .../scoringrules.crps_exponential.rst | 6 ++ .../scoringrules.crps_exponentialM.rst | 6 ++ .../generated/scoringrules.crps_gamma.rst | 6 ++ .../generated/scoringrules.crps_gev.rst | 6 ++ .../generated/scoringrules.crps_gpd.rst | 6 ++ .../scoringrules.crps_gtclogistic.rst | 6 ++ .../generated/scoringrules.crps_gtcnormal.rst | 6 ++ .../generated/scoringrules.crps_gtct.rst | 6 ++ .../scoringrules.crps_hypergeometric.rst | 6 ++ .../generated/scoringrules.crps_laplace.rst | 6 ++ .../generated/scoringrules.crps_logistic.rst | 6 ++ .../scoringrules.crps_loglaplace.rst | 6 ++ .../scoringrules.crps_loglogistic.rst | 6 ++ .../generated/scoringrules.crps_lognormal.rst | 6 ++ .../generated/scoringrules.crps_mixnorm.rst | 6 ++ .../generated/scoringrules.crps_negbinom.rst | 6 ++ .../generated/scoringrules.crps_normal.rst | 6 ++ .../generated/scoringrules.crps_poisson.rst | 6 ++ .../generated/scoringrules.crps_quantile.rst | 6 ++ .../library/generated/scoringrules.crps_t.rst | 6 ++ .../generated/scoringrules.crps_tlogistic.rst | 6 ++ .../generated/scoringrules.crps_tnormal.rst | 6 ++ .../generated/scoringrules.crps_tt.rst | 6 ++ .../generated/scoringrules.crps_uniform.rst | 6 ++ .../generated/scoringrules.es_ensemble.rst | 6 ++ .../generated/scoringrules.gksmv_ensemble.rst | 6 ++ .../generated/scoringrules.gksuv_ensemble.rst | 6 ++ .../generated/scoringrules.interval_score.rst | 6 ++ .../generated/scoringrules.log_score.rst | 6 ++ .../scoringrules.logs_2pexponential.rst | 6 ++ .../generated/scoringrules.logs_2pnormal.rst | 6 ++ .../generated/scoringrules.logs_beta.rst | 6 ++ .../generated/scoringrules.logs_binomial.rst | 6 ++ .../generated/scoringrules.logs_ensemble.rst | 6 ++ .../scoringrules.logs_exponential.rst | 6 ++ .../scoringrules.logs_exponential2.rst | 6 ++ .../generated/scoringrules.logs_gamma.rst | 6 ++ .../generated/scoringrules.logs_gev.rst | 6 ++ .../generated/scoringrules.logs_gpd.rst | 6 ++ .../scoringrules.logs_hypergeometric.rst | 6 ++ .../generated/scoringrules.logs_laplace.rst | 6 ++ .../generated/scoringrules.logs_logistic.rst | 6 ++ .../scoringrules.logs_loglaplace.rst | 6 ++ .../scoringrules.logs_loglogistic.rst | 6 ++ .../generated/scoringrules.logs_lognormal.rst | 6 ++ .../generated/scoringrules.logs_mixnorm.rst | 6 ++ .../generated/scoringrules.logs_negbinom.rst | 6 ++ .../generated/scoringrules.logs_normal.rst | 6 ++ .../generated/scoringrules.logs_poisson.rst | 6 ++ .../library/generated/scoringrules.logs_t.rst | 6 ++ .../generated/scoringrules.logs_tlogistic.rst | 6 ++ .../generated/scoringrules.logs_tnormal.rst | 6 ++ .../generated/scoringrules.logs_tt.rst | 6 ++ .../generated/scoringrules.logs_uniform.rst | 6 ++ .../scoringrules.owcrps_ensemble.rst | 6 ++ .../generated/scoringrules.owes_ensemble.rst | 6 ++ .../scoringrules.owgksmv_ensemble.rst | 6 ++ .../scoringrules.owgksuv_ensemble.rst | 6 ++ .../generated/scoringrules.owvs_ensemble.rst | 6 ++ .../generated/scoringrules.quantile_score.rst | 6 ++ .../scoringrules.register_backend.rst | 6 ++ .../generated/scoringrules.rls_score.rst | 6 ++ .../generated/scoringrules.rps_score.rst | 6 ++ .../scoringrules.twcrps_ensemble.rst | 6 ++ .../generated/scoringrules.twes_ensemble.rst | 6 ++ .../scoringrules.twgksmv_ensemble.rst | 6 ++ .../scoringrules.twgksuv_ensemble.rst | 6 ++ .../generated/scoringrules.twvs_ensemble.rst | 6 ++ .../scoringrules.vrcrps_ensemble.rst | 6 ++ .../generated/scoringrules.vres_ensemble.rst | 6 ++ .../scoringrules.vrgksmv_ensemble.rst | 6 ++ .../scoringrules.vrgksuv_ensemble.rst | 6 ++ .../generated/scoringrules.vrvs_ensemble.rst | 6 ++ .../generated/scoringrules.vs_ensemble.rst | 6 ++ .../scoringrules.weighted_interval_score.rst | 6 ++ docs/{ => library}/reference.md | 89 ++++++++++------- docs/library/user_guide.md | 95 +++++++++++++++++++ docs/user_guide.md | 75 --------------- scoringrules/_brier.py | 84 ++++++++++++---- scoringrules/core/brier.py | 12 +-- 97 files changed, 789 insertions(+), 192 deletions(-) rename docs/{ => background}/crps_estimators.md (100%) rename docs/{ => background}/forecast_dists.md (100%) rename docs/{ => background}/theory.md (100%) rename docs/{ => background}/weighted_scores.md (100%) delete mode 100644 docs/contributing.md create mode 100644 docs/library/contributing.md create mode 100644 docs/library/generated/scoringrules.brier_score.rst create mode 100644 docs/library/generated/scoringrules.clogs_ensemble.rst create mode 100644 docs/library/generated/scoringrules.crps_2pexponential.rst create mode 100644 docs/library/generated/scoringrules.crps_2pnormal.rst create mode 100644 docs/library/generated/scoringrules.crps_beta.rst create mode 100644 docs/library/generated/scoringrules.crps_binomial.rst create mode 100644 docs/library/generated/scoringrules.crps_clogistic.rst create mode 100644 docs/library/generated/scoringrules.crps_cnormal.rst create mode 100644 docs/library/generated/scoringrules.crps_ct.rst create mode 100644 docs/library/generated/scoringrules.crps_ensemble.rst create mode 100644 docs/library/generated/scoringrules.crps_exponential.rst create mode 100644 docs/library/generated/scoringrules.crps_exponentialM.rst create mode 100644 docs/library/generated/scoringrules.crps_gamma.rst create mode 100644 docs/library/generated/scoringrules.crps_gev.rst create mode 100644 docs/library/generated/scoringrules.crps_gpd.rst create mode 100644 docs/library/generated/scoringrules.crps_gtclogistic.rst create mode 100644 docs/library/generated/scoringrules.crps_gtcnormal.rst create mode 100644 docs/library/generated/scoringrules.crps_gtct.rst create mode 100644 docs/library/generated/scoringrules.crps_hypergeometric.rst create mode 100644 docs/library/generated/scoringrules.crps_laplace.rst create mode 100644 docs/library/generated/scoringrules.crps_logistic.rst create mode 100644 docs/library/generated/scoringrules.crps_loglaplace.rst create mode 100644 docs/library/generated/scoringrules.crps_loglogistic.rst create mode 100644 docs/library/generated/scoringrules.crps_lognormal.rst create mode 100644 docs/library/generated/scoringrules.crps_mixnorm.rst create mode 100644 docs/library/generated/scoringrules.crps_negbinom.rst create mode 100644 docs/library/generated/scoringrules.crps_normal.rst create mode 100644 docs/library/generated/scoringrules.crps_poisson.rst create mode 100644 docs/library/generated/scoringrules.crps_quantile.rst create mode 100644 docs/library/generated/scoringrules.crps_t.rst create mode 100644 docs/library/generated/scoringrules.crps_tlogistic.rst create mode 100644 docs/library/generated/scoringrules.crps_tnormal.rst create mode 100644 docs/library/generated/scoringrules.crps_tt.rst create mode 100644 docs/library/generated/scoringrules.crps_uniform.rst create mode 100644 docs/library/generated/scoringrules.es_ensemble.rst create mode 100644 docs/library/generated/scoringrules.gksmv_ensemble.rst create mode 100644 docs/library/generated/scoringrules.gksuv_ensemble.rst create mode 100644 docs/library/generated/scoringrules.interval_score.rst create mode 100644 docs/library/generated/scoringrules.log_score.rst create mode 100644 docs/library/generated/scoringrules.logs_2pexponential.rst create mode 100644 docs/library/generated/scoringrules.logs_2pnormal.rst create mode 100644 docs/library/generated/scoringrules.logs_beta.rst create mode 100644 docs/library/generated/scoringrules.logs_binomial.rst create mode 100644 docs/library/generated/scoringrules.logs_ensemble.rst create mode 100644 docs/library/generated/scoringrules.logs_exponential.rst create mode 100644 docs/library/generated/scoringrules.logs_exponential2.rst create mode 100644 docs/library/generated/scoringrules.logs_gamma.rst create mode 100644 docs/library/generated/scoringrules.logs_gev.rst create mode 100644 docs/library/generated/scoringrules.logs_gpd.rst create mode 100644 docs/library/generated/scoringrules.logs_hypergeometric.rst create mode 100644 docs/library/generated/scoringrules.logs_laplace.rst create mode 100644 docs/library/generated/scoringrules.logs_logistic.rst create mode 100644 docs/library/generated/scoringrules.logs_loglaplace.rst create mode 100644 docs/library/generated/scoringrules.logs_loglogistic.rst create mode 100644 docs/library/generated/scoringrules.logs_lognormal.rst create mode 100644 docs/library/generated/scoringrules.logs_mixnorm.rst create mode 100644 docs/library/generated/scoringrules.logs_negbinom.rst create mode 100644 docs/library/generated/scoringrules.logs_normal.rst create mode 100644 docs/library/generated/scoringrules.logs_poisson.rst create mode 100644 docs/library/generated/scoringrules.logs_t.rst create mode 100644 docs/library/generated/scoringrules.logs_tlogistic.rst create mode 100644 docs/library/generated/scoringrules.logs_tnormal.rst create mode 100644 docs/library/generated/scoringrules.logs_tt.rst create mode 100644 docs/library/generated/scoringrules.logs_uniform.rst create mode 100644 docs/library/generated/scoringrules.owcrps_ensemble.rst create mode 100644 docs/library/generated/scoringrules.owes_ensemble.rst create mode 100644 docs/library/generated/scoringrules.owgksmv_ensemble.rst create mode 100644 docs/library/generated/scoringrules.owgksuv_ensemble.rst create mode 100644 docs/library/generated/scoringrules.owvs_ensemble.rst create mode 100644 docs/library/generated/scoringrules.quantile_score.rst create mode 100644 docs/library/generated/scoringrules.register_backend.rst create mode 100644 docs/library/generated/scoringrules.rls_score.rst create mode 100644 docs/library/generated/scoringrules.rps_score.rst create mode 100644 docs/library/generated/scoringrules.twcrps_ensemble.rst create mode 100644 docs/library/generated/scoringrules.twes_ensemble.rst create mode 100644 docs/library/generated/scoringrules.twgksmv_ensemble.rst create mode 100644 docs/library/generated/scoringrules.twgksuv_ensemble.rst create mode 100644 docs/library/generated/scoringrules.twvs_ensemble.rst create mode 100644 docs/library/generated/scoringrules.vrcrps_ensemble.rst create mode 100644 docs/library/generated/scoringrules.vres_ensemble.rst create mode 100644 docs/library/generated/scoringrules.vrgksmv_ensemble.rst create mode 100644 docs/library/generated/scoringrules.vrgksuv_ensemble.rst create mode 100644 docs/library/generated/scoringrules.vrvs_ensemble.rst create mode 100644 docs/library/generated/scoringrules.vs_ensemble.rst create mode 100644 docs/library/generated/scoringrules.weighted_interval_score.rst rename docs/{ => library}/reference.md (82%) create mode 100644 docs/library/user_guide.md delete mode 100644 docs/user_guide.md diff --git a/docs/crps_estimators.md b/docs/background/crps_estimators.md similarity index 100% rename from docs/crps_estimators.md rename to docs/background/crps_estimators.md diff --git a/docs/forecast_dists.md b/docs/background/forecast_dists.md similarity index 100% rename from docs/forecast_dists.md rename to docs/background/forecast_dists.md diff --git a/docs/theory.md b/docs/background/theory.md similarity index 100% rename from docs/theory.md rename to docs/background/theory.md diff --git a/docs/weighted_scores.md b/docs/background/weighted_scores.md similarity index 100% rename from docs/weighted_scores.md rename to docs/background/weighted_scores.md diff --git a/docs/contributing.md b/docs/contributing.md deleted file mode 100644 index 3294e06..0000000 --- a/docs/contributing.md +++ /dev/null @@ -1,48 +0,0 @@ -# Contributing - -We welcome contributions! You can help improve the library in many ways: - -- Report issues or propose enhancements using on the [GitHub issue tracker](https://github.com/frazane/scoringrules/issues) -- Improve or extend the codebase -- Improve or extend the documentation - -## Getting started - -Fork the repository on GitHub and clone it to your computer: - -``` -git clone https://github.com//scoringrules.git -``` - -We use [uv](https://docs.astral.sh/uv/) for project management and packaging. Install it with - -``` -curl -LsSf https://astral.sh/uv/install.sh | sh -``` - -Then, you can install the library and all dependencies (including development dependencies) and install the pre-commit hooks: - -``` -uv install -uv run pre-commit install -``` - -From here you can work on your changes! Once you're satisfied with your changes, and followed the additional instructions below, push everything to your repository and open a pull request on GitHub. - - -### Contributing to the codebase -Don't forget to include new tests if necessary, then make sure that all tests are passing with - -``` -uv run pytest tests/ -``` - -### Contributing to the documentation - -You can work on the documentation by modifying files in `docs/`. The most convenient way to do it is to run - -``` -uvx --with-requirements docs/requirements.txt sphinx-autobuild docs/ docs/_build/ -``` - -and open the locally hosted documentation on your browser. It will be updated automatically every time you make changes and save. If you edit or add pieces of LaTex math, please make sure they are rendered correctly. diff --git a/docs/index.md b/docs/index.md index 739f9c0..a1174c5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -17,7 +17,6 @@ The scoring rules available in `scoringrules` include, but are not limited to, t - Logarithmic Score - (Discrete) Ranked Probability Score - Continuous Ranked Probability Score (CRPS) -- Dawid-Sebastiani Score - Energy Score - Variogram Score - Gaussian Kernel Score @@ -25,27 +24,27 @@ The scoring rules available in `scoringrules` include, but are not limited to, t - Threshold-Weighted Energy Score - Quantile Score - Interval Score +- Weighted Interval Score - - +Functionality is available for forecasts in the form of samples (i.e. ensemble forecasts) as well as popular univariate parametric distributions. ```{toctree} :hidden: :caption: Background -theory.md -forecast_dists.md -crps_estimators.md -weighted_scores.md +background/theory.md +background/forecast_dists.md +background/crps_estimators.md +background/weighted_scores.md ``` ```{toctree} :hidden: :caption: Library -user_guide.md -contributing.md -reference.md +library/user_guide.md +library/contributing.md +library/reference.md ``` diff --git a/docs/library/contributing.md b/docs/library/contributing.md new file mode 100644 index 0000000..8784c24 --- /dev/null +++ b/docs/library/contributing.md @@ -0,0 +1,49 @@ +# Contributing + +We welcome contributions! You can help improve the library in many ways. For example, by: + +- Reporting issues or proposing enhancements via the [GitHub issue tracker](https://github.com/frazane/scoringrules/issues) +- Improving or extending the codebase +- Improving or extending the documentation + +## Getting started + +To make changes to the library, fork the repository on GitHub and clone it to your computer: + +``` +git clone https://github.com//scoringrules.git +``` + +We use [uv](https://docs.astral.sh/uv/) for project management and packaging. Install it by running + +``` +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +You should then be able to install the library and all dependencies (including development dependencies), and install the pre-commit hooks: + +``` +uv install +uv run pre-commit install +``` + +From here, you can work on your changes. Once you're satisfied with your changes, and have followed the additional instructions below, push everything to your repository and open a pull request on GitHub. + + +### Contributing to the codebase + +Don't forget to include new tests if necessary. Make sure that all tests are passing with + +``` +uv run pytest tests/ +``` + +### Contributing to the documentation + +You can work on the documentation by modifying files in `docs/`. The most convenient way to do this is to run + +``` +uvx --with-requirements docs/requirements.txt sphinx-autobuild docs/ docs/_build/ +``` + +and open the locally hosted documentation on your browser. It will be updated automatically every time you make changes and save. If you edit or add pieces of LaTeX math, please make sure they are rendered correctly. diff --git a/docs/library/generated/scoringrules.brier_score.rst b/docs/library/generated/scoringrules.brier_score.rst new file mode 100644 index 0000000..c821784 --- /dev/null +++ b/docs/library/generated/scoringrules.brier_score.rst @@ -0,0 +1,6 @@ +scoringrules.brier\_score +========================= + +.. currentmodule:: scoringrules + +.. autofunction:: brier_score diff --git a/docs/library/generated/scoringrules.clogs_ensemble.rst b/docs/library/generated/scoringrules.clogs_ensemble.rst new file mode 100644 index 0000000..fcb2b79 --- /dev/null +++ b/docs/library/generated/scoringrules.clogs_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.clogs\_ensemble +============================ + +.. currentmodule:: scoringrules + +.. autofunction:: clogs_ensemble diff --git a/docs/library/generated/scoringrules.crps_2pexponential.rst b/docs/library/generated/scoringrules.crps_2pexponential.rst new file mode 100644 index 0000000..742b5a8 --- /dev/null +++ b/docs/library/generated/scoringrules.crps_2pexponential.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_2pexponential +================================ + +.. currentmodule:: scoringrules + +.. autofunction:: crps_2pexponential diff --git a/docs/library/generated/scoringrules.crps_2pnormal.rst b/docs/library/generated/scoringrules.crps_2pnormal.rst new file mode 100644 index 0000000..56e42fa --- /dev/null +++ b/docs/library/generated/scoringrules.crps_2pnormal.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_2pnormal +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_2pnormal diff --git a/docs/library/generated/scoringrules.crps_beta.rst b/docs/library/generated/scoringrules.crps_beta.rst new file mode 100644 index 0000000..fd41a3e --- /dev/null +++ b/docs/library/generated/scoringrules.crps_beta.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_beta +======================= + +.. currentmodule:: scoringrules + +.. autofunction:: crps_beta diff --git a/docs/library/generated/scoringrules.crps_binomial.rst b/docs/library/generated/scoringrules.crps_binomial.rst new file mode 100644 index 0000000..cfa1350 --- /dev/null +++ b/docs/library/generated/scoringrules.crps_binomial.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_binomial +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_binomial diff --git a/docs/library/generated/scoringrules.crps_clogistic.rst b/docs/library/generated/scoringrules.crps_clogistic.rst new file mode 100644 index 0000000..b5a658d --- /dev/null +++ b/docs/library/generated/scoringrules.crps_clogistic.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_clogistic +============================ + +.. currentmodule:: scoringrules + +.. autofunction:: crps_clogistic diff --git a/docs/library/generated/scoringrules.crps_cnormal.rst b/docs/library/generated/scoringrules.crps_cnormal.rst new file mode 100644 index 0000000..0739fbc --- /dev/null +++ b/docs/library/generated/scoringrules.crps_cnormal.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_cnormal +========================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_cnormal diff --git a/docs/library/generated/scoringrules.crps_ct.rst b/docs/library/generated/scoringrules.crps_ct.rst new file mode 100644 index 0000000..70fb9ba --- /dev/null +++ b/docs/library/generated/scoringrules.crps_ct.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_ct +===================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_ct diff --git a/docs/library/generated/scoringrules.crps_ensemble.rst b/docs/library/generated/scoringrules.crps_ensemble.rst new file mode 100644 index 0000000..557690f --- /dev/null +++ b/docs/library/generated/scoringrules.crps_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_ensemble +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_ensemble diff --git a/docs/library/generated/scoringrules.crps_exponential.rst b/docs/library/generated/scoringrules.crps_exponential.rst new file mode 100644 index 0000000..67c096b --- /dev/null +++ b/docs/library/generated/scoringrules.crps_exponential.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_exponential +============================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_exponential diff --git a/docs/library/generated/scoringrules.crps_exponentialM.rst b/docs/library/generated/scoringrules.crps_exponentialM.rst new file mode 100644 index 0000000..a13250d --- /dev/null +++ b/docs/library/generated/scoringrules.crps_exponentialM.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_exponentialM +=============================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_exponentialM diff --git a/docs/library/generated/scoringrules.crps_gamma.rst b/docs/library/generated/scoringrules.crps_gamma.rst new file mode 100644 index 0000000..ff552fc --- /dev/null +++ b/docs/library/generated/scoringrules.crps_gamma.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_gamma +======================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_gamma diff --git a/docs/library/generated/scoringrules.crps_gev.rst b/docs/library/generated/scoringrules.crps_gev.rst new file mode 100644 index 0000000..262103e --- /dev/null +++ b/docs/library/generated/scoringrules.crps_gev.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_gev +====================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_gev diff --git a/docs/library/generated/scoringrules.crps_gpd.rst b/docs/library/generated/scoringrules.crps_gpd.rst new file mode 100644 index 0000000..ff78341 --- /dev/null +++ b/docs/library/generated/scoringrules.crps_gpd.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_gpd +====================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_gpd diff --git a/docs/library/generated/scoringrules.crps_gtclogistic.rst b/docs/library/generated/scoringrules.crps_gtclogistic.rst new file mode 100644 index 0000000..3f2f82d --- /dev/null +++ b/docs/library/generated/scoringrules.crps_gtclogistic.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_gtclogistic +============================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_gtclogistic diff --git a/docs/library/generated/scoringrules.crps_gtcnormal.rst b/docs/library/generated/scoringrules.crps_gtcnormal.rst new file mode 100644 index 0000000..3e75a79 --- /dev/null +++ b/docs/library/generated/scoringrules.crps_gtcnormal.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_gtcnormal +============================ + +.. currentmodule:: scoringrules + +.. autofunction:: crps_gtcnormal diff --git a/docs/library/generated/scoringrules.crps_gtct.rst b/docs/library/generated/scoringrules.crps_gtct.rst new file mode 100644 index 0000000..89735e6 --- /dev/null +++ b/docs/library/generated/scoringrules.crps_gtct.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_gtct +======================= + +.. currentmodule:: scoringrules + +.. autofunction:: crps_gtct diff --git a/docs/library/generated/scoringrules.crps_hypergeometric.rst b/docs/library/generated/scoringrules.crps_hypergeometric.rst new file mode 100644 index 0000000..6c7cc1b --- /dev/null +++ b/docs/library/generated/scoringrules.crps_hypergeometric.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_hypergeometric +================================= + +.. currentmodule:: scoringrules + +.. autofunction:: crps_hypergeometric diff --git a/docs/library/generated/scoringrules.crps_laplace.rst b/docs/library/generated/scoringrules.crps_laplace.rst new file mode 100644 index 0000000..481596e --- /dev/null +++ b/docs/library/generated/scoringrules.crps_laplace.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_laplace +========================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_laplace diff --git a/docs/library/generated/scoringrules.crps_logistic.rst b/docs/library/generated/scoringrules.crps_logistic.rst new file mode 100644 index 0000000..687b73a --- /dev/null +++ b/docs/library/generated/scoringrules.crps_logistic.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_logistic +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_logistic diff --git a/docs/library/generated/scoringrules.crps_loglaplace.rst b/docs/library/generated/scoringrules.crps_loglaplace.rst new file mode 100644 index 0000000..d6a0bee --- /dev/null +++ b/docs/library/generated/scoringrules.crps_loglaplace.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_loglaplace +============================= + +.. currentmodule:: scoringrules + +.. autofunction:: crps_loglaplace diff --git a/docs/library/generated/scoringrules.crps_loglogistic.rst b/docs/library/generated/scoringrules.crps_loglogistic.rst new file mode 100644 index 0000000..2107a2e --- /dev/null +++ b/docs/library/generated/scoringrules.crps_loglogistic.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_loglogistic +============================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_loglogistic diff --git a/docs/library/generated/scoringrules.crps_lognormal.rst b/docs/library/generated/scoringrules.crps_lognormal.rst new file mode 100644 index 0000000..fd49df7 --- /dev/null +++ b/docs/library/generated/scoringrules.crps_lognormal.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_lognormal +============================ + +.. currentmodule:: scoringrules + +.. autofunction:: crps_lognormal diff --git a/docs/library/generated/scoringrules.crps_mixnorm.rst b/docs/library/generated/scoringrules.crps_mixnorm.rst new file mode 100644 index 0000000..8fba00b --- /dev/null +++ b/docs/library/generated/scoringrules.crps_mixnorm.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_mixnorm +========================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_mixnorm diff --git a/docs/library/generated/scoringrules.crps_negbinom.rst b/docs/library/generated/scoringrules.crps_negbinom.rst new file mode 100644 index 0000000..a1d12d2 --- /dev/null +++ b/docs/library/generated/scoringrules.crps_negbinom.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_negbinom +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_negbinom diff --git a/docs/library/generated/scoringrules.crps_normal.rst b/docs/library/generated/scoringrules.crps_normal.rst new file mode 100644 index 0000000..a670094 --- /dev/null +++ b/docs/library/generated/scoringrules.crps_normal.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_normal +========================= + +.. currentmodule:: scoringrules + +.. autofunction:: crps_normal diff --git a/docs/library/generated/scoringrules.crps_poisson.rst b/docs/library/generated/scoringrules.crps_poisson.rst new file mode 100644 index 0000000..aca7e0e --- /dev/null +++ b/docs/library/generated/scoringrules.crps_poisson.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_poisson +========================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_poisson diff --git a/docs/library/generated/scoringrules.crps_quantile.rst b/docs/library/generated/scoringrules.crps_quantile.rst new file mode 100644 index 0000000..1db2b18 --- /dev/null +++ b/docs/library/generated/scoringrules.crps_quantile.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_quantile +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_quantile diff --git a/docs/library/generated/scoringrules.crps_t.rst b/docs/library/generated/scoringrules.crps_t.rst new file mode 100644 index 0000000..c9ad9fc --- /dev/null +++ b/docs/library/generated/scoringrules.crps_t.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_t +==================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_t diff --git a/docs/library/generated/scoringrules.crps_tlogistic.rst b/docs/library/generated/scoringrules.crps_tlogistic.rst new file mode 100644 index 0000000..04699f9 --- /dev/null +++ b/docs/library/generated/scoringrules.crps_tlogistic.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_tlogistic +============================ + +.. currentmodule:: scoringrules + +.. autofunction:: crps_tlogistic diff --git a/docs/library/generated/scoringrules.crps_tnormal.rst b/docs/library/generated/scoringrules.crps_tnormal.rst new file mode 100644 index 0000000..939e73b --- /dev/null +++ b/docs/library/generated/scoringrules.crps_tnormal.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_tnormal +========================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_tnormal diff --git a/docs/library/generated/scoringrules.crps_tt.rst b/docs/library/generated/scoringrules.crps_tt.rst new file mode 100644 index 0000000..1d2dc90 --- /dev/null +++ b/docs/library/generated/scoringrules.crps_tt.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_tt +===================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_tt diff --git a/docs/library/generated/scoringrules.crps_uniform.rst b/docs/library/generated/scoringrules.crps_uniform.rst new file mode 100644 index 0000000..1ed6d9f --- /dev/null +++ b/docs/library/generated/scoringrules.crps_uniform.rst @@ -0,0 +1,6 @@ +scoringrules.crps\_uniform +========================== + +.. currentmodule:: scoringrules + +.. autofunction:: crps_uniform diff --git a/docs/library/generated/scoringrules.es_ensemble.rst b/docs/library/generated/scoringrules.es_ensemble.rst new file mode 100644 index 0000000..efbd76a --- /dev/null +++ b/docs/library/generated/scoringrules.es_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.es\_ensemble +========================= + +.. currentmodule:: scoringrules + +.. autofunction:: es_ensemble diff --git a/docs/library/generated/scoringrules.gksmv_ensemble.rst b/docs/library/generated/scoringrules.gksmv_ensemble.rst new file mode 100644 index 0000000..ae5873f --- /dev/null +++ b/docs/library/generated/scoringrules.gksmv_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.gksmv\_ensemble +============================ + +.. currentmodule:: scoringrules + +.. autofunction:: gksmv_ensemble diff --git a/docs/library/generated/scoringrules.gksuv_ensemble.rst b/docs/library/generated/scoringrules.gksuv_ensemble.rst new file mode 100644 index 0000000..1792921 --- /dev/null +++ b/docs/library/generated/scoringrules.gksuv_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.gksuv\_ensemble +============================ + +.. currentmodule:: scoringrules + +.. autofunction:: gksuv_ensemble diff --git a/docs/library/generated/scoringrules.interval_score.rst b/docs/library/generated/scoringrules.interval_score.rst new file mode 100644 index 0000000..48f50c5 --- /dev/null +++ b/docs/library/generated/scoringrules.interval_score.rst @@ -0,0 +1,6 @@ +scoringrules.interval\_score +============================ + +.. currentmodule:: scoringrules + +.. autofunction:: interval_score diff --git a/docs/library/generated/scoringrules.log_score.rst b/docs/library/generated/scoringrules.log_score.rst new file mode 100644 index 0000000..831a334 --- /dev/null +++ b/docs/library/generated/scoringrules.log_score.rst @@ -0,0 +1,6 @@ +scoringrules.log\_score +======================= + +.. currentmodule:: scoringrules + +.. autofunction:: log_score diff --git a/docs/library/generated/scoringrules.logs_2pexponential.rst b/docs/library/generated/scoringrules.logs_2pexponential.rst new file mode 100644 index 0000000..6639250 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_2pexponential.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_2pexponential +================================ + +.. currentmodule:: scoringrules + +.. autofunction:: logs_2pexponential diff --git a/docs/library/generated/scoringrules.logs_2pnormal.rst b/docs/library/generated/scoringrules.logs_2pnormal.rst new file mode 100644 index 0000000..78308b7 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_2pnormal.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_2pnormal +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_2pnormal diff --git a/docs/library/generated/scoringrules.logs_beta.rst b/docs/library/generated/scoringrules.logs_beta.rst new file mode 100644 index 0000000..fc7cf26 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_beta.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_beta +======================= + +.. currentmodule:: scoringrules + +.. autofunction:: logs_beta diff --git a/docs/library/generated/scoringrules.logs_binomial.rst b/docs/library/generated/scoringrules.logs_binomial.rst new file mode 100644 index 0000000..726d986 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_binomial.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_binomial +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_binomial diff --git a/docs/library/generated/scoringrules.logs_ensemble.rst b/docs/library/generated/scoringrules.logs_ensemble.rst new file mode 100644 index 0000000..666be79 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_ensemble +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_ensemble diff --git a/docs/library/generated/scoringrules.logs_exponential.rst b/docs/library/generated/scoringrules.logs_exponential.rst new file mode 100644 index 0000000..cbd8559 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_exponential.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_exponential +============================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_exponential diff --git a/docs/library/generated/scoringrules.logs_exponential2.rst b/docs/library/generated/scoringrules.logs_exponential2.rst new file mode 100644 index 0000000..8b9b636 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_exponential2.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_exponential2 +=============================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_exponential2 diff --git a/docs/library/generated/scoringrules.logs_gamma.rst b/docs/library/generated/scoringrules.logs_gamma.rst new file mode 100644 index 0000000..ee15b3d --- /dev/null +++ b/docs/library/generated/scoringrules.logs_gamma.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_gamma +======================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_gamma diff --git a/docs/library/generated/scoringrules.logs_gev.rst b/docs/library/generated/scoringrules.logs_gev.rst new file mode 100644 index 0000000..0284bc0 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_gev.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_gev +====================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_gev diff --git a/docs/library/generated/scoringrules.logs_gpd.rst b/docs/library/generated/scoringrules.logs_gpd.rst new file mode 100644 index 0000000..9b8e0e5 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_gpd.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_gpd +====================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_gpd diff --git a/docs/library/generated/scoringrules.logs_hypergeometric.rst b/docs/library/generated/scoringrules.logs_hypergeometric.rst new file mode 100644 index 0000000..46095fb --- /dev/null +++ b/docs/library/generated/scoringrules.logs_hypergeometric.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_hypergeometric +================================= + +.. currentmodule:: scoringrules + +.. autofunction:: logs_hypergeometric diff --git a/docs/library/generated/scoringrules.logs_laplace.rst b/docs/library/generated/scoringrules.logs_laplace.rst new file mode 100644 index 0000000..74538c6 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_laplace.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_laplace +========================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_laplace diff --git a/docs/library/generated/scoringrules.logs_logistic.rst b/docs/library/generated/scoringrules.logs_logistic.rst new file mode 100644 index 0000000..f8d0639 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_logistic.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_logistic +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_logistic diff --git a/docs/library/generated/scoringrules.logs_loglaplace.rst b/docs/library/generated/scoringrules.logs_loglaplace.rst new file mode 100644 index 0000000..3cf835b --- /dev/null +++ b/docs/library/generated/scoringrules.logs_loglaplace.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_loglaplace +============================= + +.. currentmodule:: scoringrules + +.. autofunction:: logs_loglaplace diff --git a/docs/library/generated/scoringrules.logs_loglogistic.rst b/docs/library/generated/scoringrules.logs_loglogistic.rst new file mode 100644 index 0000000..a0ea488 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_loglogistic.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_loglogistic +============================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_loglogistic diff --git a/docs/library/generated/scoringrules.logs_lognormal.rst b/docs/library/generated/scoringrules.logs_lognormal.rst new file mode 100644 index 0000000..ef016c0 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_lognormal.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_lognormal +============================ + +.. currentmodule:: scoringrules + +.. autofunction:: logs_lognormal diff --git a/docs/library/generated/scoringrules.logs_mixnorm.rst b/docs/library/generated/scoringrules.logs_mixnorm.rst new file mode 100644 index 0000000..f56a9d7 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_mixnorm.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_mixnorm +========================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_mixnorm diff --git a/docs/library/generated/scoringrules.logs_negbinom.rst b/docs/library/generated/scoringrules.logs_negbinom.rst new file mode 100644 index 0000000..8324c66 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_negbinom.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_negbinom +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_negbinom diff --git a/docs/library/generated/scoringrules.logs_normal.rst b/docs/library/generated/scoringrules.logs_normal.rst new file mode 100644 index 0000000..beb64a9 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_normal.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_normal +========================= + +.. currentmodule:: scoringrules + +.. autofunction:: logs_normal diff --git a/docs/library/generated/scoringrules.logs_poisson.rst b/docs/library/generated/scoringrules.logs_poisson.rst new file mode 100644 index 0000000..98d6ea2 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_poisson.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_poisson +========================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_poisson diff --git a/docs/library/generated/scoringrules.logs_t.rst b/docs/library/generated/scoringrules.logs_t.rst new file mode 100644 index 0000000..9b11053 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_t.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_t +==================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_t diff --git a/docs/library/generated/scoringrules.logs_tlogistic.rst b/docs/library/generated/scoringrules.logs_tlogistic.rst new file mode 100644 index 0000000..e74a1c6 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_tlogistic.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_tlogistic +============================ + +.. currentmodule:: scoringrules + +.. autofunction:: logs_tlogistic diff --git a/docs/library/generated/scoringrules.logs_tnormal.rst b/docs/library/generated/scoringrules.logs_tnormal.rst new file mode 100644 index 0000000..61df3dd --- /dev/null +++ b/docs/library/generated/scoringrules.logs_tnormal.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_tnormal +========================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_tnormal diff --git a/docs/library/generated/scoringrules.logs_tt.rst b/docs/library/generated/scoringrules.logs_tt.rst new file mode 100644 index 0000000..cf2a300 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_tt.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_tt +===================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_tt diff --git a/docs/library/generated/scoringrules.logs_uniform.rst b/docs/library/generated/scoringrules.logs_uniform.rst new file mode 100644 index 0000000..03fad84 --- /dev/null +++ b/docs/library/generated/scoringrules.logs_uniform.rst @@ -0,0 +1,6 @@ +scoringrules.logs\_uniform +========================== + +.. currentmodule:: scoringrules + +.. autofunction:: logs_uniform diff --git a/docs/library/generated/scoringrules.owcrps_ensemble.rst b/docs/library/generated/scoringrules.owcrps_ensemble.rst new file mode 100644 index 0000000..860679b --- /dev/null +++ b/docs/library/generated/scoringrules.owcrps_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.owcrps\_ensemble +============================= + +.. currentmodule:: scoringrules + +.. autofunction:: owcrps_ensemble diff --git a/docs/library/generated/scoringrules.owes_ensemble.rst b/docs/library/generated/scoringrules.owes_ensemble.rst new file mode 100644 index 0000000..9f46986 --- /dev/null +++ b/docs/library/generated/scoringrules.owes_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.owes\_ensemble +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: owes_ensemble diff --git a/docs/library/generated/scoringrules.owgksmv_ensemble.rst b/docs/library/generated/scoringrules.owgksmv_ensemble.rst new file mode 100644 index 0000000..8277dab --- /dev/null +++ b/docs/library/generated/scoringrules.owgksmv_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.owgksmv\_ensemble +============================== + +.. currentmodule:: scoringrules + +.. autofunction:: owgksmv_ensemble diff --git a/docs/library/generated/scoringrules.owgksuv_ensemble.rst b/docs/library/generated/scoringrules.owgksuv_ensemble.rst new file mode 100644 index 0000000..62a7130 --- /dev/null +++ b/docs/library/generated/scoringrules.owgksuv_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.owgksuv\_ensemble +============================== + +.. currentmodule:: scoringrules + +.. autofunction:: owgksuv_ensemble diff --git a/docs/library/generated/scoringrules.owvs_ensemble.rst b/docs/library/generated/scoringrules.owvs_ensemble.rst new file mode 100644 index 0000000..38a1765 --- /dev/null +++ b/docs/library/generated/scoringrules.owvs_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.owvs\_ensemble +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: owvs_ensemble diff --git a/docs/library/generated/scoringrules.quantile_score.rst b/docs/library/generated/scoringrules.quantile_score.rst new file mode 100644 index 0000000..560bb96 --- /dev/null +++ b/docs/library/generated/scoringrules.quantile_score.rst @@ -0,0 +1,6 @@ +scoringrules.quantile\_score +============================ + +.. currentmodule:: scoringrules + +.. autofunction:: quantile_score diff --git a/docs/library/generated/scoringrules.register_backend.rst b/docs/library/generated/scoringrules.register_backend.rst new file mode 100644 index 0000000..0820837 --- /dev/null +++ b/docs/library/generated/scoringrules.register_backend.rst @@ -0,0 +1,6 @@ +scoringrules.register\_backend +============================== + +.. currentmodule:: scoringrules + +.. autofunction:: register_backend diff --git a/docs/library/generated/scoringrules.rls_score.rst b/docs/library/generated/scoringrules.rls_score.rst new file mode 100644 index 0000000..1cc6638 --- /dev/null +++ b/docs/library/generated/scoringrules.rls_score.rst @@ -0,0 +1,6 @@ +scoringrules.rls\_score +======================= + +.. currentmodule:: scoringrules + +.. autofunction:: rls_score diff --git a/docs/library/generated/scoringrules.rps_score.rst b/docs/library/generated/scoringrules.rps_score.rst new file mode 100644 index 0000000..ac8a19e --- /dev/null +++ b/docs/library/generated/scoringrules.rps_score.rst @@ -0,0 +1,6 @@ +scoringrules.rps\_score +======================= + +.. currentmodule:: scoringrules + +.. autofunction:: rps_score diff --git a/docs/library/generated/scoringrules.twcrps_ensemble.rst b/docs/library/generated/scoringrules.twcrps_ensemble.rst new file mode 100644 index 0000000..ae01854 --- /dev/null +++ b/docs/library/generated/scoringrules.twcrps_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.twcrps\_ensemble +============================= + +.. currentmodule:: scoringrules + +.. autofunction:: twcrps_ensemble diff --git a/docs/library/generated/scoringrules.twes_ensemble.rst b/docs/library/generated/scoringrules.twes_ensemble.rst new file mode 100644 index 0000000..dd247f4 --- /dev/null +++ b/docs/library/generated/scoringrules.twes_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.twes\_ensemble +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: twes_ensemble diff --git a/docs/library/generated/scoringrules.twgksmv_ensemble.rst b/docs/library/generated/scoringrules.twgksmv_ensemble.rst new file mode 100644 index 0000000..5118306 --- /dev/null +++ b/docs/library/generated/scoringrules.twgksmv_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.twgksmv\_ensemble +============================== + +.. currentmodule:: scoringrules + +.. autofunction:: twgksmv_ensemble diff --git a/docs/library/generated/scoringrules.twgksuv_ensemble.rst b/docs/library/generated/scoringrules.twgksuv_ensemble.rst new file mode 100644 index 0000000..b8e0848 --- /dev/null +++ b/docs/library/generated/scoringrules.twgksuv_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.twgksuv\_ensemble +============================== + +.. currentmodule:: scoringrules + +.. autofunction:: twgksuv_ensemble diff --git a/docs/library/generated/scoringrules.twvs_ensemble.rst b/docs/library/generated/scoringrules.twvs_ensemble.rst new file mode 100644 index 0000000..05c0f94 --- /dev/null +++ b/docs/library/generated/scoringrules.twvs_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.twvs\_ensemble +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: twvs_ensemble diff --git a/docs/library/generated/scoringrules.vrcrps_ensemble.rst b/docs/library/generated/scoringrules.vrcrps_ensemble.rst new file mode 100644 index 0000000..c9d842d --- /dev/null +++ b/docs/library/generated/scoringrules.vrcrps_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.vrcrps\_ensemble +============================= + +.. currentmodule:: scoringrules + +.. autofunction:: vrcrps_ensemble diff --git a/docs/library/generated/scoringrules.vres_ensemble.rst b/docs/library/generated/scoringrules.vres_ensemble.rst new file mode 100644 index 0000000..5e72fda --- /dev/null +++ b/docs/library/generated/scoringrules.vres_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.vres\_ensemble +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: vres_ensemble diff --git a/docs/library/generated/scoringrules.vrgksmv_ensemble.rst b/docs/library/generated/scoringrules.vrgksmv_ensemble.rst new file mode 100644 index 0000000..326018e --- /dev/null +++ b/docs/library/generated/scoringrules.vrgksmv_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.vrgksmv\_ensemble +============================== + +.. currentmodule:: scoringrules + +.. autofunction:: vrgksmv_ensemble diff --git a/docs/library/generated/scoringrules.vrgksuv_ensemble.rst b/docs/library/generated/scoringrules.vrgksuv_ensemble.rst new file mode 100644 index 0000000..d7bae3f --- /dev/null +++ b/docs/library/generated/scoringrules.vrgksuv_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.vrgksuv\_ensemble +============================== + +.. currentmodule:: scoringrules + +.. autofunction:: vrgksuv_ensemble diff --git a/docs/library/generated/scoringrules.vrvs_ensemble.rst b/docs/library/generated/scoringrules.vrvs_ensemble.rst new file mode 100644 index 0000000..a622194 --- /dev/null +++ b/docs/library/generated/scoringrules.vrvs_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.vrvs\_ensemble +=========================== + +.. currentmodule:: scoringrules + +.. autofunction:: vrvs_ensemble diff --git a/docs/library/generated/scoringrules.vs_ensemble.rst b/docs/library/generated/scoringrules.vs_ensemble.rst new file mode 100644 index 0000000..a37cd2e --- /dev/null +++ b/docs/library/generated/scoringrules.vs_ensemble.rst @@ -0,0 +1,6 @@ +scoringrules.vs\_ensemble +========================= + +.. currentmodule:: scoringrules + +.. autofunction:: vs_ensemble diff --git a/docs/library/generated/scoringrules.weighted_interval_score.rst b/docs/library/generated/scoringrules.weighted_interval_score.rst new file mode 100644 index 0000000..37dc886 --- /dev/null +++ b/docs/library/generated/scoringrules.weighted_interval_score.rst @@ -0,0 +1,6 @@ +scoringrules.weighted\_interval\_score +====================================== + +.. currentmodule:: scoringrules + +.. autofunction:: weighted_interval_score diff --git a/docs/reference.md b/docs/library/reference.md similarity index 82% rename from docs/reference.md rename to docs/library/reference.md index b3274fd..8cad92e 100644 --- a/docs/reference.md +++ b/docs/library/reference.md @@ -1,13 +1,33 @@ # API reference This page provides a summary of scoringrules' API. All functions are -available in the top-level namespace of the package and are here -organized by category. +available in the top-level namespace of the package and are organised here by category. ```{eval-rst} .. currentmodule:: scoringrules -Ensemble forecasts +Categorical forecasts +===================== +.. autosummary:: + :toctree: generated + + brier_score + rps_score + log_score + rls_score + + +Consistent scoring functions +============================ +.. autosummary:: + :toctree: generated + + quantile_score + interval_score + weighted_interval_score + + +Scoring rules for ensemble (sample) forecasts ================== Univariate @@ -17,14 +37,8 @@ Univariate :toctree: generated crps_ensemble - twcrps_ensemble - owcrps_ensemble - vrcrps_ensemble + logs_ensemble gksuv_ensemble - twgksuv_ensemble - owgksuv_ensemble - vrgksuv_ensemble - crps_quantile Multivariate -------------------- @@ -33,22 +47,44 @@ Multivariate :toctree: generated es_ensemble + vs_ensemble + gksmv_ensemble + +Weighted scoring rules +-------------------- + +.. autosummary:: + :toctree: generated + + twcrps_ensemble + owcrps_ensemble + vrcrps_ensemble + + clogs_ensemble + + twgksuv_ensemble + owgksuv_ensemble + vrgksuv_ensemble + owes_ensemble twes_ensemble vres_ensemble - vs_ensemble owvs_ensemble twvs_ensemble vrvs_ensemble - gksmv_ensemble twgksmv_ensemble owgksmv_ensemble vrgksmv_ensemble -Parametric distributions forecasts + +Scoring rules for parametric forecast distributions ==================================== + +CRPS +-------------------- + .. autosummary:: :toctree: generated @@ -83,6 +119,14 @@ Parametric distributions forecasts crps_quantile crps_t crps_uniform + + +Log score +-------------------- + +.. autosummary:: + :toctree: generated + logs_beta logs_binomial logs_ensemble @@ -109,25 +153,6 @@ Parametric distributions forecasts logs_tt logs_uniform -Consistent scoring functions -============================ -.. autosummary:: - :toctree: generated - - interval_score - weighted_interval_score - - -Categorical forecasts -===================== -.. autosummary:: - :toctree: generated - - brier_score - rps_score - log_score - rls_score - Backends ======== diff --git a/docs/library/user_guide.md b/docs/library/user_guide.md new file mode 100644 index 0000000..cf6e821 --- /dev/null +++ b/docs/library/user_guide.md @@ -0,0 +1,95 @@ +# User guide + +## First steps +Start by importing the library using +```python +import scoringrules as sr +``` + +The library API is simple: all metrics are available under the main namespace. Some examples are given below. + +```python +import numpy as np + +## scalar data + +# Brier score (observation = 1, forecast = 0.1) +sr.brier_score(1, 0.1) + +# CRPS (observation = 0.1, forecast = normal distribution with mean 1.2 and sd 0.3) +sr.crps_normal(0.1, 1.2, 0.3) + + +## array data + +# Brier score +obs = np.random.uniform(0, 1, 100) +fct = np.random.binomial(1, 0.5, 100) +sr.brier_score(obs, fct) + +# CRPS (forecast = lognormal distribution) +obs = np.random.lognormal(0, 1, 100) +mu = np.random.randn(100) +sig = np.random.uniform(0.5, 1.5, 100) +sr.crps_lognormal(obs, mu, sig) + + +## ensemble metrics (univariate) +obs = np.random.randn(100) +fct = obs[:, None] + np.random.randn(100, 21) * 0.1 + +sr.crps_ensemble(obs, fct) # CRPS +sr.gksuv_ensemble(obs, fct) # univariate Gaussian kernel score + + +## ensemble metrics (multivariate) +obs = np.random.randn(100, 3) +fct = obs[:, None] + np.random.randn(100, 21, 3) * 0.1 + +sr.energy_score(obs, fct) # energy score +sr.variogram_score(obs, fct) # variogram score +``` + +For the ensemble metrics, the forecast should have one more dimension than the observations, containing the ensemble (i.e. sample) members. In the univariate case, this ensemble dimension is assumed to be the last axis of the input forecast array, though this can be changed manually by specifying the `m_axis` argument (default is `m_axis=-1`). In the multivariate case, the ensemble dimension is assumed to be the second last axis, with the number of variables the last axis. These can similarly be changed manually by specifying the `m_axis` and `v_axis` arguments respectively (default is `m_axis=-2` and `v_axis=-1`). + +## Backends + +The scoringrules library supports multiple backends: `numpy` (accelerated with `numba`), `torch`, `tensorflow`, and `jax`. By default, the `numpy` and `numba` backends will be registered when importing the library. You can see the list of registered backends by running + +```python +print(sr.backends) +# {'numpy': , +# 'numba': } +``` + +and the currently active backend, used by default in all metrics, by running + +```python +print(sr.backends.active) +# +``` + +The default backend can be changed manually using `sr.backends.set_active()`. For example, the following code sets the active backend to `numba`. + +```python +sr.backends.set_active("numba") +print(sr.backends.active) +# +``` +Alternatively, the `backend` argument to the score functions can be used to override the default choice. For example, + +```python +sr.crps_normal(0.1, 1.2, 0.3, backend="numba") +``` + +To register a new backend, for example `torch`, simply use + +```python +sr.register_backend("torch") +``` + +You can now use `torch` to compute scores, either by setting it as the default backend, or by specifying it manually when running the desired function: + +```python +sr.crps_normal(0.1, 1.2, 0.3, backend="torch") +``` diff --git a/docs/user_guide.md b/docs/user_guide.md deleted file mode 100644 index b200f99..0000000 --- a/docs/user_guide.md +++ /dev/null @@ -1,75 +0,0 @@ -# User guide - -## First steps -Start by importing the library in your code with -```python -import scoringrules as sr -``` - -the library API is simple: all metrics are available under the main namespace. Let's look at some examples: - -```python -import numpy as np - -# on scalars -sr.brier_score(0.1, 1) -sr.crps_normal(0.1, 1.2, 0.3) - -# on arrays -sr.brier_score(np.random.uniform(0, 1, 100), np.random.binomial(1, 0.5, 100)) -sr.crps_lognormal(np.random.lognormal(0, 1, 100), np.random.randn(100), np.random.uniform(0.5, 1.5, 100)) - -# ensemble metrics -obs = np.random.randn(100) -fct = obs[:,None] + np.random.randn(100, 21) * 0.1 - -sr.crps_ensemble(obs, fct) -sr.error_spread_score(obs, fct) - -# multivariate ensemble metrics -obs = np.random.randn(100,3) -fct = obs[:,None] + np.random.randn(100, 21, 3) * 0.1 - -sr.energy_score(obs, fct) -sr.variogram_score(obs, fct) -``` - -For the univariate ensemble metrics, the ensemble dimension is on the last axis unless you specify otherwise with the `axis` argument. For the multivariate ensemble metrics, the ensemble dimension and the variable dimension are on the second last and last axis respectively, unless specified otherwise with `m_axis` and `v_axis`. - -## Backends -Scoringrules supports multiple backends. By default, the `numpy` and `numba` backends will be registered when importing the library. You can see the list of registered backends with - -```python -print(sr.backends) -# {'numpy': , -# 'numba': } -``` - -and the currently active backend, used by default in all metrics, can be seen with - -```python -print(sr.backends.active) -# -``` - -The default backend can also be changed with - -```python -sr.backends.set_active("numba") -print(sr.backends.active) -# -``` -When computing a metric, the `backend` argument can be used to override the default choice. - - -To register a new backend, for example `torch`, simply use - -```python -sr.register_backend("torch") -``` - -You can now use `torch` to compute metrics, either by setting it as the default backend or by specifying it on a specific metric: - -```python -sr.crps_normal(0.1, 1.0, 0.0, backend="torch") -``` diff --git a/scoringrules/_brier.py b/scoringrules/_brier.py index 554f0f2..59f884d 100644 --- a/scoringrules/_brier.py +++ b/scoringrules/_brier.py @@ -15,29 +15,41 @@ def brier_score( backend: "Backend" = None, ) -> "Array": r""" - Compute the Brier Score (BS). + Brier Score - The BS is formulated as + The Brier Score is defined as .. math:: - BS(f, y) = (f - y)^2, + \text{BS}(F, y) = (F - y)^2, - where :math:`f \in [0, 1]` is the predicted probability of an event and :math:`y \in \{0, 1\}` the actual outcome. + where :math:`F \in [0, 1]` is the predicted probability of an event and :math:`y \in \{0, 1\}` + is the outcome [1]_. Parameters ---------- obs : array_like - Observed outcome, either 0 or 1. + Observed outcomes, either 0 or 1. fct : array_like - Forecasted probabilities between 0 and 1. + Forecast probabilities, between 0 and 1. backend : str - The name of the backend used for computations. Defaults to 'numpy'. + The name of the backend used for computations. Default is 'numpy'. Returns ------- - brier_score : array_like + score : array_like The computed Brier Score. + References + ---------- + .. [1] Brier, G. W. (1950). + Verification of forecasts expressed in terms of probability. + Monthly Weather Review, 78, 1-3. + + Examples + -------- + >>> import scoringrules as sr + >>> sr.brier_score(1, 0.2) + 0.64000 """ return brier.brier_score(obs=obs, fct=fct, backend=backend) @@ -48,46 +60,76 @@ def rps_score( /, k_axis: int = -1, *, + onehot: bool = False, backend: "Backend" = None, ) -> "Array": r""" - Compute the (Discrete) Ranked Probability Score (RPS). + (Discrete) Ranked Probability Score (RPS) Suppose the outcome corresponds to one of :math:`K` ordered categories. The RPS is defined as .. math:: - RPS(f, y) = \sum_{k=1}^{K}(\tilde{f}_{k} - \tilde{y}_{k})^2, + \text{RPS}(F, y) = \sum_{k=1}^{K}(\tilde{F}_{k} - \tilde{y}_{k})^2, - where :math:`f \in [0, 1]^{K}` is a vector of length :math:`K` containing forecast probabilities - that each of the :math:`K` categories will occur, and :math:`y \in \{0, 1\}^{K}` is a vector of - length :math:`K`, with the :math:`k`-th element equal to one if the :math:`k`-th category occurs. We - have :math:`\sum_{k=1}^{K} y_{k} = \sum_{k=1}^{K} f_{k} = 1`, and, for :math:`k = 1, \dots, K`, - :math:`\tilde{y}_{k} = \sum_{i=1}^{k} y_{i}` and :math:`\tilde{f}_{k} = \sum_{i=1}^{k} f_{i}`. + where :math:`F \in [0, 1]^{K}` is a vector of length :math:`K`, containing forecast probabilities + that each of the :math:`K` categories will occur, with :math:`\sum_{k=1}^{K} F_{k} = 1` and + :math:`\tilde{F}_{k} = \sum_{i=1}^{k} F_{i}` for all :math:`k = 1, \dots, K`, and where + :math:`y \in \{1, \dots, K\}` is the category that occurs, with :math:`\tilde{y}_{k} = 1\{y \le i\}` + for all :math:`k = 1, \dots, K` [1]_. + + The outcome can alternatively be interpreted as a vector :math:`y \in \{0, 1\}^K` of length :math:`K`, with the + :math:`k`-th element equal to one if the :math:`k`-th category occurs, and zero otherwise. + Using this one-hot encoding, the RPS is defined analogously to as above, but with + :math:`\tilde{y}_{k} = \sum_{i=1}^{k} y_{i}`. Parameters ---------- obs : array_like - Array of 0's and 1's corresponding to unobserved and observed categories - forecasts : + Category that occurs. Or array of 0's and 1's corresponding to unobserved and + observed categories if `onehot=True`. + fct : array Array of forecast probabilities for each category. k_axis: int - The axis corresponding to the categories. Default is the last axis. + The axis of `obs` and `fct` corresponding to the categories. Default is the last axis. + onehot: bool + Boolean indicating whether the observation is the category that occurs or a onehot + encoded vector of 0's and 1's. Default is False. backend : str - The name of the backend used for computations. Defaults to 'numpy'. + The name of the backend used for computations. Default is 'numpy'. Returns ------- - score: + score: array_like The computed Ranked Probability Score. + References + ---------- + .. [1] Epstein, E. S. (1969). + A scoring system for probability forecasts of ranked categories. + Journal of Applied Meteorology, 8, 985-987. + https://www.jstor.org/stable/26174707. + + Examples + -------- + >>> import scoringrules as sr + >>> import numpy as np + >>> fct = np.array([0.1, 0.2, 0.3, 0.4]) + >>> obs = 3 + >>> sr.rps_score(obs, fct) + 0.25999999999999995 + >>> obs = np.array([0, 0, 1, 0]) + >>> sr.rps_score(obs, fct, onehot=True) + 0.25999999999999995 """ B = backends.active if backend is None else backends[backend] fct = B.asarray(fct) if k_axis != -1: fct = B.moveaxis(fct, k_axis, -1) + if onehot: + obs = B.moveaxis(obs, k_axis, -1) - return brier.rps_score(obs=obs, fct=fct, backend=backend) + return brier.rps_score(obs=obs, fct=fct, onehot=onehot, backend=backend) def log_score( diff --git a/scoringrules/core/brier.py b/scoringrules/core/brier.py index e66fbae..a721519 100644 --- a/scoringrules/core/brier.py +++ b/scoringrules/core/brier.py @@ -21,12 +21,13 @@ def brier_score( if not set(v.item() for v in B.unique_values(obs)) <= {0, 1}: raise ValueError("Observations must be 0, 1, or NaN.") - return B.asarray((fct - obs) ** 2) + return (fct - obs) ** 2 def rps_score( obs: "ArrayLike", fct: "ArrayLike", + onehot: bool = False, backend: "Backend" = None, ) -> "Array": """Compute the Ranked Probability Score for ordinal categorical forecasts.""" @@ -36,12 +37,11 @@ def rps_score( if B.any(fct < 0.0) or B.any(fct > 1.0 + EPSILON): raise ValueError("Forecast probabilities must be between 0 and 1.") - categories = B.arange(1, fct.shape[-1] + 1) - obs_one_hot = B.where(B.expand_dims(obs, -1) == categories, 1, 0) + if not onehot: + categories = B.arange(1, fct.shape[-1] + 1) + obs = B.where(B.expand_dims(obs, -1) == categories, 1, 0) - return B.sum( - (B.cumsum(fct, axis=-1) - B.cumsum(obs_one_hot, axis=-1)) ** 2, axis=-1 - ) + return B.sum((B.cumsum(fct, axis=-1) - B.cumsum(obs, axis=-1)) ** 2, axis=-1) def log_score(obs: "ArrayLike", fct: "ArrayLike", backend: "Backend" = None) -> "Array": From ffb248bc8c8c423dedcbb5d3d1237efb62b9de96 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Mon, 4 Aug 2025 13:59:31 +0200 Subject: [PATCH 72/79] fix typo in crps closed docstrings --- scoringrules/core/crps/_closed.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scoringrules/core/crps/_closed.py b/scoringrules/core/crps/_closed.py index 5d8e561..cd98084 100644 --- a/scoringrules/core/crps/_closed.py +++ b/scoringrules/core/crps/_closed.py @@ -520,7 +520,7 @@ def logistic( sigma: "ArrayLike", backend: "Backend" = None, ) -> "Array": - """Compute the CRPS for the normal distribution.""" + """Compute the CRPS for the logistic distribution.""" B = backends.active if backend is None else backends[backend] mu, sigma, obs = map(B.asarray, (mu, sigma, obs)) ω = (obs - mu) / sigma @@ -649,7 +649,7 @@ def normal( sigma: "ArrayLike", backend: "Backend" = None, ) -> "Array": - """Compute the CRPS for the logistic distribution.""" + """Compute the CRPS for the normal distribution.""" B = backends.active if backend is None else backends[backend] mu, sigma, obs = map(B.asarray, (mu, sigma, obs)) ω = (obs - mu) / sigma From 362a546eeee84199c08ccbb553fddd66236c7895 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Tue, 5 Aug 2025 10:55:56 +0200 Subject: [PATCH 73/79] remove parametric crps gufuncs --- scoringrules/core/crps/_gufuncs.py | 52 +----------------------------- 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/scoringrules/core/crps/_gufuncs.py b/scoringrules/core/crps/_gufuncs.py index 0eb7908..be8e0d1 100644 --- a/scoringrules/core/crps/_gufuncs.py +++ b/scoringrules/core/crps/_gufuncs.py @@ -1,7 +1,5 @@ -import math - import numpy as np -from numba import guvectorize, njit, vectorize +from numba import guvectorize INV_SQRT_PI = 1 / np.sqrt(np.pi) EPSILON = 1e-6 @@ -315,51 +313,6 @@ def _vrcrps_ensemble_nrg_gufunc( out[0] = e_1 - 0.5 * e_2 + (wabs_x - wabs_y) * (wbar - ow) -@njit(["float32(float32)", "float64(float64)"]) -def _norm_cdf(x: float) -> float: - """Cumulative distribution function for the standard normal distribution.""" - out: float = (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0 - return out - - -@njit(["float32(float32)", "float64(float64)"]) -def _norm_pdf(x: float) -> float: - """Probability density function for the standard normal distribution.""" - out: float = (1 / math.sqrt(2 * math.pi)) * math.exp(-(x**2) / 2) - return out - - -@njit(["float32(float32)", "float64(float64)"]) -def _logis_cdf(x: float) -> float: - """Cumulative distribution function for the standard logistic distribution.""" - out: float = 1.0 / (1.0 + math.exp(-x)) - return out - - -@vectorize(["float32(float32, float32, float32)", "float64(float64, float64, float64)"]) -def _crps_normal_ufunc(obs: float, mu: float, sigma: float) -> float: - ω = (obs - mu) / sigma - out: float = sigma * (ω * (2 * _norm_cdf(ω) - 1) + 2 * _norm_pdf(ω) - INV_SQRT_PI) - return out - - -@vectorize(["float32(float32, float32, float32)", "float64(float64, float64, float64)"]) -def _crps_lognormal_ufunc(obs: float, mulog: float, sigmalog: float) -> float: - ω = (np.log(obs) - mulog) / sigmalog - ex = 2 * np.exp(mulog + sigmalog**2 / 2) - out: float = obs * (2 * _norm_cdf(ω) - 1) - ex * ( - _norm_cdf(ω - sigmalog) + _norm_cdf(sigmalog / np.sqrt(2)) - 1 - ) - return out - - -@vectorize(["float32(float32, float32, float32)", "float64(float64, float64, float64)"]) -def _crps_logistic_ufunc(obs: float, mu: float, sigma: float) -> float: - ω = (obs - mu) / sigma - out: float = sigma * (ω - 2 * np.log(_logis_cdf(ω)) - 1) - return out - - estimator_gufuncs = { "akr_circperm": _crps_ensemble_akr_circperm_gufunc, "akr": _crps_ensemble_akr_gufunc, @@ -380,8 +333,5 @@ def _crps_logistic_ufunc(obs: float, mu: float, sigma: float) -> float: "_crps_ensemble_nrg_gufunc", "_crps_ensemble_pwm_gufunc", "_crps_ensemble_qd_gufunc", - "_crps_normal_ufunc", - "_crps_lognormal_ufunc", - "_crps_logistic_ufunc", "quantile_pinball_gufunc", ] From 06478ad3392aaf0a880970b5213270e3e80cda37 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Tue, 5 Aug 2025 17:08:57 +0200 Subject: [PATCH 74/79] add separate crps functions for weighted ensembles --- scoringrules/_crps.py | 42 ++-- scoringrules/core/crps/__init__.py | 9 +- scoringrules/core/crps/_approx.py | 104 ++++----- scoringrules/core/crps/_approx_w.py | 159 ++++++++++++++ scoringrules/core/crps/_gufuncs.py | 152 ++++++------- scoringrules/core/crps/_gufuncs_w.py | 316 +++++++++++++++++++++++++++ tests/test_crps.py | 14 +- 7 files changed, 624 insertions(+), 172 deletions(-) create mode 100644 scoringrules/core/crps/_approx_w.py create mode 100644 scoringrules/core/crps/_gufuncs_w.py diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index 65731a2..ba63360 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -106,11 +106,23 @@ def crps_ensemble( if m_axis != -1: fct = B.moveaxis(fct, m_axis, -1) - sort_ensemble = not sorted_ensemble and estimator in ["qd", "pwm"] + sort_ensemble = not sorted_ensemble and estimator in ["qd", "pwm", "int"] + + if sort_ensemble: + fct_uns = fct + fct = B.sort(fct, axis=-1) if ens_w is None: - M = fct.shape[-1] - ens_w = B.zeros(fct.shape) + 1.0 / M + if backend == "numba": + if estimator not in crps.estimator_gufuncs: + raise ValueError( + f"{estimator} is not a valid estimator. " + f"Must be one of {crps.estimator_gufuncs.keys()}" + ) + return crps.estimator_gufuncs[estimator](obs, fct) + + return crps.ensemble(obs, fct, estimator, backend=backend) + else: ens_w = B.asarray(ens_w) if B.any(ens_w < 0): @@ -119,21 +131,17 @@ def crps_ensemble( if m_axis != -1: ens_w = B.moveaxis(ens_w, m_axis, -1) if sort_ensemble: - ind = B.argsort(fct, axis=-1) + ind = B.argsort(fct_uns, axis=-1) ens_w = B.gather(ens_w, ind, axis=-1) - - if sort_ensemble: - fct = B.sort(fct, axis=-1) - - if backend == "numba": - if estimator not in crps.estimator_gufuncs: - raise ValueError( - f"{estimator} is not a valid estimator. " - f"Must be one of {crps.estimator_gufuncs.keys()}" - ) - return crps.estimator_gufuncs[estimator](obs, fct, ens_w) - - return crps.ensemble(obs, fct, ens_w, estimator, backend=backend) + if backend == "numba": + if estimator not in crps.estimator_gufuncs: + raise ValueError( + f"{estimator} is not a valid estimator. " + f"Must be one of {crps.estimator_gufuncs.keys()}" + ) + return crps.estimator_gufuncs_w[estimator](obs, fct, ens_w) + + return crps.ensemble_w(obs, fct, ens_w, estimator, backend=backend) def twcrps_ensemble( diff --git a/scoringrules/core/crps/__init__.py b/scoringrules/core/crps/__init__.py index 74f563a..d13f562 100644 --- a/scoringrules/core/crps/__init__.py +++ b/scoringrules/core/crps/__init__.py @@ -1,4 +1,5 @@ -from ._approx import ensemble, ow_ensemble, quantile_pinball, vr_ensemble +from ._approx import ensemble, ow_ensemble, vr_ensemble, quantile_pinball +from ._approx_w import ensemble_w, ow_ensemble_w, vr_ensemble_w from ._closed import ( beta, binomial, @@ -27,14 +28,19 @@ try: from ._gufuncs import estimator_gufuncs, quantile_pinball_gufunc + from ._gufuncs_w import estimator_gufuncs_w except ImportError: estimator_gufuncs = None + estimator_gufuncs_w = None quantile_pinball_gufunc = None __all__ = [ "ensemble", + "ensemble_w", "ow_ensemble", + "ow_ensemble_w", "vr_ensemble", + "vr_ensemble_w", "beta", "binomial", "exponential", @@ -59,6 +65,7 @@ "t", "uniform", "estimator_gufuncs", + "estimator_gufuncs_w", "quantile_pinball", "quantile_pinball_gufunc", ] diff --git a/scoringrules/core/crps/_approx.py b/scoringrules/core/crps/_approx.py index 863c329..18564dc 100644 --- a/scoringrules/core/crps/_approx.py +++ b/scoringrules/core/crps/_approx.py @@ -9,23 +9,22 @@ def ensemble( obs: "ArrayLike", fct: "Array", - ens_w: "Array", estimator: str = "pwm", backend: "Backend" = None, ) -> "Array": """Compute the CRPS for a finite ensemble.""" if estimator == "nrg": - out = _crps_ensemble_nrg(obs, fct, ens_w, backend=backend) + out = _crps_ensemble_nrg(obs, fct, backend=backend) elif estimator == "pwm": - out = _crps_ensemble_pwm(obs, fct, ens_w, backend=backend) + out = _crps_ensemble_pwm(obs, fct, backend=backend) elif estimator == "fair": - out = _crps_ensemble_fair(obs, fct, ens_w, backend=backend) + out = _crps_ensemble_fair(obs, fct, backend=backend) elif estimator == "qd": - out = _crps_ensemble_qd(obs, fct, ens_w, backend=backend) + out = _crps_ensemble_qd(obs, fct, backend=backend) elif estimator == "akr": - out = _crps_ensemble_akr(obs, fct, ens_w, backend=backend) + out = _crps_ensemble_akr(obs, fct, backend=backend) elif estimator == "akr_circperm": - out = _crps_ensemble_akr_circperm(obs, fct, ens_w, backend=backend) + out = _crps_ensemble_akr_circperm(obs, fct, backend=backend) else: raise ValueError( f"{estimator} can only be used with `numpy` " @@ -36,78 +35,75 @@ def ensemble( def _crps_ensemble_fair( - obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None + obs: "Array", fct: "Array", backend: "Backend" = None ) -> "Array": """Fair version of the CRPS estimator based on the energy form.""" B = backends.active if backend is None else backends[backend] - e_1 = B.sum(B.abs(obs[..., None] - fct) * w, axis=-1) + M = fct.shape[-1] + e_1 = B.sum(B.abs(obs[..., None] - fct), axis=-1) / M e_2 = B.sum( - B.abs(fct[..., None] - fct[..., None, :]) * w[..., None] * w[..., None, :], + B.abs(fct[..., None] - fct[..., None, :]), axis=(-1, -2), - ) / (1 - B.sum(w * w, axis=-1)) + ) / (M * (M - 1)) return e_1 - 0.5 * e_2 def _crps_ensemble_nrg( - obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None + obs: "Array", fct: "Array", backend: "Backend" = None ) -> "Array": """CRPS estimator based on the energy form.""" B = backends.active if backend is None else backends[backend] - e_1 = B.sum(B.abs(obs[..., None] - fct) * w, axis=-1) - e_2 = B.sum( - B.abs(fct[..., None] - fct[..., None, :]) * w[..., None] * w[..., None, :], - (-1, -2), - ) + M: int = fct.shape[-1] + e_1 = B.sum(B.abs(obs[..., None] - fct), axis=-1) / M + e_2 = B.sum(B.abs(fct[..., None] - fct[..., None, :]), (-1, -2)) / (M**2) return e_1 - 0.5 * e_2 def _crps_ensemble_pwm( - obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None + obs: "Array", fct: "Array", backend: "Backend" = None ) -> "Array": """CRPS estimator based on the probability weighted moment (PWM) form.""" B = backends.active if backend is None else backends[backend] - w_sum = B.cumsum(w, axis=-1) - expected_diff = B.sum(B.abs(obs[..., None] - fct) * w, axis=-1) - β_0 = B.sum(fct * w * (1.0 - w), axis=-1) - β_1 = B.sum(fct * w * (w_sum - w), axis=-1) + M: int = fct.shape[-1] + expected_diff = B.sum(B.abs(obs[..., None] - fct), axis=-1) / M + β_0 = B.sum(fct, axis=-1) / M + β_1 = B.sum(fct * B.arange(0, M), axis=-1) / (M * (M - 1.0)) return expected_diff + β_0 - 2.0 * β_1 -def _crps_ensemble_qd( - obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None -) -> "Array": - """CRPS estimator based on the quantile score decomposition.""" +def _crps_ensemble_qd(obs: "Array", fct: "Array", backend: "Backend" = None) -> "Array": + """CRPS estimator based on the quantile decomposition form.""" B = backends.active if backend is None else backends[backend] - w_sum = B.cumsum(w, axis=-1) - a = w_sum - 0.5 * w - dif = fct - obs[..., None] - c = B.where(dif > 0, 1 - a, -a) - s = B.sum(w * c * dif, axis=-1) - return 2 * s + M: int = fct.shape[-1] + alpha = B.arange(1, M + 1) - 0.5 + below = (fct <= obs[..., None]) * alpha * (obs[..., None] - fct) + above = (fct > obs[..., None]) * (M - alpha) * (fct - obs[..., None]) + out = B.sum(below + above, axis=-1) / (M**2) + return 2 * out def _crps_ensemble_akr( - obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None + obs: "Array", fct: "Array", backend: "Backend" = None ) -> "Array": """CRPS estimator based on the approximate kernel representation.""" B = backends.active if backend is None else backends[backend] M = fct.shape[-1] - e_1 = B.sum(B.abs(obs[..., None] - fct) * w, axis=-1) + e_1 = B.mean(B.abs(obs[..., None] - fct), axis=-1) ind = [(i + 1) % M for i in range(M)] - e_2 = B.sum(B.abs(fct[..., ind] - fct) * w[..., ind], axis=-1) + e_2 = B.mean(B.abs(fct[..., ind] - fct)[..., ind], axis=-1) return e_1 - 0.5 * e_2 def _crps_ensemble_akr_circperm( - obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None + obs: "Array", fct: "Array", backend: "Backend" = None ) -> "Array": """CRPS estimator based on the AKR with cyclic permutation.""" B = backends.active if backend is None else backends[backend] M = fct.shape[-1] - e_1 = B.sum(B.abs(obs[..., None] - fct) * w, axis=-1) + e_1 = B.mean(B.abs(obs[..., None] - fct), axis=-1) shift = int((M - 1) / 2) ind = [(i + shift) % M for i in range(M)] - e_2 = B.sum(B.abs(fct[..., ind] - fct) * w[..., ind], axis=-1) + e_2 = B.mean(B.abs(fct[..., ind] - fct), axis=-1) return e_1 - 0.5 * e_2 @@ -126,19 +122,14 @@ def ow_ensemble( fct: "Array", ow: "Array", fw: "Array", - ens_w: "Array", backend: "Backend" = None, ) -> "Array": - """Outcome-Weighted CRPS estimator based on the energy form.""" + """Outcome-Weighted CRPS for an ensemble forecast.""" B = backends.active if backend is None else backends[backend] - wbar = B.sum(ens_w * fw, axis=-1) - e_1 = B.sum(ens_w * B.abs(obs[..., None] - fct) * fw, axis=-1) * ow / wbar - e_2 = B.sum( - ens_w[..., None] - * ens_w[..., None, :] - * B.abs(fct[..., None] - fct[..., None, :]) - * fw[..., None] - * fw[..., None, :], + wbar = B.mean(fw, axis=-1) + e_1 = B.mean(B.abs(obs[..., None] - fct) * fw, axis=-1) * ow / wbar + e_2 = B.mean( + B.abs(fct[..., None] - fct[..., None, :]) * fw[..., None] * fw[..., None, :], axis=(-1, -2), ) e_2 *= ow / (wbar**2) @@ -150,20 +141,15 @@ def vr_ensemble( fct: "Array", ow: "Array", fw: "Array", - ens_w: "Array", backend: "Backend" = None, ) -> "Array": - """Vertically Re-scaled CRPS estimator based on the energy form.""" + """Vertically Re-scaled CRPS for an ensemble forecast.""" B = backends.active if backend is None else backends[backend] - e_1 = B.sum(ens_w * B.abs(obs[..., None] - fct) * fw, axis=-1) * ow - e_2 = B.sum( - ens_w[..., None] - * ens_w[..., None, :] - * B.abs(fct[..., None] - fct[..., None, :]) - * fw[..., None] - * fw[..., None, :], + e_1 = B.mean(B.abs(obs[..., None] - fct) * fw, axis=-1) * ow + e_2 = B.mean( + B.abs(fct[..., None] - fct[..., None, :]) * fw[..., None] * fw[..., None, :], axis=(-1, -2), ) - e_3 = B.sum(ens_w * B.abs(fct) * fw, axis=-1) - B.abs(obs) * ow - e_3 *= B.sum(ens_w * fw, axis=1) - ow + e_3 = B.mean(B.abs(fct) * fw, axis=-1) - B.abs(obs) * ow + e_3 *= B.mean(fw, axis=1) - ow return e_1 - 0.5 * e_2 + e_3 diff --git a/scoringrules/core/crps/_approx_w.py b/scoringrules/core/crps/_approx_w.py new file mode 100644 index 0000000..d342da2 --- /dev/null +++ b/scoringrules/core/crps/_approx_w.py @@ -0,0 +1,159 @@ +import typing as tp + +from scoringrules.backend import backends + +if tp.TYPE_CHECKING: + from scoringrules.core.typing import Array, ArrayLike, Backend + + +def ensemble_w( + obs: "ArrayLike", + fct: "Array", + ens_w: "Array" = None, + estimator: str = "pwm", + backend: "Backend" = None, +) -> "Array": + """Compute the CRPS for a finite weighted ensemble.""" + if estimator == "nrg": + out = _crps_ensemble_nrg_w(obs, fct, ens_w, backend=backend) + elif estimator == "pwm": + out = _crps_ensemble_pwm_w(obs, fct, ens_w, backend=backend) + elif estimator == "fair": + out = _crps_ensemble_fair_w(obs, fct, ens_w, backend=backend) + elif estimator == "qd": + out = _crps_ensemble_qd_w(obs, fct, ens_w, backend=backend) + elif estimator == "akr": + out = _crps_ensemble_akr_w(obs, fct, ens_w, backend=backend) + elif estimator == "akr_circperm": + out = _crps_ensemble_akr_circperm_w(obs, fct, ens_w, backend=backend) + else: + raise ValueError( + f"{estimator} can only be used with `numpy` " + "backend and needs `numba` to be installed" + ) + + return out + + +def _crps_ensemble_fair_w( + obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None +) -> "Array": + """Fair version of the CRPS estimator based on the energy form.""" + B = backends.active if backend is None else backends[backend] + e_1 = B.sum(B.abs(obs[..., None] - fct) * w, axis=-1) + e_2 = B.sum( + B.abs(fct[..., None] - fct[..., None, :]) * w[..., None] * w[..., None, :], + axis=(-1, -2), + ) / (1 - B.sum(w * w, axis=-1)) + return e_1 - 0.5 * e_2 + + +def _crps_ensemble_nrg_w( + obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None +) -> "Array": + """CRPS estimator based on the energy form.""" + B = backends.active if backend is None else backends[backend] + e_1 = B.sum(B.abs(obs[..., None] - fct) * w, axis=-1) + e_2 = B.sum( + B.abs(fct[..., None] - fct[..., None, :]) * w[..., None] * w[..., None, :], + (-1, -2), + ) + return e_1 - 0.5 * e_2 + + +def _crps_ensemble_pwm_w( + obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None +) -> "Array": + """CRPS estimator based on the probability weighted moment (PWM) form.""" + B = backends.active if backend is None else backends[backend] + w_sum = B.cumsum(w, axis=-1) + expected_diff = B.sum(B.abs(obs[..., None] - fct) * w, axis=-1) + β_0 = B.sum(fct * w, axis=-1) + β_1 = B.sum(fct * w * (w_sum - w), axis=-1) / (1 - B.sum(w * w, axis=-1)) + return expected_diff + β_0 - 2.0 * β_1 + + +def _crps_ensemble_qd_w( + obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None +) -> "Array": + """CRPS estimator based on the quantile score decomposition.""" + B = backends.active if backend is None else backends[backend] + w_sum = B.cumsum(w, axis=-1) + a = w_sum - 0.5 * w + dif = fct - obs[..., None] + c = B.where(dif > 0, 1 - a, -a) + s = B.sum(w * c * dif, axis=-1) + return 2 * s + + +def _crps_ensemble_akr_w( + obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None +) -> "Array": + """CRPS estimator based on the approximate kernel representation.""" + B = backends.active if backend is None else backends[backend] + M = fct.shape[-1] + e_1 = B.sum(B.abs(obs[..., None] - fct) * w, axis=-1) + ind = [(i + 1) % M for i in range(M)] + e_2 = B.sum(B.abs(fct[..., ind] - fct) * w[..., ind], axis=-1) + return e_1 - 0.5 * e_2 + + +def _crps_ensemble_akr_circperm_w( + obs: "Array", fct: "Array", w: "Array", backend: "Backend" = None +) -> "Array": + """CRPS estimator based on the AKR with cyclic permutation.""" + B = backends.active if backend is None else backends[backend] + M = fct.shape[-1] + e_1 = B.sum(B.abs(obs[..., None] - fct) * w, axis=-1) + shift = int((M - 1) / 2) + ind = [(i + shift) % M for i in range(M)] + e_2 = B.sum(B.abs(fct[..., ind] - fct) * w[..., ind], axis=-1) + return e_1 - 0.5 * e_2 + + +def ow_ensemble_w( + obs: "Array", + fct: "Array", + ow: "Array", + fw: "Array", + ens_w: "Array", + backend: "Backend" = None, +) -> "Array": + """Outcome-Weighted CRPS for an ensemble forecast.""" + B = backends.active if backend is None else backends[backend] + wbar = B.sum(ens_w * fw, axis=-1) + e_1 = B.sum(ens_w * B.abs(obs[..., None] - fct) * fw, axis=-1) * ow / wbar + e_2 = B.sum( + ens_w[..., None] + * ens_w[..., None, :] + * B.abs(fct[..., None] - fct[..., None, :]) + * fw[..., None] + * fw[..., None, :], + axis=(-1, -2), + ) + e_2 *= ow / (wbar**2) + return e_1 - 0.5 * e_2 + + +def vr_ensemble_w( + obs: "Array", + fct: "Array", + ow: "Array", + fw: "Array", + ens_w: "Array", + backend: "Backend" = None, +) -> "Array": + """Vertically Re-scaled CRPS for an ensemble forecast.""" + B = backends.active if backend is None else backends[backend] + e_1 = B.sum(ens_w * B.abs(obs[..., None] - fct) * fw, axis=-1) * ow + e_2 = B.sum( + ens_w[..., None] + * ens_w[..., None, :] + * B.abs(fct[..., None] - fct[..., None, :]) + * fw[..., None] + * fw[..., None, :], + axis=(-1, -2), + ) + e_3 = B.sum(ens_w * B.abs(fct) * fw, axis=-1) - B.abs(obs) * ow + e_3 *= B.sum(ens_w * fw, axis=1) - ow + return e_1 - 0.5 * e_2 + e_3 diff --git a/scoringrules/core/crps/_gufuncs.py b/scoringrules/core/crps/_gufuncs.py index be8e0d1..abc33f0 100644 --- a/scoringrules/core/crps/_gufuncs.py +++ b/scoringrules/core/crps/_gufuncs.py @@ -26,15 +26,13 @@ def quantile_pinball_gufunc( @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:])", ], - "(),(n),(n)->()", + "(),(n)->()", ) -def _crps_ensemble_int_gufunc( - obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray -): - """CRPS estimator based on the integral form.""" # TODO: currently does not use weight argument +def _crps_ensemble_int_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): + """CRPS estimator based on the integral form.""" obs = obs[0] M = fct.shape[0] @@ -73,16 +71,15 @@ def _crps_ensemble_int_gufunc( @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:])", ], - "(),(n),(n)->()", + "(),(n)->()", ) -def _crps_ensemble_qd_gufunc( - obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray -): +def _crps_ensemble_qd_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): """CRPS estimator based on the quantile decomposition form.""" obs = obs[0] + M = fct.shape[-1] if np.isnan(obs): out[0] = np.nan @@ -90,56 +87,49 @@ def _crps_ensemble_qd_gufunc( obs_cdf = 0.0 integral = 0.0 - a = np.cumsum(w) - w / 2 for i, forecast in enumerate(fct): if obs < forecast: obs_cdf = 1.0 - integral += w[i] * (obs_cdf - a[i]) * (forecast - obs) + integral += (forecast - obs) * (M * obs_cdf - (i + 1) + 0.5) - out[0] = 2 * integral + out[0] = (2 / M**2) * integral @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:])", ], - "(),(n),(n)->()", + "(),(n)->()", ) -def _crps_ensemble_nrg_gufunc( - obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray -): +def _crps_ensemble_nrg_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): """CRPS estimator based on the energy form.""" obs = obs[0] M = fct.shape[-1] - if np.isnan(obs): out[0] = np.nan return - e_1 = 0 e_2 = 0 for i in range(M): - e_1 += abs(fct[i] - obs) * w[i] + e_1 += abs(fct[i] - obs) for j in range(i + 1, M): - e_2 += abs(fct[j] - fct[i]) * w[j] * w[i] + e_2 += 2 * abs(fct[j] - fct[i]) - out[0] = e_1 - e_2 + out[0] = e_1 / M - 0.5 * e_2 / (M**2) @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:])", ], - "(),(n),(n)->()", + "(),(n)->()", ) -def _crps_ensemble_fair_gufunc( - obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray -): +def _crps_ensemble_fair_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): """Fair version of the CRPS estimator based on the energy form.""" obs = obs[0] M = fct.shape[-1] @@ -152,27 +142,24 @@ def _crps_ensemble_fair_gufunc( e_2 = 0 for i in range(M): - e_1 += abs(fct[i] - obs) * w[i] + e_1 += abs(fct[i] - obs) for j in range(i + 1, M): - e_2 += abs(fct[j] - fct[i]) * w[j] * w[i] - - fair_c = 1 - np.sum(w**2) + e_2 += abs(fct[j] - fct[i]) - out[0] = e_1 - e_2 / fair_c + out[0] = e_1 / M - e_2 / (M * (M - 1)) @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:])", ], - "(),(n),(n)->()", + "(),(n)->()", ) -def _crps_ensemble_pwm_gufunc( - obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray -): +def _crps_ensemble_pwm_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): """CRPS estimator based on the probability weighted moment (PWM) form.""" obs = obs[0] + M = fct.shape[-1] if np.isnan(obs): out[0] = np.nan @@ -182,26 +169,22 @@ def _crps_ensemble_pwm_gufunc( β_0 = 0.0 β_1 = 0.0 - w_sum = np.cumsum(w) - for i, forecast in enumerate(fct): - expected_diff += np.abs(forecast - obs) * w[i] - β_0 += forecast * w[i] * (1.0 - w[i]) - β_1 += forecast * w[i] * (w_sum[i] - w[i]) + expected_diff += np.abs(forecast - obs) + β_0 += forecast + β_1 += forecast * i - out[0] = expected_diff + β_0 - 2 * β_1 + out[0] = expected_diff / M + β_0 / M - 2 * β_1 / (M * (M - 1)) @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:])", ], - "(),(n),(n)->()", + "(),(n)->()", ) -def _crps_ensemble_akr_gufunc( - obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray -): +def _crps_ensemble_akr_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): """CRPS estimaton based on the approximate kernel representation.""" M = fct.shape[-1] obs = obs[0] @@ -210,20 +193,21 @@ def _crps_ensemble_akr_gufunc( for i, forecast in enumerate(fct): if i == 0: i = M - 1 - e_1 += abs(forecast - obs) * w[i] - e_2 += abs(forecast - fct[i - 1]) * w[i] - out[0] = e_1 - 0.5 * e_2 + e_1 += abs(forecast - obs) + e_2 += abs(forecast - fct[i - 1]) + + out[0] = e_1 / M - 0.5 * 1 / M * e_2 @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:])", ], - "(),(n),(n)->()", + "(),(n)->()", ) def _crps_ensemble_akr_circperm_gufunc( - obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray + obs: np.ndarray, fct: np.ndarray, out: np.ndarray ): """CRPS estimaton based on the AKR with cyclic permutation.""" M = fct.shape[-1] @@ -232,85 +216,79 @@ def _crps_ensemble_akr_circperm_gufunc( e_2 = 0.0 for i, forecast in enumerate(fct): sigma_i = int((i + 1 + ((M - 1) / 2)) % M) - e_1 += abs(forecast - obs) * w[i] - e_2 += abs(forecast - fct[sigma_i]) * w[i] - out[0] = e_1 - 0.5 * e_2 + e_1 += abs(forecast - obs) + e_2 += abs(forecast - fct[sigma_i]) + out[0] = e_1 / M - 0.5 * 1 / M * e_2 @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:], float64[:])", ], - "(),(n),(),(n),(n)->()", + "(),(n),(),(n)->()", ) def _owcrps_ensemble_nrg_gufunc( obs: np.ndarray, fct: np.ndarray, ow: np.ndarray, fw: np.ndarray, - ens_w: np.ndarray, out: np.ndarray, ): """Outcome-weighted CRPS estimator based on the energy form.""" obs = obs[0] ow = ow[0] M = fct.shape[-1] - if np.isnan(obs): out[0] = np.nan return - e_1 = 0.0 e_2 = 0.0 for i in range(M): - e_1 += ens_w[i] * abs(fct[i] - obs) * fw[i] * ow + e_1 += abs(fct[i] - obs) * fw[i] * ow for j in range(i + 1, M): - e_2 += 2 * ens_w[i] * ens_w[j] * abs(fct[i] - fct[j]) * fw[i] * fw[j] * ow + e_2 += 2 * abs(fct[i] - fct[j]) * fw[i] * fw[j] * ow - wbar = np.sum(ens_w * fw) + wbar = np.mean(fw) - out[0] = e_1 / wbar - 0.5 * e_2 / (wbar**2) + out[0] = e_1 / (M * wbar) - 0.5 * e_2 / ((M * wbar) ** 2) @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:], float64[:])", ], - "(),(n),(),(n),(n)->()", + "(),(n),(),(n)->()", ) def _vrcrps_ensemble_nrg_gufunc( obs: np.ndarray, fct: np.ndarray, ow: np.ndarray, fw: np.ndarray, - ens_w: np.ndarray, out: np.ndarray, ): """Vertically re-scaled CRPS estimator based on the energy form.""" obs = obs[0] ow = ow[0] M = fct.shape[-1] - if np.isnan(obs): out[0] = np.nan return - e_1 = 0.0 e_2 = 0.0 for i in range(M): - e_1 += ens_w[i] * abs(fct[i] - obs) * fw[i] * ow + e_1 += abs(fct[i] - obs) * fw[i] * ow for j in range(i + 1, M): - e_2 += 2 * ens_w[i] * ens_w[j] * abs(fct[i] - fct[j]) * fw[i] * fw[j] + e_2 += 2 * abs(fct[i] - fct[j]) * fw[i] * fw[j] - wbar = np.sum(ens_w * fw) - wabs_x = np.sum(ens_w * np.abs(fct) * fw) + wbar = np.mean(fw) + wabs_x = np.mean(np.abs(fct) * fw) wabs_y = abs(obs) * ow - out[0] = e_1 - 0.5 * e_2 + (wabs_x - wabs_y) * (wbar - ow) + out[0] = e_1 / M - 0.5 * e_2 / (M**2) + (wabs_x - wabs_y) * (wbar - ow) estimator_gufuncs = { diff --git a/scoringrules/core/crps/_gufuncs_w.py b/scoringrules/core/crps/_gufuncs_w.py new file mode 100644 index 0000000..6baeb54 --- /dev/null +++ b/scoringrules/core/crps/_gufuncs_w.py @@ -0,0 +1,316 @@ +import numpy as np +from numba import guvectorize + +INV_SQRT_PI = 1 / np.sqrt(np.pi) +EPSILON = 1e-6 + + +@guvectorize( + [ + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", + ], + "(),(n),(n)->()", +) +def _crps_ensemble_int_w_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): + """CRPS estimator based on the integral form.""" + obs = obs[0] + + if np.isnan(obs): + out[0] = np.nan + return + + obs_cdf = 0 + forecast_cdf = 0.0 + prev_forecast = 0.0 + integral = 0.0 + + for n, forecast in enumerate(fct): + if np.isnan(forecast): + if n == 0: + integral = np.nan + forecast = prev_forecast # noqa: PLW2901 + break + + if obs_cdf == 0 and obs < forecast: + # this correctly handles the transition point of the obs CDF + integral += (obs - prev_forecast) * forecast_cdf**2 + integral += (forecast - obs) * (forecast_cdf - 1) ** 2 + obs_cdf = 1 + else: + integral += (forecast_cdf - obs_cdf) ** 2 * (forecast - prev_forecast) + + forecast_cdf += w[n] + prev_forecast = forecast + + if obs_cdf == 0: + integral += obs - forecast + + out[0] = integral + + +@guvectorize( + [ + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", + ], + "(),(n),(n)->()", +) +def _crps_ensemble_qd_w_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): + """CRPS estimator based on the quantile decomposition form.""" + obs = obs[0] + + if np.isnan(obs): + out[0] = np.nan + return + + obs_cdf = 0.0 + integral = 0.0 + a = np.cumsum(w) - w / 2 + + for i, forecast in enumerate(fct): + if obs < forecast: + obs_cdf = 1.0 + + integral += w[i] * (obs_cdf - a[i]) * (forecast - obs) + + out[0] = 2 * integral + + +@guvectorize( + [ + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", + ], + "(),(n),(n)->()", +) +def _crps_ensemble_nrg_w_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): + """CRPS estimator based on the energy form.""" + obs = obs[0] + M = fct.shape[-1] + + if np.isnan(obs): + out[0] = np.nan + return + + e_1 = 0 + e_2 = 0 + + for i in range(M): + e_1 += abs(fct[i] - obs) * w[i] + for j in range(i + 1, M): + e_2 += abs(fct[j] - fct[i]) * w[j] * w[i] + + out[0] = e_1 - e_2 + + +@guvectorize( + [ + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", + ], + "(),(n),(n)->()", +) +def _crps_ensemble_fair_w_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): + """Fair version of the CRPS estimator based on the energy form.""" + obs = obs[0] + M = fct.shape[-1] + + if np.isnan(obs): + out[0] = np.nan + return + + e_1 = 0 + e_2 = 0 + + for i in range(M): + e_1 += abs(fct[i] - obs) * w[i] + for j in range(i + 1, M): + e_2 += abs(fct[j] - fct[i]) * w[j] * w[i] + + fair_c = 1 - np.sum(w**2) + + out[0] = e_1 - e_2 / fair_c + + +@guvectorize( + [ + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", + ], + "(),(n),(n)->()", +) +def _crps_ensemble_pwm_w_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): + """CRPS estimator based on the probability weighted moment (PWM) form.""" + obs = obs[0] + + if np.isnan(obs): + out[0] = np.nan + return + + expected_diff = 0.0 + β_0 = 0.0 + β_1 = 0.0 + + w_sum = np.cumsum(w) + + for i, forecast in enumerate(fct): + expected_diff += np.abs(forecast - obs) * w[i] + β_0 += forecast * w[i] * (1.0 - w[i]) + β_1 += forecast * w[i] * (w_sum[i] - w[i]) + + out[0] = expected_diff + β_0 - 2 * β_1 + + +@guvectorize( + [ + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", + ], + "(),(n),(n)->()", +) +def _crps_ensemble_akr_w_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): + """CRPS estimaton based on the approximate kernel representation.""" + M = fct.shape[-1] + obs = obs[0] + e_1 = 0 + e_2 = 0 + for i, forecast in enumerate(fct): + if i == 0: + i = M - 1 + e_1 += abs(forecast - obs) * w[i] + e_2 += abs(forecast - fct[i - 1]) * w[i] + out[0] = e_1 - 0.5 * e_2 + + +@guvectorize( + [ + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", + ], + "(),(n),(n)->()", +) +def _crps_ensemble_akr_circperm_w_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): + """CRPS estimaton based on the AKR with cyclic permutation.""" + M = fct.shape[-1] + obs = obs[0] + e_1 = 0.0 + e_2 = 0.0 + for i, forecast in enumerate(fct): + sigma_i = int((i + 1 + ((M - 1) / 2)) % M) + e_1 += abs(forecast - obs) * w[i] + e_2 += abs(forecast - fct[sigma_i]) * w[i] + out[0] = e_1 - 0.5 * e_2 + + +@guvectorize( + [ + "void(float32[:], float32[:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:], float64[:], float64[:])", + ], + "(),(n),(),(n),(n)->()", +) +def _owcrps_ensemble_nrg_w_gufunc( + obs: np.ndarray, + fct: np.ndarray, + ow: np.ndarray, + fw: np.ndarray, + ens_w: np.ndarray, + out: np.ndarray, +): + """Outcome-weighted CRPS estimator based on the energy form.""" + obs = obs[0] + ow = ow[0] + M = fct.shape[-1] + + if np.isnan(obs): + out[0] = np.nan + return + + e_1 = 0.0 + e_2 = 0.0 + + for i in range(M): + e_1 += ens_w[i] * abs(fct[i] - obs) * fw[i] * ow + for j in range(i + 1, M): + e_2 += 2 * ens_w[i] * ens_w[j] * abs(fct[i] - fct[j]) * fw[i] * fw[j] * ow + + wbar = np.sum(ens_w * fw) + + out[0] = e_1 / wbar - 0.5 * e_2 / (wbar**2) + + +@guvectorize( + [ + "void(float32[:], float32[:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:], float64[:], float64[:])", + ], + "(),(n),(),(n),(n)->()", +) +def _vrcrps_ensemble_nrg_w_gufunc( + obs: np.ndarray, + fct: np.ndarray, + ow: np.ndarray, + fw: np.ndarray, + ens_w: np.ndarray, + out: np.ndarray, +): + """Vertically re-scaled CRPS estimator based on the energy form.""" + obs = obs[0] + ow = ow[0] + M = fct.shape[-1] + + if np.isnan(obs): + out[0] = np.nan + return + + e_1 = 0.0 + e_2 = 0.0 + + for i in range(M): + e_1 += ens_w[i] * abs(fct[i] - obs) * fw[i] * ow + for j in range(i + 1, M): + e_2 += 2 * ens_w[i] * ens_w[j] * abs(fct[i] - fct[j]) * fw[i] * fw[j] + + wbar = np.sum(ens_w * fw) + wabs_x = np.sum(ens_w * np.abs(fct) * fw) + wabs_y = abs(obs) * ow + + out[0] = e_1 - 0.5 * e_2 + (wabs_x - wabs_y) * (wbar - ow) + + +estimator_gufuncs_w = { + "akr_circperm": _crps_ensemble_akr_circperm_w_gufunc, + "akr": _crps_ensemble_akr_w_gufunc, + "fair": _crps_ensemble_fair_w_gufunc, + "int": _crps_ensemble_int_w_gufunc, + "nrg": _crps_ensemble_nrg_w_gufunc, + "pwm": _crps_ensemble_pwm_w_gufunc, + "qd": _crps_ensemble_qd_w_gufunc, + "ownrg": _owcrps_ensemble_nrg_w_gufunc, + "vrnrg": _vrcrps_ensemble_nrg_w_gufunc, +} + +__all__ = [ + "_crps_ensemble_akr_circperm_w_gufunc", + "_crps_ensemble_akr_w_gufunc", + "_crps_ensemble_fair_w_gufunc", + "_crps_ensemble_int_w_gufunc", + "_crps_ensemble_nrg_w_gufunc", + "_crps_ensemble_pwm_w_gufunc", + "_crps_ensemble_qd_w_gufunc", +] diff --git a/tests/test_crps.py b/tests/test_crps.py index a2340b1..9abee88 100644 --- a/tests/test_crps.py +++ b/tests/test_crps.py @@ -64,24 +64,22 @@ def test_crps_ensemble_corr(backend): # test equivalence of different estimators res_nrg = sr.crps_ensemble(obs, fct, estimator="nrg", backend=backend) - res_pwm = sr.crps_ensemble(obs, fct, estimator="pwm", backend=backend) res_qd = sr.crps_ensemble(obs, fct, estimator="qd", backend=backend) + res_fair = sr.crps_ensemble(obs, fct, estimator="fair", backend=backend) + res_pwm = sr.crps_ensemble(obs, fct, estimator="pwm", backend=backend) if backend in ["torch", "jax"]: - assert np.allclose(res_nrg, res_pwm, rtol=1e-03) assert np.allclose(res_nrg, res_qd, rtol=1e-03) + assert np.allclose(res_fair, res_pwm, rtol=1e-03) else: - assert np.allclose(res_nrg, res_pwm) assert np.allclose(res_nrg, res_qd) + assert np.allclose(res_fair, res_pwm) w = np.abs(np.random.randn(N, ENSEMBLE_SIZE) * sigma[..., None]) res_nrg = sr.crps_ensemble(obs, fct, ens_w=w, estimator="nrg", backend=backend) - res_pwm = sr.crps_ensemble(obs, fct, ens_w=w, estimator="pwm", backend=backend) res_qd = sr.crps_ensemble(obs, fct, ens_w=w, estimator="qd", backend=backend) if backend in ["torch", "jax"]: - assert np.allclose(res_nrg, res_pwm, rtol=1e-03) assert np.allclose(res_nrg, res_qd, rtol=1e-03) else: - assert np.allclose(res_nrg, res_pwm) assert np.allclose(res_nrg, res_qd) # test correctness @@ -100,11 +98,11 @@ def test_crps_ensemble_corr(backend): 0.9064937, ] ) - res = sr.crps_ensemble(obs, fct) + res = sr.crps_ensemble(obs, fct, estimator="qd") assert np.isclose(res, 0.6126602) w = np.arange(10) - res = sr.crps_ensemble(obs, fct, ens_w=w) + res = sr.crps_ensemble(obs, fct, ens_w=w, estimator="qd") assert np.isclose(res, 0.4923673) From fa5f4f9a66fa0301546de9caa085bee1a02fe7d3 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Tue, 5 Aug 2025 17:20:12 +0200 Subject: [PATCH 75/79] add separate weighted crps functions for weighted ensembles --- scoringrules/_crps.py | 73 +++++++++++++++++++++++++------------------ tests/test_wcrps.py | 2 +- 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/scoringrules/_crps.py b/scoringrules/_crps.py index ba63360..1550e90 100644 --- a/scoringrules/_crps.py +++ b/scoringrules/_crps.py @@ -133,6 +133,7 @@ def crps_ensemble( if sort_ensemble: ind = B.argsort(fct_uns, axis=-1) ens_w = B.gather(ens_w, ind, axis=-1) + if backend == "numba": if estimator not in crps.estimator_gufuncs: raise ValueError( @@ -339,18 +340,8 @@ def owcrps_ensemble( B = backends.active if backend is None else backends[backend] obs, fct = map(B.asarray, (obs, fct)) - if ens_w is None: - M = fct.shape[m_axis] - ens_w = B.zeros(fct.shape) + 1.0 / M - else: - ens_w = B.asarray(ens_w) - if B.any(ens_w < 0): - raise ValueError("`ens_w` contains negative entries") - ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) - if m_axis != -1: fct = B.moveaxis(fct, m_axis, -1) - ens_w = B.moveaxis(ens_w, m_axis, -1) if w_func is None: @@ -361,12 +352,28 @@ def w_func(x): if B.any(obs_weights < 0) or B.any(fct_weights < 0): raise ValueError("`w_func` returns negative values") - if backend == "numba": - return crps.estimator_gufuncs["ownrg"]( - obs, fct, obs_weights, fct_weights, ens_w - ) + if ens_w is None: + if backend == "numba": + return crps.estimator_gufuncs["ownrg"](obs, fct, obs_weights, fct_weights) + + return crps.ow_ensemble(obs, fct, obs_weights, fct_weights, backend=backend) - return crps.ow_ensemble(obs, fct, obs_weights, fct_weights, ens_w, backend=backend) + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) + if m_axis != -1: + ens_w = B.moveaxis(ens_w, m_axis, -1) + + if backend == "numba": + return crps.estimator_gufuncs_w["ownrg"]( + obs, fct, obs_weights, fct_weights, ens_w + ) + + return crps.ow_ensemble_w( + obs, fct, obs_weights, fct_weights, ens_w, backend=backend + ) def vrcrps_ensemble( @@ -455,18 +462,8 @@ def vrcrps_ensemble( B = backends.active if backend is None else backends[backend] obs, fct = map(B.asarray, (obs, fct)) - if ens_w is None: - M = fct.shape[m_axis] - ens_w = B.zeros(fct.shape) + 1.0 / M - else: - ens_w = B.asarray(ens_w) - if B.any(ens_w < 0): - raise ValueError("`ens_w` contains negative entries") - ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) - if m_axis != -1: fct = B.moveaxis(fct, m_axis, -1) - ens_w = B.moveaxis(ens_w, m_axis, -1) if w_func is None: @@ -477,12 +474,28 @@ def w_func(x): if B.any(obs_weights < 0) or B.any(fct_weights < 0): raise ValueError("`w_func` returns negative values") - if backend == "numba": - return crps.estimator_gufuncs["vrnrg"]( - obs, fct, obs_weights, fct_weights, ens_w - ) + if ens_w is None: + if backend == "numba": + return crps.estimator_gufuncs["vrnrg"](obs, fct, obs_weights, fct_weights) + + return crps.vr_ensemble(obs, fct, obs_weights, fct_weights, backend=backend) - return crps.vr_ensemble(obs, fct, obs_weights, fct_weights, ens_w, backend=backend) + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) + if m_axis != -1: + ens_w = B.moveaxis(ens_w, m_axis, -1) + + if backend == "numba": + return crps.estimator_gufuncs_w["vrnrg"]( + obs, fct, obs_weights, fct_weights, ens_w + ) + + return crps.vr_ensemble_w( + obs, fct, obs_weights, fct_weights, ens_w, backend=backend + ) def crps_quantile( diff --git a/tests/test_wcrps.py b/tests/test_wcrps.py index da3c779..10d951f 100644 --- a/tests/test_wcrps.py +++ b/tests/test_wcrps.py @@ -77,7 +77,7 @@ def test_owcrps_vs_crps(backend): sigma = abs(np.random.randn(N)) * 0.5 fct = np.random.randn(N, M) * sigma[..., None] + mu[..., None] - res = sr.crps_ensemble(obs, fct, backend=backend) + res = sr.crps_ensemble(obs, fct, estimator="qd", backend=backend) # no argument given resw = sr.owcrps_ensemble(obs, fct, backend=backend) From 5a50e9c655c198a534363490bc262f0706901286 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Tue, 5 Aug 2025 17:47:38 +0200 Subject: [PATCH 76/79] add separate energy score functions for weighted ensembles --- scoringrules/_energy.py | 68 +++++++++------- scoringrules/core/energy/__init__.py | 25 ++++++ scoringrules/core/energy/_gufuncs.py | 55 +++++-------- scoringrules/core/energy/_gufuncs_w.py | 103 +++++++++++++++++++++++++ scoringrules/core/energy/_score.py | 29 +++---- scoringrules/core/energy/_score_w.py | 82 ++++++++++++++++++++ tests/test_wenergy.py | 4 +- 7 files changed, 285 insertions(+), 81 deletions(-) create mode 100644 scoringrules/core/energy/_gufuncs_w.py create mode 100644 scoringrules/core/energy/_score_w.py diff --git a/scoringrules/_energy.py b/scoringrules/_energy.py index 72fcbe9..7732099 100644 --- a/scoringrules/_energy.py +++ b/scoringrules/_energy.py @@ -67,15 +67,19 @@ def es_ensemble( obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) if ens_w is None: - M = fct.shape[-2] - ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M - else: - ens_w = B.moveaxis(ens_w, m_axis, -2) + if backend == "numba": + return energy._energy_score_gufunc(obs, fct) - if backend == "numba": - return energy._energy_score_gufunc(obs, fct, ens_w) + return energy.nrg(obs, fct, backend=backend) + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=-1, keepdims=True) + if backend == "numba": + return energy._energy_score_gufunc_w(obs, fct, ens_w) - return energy.nrg(obs, fct, ens_w, backend=backend) + return energy.nrg_w(obs, fct, ens_w, backend=backend) def twes_ensemble( @@ -192,17 +196,23 @@ def owes_ensemble( obs_weights = B.apply_along_axis(w_func, obs, -1) if ens_w is None: - M = fct.shape[-2] - ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M - else: - ens_w = B.moveaxis(ens_w, m_axis, -2) + if B.name == "numba": + return energy._owenergy_score_gufunc(obs, fct, obs_weights, fct_weights) - if B.name == "numba": - return energy._owenergy_score_gufunc(obs, fct, obs_weights, fct_weights, ens_w) + return energy.ownrg(obs, fct, obs_weights, fct_weights, backend=backend) + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=-1, keepdims=True) + if B.name == "numba": + return energy._owenergy_score_gufunc_w( + obs, fct, obs_weights, fct_weights, ens_w + ) - return energy.ownrg( - obs, fct, obs_weights, fct_weights, ens_w=ens_w, backend=backend - ) + return energy.ownrg_w( + obs, fct, obs_weights, fct_weights, ens_w=ens_w, backend=backend + ) def vres_ensemble( @@ -265,14 +275,20 @@ def vres_ensemble( obs_weights = B.apply_along_axis(w_func, obs, -1) if ens_w is None: - M = fct.shape[-2] - ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M - else: - ens_w = B.moveaxis(ens_w, m_axis, -2) - - if backend == "numba": - return energy._vrenergy_score_gufunc(obs, fct, obs_weights, fct_weights, ens_w) + if backend == "numba": + return energy._vrenergy_score_gufunc(obs, fct, obs_weights, fct_weights) - return energy.vrnrg( - obs, fct, obs_weights, fct_weights, ens_w=ens_w, backend=backend - ) + return energy.vrnrg(obs, fct, obs_weights, fct_weights, backend=backend) + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=-1, keepdims=True) + if backend == "numba": + return energy._vrenergy_score_gufunc_w( + obs, fct, obs_weights, fct_weights, ens_w + ) + + return energy.vrnrg_w( + obs, fct, obs_weights, fct_weights, ens_w=ens_w, backend=backend + ) diff --git a/scoringrules/core/energy/__init__.py b/scoringrules/core/energy/__init__.py index b9bc987..134bb2b 100644 --- a/scoringrules/core/energy/__init__.py +++ b/scoringrules/core/energy/__init__.py @@ -9,15 +9,40 @@ _owenergy_score_gufunc = None _vrenergy_score_gufunc = None +try: + from ._gufuncs_w import ( + _energy_score_gufunc_w, + _owenergy_score_gufunc_w, + _vrenergy_score_gufunc_w, + ) + +except ImportError: + _energy_score_gufunc = None + _energy_score_gufunc_w = None + _owenergy_score_gufunc = None + _owenergy_score_gufunc_w = None + _vrenergy_score_gufunc = None + _vrenergy_score_gufunc_w = None + from ._score import es_ensemble as nrg from ._score import owes_ensemble as ownrg from ._score import vres_ensemble as vrnrg +from ._score_w import es_ensemble_w as nrg_w +from ._score_w import owes_ensemble_w as ownrg_w +from ._score_w import vres_ensemble_w as vrnrg_w + __all__ = [ "nrg", + "nrg_w", "ownrg", + "ownrg_w", "vrnrg", + "vrnrg_w", "_energy_score_gufunc", + "_energy_score_gufunc_w", "_owenergy_score_gufunc", + "_owenergy_score_gufunc_w", "_vrenergy_score_gufunc", + "_vrenergy_score_gufunc_w", ] diff --git a/scoringrules/core/energy/_gufuncs.py b/scoringrules/core/energy/_gufuncs.py index fc95a39..9a29659 100644 --- a/scoringrules/core/energy/_gufuncs.py +++ b/scoringrules/core/energy/_gufuncs.py @@ -4,43 +4,40 @@ @guvectorize( [ - "void(float32[:], float32[:,:], float32[:], float32[:])", - "void(float64[:], float64[:,:], float64[:], float64[:])", + "void(float32[:], float32[:,:], float32[:])", + "void(float64[:], float64[:,:], float64[:])", ], - "(d),(m,d),(m)->()", + "(d),(m,d)->()", ) def _energy_score_gufunc( obs: np.ndarray, fct: np.ndarray, - ens_w: np.ndarray, out: np.ndarray, ): """Compute the Energy Score for a finite ensemble.""" M = fct.shape[0] - e_1 = 0.0 e_2 = 0.0 for i in range(M): - e_1 += float(np.linalg.norm(fct[i] - obs)) * ens_w[i] + e_1 += float(np.linalg.norm(fct[i] - obs)) for j in range(i + 1, M): - e_2 += 2 * float(np.linalg.norm(fct[i] - fct[j])) * ens_w[i] * ens_w[j] + e_2 += 2 * float(np.linalg.norm(fct[i] - fct[j])) - out[0] = e_1 - 0.5 * e_2 + out[0] = e_1 / M - 0.5 / (M**2) * e_2 @guvectorize( [ - "void(float32[:], float32[:,:], float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:,:], float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:,:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:], float64[:])", ], - "(d),(m,d),(),(m),(m)->()", + "(d),(m,d),(),(m)->()", ) def _owenergy_score_gufunc( obs: np.ndarray, fct: np.ndarray, ow: np.ndarray, fw: np.ndarray, - ens_w: np.ndarray, out: np.ndarray, ): """Compute the Outcome-Weighted Energy Score for a finite ensemble.""" @@ -50,33 +47,27 @@ def _owenergy_score_gufunc( e_1 = 0.0 e_2 = 0.0 for i in range(M): - e_1 += float(np.linalg.norm(fct[i] - obs) * fw[i] * ow) * ens_w[i] + e_1 += float(np.linalg.norm(fct[i] - obs) * fw[i] * ow) for j in range(i + 1, M): - e_2 += ( - 2 - * float(np.linalg.norm(fct[i] - fct[j]) * fw[i] * fw[j] * ow) - * ens_w[i] - * ens_w[j] - ) + e_2 += 2 * float(np.linalg.norm(fct[i] - fct[j]) * fw[i] * fw[j] * ow) wbar = np.mean(fw) - out[0] = e_1 / (wbar) - 0.5 * e_2 / (wbar**2) + out[0] = e_1 / (M * wbar) - 0.5 * e_2 / (M**2 * wbar**2) @guvectorize( [ - "void(float32[:], float32[:,:], float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:,:], float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:,:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:], float64[:])", ], - "(d),(m,d),(),(m),(m)->()", + "(d),(m,d),(),(m)->()", ) def _vrenergy_score_gufunc( obs: np.ndarray, fct: np.ndarray, ow: np.ndarray, fw: np.ndarray, - ens_w: np.ndarray, out: np.ndarray, ): """Compute the Vertically Re-scaled Energy Score for a finite ensemble.""" @@ -87,17 +78,13 @@ def _vrenergy_score_gufunc( e_2 = 0.0 wabs_x = 0.0 for i in range(M): - e_1 += float(np.linalg.norm(fct[i] - obs) * fw[i] * ow) * ens_w[i] - wabs_x += np.linalg.norm(fct[i]) * fw[i] * ens_w[i] + e_1 += float(np.linalg.norm(fct[i] - obs) * fw[i] * ow) + wabs_x += np.linalg.norm(fct[i]) * fw[i] for j in range(i + 1, M): - e_2 += ( - 2 - * float(np.linalg.norm(fct[i] - fct[j]) * fw[i] * fw[j]) - * ens_w[i] - * ens_w[j] - ) + e_2 += 2 * float(np.linalg.norm(fct[i] - fct[j]) * fw[i] * fw[j]) - wbar = np.sum(fw * ens_w) + wabs_x = wabs_x / M + wbar = np.mean(fw) wabs_y = np.linalg.norm(obs) * ow - out[0] = e_1 - 0.5 * e_2 + (wabs_x - wabs_y) * (wbar - ow) + out[0] = e_1 / M - 0.5 * e_2 / (M**2) + (wabs_x - wabs_y) * (wbar - ow) diff --git a/scoringrules/core/energy/_gufuncs_w.py b/scoringrules/core/energy/_gufuncs_w.py new file mode 100644 index 0000000..19e443c --- /dev/null +++ b/scoringrules/core/energy/_gufuncs_w.py @@ -0,0 +1,103 @@ +import numpy as np +from numba import guvectorize + + +@guvectorize( + [ + "void(float32[:], float32[:,:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:])", + ], + "(d),(m,d),(m)->()", +) +def _energy_score_gufunc_w( + obs: np.ndarray, + fct: np.ndarray, + ens_w: np.ndarray, + out: np.ndarray, +): + """Compute the Energy Score for a finite ensemble.""" + M = fct.shape[0] + + e_1 = 0.0 + e_2 = 0.0 + for i in range(M): + e_1 += float(np.linalg.norm(fct[i] - obs)) * ens_w[i] + for j in range(i + 1, M): + e_2 += 2 * float(np.linalg.norm(fct[i] - fct[j])) * ens_w[i] * ens_w[j] + + out[0] = e_1 - 0.5 * e_2 + + +@guvectorize( + [ + "void(float32[:], float32[:,:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:], float64[:], float64[:])", + ], + "(d),(m,d),(),(m),(m)->()", +) +def _owenergy_score_gufunc_w( + obs: np.ndarray, + fct: np.ndarray, + ow: np.ndarray, + fw: np.ndarray, + ens_w: np.ndarray, + out: np.ndarray, +): + """Compute the Outcome-Weighted Energy Score for a finite ensemble.""" + M = fct.shape[0] + ow = ow[0] + + e_1 = 0.0 + e_2 = 0.0 + for i in range(M): + e_1 += float(np.linalg.norm(fct[i] - obs) * fw[i] * ow) * ens_w[i] + for j in range(i + 1, M): + e_2 += ( + 2 + * float(np.linalg.norm(fct[i] - fct[j]) * fw[i] * fw[j] * ow) + * ens_w[i] + * ens_w[j] + ) + + wbar = np.mean(fw) + + out[0] = e_1 / (wbar) - 0.5 * e_2 / (wbar**2) + + +@guvectorize( + [ + "void(float32[:], float32[:,:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:], float64[:], float64[:])", + ], + "(d),(m,d),(),(m),(m)->()", +) +def _vrenergy_score_gufunc_w( + obs: np.ndarray, + fct: np.ndarray, + ow: np.ndarray, + fw: np.ndarray, + ens_w: np.ndarray, + out: np.ndarray, +): + """Compute the Vertically Re-scaled Energy Score for a finite ensemble.""" + M = fct.shape[0] + ow = ow[0] + + e_1 = 0.0 + e_2 = 0.0 + wabs_x = 0.0 + for i in range(M): + e_1 += float(np.linalg.norm(fct[i] - obs) * fw[i] * ow) * ens_w[i] + wabs_x += np.linalg.norm(fct[i]) * fw[i] * ens_w[i] + for j in range(i + 1, M): + e_2 += ( + 2 + * float(np.linalg.norm(fct[i] - fct[j]) * fw[i] * fw[j]) + * ens_w[i] + * ens_w[j] + ) + + wbar = np.sum(fw * ens_w) + wabs_y = np.linalg.norm(obs) * ow + + out[0] = e_1 - 0.5 * e_2 + (wabs_x - wabs_y) * (wbar - ow) diff --git a/scoringrules/core/energy/_score.py b/scoringrules/core/energy/_score.py index ff5b615..8bf1059 100644 --- a/scoringrules/core/energy/_score.py +++ b/scoringrules/core/energy/_score.py @@ -6,7 +6,7 @@ from scoringrules.core.typing import Array, Backend -def es_ensemble(obs: "Array", fct: "Array", ens_w: "Array", backend=None) -> "Array": +def es_ensemble(obs: "Array", fct: "Array", backend=None) -> "Array": """ Compute the energy score based on a finite ensemble. @@ -15,12 +15,10 @@ def es_ensemble(obs: "Array", fct: "Array", ens_w: "Array", backend=None) -> "Ar B = backends.active if backend is None else backends[backend] err_norm = B.norm(fct - B.expand_dims(obs, -2), -1) - E_1 = B.sum(err_norm * ens_w, -1) + E_1 = B.mean(err_norm, axis=-1) spread_norm = B.norm(B.expand_dims(fct, -3) - B.expand_dims(fct, -2), -1) - E_2 = B.sum( - spread_norm * B.expand_dims(ens_w, -1) * B.expand_dims(ens_w, -2), (-2, -1) - ) + E_2 = B.mean(spread_norm, axis=(-2, -1)) return E_1 - 0.5 * E_2 @@ -30,24 +28,21 @@ def owes_ensemble( fct: "Array", # (... M D) ow: "Array", # (...) fw: "Array", # (... M) - ens_w: "Array", # (... M) backend: "Backend" = None, ) -> "Array": """Compute the outcome-weighted energy score based on a finite ensemble.""" B = backends.active if backend is None else backends[backend] - wbar = B.sum(fw * ens_w, -1) + wbar = B.mean(fw, axis=-1) err_norm = B.norm(fct - B.expand_dims(obs, -2), -1) # (... M) - E_1 = B.sum(err_norm * fw * B.expand_dims(ow, -1) * ens_w, -1) / wbar # (...) + E_1 = B.mean(err_norm * fw * B.expand_dims(ow, -1), axis=-1) / wbar # (...) spread_norm = B.norm( B.expand_dims(fct, -2) - B.expand_dims(fct, -3), -1 ) # (... M M) fw_prod = B.expand_dims(fw, -1) * B.expand_dims(fw, -2) # (... M M) spread_norm *= fw_prod * B.expand_dims(ow, (-2, -1)) # (... M M) - E_2 = B.sum( - spread_norm * B.expand_dims(ens_w, -1) * B.expand_dims(ens_w, -2), (-2, -1) - ) / (wbar**2) # (...) + E_2 = B.mean(spread_norm, axis=(-2, -1)) / (wbar**2) # (...) return E_1 - 0.5 * E_2 @@ -57,26 +52,22 @@ def vres_ensemble( fct: "Array", ow: "Array", fw: "Array", - ens_w: "Array", backend: "Backend" = None, ) -> "Array": """Compute the vertically re-scaled energy score based on a finite ensemble.""" B = backends.active if backend is None else backends[backend] - wbar = B.sum(fw * ens_w, -1) + wbar = B.mean(fw, axis=-1) err_norm = B.norm(fct - B.expand_dims(obs, -2), -1) # (... M) err_norm *= fw * B.expand_dims(ow, -1) # (... M) - E_1 = B.sum(err_norm * ens_w, -1) # (...) + E_1 = B.mean(err_norm, axis=-1) # (...) spread_norm = B.norm( B.expand_dims(fct, -2) - B.expand_dims(fct, -3), -1 ) # (... M M) fw_prod = B.expand_dims(fw, -2) * B.expand_dims(fw, -1) # (... M M) - E_2 = B.sum( - spread_norm * fw_prod * B.expand_dims(ens_w, -1) * B.expand_dims(ens_w, -2), - (-2, -1), - ) # (...) + E_2 = B.mean(spread_norm * fw_prod, axis=(-2, -1)) # (...) - rhobar = B.sum(B.norm(fct, -1) * fw * ens_w, -1) # (...) + rhobar = B.mean(B.norm(fct, -1) * fw, axis=-1) # (...) E_3 = (rhobar - B.norm(obs, -1) * ow) * (wbar - ow) # (...) return E_1 - 0.5 * E_2 + E_3 diff --git a/scoringrules/core/energy/_score_w.py b/scoringrules/core/energy/_score_w.py new file mode 100644 index 0000000..3cd1759 --- /dev/null +++ b/scoringrules/core/energy/_score_w.py @@ -0,0 +1,82 @@ +import typing as tp + +from scoringrules.backend import backends + +if tp.TYPE_CHECKING: + from scoringrules.core.typing import Array, Backend + + +def es_ensemble_w(obs: "Array", fct: "Array", ens_w: "Array", backend=None) -> "Array": + """ + Compute the energy score based on a finite ensemble. + + The ensemble and variables axes are on the second last and last dimensions respectively. + """ + B = backends.active if backend is None else backends[backend] + + err_norm = B.norm(fct - B.expand_dims(obs, -2), -1) + E_1 = B.sum(err_norm * ens_w, -1) + + spread_norm = B.norm(B.expand_dims(fct, -3) - B.expand_dims(fct, -2), -1) + E_2 = B.sum( + spread_norm * B.expand_dims(ens_w, -1) * B.expand_dims(ens_w, -2), (-2, -1) + ) + + return E_1 - 0.5 * E_2 + + +def owes_ensemble_w( + obs: "Array", # (... D) + fct: "Array", # (... M D) + ow: "Array", # (...) + fw: "Array", # (... M) + ens_w: "Array", # (... M) + backend: "Backend" = None, +) -> "Array": + """Compute the outcome-weighted energy score based on a finite ensemble.""" + B = backends.active if backend is None else backends[backend] + wbar = B.sum(fw * ens_w, -1) + + err_norm = B.norm(fct - B.expand_dims(obs, -2), -1) # (... M) + E_1 = B.sum(err_norm * fw * B.expand_dims(ow, -1) * ens_w, -1) / wbar # (...) + + spread_norm = B.norm( + B.expand_dims(fct, -2) - B.expand_dims(fct, -3), -1 + ) # (... M M) + fw_prod = B.expand_dims(fw, -1) * B.expand_dims(fw, -2) # (... M M) + spread_norm *= fw_prod * B.expand_dims(ow, (-2, -1)) # (... M M) + E_2 = B.sum( + spread_norm * B.expand_dims(ens_w, -1) * B.expand_dims(ens_w, -2), (-2, -1) + ) / (wbar**2) # (...) + + return E_1 - 0.5 * E_2 + + +def vres_ensemble_w( + obs: "Array", + fct: "Array", + ow: "Array", + fw: "Array", + ens_w: "Array", + backend: "Backend" = None, +) -> "Array": + """Compute the vertically re-scaled energy score based on a finite ensemble.""" + B = backends.active if backend is None else backends[backend] + wbar = B.sum(fw * ens_w, -1) + + err_norm = B.norm(fct - B.expand_dims(obs, -2), -1) # (... M) + err_norm *= fw * B.expand_dims(ow, -1) # (... M) + E_1 = B.sum(err_norm * ens_w, -1) # (...) + + spread_norm = B.norm( + B.expand_dims(fct, -2) - B.expand_dims(fct, -3), -1 + ) # (... M M) + fw_prod = B.expand_dims(fw, -2) * B.expand_dims(fw, -1) # (... M M) + E_2 = B.sum( + spread_norm * fw_prod * B.expand_dims(ens_w, -1) * B.expand_dims(ens_w, -2), + (-2, -1), + ) # (...) + + rhobar = B.sum(B.norm(fct, -1) * fw * ens_w, -1) # (...) + E_3 = (rhobar - B.norm(obs, -1) * ow) * (wbar - ow) # (...) + return E_1 - 0.5 * E_2 + E_3 diff --git a/tests/test_wenergy.py b/tests/test_wenergy.py index ed1a100..4931f9d 100644 --- a/tests/test_wenergy.py +++ b/tests/test_wenergy.py @@ -59,13 +59,13 @@ def test_owenergy_score_correctness(backend): obs = np.array([0.2743836, 0.8146400]) def w_func(x): - return backends[backend].all(x > 0.2) + return backends[backend].all(x > 0.2) * 1.0 res = sr.owes_ensemble(obs, fct, w_func, backend=backend) np.testing.assert_allclose(res, 0.2274243, rtol=1e-6) def w_func(x): - return backends[backend].all(x < 1.0) + return backends[backend].all(x < 1.0) * 1.0 res = sr.owes_ensemble(obs, fct, w_func, backend=backend) np.testing.assert_allclose(res, 0.3345418, rtol=1e-6) From bc4d2a6bfc7809fe2f859ea23af911a39961f790 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Tue, 5 Aug 2025 18:52:09 +0200 Subject: [PATCH 77/79] add separate variogram functions for weighted ensemble members --- scoringrules/_variogram.py | 93 ++++++++++++------- scoringrules/core/variogram/__init__.py | 21 +++++ scoringrules/core/variogram/_gufuncs.py | 56 +++++------- scoringrules/core/variogram/_gufuncs_w.py | 101 +++++++++++++++++++++ scoringrules/core/variogram/_score.py | 25 ++---- scoringrules/core/variogram/_score_w.py | 104 ++++++++++++++++++++++ tests/test_wvariogram.py | 4 +- 7 files changed, 319 insertions(+), 85 deletions(-) create mode 100644 scoringrules/core/variogram/_gufuncs_w.py create mode 100644 scoringrules/core/variogram/_score_w.py diff --git a/scoringrules/_variogram.py b/scoringrules/_variogram.py index cdf6963..0d88a9d 100644 --- a/scoringrules/_variogram.py +++ b/scoringrules/_variogram.py @@ -77,20 +77,26 @@ def vs_ensemble( B = backends.active if backend is None else backends[backend] obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) - if ens_w is None: - M = fct.shape[-2] - ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M - else: - ens_w = B.moveaxis(ens_w, m_axis, -2) - if w is None: D = fct.shape[-1] w = B.zeros(obs.shape + (D,)) + 1.0 - if backend == "numba": - return variogram._variogram_score_gufunc(obs, fct, w, ens_w, p) + if ens_w is None: + if backend == "numba": + return variogram._variogram_score_gufunc(obs, fct, w, p) + + return variogram.vs(obs, fct, w, p, backend=backend) + + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=-1, keepdims=True) + + if backend == "numba": + return variogram._variogram_score_gufunc_w(obs, fct, w, ens_w, p) - return variogram.vs(obs, fct, w, ens_w, p, backend=backend) + return variogram.vs_w(obs, fct, w, ens_w, p, backend=backend) def twvs_ensemble( @@ -241,24 +247,34 @@ def owvs_ensemble( obs_weights = B.apply_along_axis(w_func, obs, -1) fct_weights = B.apply_along_axis(w_func, fct, -1) - if ens_w is None: - M = fct.shape[-2] - ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M - else: - ens_w = B.moveaxis(ens_w, m_axis, -2) - if w is None: D = fct.shape[-1] w = B.zeros(obs.shape + (D,)) + 1.0 - if backend == "numba": - return variogram._owvariogram_score_gufunc( - obs, fct, w, obs_weights, fct_weights, ens_w, p + if ens_w is None: + if backend == "numba": + return variogram._owvariogram_score_gufunc( + obs, fct, w, obs_weights, fct_weights, p + ) + + return variogram.owvs( + obs, fct, w, obs_weights, fct_weights, p=p, backend=backend ) - return variogram.owvs( - obs, fct, w, obs_weights, fct_weights, ens_w=ens_w, p=p, backend=backend - ) + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=-1, keepdims=True) + + if backend == "numba": + return variogram._owvariogram_score_gufunc_w( + obs, fct, w, obs_weights, fct_weights, ens_w, p + ) + + return variogram.owvs_w( + obs, fct, w, obs_weights, fct_weights, ens_w=ens_w, p=p, backend=backend + ) def vrvs_ensemble( @@ -332,27 +348,36 @@ def vrvs_ensemble( array([46.48256493, 57.90759816, 92.37153472]) """ B = backends.active if backend is None else backends[backend] - obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) obs_weights = B.apply_along_axis(w_func, obs, -1) fct_weights = B.apply_along_axis(w_func, fct, -1) - if ens_w is None: - M = fct.shape[-2] - ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M - else: - ens_w = B.moveaxis(ens_w, m_axis, -2) - if w is None: D = fct.shape[-1] w = B.zeros(obs.shape + (D,)) + 1.0 - if backend == "numba": - return variogram._vrvariogram_score_gufunc( - obs, fct, w, obs_weights, fct_weights, ens_w, p + if ens_w is None: + if backend == "numba": + return variogram._vrvariogram_score_gufunc( + obs, fct, w, obs_weights, fct_weights, p + ) + + return variogram.vrvs( + obs, fct, w, obs_weights, fct_weights, p=p, backend=backend ) - return variogram.vrvs( - obs, fct, w, obs_weights, fct_weights, ens_w=ens_w, p=p, backend=backend - ) + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=-1, keepdims=True) + + if backend == "numba": + return variogram._vrvariogram_score_gufunc_w( + obs, fct, w, obs_weights, fct_weights, ens_w, p + ) + + return variogram.vrvs_w( + obs, fct, w, obs_weights, fct_weights, ens_w=ens_w, p=p, backend=backend + ) diff --git a/scoringrules/core/variogram/__init__.py b/scoringrules/core/variogram/__init__.py index 1e74527..8b9749b 100644 --- a/scoringrules/core/variogram/__init__.py +++ b/scoringrules/core/variogram/__init__.py @@ -9,15 +9,36 @@ _variogram_score_gufunc = None _vrvariogram_score_gufunc = None +try: + from ._gufuncs_w import ( + _owvariogram_score_gufunc_w, + _variogram_score_gufunc_w, + _vrvariogram_score_gufunc_w, + ) +except ImportError: + _owvariogram_score_gufunc_w = None + _variogram_score_gufunc_w = None + _vrvariogram_score_gufunc_w = None + from ._score import owvs_ensemble as owvs from ._score import vs_ensemble as vs from ._score import vrvs_ensemble as vrvs +from ._score_w import owvs_ensemble_w as owvs_w +from ._score_w import vs_ensemble_w as vs_w +from ._score_w import vrvs_ensemble_w as vrvs_w + __all__ = [ "vs", + "vs_w", "owvs", + "owvs_w", "vrvs", + "vrvs_w", "_variogram_score_gufunc", + "_variogram_score_gufunc_w", "_owvariogram_score_gufunc", + "_owvariogram_score_gufunc_w", "_vrvariogram_score_gufunc", + "_vrvariogram_score_gufunc_w", ] diff --git a/scoringrules/core/variogram/_gufuncs.py b/scoringrules/core/variogram/_gufuncs.py index 46b8a96..1fba6df 100644 --- a/scoringrules/core/variogram/_gufuncs.py +++ b/scoringrules/core/variogram/_gufuncs.py @@ -4,12 +4,12 @@ @guvectorize( [ - "void(float32[:], float32[:,:], float32[:,:], float32[:], float32, float32[:])", - "void(float64[:], float64[:,:], float64[:,:], float64[:], float64, float64[:])", + "void(float32[:], float32[:,:], float32[:,:], float32, float32[:])", + "void(float64[:], float64[:,:], float64[:,:], float64, float64[:])", ], - "(d),(m,d),(d,d),(m),()->()", + "(d),(m,d),(d,d),()->()", ) -def _variogram_score_gufunc(obs, fct, w, ens_w, p, out): +def _variogram_score_gufunc(obs, fct, w, p, out): M = fct.shape[-2] D = fct.shape[-1] out[0] = 0.0 @@ -17,19 +17,20 @@ def _variogram_score_gufunc(obs, fct, w, ens_w, p, out): for j in range(D): vfct = 0.0 for m in range(M): - vfct += ens_w[m] * abs(fct[m, i] - fct[m, j]) ** p + vfct += abs(fct[m, i] - fct[m, j]) ** p + vfct = vfct / M vobs = abs(obs[i] - obs[j]) ** p out[0] += w[i, j] * (vobs - vfct) ** 2 @guvectorize( [ - "void(float32[:], float32[:,:], float32[:,:], float32, float32[:], float32[:], float32, float32[:])", - "void(float64[:], float64[:,:], float64[:,:], float64, float64[:], float64[:], float64, float64[:])", + "void(float32[:], float32[:,:], float32[:,:], float32, float32[:], float32, float32[:])", + "void(float64[:], float64[:,:], float64[:,:], float64, float64[:], float64, float64[:])", ], - "(d),(m,d),(d,d),(),(m),(m),()->()", + "(d),(m,d),(d,d),(),(m),()->()", ) -def _owvariogram_score_gufunc(obs, fct, w, ow, fw, ens_w, p, out): +def _owvariogram_score_gufunc(obs, fct, w, ow, fw, p, out): M = fct.shape[-2] D = fct.shape[-1] @@ -40,35 +41,27 @@ def _owvariogram_score_gufunc(obs, fct, w, ow, fw, ens_w, p, out): for j in range(D): rho1 = abs(fct[k, i] - fct[k, j]) ** p rho2 = abs(obs[i] - obs[j]) ** p - e_1 += w[i, j] * (rho1 - rho2) ** 2 * fw[k] * ow * ens_w[k] + e_1 += w[i, j] * (rho1 - rho2) ** 2 * fw[k] * ow for m in range(k + 1, M): for i in range(D): for j in range(D): rho1 = abs(fct[k, i] - fct[k, j]) ** p rho2 = abs(fct[m, i] - fct[m, j]) ** p - e_2 += w[i, j] * ( - 2 - * ((rho1 - rho2) ** 2) - * fw[k] - * fw[m] - * ow - * ens_w[k] - * ens_w[m] - ) + e_2 += 2 * w[i, j] * ((rho1 - rho2) ** 2) * fw[k] * fw[m] * ow - wbar = np.sum(fw * ens_w) + wbar = np.mean(fw) - out[0] = e_1 / wbar - 0.5 * e_2 / (wbar**2) + out[0] = e_1 / (M * wbar) - 0.5 * e_2 / (M**2 * wbar**2) @guvectorize( [ - "void(float32[:], float32[:,:], float32[:,:], float32, float32[:], float32[:], float32, float32[:])", - "void(float64[:], float64[:,:], float64[:,:], float64, float64[:], float64[:], float64, float64[:])", + "void(float32[:], float32[:,:], float32[:,:], float32, float32[:], float32, float32[:])", + "void(float64[:], float64[:,:], float64[:,:], float64, float64[:], float64, float64[:])", ], - "(d),(m,d),(d,d),(),(m),(m),()->()", + "(d),(m,d),(d,d),(),(m),()->()", ) -def _vrvariogram_score_gufunc(obs, fct, w, ow, fw, ens_w, p, out): +def _vrvariogram_score_gufunc(obs, fct, w, ow, fw, p, out): M = fct.shape[-2] D = fct.shape[-1] @@ -80,22 +73,21 @@ def _vrvariogram_score_gufunc(obs, fct, w, ow, fw, ens_w, p, out): for j in range(D): rho1 = abs(fct[k, i] - fct[k, j]) ** p rho2 = abs(obs[i] - obs[j]) ** p - e_1 += w[i, j] * (rho1 - rho2) ** 2 * fw[k] * ow * ens_w[k] - e_3_x += w[i, j] * (rho1) ** 2 * fw[k] * ens_w[k] + e_1 += w[i, j] * (rho1 - rho2) ** 2 * fw[k] * ow + e_3_x += w[i, j] * (rho1) ** 2 * fw[k] for m in range(k + 1, M): for i in range(D): for j in range(D): rho1 = abs(fct[k, i] - fct[k, j]) ** p rho2 = abs(fct[m, i] - fct[m, j]) ** p - e_2 += w[i, j] * ( - 2 * ((rho1 - rho2) ** 2) * fw[k] * fw[m] * ens_w[k] * ens_w[m] - ) + e_2 += 2 * w[i, j] * ((rho1 - rho2) ** 2) * fw[k] * fw[m] - wbar = np.sum(fw * ens_w) + e_3_x *= 1 / M + wbar = np.mean(fw) e_3_y = 0.0 for i in range(D): for j in range(D): rho1 = abs(obs[i] - obs[j]) ** p e_3_y += w[i, j] * (rho1) ** 2 * ow - out[0] = e_1 - 0.5 * e_2 + (e_3_x - e_3_y) * (wbar - ow) + out[0] = e_1 / M - 0.5 * e_2 / (M**2) + (e_3_x - e_3_y) * (wbar - ow) diff --git a/scoringrules/core/variogram/_gufuncs_w.py b/scoringrules/core/variogram/_gufuncs_w.py new file mode 100644 index 0000000..30a7dcc --- /dev/null +++ b/scoringrules/core/variogram/_gufuncs_w.py @@ -0,0 +1,101 @@ +import numpy as np +from numba import guvectorize + + +@guvectorize( + [ + "void(float32[:], float32[:,:], float32[:,:], float32[:], float32, float32[:])", + "void(float64[:], float64[:,:], float64[:,:], float64[:], float64, float64[:])", + ], + "(d),(m,d),(d,d),(m),()->()", +) +def _variogram_score_gufunc_w(obs, fct, w, ens_w, p, out): + M = fct.shape[-2] + D = fct.shape[-1] + out[0] = 0.0 + for i in range(D): + for j in range(D): + vfct = 0.0 + for m in range(M): + vfct += ens_w[m] * abs(fct[m, i] - fct[m, j]) ** p + vobs = abs(obs[i] - obs[j]) ** p + out[0] += w[i, j] * (vobs - vfct) ** 2 + + +@guvectorize( + [ + "void(float32[:], float32[:,:], float32[:,:], float32, float32[:], float32[:], float32, float32[:])", + "void(float64[:], float64[:,:], float64[:,:], float64, float64[:], float64[:], float64, float64[:])", + ], + "(d),(m,d),(d,d),(),(m),(m),()->()", +) +def _owvariogram_score_gufunc_w(obs, fct, w, ow, fw, ens_w, p, out): + M = fct.shape[-2] + D = fct.shape[-1] + + e_1 = 0.0 + e_2 = 0.0 + for k in range(M): + for i in range(D): + for j in range(D): + rho1 = abs(fct[k, i] - fct[k, j]) ** p + rho2 = abs(obs[i] - obs[j]) ** p + e_1 += w[i, j] * (rho1 - rho2) ** 2 * fw[k] * ow * ens_w[k] + for m in range(k + 1, M): + for i in range(D): + for j in range(D): + rho1 = abs(fct[k, i] - fct[k, j]) ** p + rho2 = abs(fct[m, i] - fct[m, j]) ** p + e_2 += w[i, j] * ( + 2 + * ((rho1 - rho2) ** 2) + * fw[k] + * fw[m] + * ow + * ens_w[k] + * ens_w[m] + ) + + wbar = np.sum(fw * ens_w) + + out[0] = e_1 / wbar - 0.5 * e_2 / (wbar**2) + + +@guvectorize( + [ + "void(float32[:], float32[:,:], float32[:,:], float32, float32[:], float32[:], float32, float32[:])", + "void(float64[:], float64[:,:], float64[:,:], float64, float64[:], float64[:], float64, float64[:])", + ], + "(d),(m,d),(d,d),(),(m),(m),()->()", +) +def _vrvariogram_score_gufunc_w(obs, fct, w, ow, fw, ens_w, p, out): + M = fct.shape[-2] + D = fct.shape[-1] + + e_1 = 0.0 + e_2 = 0.0 + e_3_x = 0.0 + for k in range(M): + for i in range(D): + for j in range(D): + rho1 = abs(fct[k, i] - fct[k, j]) ** p + rho2 = abs(obs[i] - obs[j]) ** p + e_1 += w[i, j] * (rho1 - rho2) ** 2 * fw[k] * ow * ens_w[k] + e_3_x += w[i, j] * (rho1) ** 2 * fw[k] * ens_w[k] + for m in range(k + 1, M): + for i in range(D): + for j in range(D): + rho1 = abs(fct[k, i] - fct[k, j]) ** p + rho2 = abs(fct[m, i] - fct[m, j]) ** p + e_2 += w[i, j] * ( + 2 * ((rho1 - rho2) ** 2) * fw[k] * fw[m] * ens_w[k] * ens_w[m] + ) + + wbar = np.sum(fw * ens_w) + e_3_y = 0.0 + for i in range(D): + for j in range(D): + rho1 = abs(obs[i] - obs[j]) ** p + e_3_y += w[i, j] * (rho1) ** 2 * ow + + out[0] = e_1 - 0.5 * e_2 + (e_3_x - e_3_y) * (wbar - ow) diff --git a/scoringrules/core/variogram/_score.py b/scoringrules/core/variogram/_score.py index 46a7205..e232f94 100644 --- a/scoringrules/core/variogram/_score.py +++ b/scoringrules/core/variogram/_score.py @@ -10,16 +10,13 @@ def vs_ensemble( obs: "Array", # (... D) fct: "Array", # (... M D) w: "Array", # (..., D D) - ens_w: "Array", # (... M) p: float = 1, backend: "Backend" = None, ) -> "Array": """Compute the Variogram Score for a multivariate finite ensemble.""" B = backends.active if backend is None else backends[backend] fct_diff = B.expand_dims(fct, -2) - B.expand_dims(fct, -1) # (... M D D) - vfct = B.sum( - B.expand_dims(ens_w, (-1, -2)) * B.abs(fct_diff) ** p, axis=-3 - ) # (... D D) + vfct = B.mean(B.abs(fct_diff) ** p, axis=-3) # (... D D) obs_diff = B.expand_dims(obs, -2) - B.expand_dims(obs, -1) # (... D D) vobs = B.abs(obs_diff) ** p # (... D D) return B.sum(w * (vobs - vfct) ** 2, axis=(-2, -1)) # (...) @@ -31,13 +28,12 @@ def owvs_ensemble( w: "Array", ow: "Array", fw: "Array", - ens_w: "Array", p: float = 1, backend: "Backend" = None, ) -> "Array": """Compute the Outcome-Weighted Variogram Score for a multivariate finite ensemble.""" B = backends.active if backend is None else backends[backend] - wbar = B.sum(fw * ens_w, axis=-1) + wbar = B.mean(fw, axis=-1) fct_diff = B.expand_dims(fct, -2) - B.expand_dims(fct, -1) # (... M D D) fct_diff = B.abs(fct_diff) ** p # (... M D D) @@ -48,18 +44,17 @@ def owvs_ensemble( E_1 = (fct_diff - B.expand_dims(obs_diff, -3)) ** 2 # (... M D D) E_1 = B.sum(B.expand_dims(w, -3) * E_1, axis=(-2, -1)) # (... M) - E_1 = B.sum(E_1 * fw * B.expand_dims(ow, -1) * ens_w, axis=-1) / wbar # (...) + E_1 = B.mean(E_1 * fw * B.expand_dims(ow, -1), axis=-1) / wbar # (...) fct_diff_spread = B.expand_dims(fct_diff, -3) - B.expand_dims( fct_diff, -4 ) # (... M M D D) fw_prod = B.expand_dims(fw, -2) * B.expand_dims(fw, -1) # (... M M) - ew_prod = B.expand_dims(ens_w, -2) * B.expand_dims(ens_w, -1) # (... M M) E_2 = B.sum( B.expand_dims(w, (-3, -4)) * fct_diff_spread**2, axis=(-2, -1) ) # (... M M) - E_2 *= fw_prod * B.expand_dims(ow, (-2, -1)) * ew_prod # (... M M) - E_2 = B.sum(E_2, axis=(-2, -1)) / (wbar**2) # (...) + E_2 *= fw_prod * B.expand_dims(ow, (-2, -1)) # (... M M) + E_2 = B.mean(E_2, axis=(-2, -1)) / (wbar**2) # (...) return E_1 - 0.5 * E_2 @@ -70,13 +65,12 @@ def vrvs_ensemble( w: "Array", ow: "Array", fw: "Array", - ens_w: "Array", p: float = 1, backend: "Backend" = None, ) -> "Array": """Compute the Vertically Re-scaled Variogram Score for a multivariate finite ensemble.""" B = backends.active if backend is None else backends[backend] - wbar = B.sum(fw * ens_w, axis=-1) + wbar = B.mean(fw, axis=-1) fct_diff = ( B.abs(B.expand_dims(fct, -2) - B.expand_dims(fct, -1)) ** p @@ -85,7 +79,7 @@ def vrvs_ensemble( E_1 = (fct_diff - B.expand_dims(obs_diff, axis=-3)) ** 2 # (... M D D) E_1 = B.sum(B.expand_dims(w, -3) * E_1, axis=(-2, -1)) # (... M) - E_1 = B.sum(E_1 * fw * B.expand_dims(ow, axis=-1) * ens_w, axis=-1) # (...) + E_1 = B.mean(E_1 * fw * B.expand_dims(ow, axis=-1), axis=-1) # (...) E_2 = ( B.expand_dims(fct_diff, -3) - B.expand_dims(fct_diff, -4) @@ -93,11 +87,10 @@ def vrvs_ensemble( E_2 = B.sum(B.expand_dims(w, (-3, -4)) * E_2, axis=(-2, -1)) # (... M M) fw_prod = B.expand_dims(fw, axis=-2) * B.expand_dims(fw, axis=-1) # (... M M) - ew_prod = B.expand_dims(ens_w, -2) * B.expand_dims(ens_w, -1) # (... M M) - E_2 = B.sum(E_2 * fw_prod * ew_prod, axis=(-2, -1)) # (...) + E_2 = B.mean(E_2 * fw_prod, axis=(-2, -1)) # (...) E_3 = B.sum(B.expand_dims(w, -3) * fct_diff**2, axis=(-2, -1)) # (... M) - E_3 = B.sum(E_3 * fw * ens_w, axis=-1) # (...) + E_3 = B.mean(E_3 * fw, axis=-1) # (...) E_3 -= B.sum(w * obs_diff**2, axis=(-2, -1)) * ow # (...) E_3 *= wbar - ow # (...) diff --git a/scoringrules/core/variogram/_score_w.py b/scoringrules/core/variogram/_score_w.py new file mode 100644 index 0000000..b8dad99 --- /dev/null +++ b/scoringrules/core/variogram/_score_w.py @@ -0,0 +1,104 @@ +import typing as tp + +from scoringrules.backend import backends + +if tp.TYPE_CHECKING: + from scoringrules.core.typing import Array, Backend + + +def vs_ensemble_w( + obs: "Array", # (... D) + fct: "Array", # (... M D) + w: "Array", # (..., D D) + ens_w: "Array", # (... M) + p: float = 1, + backend: "Backend" = None, +) -> "Array": + """Compute the Variogram Score for a multivariate finite ensemble.""" + B = backends.active if backend is None else backends[backend] + fct_diff = B.expand_dims(fct, -2) - B.expand_dims(fct, -1) # (... M D D) + vfct = B.sum( + B.expand_dims(ens_w, (-1, -2)) * B.abs(fct_diff) ** p, axis=-3 + ) # (... D D) + obs_diff = B.expand_dims(obs, -2) - B.expand_dims(obs, -1) # (... D D) + vobs = B.abs(obs_diff) ** p # (... D D) + return B.sum(w * (vobs - vfct) ** 2, axis=(-2, -1)) # (...) + + +def owvs_ensemble_w( + obs: "Array", + fct: "Array", + w: "Array", + ow: "Array", + fw: "Array", + ens_w: "Array", + p: float = 1, + backend: "Backend" = None, +) -> "Array": + """Compute the Outcome-Weighted Variogram Score for a multivariate finite ensemble.""" + B = backends.active if backend is None else backends[backend] + wbar = B.sum(fw * ens_w, axis=-1) + + fct_diff = B.expand_dims(fct, -2) - B.expand_dims(fct, -1) # (... M D D) + fct_diff = B.abs(fct_diff) ** p # (... M D D) + + obs_diff = B.expand_dims(obs, -2) - B.expand_dims(obs, -1) # (... D D) + obs_diff = B.abs(obs_diff) ** p # (... D D) + del obs, fct + + E_1 = (fct_diff - B.expand_dims(obs_diff, -3)) ** 2 # (... M D D) + E_1 = B.sum(B.expand_dims(w, -3) * E_1, axis=(-2, -1)) # (... M) + E_1 = B.sum(E_1 * fw * B.expand_dims(ow, -1) * ens_w, axis=-1) / wbar # (...) + + fct_diff_spread = B.expand_dims(fct_diff, -3) - B.expand_dims( + fct_diff, -4 + ) # (... M M D D) + fw_prod = B.expand_dims(fw, -2) * B.expand_dims(fw, -1) # (... M M) + ew_prod = B.expand_dims(ens_w, -2) * B.expand_dims(ens_w, -1) # (... M M) + E_2 = B.sum( + B.expand_dims(w, (-3, -4)) * fct_diff_spread**2, axis=(-2, -1) + ) # (... M M) + E_2 *= fw_prod * B.expand_dims(ow, (-2, -1)) * ew_prod # (... M M) + E_2 = B.sum(E_2, axis=(-2, -1)) / (wbar**2) # (...) + + return E_1 - 0.5 * E_2 + + +def vrvs_ensemble_w( + obs: "Array", + fct: "Array", + w: "Array", + ow: "Array", + fw: "Array", + ens_w: "Array", + p: float = 1, + backend: "Backend" = None, +) -> "Array": + """Compute the Vertically Re-scaled Variogram Score for a multivariate finite ensemble.""" + B = backends.active if backend is None else backends[backend] + wbar = B.sum(fw * ens_w, axis=-1) + + fct_diff = ( + B.abs(B.expand_dims(fct, -2) - B.expand_dims(fct, -1)) ** p + ) # (... M D D) + obs_diff = B.abs(B.expand_dims(obs, -2) - B.expand_dims(obs, -1)) ** p # (... D D) + + E_1 = (fct_diff - B.expand_dims(obs_diff, axis=-3)) ** 2 # (... M D D) + E_1 = B.sum(B.expand_dims(w, -3) * E_1, axis=(-2, -1)) # (... M) + E_1 = B.sum(E_1 * fw * B.expand_dims(ow, axis=-1) * ens_w, axis=-1) # (...) + + E_2 = ( + B.expand_dims(fct_diff, -3) - B.expand_dims(fct_diff, -4) + ) ** 2 # (... M M D D) + E_2 = B.sum(B.expand_dims(w, (-3, -4)) * E_2, axis=(-2, -1)) # (... M M) + + fw_prod = B.expand_dims(fw, axis=-2) * B.expand_dims(fw, axis=-1) # (... M M) + ew_prod = B.expand_dims(ens_w, -2) * B.expand_dims(ens_w, -1) # (... M M) + E_2 = B.sum(E_2 * fw_prod * ew_prod, axis=(-2, -1)) # (...) + + E_3 = B.sum(B.expand_dims(w, -3) * fct_diff**2, axis=(-2, -1)) # (... M) + E_3 = B.sum(E_3 * fw * ens_w, axis=-1) # (...) + E_3 -= B.sum(w * obs_diff**2, axis=(-2, -1)) * ow # (...) + E_3 *= wbar - ow # (...) + + return E_1 - 0.5 * E_2 + E_3 diff --git a/tests/test_wvariogram.py b/tests/test_wvariogram.py index 688db83..5592a1d 100644 --- a/tests/test_wvariogram.py +++ b/tests/test_wvariogram.py @@ -23,9 +23,7 @@ def test_owvs_vs_vs(backend): lambda x: backends[backend].mean(x) * 0.0 + 1.0, backend=backend, ) - np.testing.assert_allclose( - res, resw, rtol=1e-3 - ) # TODO: not sure why tolerance must be so high + np.testing.assert_allclose(res, resw, rtol=1e-3) @pytest.mark.parametrize("backend", BACKENDS) From 6c8ed6f020ce8e7d10a9b0ee772f9136c99f356b Mon Sep 17 00:00:00 2001 From: sallen12 Date: Fri, 15 Aug 2025 12:21:40 +0200 Subject: [PATCH 78/79] add separate kernel score functions for when ensemble weights are given --- scoringrules/_kernels.py | 214 +++++++++++------- scoringrules/core/kernels/__init__.py | 28 ++- scoringrules/core/kernels/_approx.py | 87 ++++--- scoringrules/core/kernels/_approx_w.py | 207 +++++++++++++++++ scoringrules/core/kernels/_gufuncs.py | 126 +++++------ scoringrules/core/kernels/_gufuncs_w.py | 288 ++++++++++++++++++++++++ 6 files changed, 743 insertions(+), 207 deletions(-) create mode 100644 scoringrules/core/kernels/_approx_w.py create mode 100644 scoringrules/core/kernels/_gufuncs_w.py diff --git a/scoringrules/_kernels.py b/scoringrules/_kernels.py index 65482ec..3596b7b 100644 --- a/scoringrules/_kernels.py +++ b/scoringrules/_kernels.py @@ -64,20 +64,14 @@ def gksuv_ensemble( B = backends.active if backend is None else backends[backend] obs, fct = map(B.asarray, (obs, fct)) - if ens_w is None: - M = fct.shape[m_axis] - ens_w = B.zeros(fct.shape) + 1.0 / M - else: - ens_w = B.asarray(ens_w) - if B.any(ens_w < 0): - raise ValueError("`ens_w` contains negative entries") - ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) + if m_axis != -1: + fct = B.moveaxis(fct, m_axis, -1) if backend == "numba": - if estimator not in kernels.estimator_gufuncs: + if estimator not in kernels.estimator_gufuncs_uv: raise ValueError( f"{estimator} is not a valid estimator. " - f"Must be one of {kernels.estimator_gufuncs.keys()}" + f"Must be one of {kernels.estimator_gufuncs_uv.keys()}" ) else: if estimator not in ["fair", "nrg"]: @@ -86,16 +80,26 @@ def gksuv_ensemble( f"Must be one of ['fair', 'nrg']" ) - if m_axis != -1: - fct = B.moveaxis(fct, m_axis, -1) - ens_w = B.moveaxis(ens_w, m_axis, -1) + if ens_w is None: + if backend == "numba": + return kernels.estimator_gufuncs_uv[estimator](obs, fct) - if backend == "numba": - return kernels.estimator_gufuncs[estimator](obs, fct, ens_w) + return kernels.ensemble_uv(obs, fct, estimator=estimator, backend=backend) - return kernels.ensemble_uv( - obs, fct, ens_w=ens_w, estimator=estimator, backend=backend - ) + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) + if m_axis != -1: + ens_w = B.moveaxis(ens_w, m_axis, -1) + + if backend == "numba": + return kernels.estimator_gufuncs_uv_w[estimator](obs, fct, ens_w) + + return kernels.ensemble_uv_w( + obs, fct, ens_w=ens_w, estimator=estimator, backend=backend + ) def twgksuv_ensemble( @@ -257,18 +261,8 @@ def owgksuv_ensemble( B = backends.active if backend is None else backends[backend] obs, fct = map(B.asarray, (obs, fct)) - if ens_w is None: - M = fct.shape[m_axis] - ens_w = B.zeros(fct.shape) + 1.0 / M - else: - ens_w = B.asarray(ens_w) - if B.any(ens_w < 0): - raise ValueError("`ens_w` contains negative entries") - ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) - if m_axis != -1: fct = B.moveaxis(fct, m_axis, -1) - ens_w = B.moveaxis(ens_w, m_axis, -1) if w_func is None: @@ -277,15 +271,35 @@ def w_func(x): obs_weights, fct_weights = map(w_func, (obs, fct)) obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) + if B.any(obs_weights < 0) or B.any(fct_weights < 0): + raise ValueError("`w_func` returns negative values") - if backend == "numba": - return kernels.estimator_gufuncs["ow"]( - obs, fct, obs_weights, fct_weights, ens_w + if ens_w is None: + if backend == "numba": + return kernels.estimator_gufuncs_uv["ow"]( + obs, fct, obs_weights, fct_weights + ) + + return kernels.ow_ensemble_uv( + obs, fct, obs_weights, fct_weights, backend=backend ) - return kernels.ow_ensemble_uv( - obs, fct, obs_weights, fct_weights, ens_w=ens_w, backend=backend - ) + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) + if m_axis != -1: + ens_w = B.moveaxis(ens_w, m_axis, -1) + + if backend == "numba": + return kernels.estimator_gufuncs_uv_w["ow"]( + obs, fct, obs_weights, fct_weights, ens_w + ) + + return kernels.ow_ensemble_uv_w( + obs, fct, obs_weights, fct_weights, ens_w, backend=backend + ) def vrgksuv_ensemble( @@ -358,18 +372,8 @@ def vrgksuv_ensemble( B = backends.active if backend is None else backends[backend] obs, fct = map(B.asarray, (obs, fct)) - if ens_w is None: - M = fct.shape[m_axis] - ens_w = B.zeros(fct.shape) + 1.0 / M - else: - ens_w = B.asarray(ens_w) - if B.any(ens_w < 0): - raise ValueError("`ens_w` contains negative entries") - ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) - if m_axis != -1: fct = B.moveaxis(fct, m_axis, -1) - ens_w = B.moveaxis(ens_w, m_axis, -1) if w_func is None: @@ -378,15 +382,35 @@ def w_func(x): obs_weights, fct_weights = map(w_func, (obs, fct)) obs_weights, fct_weights = map(B.asarray, (obs_weights, fct_weights)) + if B.any(obs_weights < 0) or B.any(fct_weights < 0): + raise ValueError("`w_func` returns negative values") - if backend == "numba": - return kernels.estimator_gufuncs["vr"]( - obs, fct, obs_weights, fct_weights, ens_w + if ens_w is None: + if backend == "numba": + return kernels.estimator_gufuncs_uv["vr"]( + obs, fct, obs_weights, fct_weights + ) + + return kernels.vr_ensemble_uv( + obs, fct, obs_weights, fct_weights, backend=backend ) - return kernels.vr_ensemble_uv( - obs, fct, obs_weights, fct_weights, ens_w, backend=backend - ) + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=m_axis, keepdims=True) + if m_axis != -1: + ens_w = B.moveaxis(ens_w, m_axis, -1) + + if backend == "numba": + return kernels.estimator_gufuncs_uv_w["vr"]( + obs, fct, obs_weights, fct_weights, ens_w + ) + + return kernels.vr_ensemble_uv_w( + obs, fct, obs_weights, fct_weights, ens_w, backend=backend + ) def gksmv_ensemble( @@ -447,24 +471,28 @@ def gksmv_ensemble( B = backends.active if backend is None else backends[backend] obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) - if ens_w is None: - M = fct.shape[-2] - ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M - else: - ens_w = B.moveaxis(ens_w, m_axis, -2) - if estimator not in kernels.estimator_gufuncs_mv: raise ValueError( f"{estimator} is not a valid estimator. " f"Must be one of {kernels.estimator_gufuncs_mv.keys()}" ) - if backend == "numba": - return kernels.estimator_gufuncs_mv[estimator](obs, fct, ens_w) + if ens_w is None: + if backend == "numba": + return kernels.estimator_gufuncs_mv[estimator](obs, fct) - return kernels.ensemble_mv( - obs, fct, ens_w=ens_w, estimator=estimator, backend=backend - ) + return kernels.ensemble_mv(obs, fct, estimator=estimator, backend=backend) + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=-1, keepdims=True) + if backend == "numba": + return kernels.estimator_gufuncs_mv_w[estimator](obs, fct, ens_w) + + return kernels.ensemble_mv_w( + obs, fct, ens_w, estimator=estimator, backend=backend + ) def twgksmv_ensemble( @@ -602,23 +630,31 @@ def owgksmv_ensemble( B = backends.active if backend is None else backends[backend] obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) - if ens_w is None: - M = fct.shape[-2] - ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M - else: - ens_w = B.moveaxis(ens_w, m_axis, -2) - fct_weights = B.apply_along_axis(w_func, fct, -1) obs_weights = B.apply_along_axis(w_func, obs, -1) - if B.name == "numba": - return kernels.estimator_gufuncs_mv["ow"]( - obs, fct, obs_weights, fct_weights, ens_w + if ens_w is None: + if backend == "numba": + return kernels.estimator_gufuncs_mv["ow"]( + obs, fct, obs_weights, fct_weights + ) + + return kernels.ow_ensemble_mv( + obs, fct, obs_weights, fct_weights, backend=backend ) + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=-1, keepdims=True) + if backend == "numba": + return kernels.estimator_gufuncs_mv_w["ow"]( + obs, fct, obs_weights, fct_weights, ens_w + ) - return kernels.ow_ensemble_mv( - obs, fct, obs_weights, fct_weights, ens_w=ens_w, backend=backend - ) + return kernels.ow_ensemble_mv_w( + obs, fct, obs_weights, fct_weights, ens_w, backend=backend + ) def vrgksmv_ensemble( @@ -677,23 +713,31 @@ def vrgksmv_ensemble( B = backends.active if backend is None else backends[backend] obs, fct = multivariate_array_check(obs, fct, m_axis, v_axis, backend=backend) - if ens_w is None: - M = fct.shape[-2] - ens_w = B.zeros(fct.shape[:-1]) + 1.0 / M - else: - ens_w = B.moveaxis(ens_w, m_axis, -2) - fct_weights = B.apply_along_axis(w_func, fct, -1) obs_weights = B.apply_along_axis(w_func, obs, -1) - if B.name == "numba": - return kernels.estimator_gufuncs_mv["vr"]( - obs, fct, obs_weights, fct_weights, ens_w + if ens_w is None: + if backend == "numba": + return kernels.estimator_gufuncs_mv["vr"]( + obs, fct, obs_weights, fct_weights + ) + + return kernels.vr_ensemble_mv( + obs, fct, obs_weights, fct_weights, backend=backend ) + else: + ens_w = B.asarray(ens_w) + if B.any(ens_w < 0): + raise ValueError("`ens_w` contains negative entries") + ens_w = ens_w / B.sum(ens_w, axis=-1, keepdims=True) + if backend == "numba": + return kernels.estimator_gufuncs_mv_w["vr"]( + obs, fct, obs_weights, fct_weights, ens_w + ) - return kernels.vr_ensemble_mv( - obs, fct, obs_weights, fct_weights, ens_w=ens_w, backend=backend - ) + return kernels.vr_ensemble_mv_w( + obs, fct, obs_weights, fct_weights, ens_w, backend=backend + ) __all__ = [ diff --git a/scoringrules/core/kernels/__init__.py b/scoringrules/core/kernels/__init__.py index d0377b7..f7ac807 100644 --- a/scoringrules/core/kernels/__init__.py +++ b/scoringrules/core/kernels/__init__.py @@ -7,15 +7,23 @@ vr_ensemble_mv, ) -try: - from ._gufuncs import estimator_gufuncs -except ImportError: - estimator_gufuncs = None +from ._approx_w import ( + ensemble_uv_w, + ow_ensemble_uv_w, + vr_ensemble_uv_w, + ensemble_mv_w, + ow_ensemble_mv_w, + vr_ensemble_mv_w, +) try: - from ._gufuncs import estimator_gufuncs_mv + from ._gufuncs import estimator_gufuncs_uv, estimator_gufuncs_mv + from ._gufuncs_w import estimator_gufuncs_uv_w, estimator_gufuncs_mv_w except ImportError: + estimator_gufuncs_uv = None estimator_gufuncs_mv = None + estimator_gufuncs_uv_w = None + estimator_gufuncs_mv_w = None __all__ = [ "ensemble_uv", @@ -24,6 +32,14 @@ "ensemble_mv", "ow_ensemble_mv", "vr_ensemble_mv", - "estimator_gufuncs", + "ensemble_uv_w", + "ow_ensemble_uv_w", + "vr_ensemble_uv_w", + "ensemble_mv_w", + "ow_ensemble_mv_w", + "vr_ensemble_mv_w", + "estimator_gufuncs_uv", "estimator_gufuncs_mv", + "estimator_gufuncs_uv_w", + "estimator_gufuncs_mv_w", ] diff --git a/scoringrules/core/kernels/_approx.py b/scoringrules/core/kernels/_approx.py index 42d5f5f..1d581bf 100644 --- a/scoringrules/core/kernels/_approx.py +++ b/scoringrules/core/kernels/_approx.py @@ -25,33 +25,30 @@ def gauss_kern_mv( def ensemble_uv( obs: "ArrayLike", fct: "Array", - ens_w: "Array", estimator: str = "nrg", backend: "Backend" = None, ) -> "Array": """Compute a kernel score for a finite ensemble.""" B = backends.active if backend is None else backends[backend] - - e_1 = B.sum(gauss_kern_uv(obs[..., None], fct, backend=backend) * ens_w, axis=-1) + M: int = fct.shape[-1] + e_1 = B.sum(gauss_kern_uv(obs[..., None], fct, backend=backend), axis=-1) / M e_2 = B.sum( - gauss_kern_uv(fct[..., None], fct[..., None, :], backend=backend) - * ens_w[..., None] - * ens_w[..., None, :], + gauss_kern_uv(fct[..., None], fct[..., None, :], backend=backend), axis=(-1, -2), - ) + ) / (M**2) e_3 = gauss_kern_uv(obs, obs) - if estimator == "fair": - e_2 = e_2 / (1 - B.sum(ens_w * ens_w, axis=-1)) + if estimator == "nrg": + out = e_1 - 0.5 * e_2 - 0.5 * e_3 + elif estimator == "fair": + out = e_1 - 0.5 * e_2 * (M / (M - 1)) - 0.5 * e_3 - out = e_1 - 0.5 * e_2 - 0.5 * e_3 return -out def ensemble_mv( obs: "ArrayLike", fct: "Array", - ens_w: "Array", estimator: str = "nrg", backend: "Backend" = None, ) -> "Array": @@ -60,22 +57,22 @@ def ensemble_mv( The ensemble and variables axes are on the second last and last dimensions respectively. """ B = backends.active if backend is None else backends[backend] + M: int = fct.shape[-2] - e_1 = B.sum( - ens_w * gauss_kern_mv(B.expand_dims(obs, -2), fct, backend=backend), axis=-1 + e_1 = ( + B.sum(gauss_kern_mv(B.expand_dims(obs, -2), fct, backend=backend), axis=-1) / M ) e_2 = B.sum( - gauss_kern_mv(B.expand_dims(fct, -3), B.expand_dims(fct, -2), backend=backend) - * B.expand_dims(ens_w, -1) - * B.expand_dims(ens_w, -2), + gauss_kern_mv(B.expand_dims(fct, -3), B.expand_dims(fct, -2), backend=backend), axis=(-2, -1), - ) + ) / (M**2) e_3 = gauss_kern_mv(obs, obs) - if estimator == "fair": - e_2 = e_2 / (1 - B.sum(ens_w * ens_w, axis=-1)) + if estimator == "nrg": + out = e_1 - 0.5 * e_2 - 0.5 * e_3 + elif estimator == "fair": + out = e_1 - 0.5 * e_2 * (M / (M - 1)) - 0.5 * e_3 - out = e_1 - 0.5 * e_2 - 0.5 * e_3 return -out @@ -84,26 +81,24 @@ def ow_ensemble_uv( fct: "Array", ow: "Array", fw: "Array", - ens_w: "Array", backend: "Backend" = None, ) -> "Array": """Compute an outcome-weighted kernel score for a finite univariate ensemble.""" B = backends.active if backend is None else backends[backend] - wbar = B.sum(ens_w * fw, axis=-1) + M: int = fct.shape[-1] + wbar = B.mean(fw, axis=-1) e_1 = ( - B.sum(ens_w * gauss_kern_uv(obs[..., None], fct, backend=backend) * fw, axis=-1) + B.sum(gauss_kern_uv(obs[..., None], fct, backend=backend) * fw, axis=-1) * ow - / wbar + / (M * wbar) ) e_2 = B.sum( - ens_w[..., None] - * ens_w[..., None, :] - * gauss_kern_uv(fct[..., None], fct[..., None, :], backend=backend) + gauss_kern_uv(fct[..., None], fct[..., None, :], backend=backend) * fw[..., None] * fw[..., None, :], axis=(-1, -2), ) - e_2 *= ow / (wbar**2) + e_2 *= ow / (M**2 * wbar**2) e_3 = gauss_kern_uv(obs, obs, backend=backend) * ow out = e_1 - 0.5 * e_2 - 0.5 * e_3 @@ -115,7 +110,6 @@ def ow_ensemble_mv( fct: "Array", ow: "Array", fw: "Array", - ens_w: "Array", backend: "Backend" = None, ) -> "Array": """Compute an outcome-weighted kernel score for a finite multivariate ensemble. @@ -123,19 +117,18 @@ def ow_ensemble_mv( The ensemble and variables axes are on the second last and last dimensions respectively. """ B = backends.active if backend is None else backends[backend] - wbar = B.sum(fw * ens_w, -1) + M: int = fct.shape[-2] + wbar = B.sum(fw, -1) / M err_kern = gauss_kern_mv(B.expand_dims(obs, -2), fct, backend=backend) - E_1 = B.sum(err_kern * fw * B.expand_dims(ow, -1) * ens_w, axis=-1) / wbar + E_1 = B.sum(err_kern * fw * B.expand_dims(ow, -1), axis=-1) / (M * wbar) spread_kern = gauss_kern_mv( B.expand_dims(fct, -3), B.expand_dims(fct, -2), backend=backend ) fw_prod = B.expand_dims(fw, -1) * B.expand_dims(fw, -2) spread_kern *= fw_prod * B.expand_dims(ow, (-2, -1)) - E_2 = B.sum( - spread_kern * B.expand_dims(ens_w, -1) * B.expand_dims(ens_w, -2), (-2, -1) - ) / (wbar**2) + E_2 = B.sum(spread_kern, (-2, -1)) / (M**2 * wbar**2) E_3 = gauss_kern_mv(obs, obs, backend=backend) * ow @@ -149,24 +142,26 @@ def vr_ensemble_uv( fct: "Array", ow: "Array", fw: "Array", - ens_w: "Array", backend: "Backend" = None, ) -> "Array": """Compute a vertically re-scaled kernel score for a finite univariate ensemble.""" B = backends.active if backend is None else backends[backend] + M: int = fct.shape[-1] e_1 = ( - B.sum(ens_w * gauss_kern_uv(obs[..., None], fct, backend=backend) * fw, axis=-1) + B.sum(gauss_kern_uv(obs[..., None], fct, backend=backend) * fw, axis=-1) * ow + / M ) e_2 = B.sum( - ens_w[..., None] - * ens_w[..., None, :] - * gauss_kern_uv(fct[..., None], fct[..., None, :], backend=backend) - * fw[..., None] - * fw[..., None, :], + gauss_kern_uv( + B.expand_dims(fct, axis=-1), + B.expand_dims(fct, axis=-2), + backend=backend, + ) + * (B.expand_dims(fw, axis=-1) * B.expand_dims(fw, axis=-2)), axis=(-1, -2), - ) + ) / (M**2) e_3 = gauss_kern_uv(obs, obs, backend=backend) * ow * ow out = e_1 - 0.5 * e_2 - 0.5 * e_3 @@ -179,7 +174,6 @@ def vr_ensemble_mv( fct: "Array", ow: "Array", fw: "Array", - ens_w: "Array", backend: "Backend" = None, ) -> "Array": """Compute a vertically re-scaled kernel score for a finite multivariate ensemble. @@ -187,18 +181,17 @@ def vr_ensemble_mv( The ensemble and variables axes are on the second last and last dimensions respectively. """ B = backends.active if backend is None else backends[backend] + M: int = fct.shape[-2] err_kern = gauss_kern_mv(B.expand_dims(obs, -2), fct, backend=backend) - E_1 = B.sum(err_kern * fw * B.expand_dims(ow, -1) * ens_w, axis=-1) + E_1 = B.sum(err_kern * fw * B.expand_dims(ow, -1), axis=-1) / M spread_kern = gauss_kern_mv( B.expand_dims(fct, -3), B.expand_dims(fct, -2), backend=backend ) fw_prod = B.expand_dims(fw, -1) * B.expand_dims(fw, -2) spread_kern *= fw_prod - E_2 = B.sum( - spread_kern * B.expand_dims(ens_w, -1) * B.expand_dims(ens_w, -2), (-2, -1) - ) + E_2 = B.sum(spread_kern, (-2, -1)) / (M**2) E_3 = gauss_kern_mv(obs, obs, backend=backend) * ow * ow diff --git a/scoringrules/core/kernels/_approx_w.py b/scoringrules/core/kernels/_approx_w.py new file mode 100644 index 0000000..077541d --- /dev/null +++ b/scoringrules/core/kernels/_approx_w.py @@ -0,0 +1,207 @@ +import typing as tp + +from scoringrules.backend import backends + +if tp.TYPE_CHECKING: + from scoringrules.core.typing import Array, ArrayLike, Backend + + +def gauss_kern_uv( + x1: "ArrayLike", x2: "ArrayLike", backend: "Backend" = None +) -> "Array": + """Compute the gaussian kernel evaluated at x1 and x2.""" + B = backends.active if backend is None else backends[backend] + return B.exp(-0.5 * (x1 - x2) ** 2) + + +def gauss_kern_mv( + x1: "ArrayLike", x2: "ArrayLike", backend: "Backend" = None +) -> "Array": + """Compute the gaussian kernel evaluated at vectors x1 and x2.""" + B = backends.active if backend is None else backends[backend] + return B.exp(-0.5 * B.norm(x1 - x2, -1) ** 2) + + +def ensemble_uv_w( + obs: "ArrayLike", + fct: "Array", + ens_w: "Array", + estimator: str = "nrg", + backend: "Backend" = None, +) -> "Array": + """Compute a kernel score for a finite ensemble.""" + B = backends.active if backend is None else backends[backend] + + e_1 = B.sum(gauss_kern_uv(obs[..., None], fct, backend=backend) * ens_w, axis=-1) + e_2 = B.sum( + gauss_kern_uv(fct[..., None], fct[..., None, :], backend=backend) + * ens_w[..., None] + * ens_w[..., None, :], + axis=(-1, -2), + ) + e_3 = gauss_kern_uv(obs, obs) + + if estimator == "fair": + e_2 = e_2 / (1 - B.sum(ens_w * ens_w, axis=-1)) + + out = e_1 - 0.5 * e_2 - 0.5 * e_3 + return -out + + +def ensemble_mv_w( + obs: "ArrayLike", + fct: "Array", + ens_w: "Array", + estimator: str = "nrg", + backend: "Backend" = None, +) -> "Array": + """Compute a kernel score for a finite multivariate ensemble. + + The ensemble and variables axes are on the second last and last dimensions respectively. + """ + B = backends.active if backend is None else backends[backend] + + e_1 = B.sum( + ens_w * gauss_kern_mv(B.expand_dims(obs, -2), fct, backend=backend), axis=-1 + ) + e_2 = B.sum( + gauss_kern_mv(B.expand_dims(fct, -3), B.expand_dims(fct, -2), backend=backend) + * B.expand_dims(ens_w, -1) + * B.expand_dims(ens_w, -2), + axis=(-2, -1), + ) + e_3 = gauss_kern_mv(obs, obs) + + if estimator == "fair": + e_2 = e_2 / (1 - B.sum(ens_w * ens_w, axis=-1)) + + out = e_1 - 0.5 * e_2 - 0.5 * e_3 + return -out + + +def ow_ensemble_uv_w( + obs: "ArrayLike", + fct: "Array", + ow: "Array", + fw: "Array", + ens_w: "Array", + backend: "Backend" = None, +) -> "Array": + """Compute an outcome-weighted kernel score for a finite univariate ensemble.""" + B = backends.active if backend is None else backends[backend] + wbar = B.sum(ens_w * fw, axis=-1) + e_1 = ( + B.sum(ens_w * gauss_kern_uv(obs[..., None], fct, backend=backend) * fw, axis=-1) + * ow + / wbar + ) + e_2 = B.sum( + ens_w[..., None] + * ens_w[..., None, :] + * gauss_kern_uv(fct[..., None], fct[..., None, :], backend=backend) + * fw[..., None] + * fw[..., None, :], + axis=(-1, -2), + ) + e_2 *= ow / (wbar**2) + e_3 = gauss_kern_uv(obs, obs, backend=backend) * ow + + out = e_1 - 0.5 * e_2 - 0.5 * e_3 + return -out + + +def ow_ensemble_mv_w( + obs: "ArrayLike", + fct: "Array", + ow: "Array", + fw: "Array", + ens_w: "Array", + backend: "Backend" = None, +) -> "Array": + """Compute an outcome-weighted kernel score for a finite multivariate ensemble. + + The ensemble and variables axes are on the second last and last dimensions respectively. + """ + B = backends.active if backend is None else backends[backend] + wbar = B.sum(fw * ens_w, -1) + + err_kern = gauss_kern_mv(B.expand_dims(obs, -2), fct, backend=backend) + E_1 = B.sum(err_kern * fw * B.expand_dims(ow, -1) * ens_w, axis=-1) / wbar + + spread_kern = gauss_kern_mv( + B.expand_dims(fct, -3), B.expand_dims(fct, -2), backend=backend + ) + fw_prod = B.expand_dims(fw, -1) * B.expand_dims(fw, -2) + spread_kern *= fw_prod * B.expand_dims(ow, (-2, -1)) + E_2 = B.sum( + spread_kern * B.expand_dims(ens_w, -1) * B.expand_dims(ens_w, -2), (-2, -1) + ) / (wbar**2) + + E_3 = gauss_kern_mv(obs, obs, backend=backend) * ow + + out = E_1 - 0.5 * E_2 - 0.5 * E_3 + out = -out + return out + + +def vr_ensemble_uv_w( + obs: "ArrayLike", + fct: "Array", + ow: "Array", + fw: "Array", + ens_w: "Array", + backend: "Backend" = None, +) -> "Array": + """Compute a vertically re-scaled kernel score for a finite univariate ensemble.""" + B = backends.active if backend is None else backends[backend] + + e_1 = ( + B.sum(ens_w * gauss_kern_uv(obs[..., None], fct, backend=backend) * fw, axis=-1) + * ow + ) + e_2 = B.sum( + ens_w[..., None] + * ens_w[..., None, :] + * gauss_kern_uv(fct[..., None], fct[..., None, :], backend=backend) + * fw[..., None] + * fw[..., None, :], + axis=(-1, -2), + ) + e_3 = gauss_kern_uv(obs, obs, backend=backend) * ow * ow + + out = e_1 - 0.5 * e_2 - 0.5 * e_3 + out = -out + return out + + +def vr_ensemble_mv_w( + obs: "ArrayLike", + fct: "Array", + ow: "Array", + fw: "Array", + ens_w: "Array", + backend: "Backend" = None, +) -> "Array": + """Compute a vertically re-scaled kernel score for a finite multivariate ensemble. + + The ensemble and variables axes are on the second last and last dimensions respectively. + """ + B = backends.active if backend is None else backends[backend] + + err_kern = gauss_kern_mv(B.expand_dims(obs, -2), fct, backend=backend) + E_1 = B.sum(err_kern * fw * B.expand_dims(ow, -1) * ens_w, axis=-1) + + spread_kern = gauss_kern_mv( + B.expand_dims(fct, -3), B.expand_dims(fct, -2), backend=backend + ) + fw_prod = B.expand_dims(fw, -1) * B.expand_dims(fw, -2) + spread_kern *= fw_prod + E_2 = B.sum( + spread_kern * B.expand_dims(ens_w, -1) * B.expand_dims(ens_w, -2), (-2, -1) + ) + + E_3 = gauss_kern_mv(obs, obs, backend=backend) * ow * ow + + out = E_1 - 0.5 * E_2 - 0.5 * E_3 + out = -out + return out diff --git a/scoringrules/core/kernels/_gufuncs.py b/scoringrules/core/kernels/_gufuncs.py index 24a3666..5250a77 100644 --- a/scoringrules/core/kernels/_gufuncs.py +++ b/scoringrules/core/kernels/_gufuncs.py @@ -20,14 +20,12 @@ def _gauss_kern_mv(x1: float, x2: float) -> float: @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:])", ], - "(),(n),(n)->()", + "(),(n)->()", ) -def _ks_ensemble_uv_nrg_gufunc( - obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray -): +def _ks_ensemble_uv_nrg_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): """Standard version of the kernel score.""" obs = obs[0] M = fct.shape[-1] @@ -40,24 +38,22 @@ def _ks_ensemble_uv_nrg_gufunc( e_2 = 0 for i in range(M): - e_1 += _gauss_kern_uv(fct[i], obs) * w[i] + e_1 += _gauss_kern_uv(fct[i], obs) for j in range(M): - e_2 += _gauss_kern_uv(fct[i], fct[j]) * w[i] * w[j] + e_2 += _gauss_kern_uv(fct[i], fct[j]) e_3 = _gauss_kern_uv(obs, obs) - out[0] = -(e_1 - 0.5 * e_2 - 0.5 * e_3) + out[0] = -((e_1 / M) - 0.5 * (e_2 / (M**2)) - 0.5 * e_3) @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:])", ], - "(),(n),(n)->()", + "(),(n)->()", ) -def _ks_ensemble_uv_fair_gufunc( - obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray -): +def _ks_ensemble_uv_fair_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): """Fair version of the kernel score.""" obs = obs[0] M = fct.shape[-1] @@ -70,30 +66,30 @@ def _ks_ensemble_uv_fair_gufunc( e_2 = 0 for i in range(M): - e_1 += _gauss_kern_uv(fct[i], obs) * w[i] + e_1 += _gauss_kern_uv(fct[i], obs) for j in range(i + 1, M): - e_2 += _gauss_kern_uv(fct[i], fct[j]) * w[i] * w[j] + e_2 += _gauss_kern_uv(fct[i], fct[j]) e_3 = _gauss_kern_uv(obs, obs) - out[0] = -(e_1 - e_2 / (1 - np.sum(w * w)) - 0.5 * e_3) + out[0] = -((e_1 / M) - e_2 / (M * (M - 1)) - 0.5 * e_3) @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:], float64[:])", ], - "(),(n),(),(n),(n)->()", + "(),(n),(),(n)->()", ) def _owks_ensemble_uv_gufunc( obs: np.ndarray, fct: np.ndarray, ow: np.ndarray, fw: np.ndarray, - w: np.ndarray, out: np.ndarray, ): """Outcome-weighted kernel score for univariate ensembles.""" + M = fct.shape[-1] obs = obs[0] ow = ow[0] @@ -105,32 +101,32 @@ def _owks_ensemble_uv_gufunc( e_2 = 0.0 for i, x_i in enumerate(fct): - e_1 += _gauss_kern_uv(x_i, obs) * fw[i] * ow * w[i] + e_1 += _gauss_kern_uv(x_i, obs) * fw[i] * ow for j, x_j in enumerate(fct): - e_2 += _gauss_kern_uv(x_i, x_j) * fw[i] * fw[j] * ow * w[i] * w[j] + e_2 += _gauss_kern_uv(x_i, x_j) * fw[i] * fw[j] * ow e_3 = _gauss_kern_uv(obs, obs) * ow - wbar = np.sum(fw * w) + wbar = np.mean(fw) - out[0] = -(e_1 / wbar - 0.5 * e_2 / (wbar**2) - 0.5 * e_3) + out[0] = -(e_1 / (M * wbar) - 0.5 * e_2 / (M**2 * wbar**2) - 0.5 * e_3) @guvectorize( [ - "void(float32[:], float32[:], float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:], float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:], float64[:])", ], - "(),(n),(),(n),(n)->()", + "(),(n),(),(n)->()", ) def _vrks_ensemble_uv_gufunc( obs: np.ndarray, fct: np.ndarray, ow: np.ndarray, fw: np.ndarray, - w: np.ndarray, out: np.ndarray, ): """Vertically re-scaled kernel score for univariate ensembles.""" + M = fct.shape[-1] obs = obs[0] ow = ow[0] @@ -142,75 +138,70 @@ def _vrks_ensemble_uv_gufunc( e_2 = 0.0 for i, x_i in enumerate(fct): - e_1 += _gauss_kern_uv(x_i, obs) * fw[i] * ow * w[i] + e_1 += _gauss_kern_uv(x_i, obs) * fw[i] * ow for j, x_j in enumerate(fct): - e_2 += _gauss_kern_uv(x_i, x_j) * fw[i] * fw[j] * w[i] * w[j] + e_2 += _gauss_kern_uv(x_i, x_j) * fw[i] * fw[j] e_3 = _gauss_kern_uv(obs, obs) * ow * ow - out[0] = -(e_1 - 0.5 * e_2 - 0.5 * e_3) + out[0] = -((e_1 / M) - 0.5 * (e_2 / (M**2)) - 0.5 * e_3) @guvectorize( [ - "void(float32[:], float32[:,:], float32[:], float32[:])", - "void(float64[:], float64[:,:], float64[:], float64[:])", + "void(float32[:], float32[:,:], float32[:])", + "void(float64[:], float64[:,:], float64[:])", ], - "(d),(m,d),(m)->()", + "(d),(m,d)->()", ) -def _ks_ensemble_mv_nrg_gufunc( - obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray -): +def _ks_ensemble_mv_nrg_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): """Standard version of the multivariate kernel score.""" M = fct.shape[0] e_1 = 0.0 e_2 = 0.0 for i in range(M): - e_1 += float(_gauss_kern_mv(fct[i], obs)) * w[i] + e_1 += float(_gauss_kern_mv(fct[i], obs)) for j in range(M): - e_2 += float(_gauss_kern_mv(fct[i], fct[j])) * w[i] * w[j] + e_2 += float(_gauss_kern_mv(fct[i], fct[j])) e_3 = float(_gauss_kern_mv(obs, obs)) - out[0] = -(e_1 - 0.5 * e_2 - 0.5 * e_3) + out[0] = -((e_1 / M) - 0.5 * (e_2 / (M**2)) - 0.5 * e_3) @guvectorize( [ - "void(float32[:], float32[:,:], float32[:], float32[:])", - "void(float64[:], float64[:,:], float64[:], float64[:])", + "void(float32[:], float32[:,:], float32[:])", + "void(float64[:], float64[:,:], float64[:])", ], - "(d),(m,d),(m)->()", + "(d),(m,d)->()", ) -def _ks_ensemble_mv_fair_gufunc( - obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray -): +def _ks_ensemble_mv_fair_gufunc(obs: np.ndarray, fct: np.ndarray, out: np.ndarray): """Fair version of the multivariate kernel score.""" M = fct.shape[0] e_1 = 0.0 e_2 = 0.0 for i in range(M): - e_1 += float(_gauss_kern_mv(fct[i], obs) * w[i]) + e_1 += float(_gauss_kern_mv(fct[i], obs)) for j in range(i + 1, M): - e_2 += float(_gauss_kern_mv(fct[i], fct[j]) * w[i] * w[j]) + e_2 += float(_gauss_kern_mv(fct[i], fct[j])) e_3 = float(_gauss_kern_mv(obs, obs)) - out[0] = -(e_1 - e_2 / (1 - np.sum(w * w)) - 0.5 * e_3) + out[0] = -((e_1 / M) - (e_2 / (M * (M - 1))) - 0.5 * e_3) @guvectorize( [ - "void(float32[:], float32[:,:], float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:,:], float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:,:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:], float64[:])", ], - "(d),(m,d),(),(m),(m)->()", + "(d),(m,d),(),(m)->()", ) def _owks_ensemble_mv_gufunc( obs: np.ndarray, fct: np.ndarray, ow: np.ndarray, fw: np.ndarray, - w: np.ndarray, out: np.ndarray, ): """Outcome-weighted kernel score for multivariate ensembles.""" @@ -220,31 +211,28 @@ def _owks_ensemble_mv_gufunc( e_1 = 0.0 e_2 = 0.0 for i in range(M): - e_1 += float(_gauss_kern_mv(fct[i], obs) * fw[i] * ow * w[i]) + e_1 += float(_gauss_kern_mv(fct[i], obs) * fw[i] * ow) for j in range(M): - e_2 += float( - _gauss_kern_mv(fct[i], fct[j]) * fw[i] * fw[j] * ow * w[i] * w[j] - ) + e_2 += float(_gauss_kern_mv(fct[i], fct[j]) * fw[i] * fw[j] * ow) e_3 = float(_gauss_kern_mv(obs, obs)) * ow - wbar = np.sum(fw * w) + wbar = np.mean(fw) - out[0] = -(e_1 / wbar - 0.5 * e_2 / (wbar**2) - 0.5 * e_3) + out[0] = -(e_1 / (M * wbar) - 0.5 * e_2 / (M**2 * wbar**2) - 0.5 * e_3) @guvectorize( [ - "void(float32[:], float32[:,:], float32[:], float32[:], float32[:], float32[:])", - "void(float64[:], float64[:,:], float64[:], float64[:], float64[:], float64[:])", + "void(float32[:], float32[:,:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:], float64[:])", ], - "(d),(m,d),(),(m),(m)->()", + "(d),(m,d),(),(m)->()", ) def _vrks_ensemble_mv_gufunc( obs: np.ndarray, fct: np.ndarray, ow: np.ndarray, fw: np.ndarray, - w: np.ndarray, out: np.ndarray, ): """Vertically re-scaled kernel score for multivariate ensembles.""" @@ -254,15 +242,15 @@ def _vrks_ensemble_mv_gufunc( e_1 = 0.0 e_2 = 0.0 for i in range(M): - e_1 += float(_gauss_kern_mv(fct[i], obs) * fw[i] * ow * w[i]) + e_1 += float(_gauss_kern_mv(fct[i], obs) * fw[i] * ow) for j in range(M): - e_2 += float(_gauss_kern_mv(fct[i], fct[j]) * fw[i] * fw[j] * w[i] * w[j]) + e_2 += float(_gauss_kern_mv(fct[i], fct[j]) * fw[i] * fw[j]) e_3 = float(_gauss_kern_mv(obs, obs)) * ow * ow - out[0] = -(e_1 - 0.5 * e_2 - 0.5 * e_3) + out[0] = -((e_1 / M) - 0.5 * (e_2 / (M**2)) - 0.5 * e_3) -estimator_gufuncs = { +estimator_gufuncs_uv = { "fair": _ks_ensemble_uv_fair_gufunc, "nrg": _ks_ensemble_uv_nrg_gufunc, "ow": _owks_ensemble_uv_gufunc, diff --git a/scoringrules/core/kernels/_gufuncs_w.py b/scoringrules/core/kernels/_gufuncs_w.py new file mode 100644 index 0000000..0832ca3 --- /dev/null +++ b/scoringrules/core/kernels/_gufuncs_w.py @@ -0,0 +1,288 @@ +import math + +import numpy as np +from numba import njit, guvectorize + + +@njit(["float32(float32, float32)", "float64(float64, float64)"]) +def _gauss_kern_uv(x1: float, x2: float) -> float: + """Gaussian kernel evaluated at x1 and x2.""" + out: float = math.exp(-0.5 * (x1 - x2) ** 2) + return out + + +@njit(["float32(float32[:], float32[:])", "float64(float64[:], float64[:])"]) +def _gauss_kern_mv(x1: float, x2: float) -> float: + """Gaussian kernel evaluated at x1 and x2.""" + out: float = math.exp(-0.5 * np.linalg.norm(x1 - x2) ** 2) + return out + + +@guvectorize( + [ + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", + ], + "(),(n),(n)->()", +) +def _ks_ensemble_uv_w_nrg_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): + """Standard version of the kernel score.""" + obs = obs[0] + M = fct.shape[-1] + + if np.isnan(obs): + out[0] = np.nan + return + + e_1 = 0 + e_2 = 0 + + for i in range(M): + e_1 += _gauss_kern_uv(fct[i], obs) * w[i] + for j in range(M): + e_2 += _gauss_kern_uv(fct[i], fct[j]) * w[i] * w[j] + e_3 = _gauss_kern_uv(obs, obs) + + out[0] = -(e_1 - 0.5 * e_2 - 0.5 * e_3) + + +@guvectorize( + [ + "void(float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:])", + ], + "(),(n),(n)->()", +) +def _ks_ensemble_uv_w_fair_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): + """Fair version of the kernel score.""" + obs = obs[0] + M = fct.shape[-1] + + if np.isnan(obs): + out[0] = np.nan + return + + e_1 = 0 + e_2 = 0 + + for i in range(M): + e_1 += _gauss_kern_uv(fct[i], obs) * w[i] + for j in range(i + 1, M): + e_2 += _gauss_kern_uv(fct[i], fct[j]) * w[i] * w[j] + e_3 = _gauss_kern_uv(obs, obs) + + out[0] = -(e_1 - e_2 / (1 - np.sum(w * w)) - 0.5 * e_3) + + +@guvectorize( + [ + "void(float32[:], float32[:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:], float64[:], float64[:])", + ], + "(),(n),(),(n),(n)->()", +) +def _owks_ensemble_uv_w_gufunc( + obs: np.ndarray, + fct: np.ndarray, + ow: np.ndarray, + fw: np.ndarray, + w: np.ndarray, + out: np.ndarray, +): + """Outcome-weighted kernel score for univariate ensembles.""" + obs = obs[0] + ow = ow[0] + + if np.isnan(obs): + out[0] = np.nan + return + + e_1 = 0.0 + e_2 = 0.0 + + for i, x_i in enumerate(fct): + e_1 += _gauss_kern_uv(x_i, obs) * fw[i] * ow * w[i] + for j, x_j in enumerate(fct): + e_2 += _gauss_kern_uv(x_i, x_j) * fw[i] * fw[j] * ow * w[i] * w[j] + e_3 = _gauss_kern_uv(obs, obs) * ow + + wbar = np.sum(fw * w) + + out[0] = -(e_1 / wbar - 0.5 * e_2 / (wbar**2) - 0.5 * e_3) + + +@guvectorize( + [ + "void(float32[:], float32[:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:], float64[:], float64[:], float64[:], float64[:])", + ], + "(),(n),(),(n),(n)->()", +) +def _vrks_ensemble_uv_w_gufunc( + obs: np.ndarray, + fct: np.ndarray, + ow: np.ndarray, + fw: np.ndarray, + w: np.ndarray, + out: np.ndarray, +): + """Vertically re-scaled kernel score for univariate ensembles.""" + obs = obs[0] + ow = ow[0] + + if np.isnan(obs): + out[0] = np.nan + return + + e_1 = 0.0 + e_2 = 0.0 + + for i, x_i in enumerate(fct): + e_1 += _gauss_kern_uv(x_i, obs) * fw[i] * ow * w[i] + for j, x_j in enumerate(fct): + e_2 += _gauss_kern_uv(x_i, x_j) * fw[i] * fw[j] * w[i] * w[j] + e_3 = _gauss_kern_uv(obs, obs) * ow * ow + + out[0] = -(e_1 - 0.5 * e_2 - 0.5 * e_3) + + +@guvectorize( + [ + "void(float32[:], float32[:,:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:])", + ], + "(d),(m,d),(m)->()", +) +def _ks_ensemble_mv_w_nrg_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): + """Standard version of the multivariate kernel score.""" + M = fct.shape[0] + + e_1 = 0.0 + e_2 = 0.0 + for i in range(M): + e_1 += float(_gauss_kern_mv(fct[i], obs)) * w[i] + for j in range(M): + e_2 += float(_gauss_kern_mv(fct[i], fct[j])) * w[i] * w[j] + e_3 = float(_gauss_kern_mv(obs, obs)) + + out[0] = -(e_1 - 0.5 * e_2 - 0.5 * e_3) + + +@guvectorize( + [ + "void(float32[:], float32[:,:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:])", + ], + "(d),(m,d),(m)->()", +) +def _ks_ensemble_mv_w_fair_gufunc( + obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray +): + """Fair version of the multivariate kernel score.""" + M = fct.shape[0] + + e_1 = 0.0 + e_2 = 0.0 + for i in range(M): + e_1 += float(_gauss_kern_mv(fct[i], obs) * w[i]) + for j in range(i + 1, M): + e_2 += float(_gauss_kern_mv(fct[i], fct[j]) * w[i] * w[j]) + e_3 = float(_gauss_kern_mv(obs, obs)) + + out[0] = -(e_1 - e_2 / (1 - np.sum(w * w)) - 0.5 * e_3) + + +@guvectorize( + [ + "void(float32[:], float32[:,:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:], float64[:], float64[:])", + ], + "(d),(m,d),(),(m),(m)->()", +) +def _owks_ensemble_mv_w_gufunc( + obs: np.ndarray, + fct: np.ndarray, + ow: np.ndarray, + fw: np.ndarray, + w: np.ndarray, + out: np.ndarray, +): + """Outcome-weighted kernel score for multivariate ensembles.""" + M = fct.shape[0] + ow = ow[0] + + e_1 = 0.0 + e_2 = 0.0 + for i in range(M): + e_1 += float(_gauss_kern_mv(fct[i], obs) * fw[i] * ow * w[i]) + for j in range(M): + e_2 += float( + _gauss_kern_mv(fct[i], fct[j]) * fw[i] * fw[j] * ow * w[i] * w[j] + ) + e_3 = float(_gauss_kern_mv(obs, obs)) * ow + + wbar = np.sum(fw * w) + + out[0] = -(e_1 / wbar - 0.5 * e_2 / (wbar**2) - 0.5 * e_3) + + +@guvectorize( + [ + "void(float32[:], float32[:,:], float32[:], float32[:], float32[:], float32[:])", + "void(float64[:], float64[:,:], float64[:], float64[:], float64[:], float64[:])", + ], + "(d),(m,d),(),(m),(m)->()", +) +def _vrks_ensemble_mv_w_gufunc( + obs: np.ndarray, + fct: np.ndarray, + ow: np.ndarray, + fw: np.ndarray, + w: np.ndarray, + out: np.ndarray, +): + """Vertically re-scaled kernel score for multivariate ensembles.""" + M = fct.shape[0] + ow = ow[0] + + e_1 = 0.0 + e_2 = 0.0 + for i in range(M): + e_1 += float(_gauss_kern_mv(fct[i], obs) * fw[i] * ow * w[i]) + for j in range(M): + e_2 += float(_gauss_kern_mv(fct[i], fct[j]) * fw[i] * fw[j] * w[i] * w[j]) + e_3 = float(_gauss_kern_mv(obs, obs)) * ow * ow + + out[0] = -(e_1 - 0.5 * e_2 - 0.5 * e_3) + + +estimator_gufuncs_uv_w = { + "fair": _ks_ensemble_uv_w_fair_gufunc, + "nrg": _ks_ensemble_uv_w_nrg_gufunc, + "ow": _owks_ensemble_uv_w_gufunc, + "vr": _vrks_ensemble_uv_w_gufunc, +} + +estimator_gufuncs_mv_w = { + "fair": _ks_ensemble_mv_w_fair_gufunc, + "nrg": _ks_ensemble_mv_w_nrg_gufunc, + "ow": _owks_ensemble_mv_w_gufunc, + "vr": _vrks_ensemble_mv_w_gufunc, +} + +__all__ = [ + "_ks_ensemble_uv_w_fair_gufunc", + "_ks_ensemble_uv_w_nrg_gufunc", + "_owks_ensemble_uv_w_gufunc", + "_vrks_ensemble_uv_w_gufunc", + "_ks_ensemble_mv_w_fair_gufunc", + "_ks_ensemble_mv_w_nrg_gufunc", + "_owks_ensemble_mv_w_gufunc", + "_vrks_ensemble_mv_w_gufunc", +] From 03c470daf0807d905b986e827fc6c53059772207 Mon Sep 17 00:00:00 2001 From: sallen12 Date: Fri, 15 Aug 2025 12:49:14 +0200 Subject: [PATCH 79/79] fix bugs in weighted gufuncs --- scoringrules/core/crps/_gufuncs_w.py | 11 ----------- scoringrules/core/kernels/_gufuncs_w.py | 6 ------ 2 files changed, 17 deletions(-) diff --git a/scoringrules/core/crps/_gufuncs_w.py b/scoringrules/core/crps/_gufuncs_w.py index 106c602..f4ccaad 100644 --- a/scoringrules/core/crps/_gufuncs_w.py +++ b/scoringrules/core/crps/_gufuncs_w.py @@ -12,7 +12,6 @@ def _crps_ensemble_int_w_gufunc( obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray ): """CRPS estimator based on the integral form.""" - obs = obs[0] if np.isnan(obs): out[0] = np.nan @@ -52,7 +51,6 @@ def _crps_ensemble_qd_w_gufunc( obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray ): """CRPS estimator based on the quantile decomposition form.""" - obs = obs[0] if np.isnan(obs): out[0] = np.nan @@ -76,7 +74,6 @@ def _crps_ensemble_nrg_w_gufunc( obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray ): """CRPS estimator based on the energy form.""" - obs = obs[0] M = fct.shape[-1] if np.isnan(obs): @@ -99,7 +96,6 @@ def _crps_ensemble_fair_w_gufunc( obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray ): """Fair version of the CRPS estimator based on the energy form.""" - obs = obs[0] M = fct.shape[-1] if np.isnan(obs): @@ -124,7 +120,6 @@ def _crps_ensemble_pwm_w_gufunc( obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray ): """CRPS estimator based on the probability weighted moment (PWM) form.""" - obs = obs[0] if np.isnan(obs): out[0] = np.nan @@ -150,7 +145,6 @@ def _crps_ensemble_akr_w_gufunc( ): """CRPS estimaton based on the approximate kernel representation.""" M = fct.shape[-1] - obs = obs[0] e_1 = 0 e_2 = 0 for i, forecast in enumerate(fct): @@ -167,7 +161,6 @@ def _crps_ensemble_akr_circperm_w_gufunc( ): """CRPS estimaton based on the AKR with cyclic permutation.""" M = fct.shape[-1] - obs = obs[0] e_1 = 0.0 e_2 = 0.0 for i, forecast in enumerate(fct): @@ -187,8 +180,6 @@ def _owcrps_ensemble_nrg_w_gufunc( out: np.ndarray, ): """Outcome-weighted CRPS estimator based on the energy form.""" - obs = obs[0] - ow = ow[0] M = fct.shape[-1] if np.isnan(obs): @@ -218,8 +209,6 @@ def _vrcrps_ensemble_nrg_w_gufunc( out: np.ndarray, ): """Vertically re-scaled CRPS estimator based on the energy form.""" - obs = obs[0] - ow = ow[0] M = fct.shape[-1] if np.isnan(obs): diff --git a/scoringrules/core/kernels/_gufuncs_w.py b/scoringrules/core/kernels/_gufuncs_w.py index e359fcc..8c9a0ee 100644 --- a/scoringrules/core/kernels/_gufuncs_w.py +++ b/scoringrules/core/kernels/_gufuncs_w.py @@ -25,7 +25,6 @@ def _ks_ensemble_uv_w_nrg_gufunc( obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray ): """Standard version of the kernel score.""" - obs = obs[0] M = fct.shape[-1] if np.isnan(obs): @@ -49,7 +48,6 @@ def _ks_ensemble_uv_w_fair_gufunc( obs: np.ndarray, fct: np.ndarray, w: np.ndarray, out: np.ndarray ): """Fair version of the kernel score.""" - obs = obs[0] M = fct.shape[-1] if np.isnan(obs): @@ -78,8 +76,6 @@ def _owks_ensemble_uv_w_gufunc( out: np.ndarray, ): """Outcome-weighted kernel score for univariate ensembles.""" - obs = obs[0] - ow = ow[0] if np.isnan(obs): out[0] = np.nan @@ -109,8 +105,6 @@ def _vrks_ensemble_uv_w_gufunc( out: np.ndarray, ): """Vertically re-scaled kernel score for univariate ensembles.""" - obs = obs[0] - ow = ow[0] if np.isnan(obs): out[0] = np.nan