From 85b84915e71f473d441ebf38e34029f290332d81 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Mon, 20 Nov 2023 14:04:22 +0100 Subject: [PATCH 01/40] make input of set_forecast_unit() a data.table --- DESCRIPTION | 2 +- NEWS.md | 6 ++++++ R/convenience-functions.R | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 3e876aaf7..2fabe6a00 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: scoringutils Title: Utilities for Scoring and Assessing Predictions -Version: 1.2.1 +Version: 1.2.2 Language: en-GB Authors@R: c( person(given = "Nikos", diff --git a/NEWS.md b/NEWS.md index eb47b1cb4..4f35ea6d5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,9 @@ +# scoringutils 1.2.2 + +## Bug fixes +- fixes a bug with `set_forecast_unit()` where the function only workded with a data.table, but not a data.frame as an input. + + # scoringutils 1.2.1 ## Package updates diff --git a/R/convenience-functions.R b/R/convenience-functions.R index f21aeefc8..41592d7d4 100644 --- a/R/convenience-functions.R +++ b/R/convenience-functions.R @@ -235,7 +235,7 @@ log_shift <- function(x, offset = 0, base = exp(1)) { #' ) set_forecast_unit <- function(data, forecast_unit) { - + data <- as.data.table(data) datacols <- colnames(data) missing <- forecast_unit[!(forecast_unit %in% datacols)] From c8fdad59c2c1a93b93086240feabf529650319dc Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Mon, 20 Nov 2023 14:30:48 +0100 Subject: [PATCH 02/40] Update to R version 3.6 to fix failing gh action --- .github/workflows/R-CMD-check.yaml | 2 +- DESCRIPTION | 4 ++-- NEWS.md | 5 +++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index f2a50448b..cda899829 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -23,7 +23,7 @@ jobs: - {os: ubuntu-latest, r: 'devel', http-user-agent: 'release'} - {os: ubuntu-latest, r: 'release'} - {os: ubuntu-latest, r: 'oldrel-1'} - - {os: ubuntu-latest, r: '3.5'} + - {os: ubuntu-latest, r: '3.6'} env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} diff --git a/DESCRIPTION b/DESCRIPTION index 3e876aaf7..4bd770f5c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: scoringutils Title: Utilities for Scoring and Assessing Predictions -Version: 1.2.1 +Version: 1.2.2 Language: en-GB Authors@R: c( person(given = "Nikos", @@ -71,5 +71,5 @@ URL: https://doi.org/10.48550/arXiv.2205.07090, https://epiforecasts.io/scoringu BugReports: https://github.com/epiforecasts/scoringutils/issues VignetteBuilder: knitr Depends: - R (>= 3.5) + R (>= 3.6) Roxygen: list(markdown = TRUE) diff --git a/NEWS.md b/NEWS.md index eb47b1cb4..5b8d2810e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +# scoringutils 1.2.2 + +## Package updates +- `scoringutils` now depends on R 3.6. The change was made since packages `testthat` and `lifecycle`, which are used in `scoringutils` now require R 3.6. We also updated the Github action CI check to work with R 3.6 now. + # scoringutils 1.2.1 ## Package updates From f0f0cc0080e992ca1b8c8fe32be034f544342f8e Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Mon, 20 Nov 2023 14:40:43 +0100 Subject: [PATCH 03/40] Update test snapshot --- tests/testthat/_snaps/summarise_scores.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/testthat/_snaps/summarise_scores.md b/tests/testthat/_snaps/summarise_scores.md index fdc138864..1ecd8e86b 100644 --- a/tests/testthat/_snaps/summarise_scores.md +++ b/tests/testthat/_snaps/summarise_scores.md @@ -2,7 +2,8 @@ Code x <- summarise_scores(scores, by = "model", metric = "auto", relative_skill = TRUE) - Warning + Condition + Warning: The `metric` argument of `summarise_scores()` is deprecated as of scoringutils 1.1.0. i Please use the `relative_skill_metric` argument instead. From 6d72ef3e640b56697c8fb669efc6eeca48aa2486 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Mon, 20 Nov 2023 17:23:11 +0100 Subject: [PATCH 04/40] Fix errors in the metrics table --- inst/create-metric-tables.R | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/inst/create-metric-tables.R b/inst/create-metric-tables.R index 520e09ad7..7ae4d8f3e 100644 --- a/inst/create-metric-tables.R +++ b/inst/create-metric-tables.R @@ -49,7 +49,7 @@ log_score <- list( `C` = r"($\checkmark$)", `B` = r"($\checkmark$)", `Q` = r"($-$)", - `Properties` = "Proper scoring rule, smaller is better, only evaluates predictive density at observed value (local), penalises over-confidence severely, susceptible to outliers", + `Properties` = "Proper scoring rule, smaller is better, equals negative log of the predictive density at observed value (local), penalises over-confidence severely, susceptible to outliers", `References` = "" ) @@ -266,33 +266,33 @@ crps <- list( `Metric` = "CRPS (Continuous) ranked probability score", `Explanation` = r"(The crps is a proper scoring rule that generalises the absolute error to probabilistic forecasts. It measures the 'distance' of the predictive distribution to the observed data-generating distribution. The CRPS is given as $$\text{CRPS}(F, y) = \int_{-\infty}^\infty \left( F(x) - 1(x \geq y) \right)^2 dx,$$ - where y is the true observed value and F the CDF of predictive distribution. Often An alternative representation is used: - $$ \text{CRPS}(F, y) = \frac{1}{2} \mathbb{E}_{F} |X - X'| - \mathbb{E}_P |X - y|,$$ where $X$ and $X'$ are independent realisations from the predictive distributions $F$ with finite first moment and $y$ is the true value. In this represenation we can simply replace $X$ and $X'$ by samples sum over all possible combinations to obtain the CRPS. + where y is the observed value and F the CDF of predictive distribution. Often An alternative representation is used: + $$ \text{CRPS}(F, y) = \frac{1}{2} \mathbb{E}_{F} |X - X'| - \mathbb{E}_P |X - y|,$$ where $X$ and $X'$ are independent realisations from the predictive distributions $F$ with finite first moment and $y$ is the observed value. In this representation we can simply replace $X$ and $X'$ by samples and sum over all possible combinations to obtain the CRPS. For integer-valued forecasts, the RPS is given as $$ \text{RPS}(F, y) = \sum_{x = 0}^\infty (F(x) - 1(x \geq y))^2. $$ **Usage and caveats**: - Smaller values are better. The crps is a good choice for most practical purposes that involve decision making, as it takes the entire predictive distribution into account. If two forecasters assign the same probability to the true event $y$, then the forecaster who assigned high probability to events far away from $y$ will still get a worse score. The crps (in contrast to the log score) can at times be quite lenient towards extreme mispredictions. Also, due to it's similarity to the absolute error, the level of scores depend a lot on the absolute value of what is predicted, which makes it hard to compare scores of forecasts for quantities that are orders of magnitude apart.)" + Smaller values are better. The crps is a good choice for most practical purposes that involve decision making, as it takes the entire predictive distribution into account. If two forecasters assign the same probability to the observed event $y$, then the forecaster who assigned high probability to events far away from $y$ will still get a worse score. The crps (in contrast to the log score) can at times be quite lenient towards very poor predictions. Also, due to it's similarity to the absolute error, the level of scores depend a lot on the absolute value of what is predicted, which makes it hard to compare scores of forecasts for quantities that are orders of magnitude apart.)" ) log_score <- list( `Metric` = "Log score", - `Explanation` = r"(The Log score is a proper scoring rule that is simply compuated as the log of the predictive density evaluated at the true observed value. It is given as - $$ \text{log score} = \log f(y), $$ - where $f$ is the predictive density function and y is the true value. For integer-valued forecasts, the log score can be computed as - $$ \text{log score} = \log p_y, $$ + `Explanation` = r"(The Log score is a proper scoring rule that is computed as the negative log of the predictive density evaluated at the observed value. It is given as + $$ \text{log score} = -\log f(y), $$ + where $f$ is the predictive density function and y is the observed value. For integer-valued forecasts, the log score can be computed as + $$ \text{log score} = -\log p_y, $$ where $p_y$ is the probability assigned to outcome p by the forecast F. **Usage and caveats**: - Larger values are better, but sometimes the sign is reversed. The log score is ensitive to outliers, as individual negative log score contributions quickly can become very large if the event falls in the tails of the predictive distribution, where $f(y)$ (or $p_y$) is close to zero. Whether or not that is desirable depends ont the application. In scoringutils, the log score cannot be used for integer-valued forecasts, as the implementation requires a predictive density. In contrast to the crps, the log score is a local scoring rule: it's value only depends only on the probability that was assigned to the actual outcome. This property may be desirable for inferential purposes, for example in a Bayesian context (Winkler et al., 1996). In settings where forecasts inform decision making, it may be more appropriate to score forecasts based on the entire predictive distribution.)" + Smaller values are better, but sometimes the sign is reversed. The log score is sensitive to outliers, as individual log score contributions can become very large if the event falls in a range of the predictive distribution where $f(y)$ (or $p_y$) is close to zero. Whether or not that is desirable depends ont the application. In scoringutils, the log score cannot be used for integer-valued forecasts, as the implementation requires a predictive density. In contrast to the crps, the log score is a local scoring rule: it's value only depends only on the probability that was assigned to the actual outcome. This property may be desirable for inferential purposes, for example in a Bayesian context (Winkler et al., 1996). In settings where forecasts inform decision making, it may be more appropriate to score forecasts based on the entire predictive distribution.)" ) wis <- list( Metric = "WIS (Weighted) interval score", `Explanation` = r"(The (weighted) interval score is a proper scoring rule for quantile forecasts that converges to the crps for an increasing number of intervals. The score can be decomposed into a sharpness (uncertainty) component and penalties for over- and underprediction. For a single interval, the score is computed as $$IS_\alpha(F,y) = (u-l) + \frac{2}{\alpha} \cdot (l-y) \cdot \mathbf{1}(y \leq l) + \frac{2}{\alpha} \cdot (y-u) \cdot \mathbf{1}(y \geq u), $$ - where $\mathbf{1}()$ is the indicator function, $y$ is the true value, and $l$ and $u$ are the $\frac{\alpha}{2}$ and $1 - \frac{\alpha}{2}$ quantiles of the predictive distribution $F$, i.e. the lower and upper bound of a single prediction interval. For a set of $K$ prediction intervals and the median $m$, the score is computed as a weighted sum, + where $\mathbf{1}()$ is the indicator function, $y$ is the observed value, and $l$ and $u$ are the $\frac{\alpha}{2}$ and $1 - \frac{\alpha}{2}$ quantiles of the predictive distribution $F$, i.e. the lower and upper bound of a single prediction interval. For a set of $K$ prediction intervals and the median $m$, the score is computed as a weighted sum, $$WIS = \frac{1}{K + 0.5} \cdot \left(w_0 \cdot |y - m| + \sum_{k = 1}^{K} w_k \cdot IS_{\alpha}(F, y)\right),$$ where $w_k$ is a weight for every interval. Usually, $w_k = \frac{\alpha_k}{2}$ and $w_0 = 0.5$. @@ -308,7 +308,7 @@ dss <- list( `Explanation` = r"(The Dawid-Sebastiani-Score is a proper scoring rule proposed that only relies on the first moments of the predictive distribution and is therefore easy to compute. It is given as $$\text{dss}(F, y) = \left( \frac{y - \mu}{\sigma} \right)^2 + 2 \cdot \log \sigma,$$ - where $F$ is the predictive distribution with mean $\mu$ and standard deviation $\sigma$ and $y$ is the true observed value. + where $F$ is the predictive distribution with mean $\mu$ and standard deviation $\sigma$ and $y$ is the observed value. **Usage and caveats**: The dss is applicable to continuous and integer forecasts and easy to compute. Apart from the ease of computation we see little advantage in using it over other scores.)" @@ -327,7 +327,7 @@ brier_score <- list( interval_coverage <- list( `Metric` = "Interval coverage", `Explanation` = r"(Interval coverage measures the proportion of observed values that fall in a given prediction interval range. Interval coverage for a single prediction interval range can be calculated as $$IC_{\alpha} = \text{nominal coverage} - \text{empirical coverage},$$ - where nominal coverage is $1 - \alpha$ and empirical coverage is the proportion of true values actually covered by all $1 - \alpha$ prediction intervals. + where nominal coverage is $1 - \alpha$ and empirical coverage is the proportion of observed values actually covered by all $1 - \alpha$ prediction intervals. To summarise interval coverage over different over multiple interval ranges, we can compute coverage deviation defined as the mean interval coverage over all $K$ interval ranges $\alpha_k$ with $k = 1, \dots, K$: $$\text{Coverage deviation} = \frac{1}{K} \sum_{k = 1}^{K} \text{IC}_{\alpha_k}$$ @@ -338,7 +338,7 @@ interval_coverage <- list( quantile_coverage <- list( `Metric` = "Quantile coverage", - `Explanation` = r"(Quantile coverage for a given quantile level is the proportion of true values smaller than the predictions corresponding to that quantile level. + `Explanation` = r"(Quantile coverage for a given quantile level is the proportion of observed values smaller than the predictions corresponding to that quantile level. **Usage**: Quantile coverage is similar to interval coverage, but conveys more information. For example, it allows us to look at the 5\% and 95\% quantile separately, instead of jointly at the 90\% prediction interval). This helps to diagnose whether it is the upper or lower end of a prediction interval that is causing problems. Plots of quantile coverage are conceptually very similar to PIT histograms.)" @@ -365,7 +365,7 @@ bias <- list( $$B(P, y) = 1 - (P(y) + P(y + 1)), $$ where $P(y)$ is the cumulative probability assigned to all outcomes smaller or equal to $y$, i.e. the cumulative probability mass function corresponding to $p(y)$. - For quantile forecasts, Bias can be calculated as the maximum percentile rank for which the prediction is smaller than $y$, if the true value is smaller than the median of the predictive distribution. If the true value is above the median of the predictive distribution, then bias is the minimum percentile rank for which the corresponding quantile is still larger than the true value. If the true value is exactly the median, bias is zero. For a large enough number of quantiles, the percentile rank will equal the proportion of predictive samples below the observed true value, and this metric coincides with the one for continuous forecasts. + For quantile forecasts, Bias can be calculated as the maximum percentile rank for which the prediction is smaller than $y$, if the observed value is smaller than the median of the predictive distribution. If the observed value is above the median of the predictive distribution, then bias is the minimum percentile rank for which the corresponding quantile is still larger than the observed value. If the observed value is exactly the median, bias is zero. For a large enough number of quantiles, the percentile rank will equal the proportion of predictive samples below the observed value, and this metric coincides with the one for continuous forecasts. **Usage**: In contrast to the over- and underprediction penalties of the interval score it is bound between -1 and 1 and represents the tendency of forecasts to be biased rather than the absolute amount of over- and underprediction. It is therefore a more robust measurement, but harder to interpet. It largely depends on the application whether one is more interested in the tendency to be biased or in the absolute value of over- and underpredictions.)" @@ -375,7 +375,7 @@ pit <- list( `Metric` = "Probability integral transform (PIT)", `Explanation` = r"(The probability integral transform (PIT, Dawid 1984) represents a succinct way to visualise deviations between the predictive distribution $F$ and the true data-generating distribution $G$. The idea is to transform the observed values such that agreement between forecasts and data can then be examined by observing whether or not the transformed values follow a uniform distribution. The PIT is given by $$u = F (y),$$ - where $u$ is the transformed variable and $F(y)$ is the predictive distribution $F$ evaluated at the true observed value $y$. If $F = G$, then $u$ follows a uniform distribution. + where $u$ is the transformed variable and $F(y)$ is the predictive distribution $F$ evaluated at the observed value $y$. If $F = G$, then $u$ follows a uniform distribution. For integer outcomes, the PIT is no longer uniform even when forecasts are ideal. Instead, a randomised PIT can be used: $$u = P(y) + v \cdot (P(y) - P(y - 1) ),$$ From 6d556f1b1ceda1519a620a59ec265d2f962e5b8b Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Mon, 20 Nov 2023 17:25:26 +0100 Subject: [PATCH 05/40] Update stored metrics --- data/metrics.rda | Bin 1694 -> 1707 bytes inst/metrics-overview/metrics-detailed.Rda | Bin 5020 -> 4985 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/data/metrics.rda b/data/metrics.rda index e378f5ddea63df7c8428c42cb32e3064056a64a2..64a94e6cf0849927367c1d706e2c1e82cd9d4e32 100644 GIT binary patch delta 1706 zcmV;b237f<4XX`WLRx4!F+o`-Q&~nF_09kU?En4$`}?KG?{FTyC;q{|@BiUI0uT@Y z1V8`)5C9MW0pJIJJtP5*fB*!50)Z+bYJQYqJxmj6GBh$^hKxp-CYk~0fsGNM9;1O${{BriLH}BNGq?Kw@Ya13)+$ zX{LiiLqj7#G62xf8feG>0ffW`f<%I7(E&6vBU3a}$vlL}9;d0b28{uT^hc;YH9bwJ zgkl0lyXhkakZz#@N7lqYbN_v;`nNc$OMm%t#0exL8G2*av;bMN-Q|k?Fd+$WVULh+ z;BloJV`E9SG{(k;F{U)yWRV82A{k$>XVOBknY@l={w95FSD!PRoW2(gzD&BRM)A>x(3V>h48i^)8t%}36qoCU!V>G)(FI)sxh=Zp8M!WVFHEP$a`sJL35uu|v z>+CdYsDYS0N#dHtrMoOCjEY&hN}CEIYFYncAf6X1U2!8;NUjAmM>MFX1Py+MwGTsc z$4VQkLI}kXc0pJ${^e-t`{IJ(w|_j|V`Lbbg(tTYaabUNhFa~r2%U8bx%#hbQ3bOJ zm)j`=3|K)s;H+HU$XL;1fsu(T`fEb!xss!?)QWeZL7NeFXCep~fsCTXhoOk@2OQ!C zi!?(A^JxsyXE4!L3uw^xeBL1-;$R|afso8wVwS+%NSVIEK~!wj2^y7KYk!_0B@%TY zNN&#CE5RYolMExIW3aMmu|mOFNFO_3TI2D-e0?m|88DhjEe@c@hKAoX#AhN3AfG9K z&^YwBmi$sDy&Q?u*ggF*u&C}-65)AIz+@d1j6#ishVs*&ug6e8B(GgbA=>BcIEy>? z0iJ$AiX1v;ZjO^!LN6l!_kYsXaY;O+=P^iEkX;|HUh1)>Z?wS7+6EqUdjl3Y&^LaIp|Vvn)a`Y(aF*lKA^X%AGP^MANxIIJj4Y-2?o zJ4q9JfrMlX2cy9uI*kPWsHxA@nBd-Wo0}RRT)MjJ+>{dNlHM}2wOl0zGengD)aFjo zQXB;q6g#m6L5yr;2Ec75A4W%#%#-#)u$p?Eam`v4@`>O@VTqlusj8YFd zvMg@5>B@(Nl`xu-(SQ7UwY(HY;I0UTc+AIi5>QEPMtp3GBI?5!#>Py5UOGh`EEQA* zK`jGuHO)DOu2kufgyMra$%dq^P|Wc=oBIJO?bXaF31Ul5u4-anj9ME99YQd*PWeii zlWl@J7H2i>tP1>lmq3{Z?t;jDj;2i~dw5UN)k}wz?&&tIdmnaGn zI7C7lG*HnXimM{9v{5YPiU7dS<-f|yP-9J_pi#|`;gWs&??km(cqp0Xf*3MIq`%g+ zR4Wk$!~!JSV1G6e4h^*+Fk@sTib+IF77U)ng0C=_5|d3y{2;2>FahzB5}g{Q5`uU& zNEr!Lf}ue*Ni|qf?+|mNy4Hcbz#niR5|3+KlHe-!_k8dRG$R(8DxhsP@Ck&GA_^yh zOFI6(f`1}FKunVZttJvS*kM2jFhp0m z7|Q5#V6B_Vi^T<-u4E4fMbzP7oQLDF#^7=h0Up*po8_asBNIpsvO&OY7lZOMV)x2xJd^8Q@K87wIE-yUUO-NI58z>8SO~%Z ze-9zSA(L={iVH5sdBHE-K_aje@XXghowi(JuiwU$= zsK-QBi4hdIg_;{LrEsW(N?EUR_a=_id*S0cO-D$-^(B%%=L%~;gvk+zx>AK zwltIsqF&*P(AEXw0+4a&G04dho&=24$Z?+u$om0%ESLUZ$N$CLkxmpO5r;ir2>TjGz=va6!M!ydYLjjrbdHJ0MN+900Yzkr>Fq+JwV`TrkV{6 z4GfI{$OA(_X`>(j1``k(91S$nL7}0ck)Rm>XlM;IWB>rdVgo@Yq==^4Nt4uifCicX z003wJ000Jn91S$nL7}0ck)Rm>XlM;IWB>rdVgo@UAv6dRBSH#nDdiqQWE)e|Xf$XC zsCt9c9*}uOLqG`)EHrFmWDM7oNPChG_4{h4t20=no^H%Ql!RkGxc2V=7L51#;@+qb zgt;)s%s245WTizJ6&Q*zL}D>wRa6b2L^9vZXQ+j9L!gc;`%wCgF}p2mVM2`UbU(Vj z>SGvs-W_>OsN({#iC|1;KPS6OSAgd6q_&9J|J(1BQL~{G|Cv_rDqkMV|67Y(Bun}sa4tf6Vx%bPPHf`E}Dbhs| zsjE8n_sv?eAZ8C@c&4#waLWqgHM@14ZMd>3r(ftq6uyP6c#~#Gwgq)Z8mXZJUx}@Q z)Zvb`I9kLJiX%2bXfXbzs{c!%wB^5DUZZ3fmK44C&jQjx1etBW3?i1?D{1V`_o@r% z6f@s43^8E^Gr?TU=wvKvV}X&0Ir{BWHEjv2x!Q{S6)4J(!7|vQgn)^B&_h`v!41Y* zp@rHZgZedwYeR_9W(#=G_kA8gAnPy@L4lDFV%rpM4g{1sE+iI4?U0eNS+=@kVx%)B z;=y+%)@KvzLp&)c-zjvzZ#vm!#2Sv+#^c08H@Ka$I&~MyZ4se)A0TP==*2*AoP4a; z!YOJjvN)CHq)_94CH<=aSdYmHm>sqHPZBZ2@EkFeGgz$iI@SaQEm}&dqm{eXY8N=L z1Bz>cN$b&4GWYz|h_X9+c(?8)I2WzM5a- z8JM~my#yJBgK%fpT+QWKP@Nx1n$XELyo`eme@L+mn%>g+S@=Q!*cL>!?2CuBRh3GO z1w5xZEW-|wIiT0GxuTT0oO)- z%m`LMKy+Mk5yUVl`{Kt>W`l#E@X^tY54kXpzk)V-V~1~f#K17o2}$s>TO z!l!y5urZB{V4y6BdNNA7rg)&3@nzVCyDdc<%Lb-RJa-#1Uh6@a?S(QkXvHA(o1*3) zj~X5}RJ9bHU$3`g!9;HhV2D}AQI7~EK_>KR*K}POtBhkC8A1VhYZZ04SyUB7wGIUr z8c;SzWqHuTkzt)B17caKnjS})(cef`-?NCMOA>YM<6KMxV!`%^!uC7{7Bnc^NII5> z9PhLXe4H0BlXBXE1}h{T%<;pZC|O-xFP`p7g3vjJt9gErd2E-8;R4yDbD_{+VGMC1 zkr<-VI}W5!T+Rs18G*u$ruLL2EA|F-Gf8z8G%G5B1X7B)l$0VdpmzDF6IKzK5}Ilw+(4$61LPn|Xw@=lK};H~41}z~RH&M) zwP;iD5OpKG)}hc~54sSLzvQmexeDFhewYQK5sOV0P&S)*1p<^s2`ZimEks-6D1;IW zLkpf{eQInuE2K&?Ys0rqY#%l#CPR|sh>##CO9Qx*2{+tfPzW#xw|CB9l||OY0vZwG zV7mN0SxEBat_LRhz$IQ#8XVFLv9Z&twngzMXbUm@eM=y?h8EQHJ&*)5pgdJH8bN`z z5FphV1a&)YJBEd9W_AT}Mj;LPP6UeMs|=m+E=$1Wl`)Cr2j|FzwI|~g6=Bj(JddO# zVnZE~*@zTHR32d2QS{Fn!8BT?l|rG&xz;hY1-k&HH6Oyl;O2DH- zWs=NNK?zQa-h0zWgkNAhr!!(R{v_m&bA?5yLS&JMxupu?TNP9rH!&y~RK4RD&ejHT o0a!ZvjB+wW-Hfr8T(s6)_gMFY*Vsw^K_B?Lk}1N3f%rg1*a%qf6951J diff --git a/inst/metrics-overview/metrics-detailed.Rda b/inst/metrics-overview/metrics-detailed.Rda index 69d0b6363123604ee5154f265dafa1467d468aa9..cf8f9683213a91cf3d8aec0c90d4a190be3d83f0 100644 GIT binary patch literal 4985 zcmV-<6Nc;`iwFP!000001HD^Ij~qvGrbKBC(GTpxHv*`jyE`@YXx3wT+QSC?gM9OE@X5c#fR8>~{vskXE33MDM$o|rQ8QJQ znemP|LETRPx0fuPrrL}a`byV`+NNIG5-4{{`;GIpWOQl|9|xS z+tV{Od|u?W&6}dBMyk~LrAgGPET(#DQ(JFTH7iQvR($s7V$pf<>$fkl!&_q)%i1I( zW%JsUS32!{^bar2&eRLNw#m_%nd+*xI=7uK{`%|EnzHlgS1dJ$gybE83U zYWgn*rxDde$fCsR8Z1@wqRiCr^yQn8XXI?D%T;cw%02kxYpeawzkF?UF2q0}wJqH9 zzx&pt`1Q)D>e8lZSDL?g`(_D}%2lOor8J3fU4@u1wXg4zmt(vHpwaSdHn$j(OnBWtg&&)$r%=yWxG>M%tjr6Xx z(+2eCv@5+XrWIrg1t+@JM?y(W?B;lT^JJ^eGhHvI)0;nRFK$k@>cczm^6udW{2RSFmFJre;KuPXcsF3W z2k#!@2RyKOVpazK4sjWde$R)3dh%Y4NIeERg)?5Hi$M=tD>sxlpQde+T3vh z6Dy{*wz=pEGMzxkCopODMN=LzQAumiWBziVojJvH^#U3$Zm#ju<6vdOUHgJzO|ZiQ z?DOc+cNM||GceOvM%UG&N2K|)3~YcICa7WsO-&7(5z}v-cu^Esx-2Zj1F|7?h<*k8 zfMX)IteO(CU}uD5+x)6XNquH!D-v_2FX3BbFr{p*YJKV47J9JdVPA!~3Z-YWqRDHh z5RSbrg3wF}x*-0Ah#5LEooL;6baO>!0uy0j%c6EKm}}v-!?#N~W`8qY4>Q$J{aSCt zUdiIth=2+K3jZz)3{kC%vN94iBQXwP4>Kf~66bm1QlCUdY=#hBZ8OMbYSjCNOguHY zC9>;ct;?iRFuV<{4hrxxgTYPIv$QJ4DnX3E#yFDE3zrKJ=O-3XDfQo39Po> zaOm5jOh~o(Hq#6Ed6R$!>{3r8y?pvR5@|@kcjOqnB_cq7Le@tNGCTHEU#~#QBlb;Y$0wru; zQxgm04vJS7n=Zk45Ru&aM(`SndrgfK@SehE>|yF;^78`GSRojw3Y&^+u&Od?PJnaMeO5r5!VoC;vkx9NAm=ZX8%-8cW@!+b$eZ;Z!({^cQI5i2 zpAh!iw3s=7?49;YiC!+$iFr?-;P8XFk&!Q zIRj|JAI>5uzLO#mJsoWs$qC8@$)!agBF02`?JlSxz_Am{?b+0a&;KA_8Lkf$Du=^q z8?V!t0+v4n0Y_PwDponK(3Hhz0D2W|lsDOwOnV+qt;BabF)AU~BjG0w0jSE?C~Ann z2n7OM)xbjhUGCu@3;MsPZPHxLRPd7#vNa!jy)=an~(>aJJ_91lC6 zz$YaptO-q%o>pXZQ=q+swg?((RTq>MbO?{br-N66eP0P;r$aKlgK-VA!C^HMa&ulm zD1Xdqr>Z`xgkC{yk0)PvRY`5PzIg0@|FD5%vJa$N#^{$=d%XSr=G9iMFD~6jup}=x zf%L8pj7z`)J|BoL3QZD)I2*Mx_^v{PODRp$?q79ua&Z|cj41qy2p{~+EDVS#S%yL- zxg@~uZZH`CTxkEN2HC=;n>JglEf*gl#)XQXjH82QIcp*`a~FgHjh>Y#%~aMP9#Gt_ zDT5P%2(OISDa9HhoC8jgQSpXT2(`$kQfX2lK-Ss8z%FL_FlAeYLYx{O4nK^Ni4JhX z;GFe|_qB(~yr^Oq%v`aR0Hz}km)Mmu@kB{5|M*WtNWY4Dr&{S8E)s0l}$MeWLcSyR+3AD#L7B1F?~wk}TQE>7aXaDI_f59uQB21GnY z>}I_U`17$khYi5fzG)Lys7M_|OrzS8V@pwjRX~P>Sd)sLG9!*WhjfOI0o_KfqDNc+ z(xtvB3W&U;WKryHMBEXP$n^ey?R0Rv-Ai`Jl641Fc8$wXPxXeNPw}HY*-i=(LY@7? z<}uzmxH6Q8I-NDCc1aP)=XuCiq&(JW&JnXcVoUDIZ%nqbCHI8FyI7C?g=92>C-?;M zwB0?Ty7LxE3FJpO<7E=gNp773_TdCp?wf+rex^%XwWzSAfITL$b14zz*N|Vjl7vmU z#-j6wc9%A`BxY{&wkyHG=)kGW10YWC6oe<2>Rn%SaM^K;SA#F(UiA(M9bbI4-Nh`j z@a6OEMGGoA4MlN;g9azyY_>$QiG)2?Lo-=S)EAG>f5Ym2iXTR7*P+ws0Ibt9bg~eu z(W#x5I?g*gS#+YlHEbjB@I7K*I(fHYk=kEQ@4O~DgB{NKjnppEj!3!JNlgi#9rK2! z56tLHH{f)x3nlOeJkw?uX*|*kXaezHpCr5v^KA5qqz?F zp#ULpYPXiGWuFs|T#3Il>52~o;$P&{T3Ne}imkn2)^_4DZH*`PA{# z-~ZF7ooSH!RLyLLiko^O>=##-iUbO2!3U_nT(6DEZ-q#NqI~&OmyNk~QlfR;b{F$U2fJXMF7deEId`IeZgvK+TLd1y|r{T>95h|!KZO3`piFoQjGEFDlP zp{S&w-5ic+r;8-KQ-}f3{mM5v`1d1vOjE0o>Izn%wqE!lT3-~WQL|&#WR06fA8Jce z0`0|73RNE7(Q!j}^6BvCn5vP-Jo)1!ZKZ+2+PrdAc_x43^HC8y(HI-C4KPbq#7Qqq zjOwCL0N!P>=0Oy6uxzr`p8XuEaHFAkFV}LVoe)<6qUjO@ml5qZTop9Za`8klK!Cb5 zL4=(XCqYQ8p4wFPUOD8HLdj^yu0V@h&YA}KG!lbKU+6OB43C{8-Hp`}DeQ`X33mrr z;8VekAd^gK#$>u#tpp`@?JDKQQlA)P#<{_o4%VVh4zyJ*xhG3qDeYSCVdlpfu0QzW z&P7E54N~KVxK0x1Pu#h2hqe*MX@U`r6d^-y6~ zL%j<4ZY3-_4>%yvwDln4b06mMI$?EO!!*!^H<pN%rtU zNvC(POF^DLDRgp53htJ~9YDjK?Jg1djXK}CeAB9igP%)Y)IZ6+s0eaWtj819iInED4L%utu+BNiWho`fOi)z1>P3mO`05d}~%5;A>z};i}Z%P6p zYj3M=bX}d7DV>`HUx&6R^)7tFbI1_1XH7dA zy{lV$dn5DqO9h#}wpo+$KEUAoGc&*q2r5D;cM#;tH-aoyRKD?yvXJj&l(>VfrK`vG^BR2ds<7^TRGm!&7E#TfiY&>ZxAl#`=EIL(F4FPzrCjwxO4mC@F1+6*E{|8uOcwMka@803>+QvR`PGu_~=LF=z;>{;i=$xV4P*F z(SQha#kQH(UDGc-mG* z&gSGgZGzndfp_@ru*l)N%Urm*caXOLWB=`q??p$qzWrU{7}?WROo#405~Y#@T#zrh z)^EU$eyG~|-?X3!oNTbvS4Loc+=7#4%syK7M7Z`VJzH9HeNP2Ku44l-ng3lb1;puA#QV;a9o90gi5-Z7*)@ z4{l`d2M6^&5e&}pxYqu2KWYol)K~5%fWx6-xkzEFtb?eeI(N4_qF{S?;SaYjoNzAP ztF{@BHy}!iFjUvZml5Tr5K?P7wFn-i`&*0M_|pD8LM8i3OUhT}kzqI=@K>}bGHP3{ zsiF}RwkZE94?n`N!s*qGGPN5bQ3r#s71TS7RRO&FyWq5UzNp%pd=V~i8D-~wML!r6 zr41METyVrA8G^{+-) z+16hqxHTBq7OK9pCf% zqDGb^FK)Hs|C~fZZ_Uc|ByS^q+K#t7^Dlj+Z7Q>H7wSW6gD(FSIlpt_K=w^i0%=v% z35?P&YaNLmGs2^Y;7c&9hgUW;>N-_b^ zU%B*Fe^wUjiQnOGX@`HsKR^EXyf**< DHOrCr literal 5020 zcmV;N6JzWjiwFP!000001HD^Ik0VEN9=+Dk_<=QiTfjgbXbdT_>1n~*T?x=(G_x&? zMd|bvhk$>Oe}Jz(`&anjf8ev_FCsFtvWjH4;DZkubyrnp z#yh@<$bR$i!Gj0yJ$V1UcOJa=0e*b&!S}DvPX35ze}iA%#ecuWe}D1d{Rbc7{~ta5 z?);S+KP`*K7H!$q6IJQr%A{&tm2*9}nQgYJPRh!-6(4?GE_)CD_J`-#;Rj=vtHz`g zWsAmC*E;Kc^t)%TUa4n#W7CsYX0GeT>caNE`13C-YpUL(zkcqwO3G_f>1BBFPpt;Q znHjzqokvs?A949 z|J8RU!>`vy)mJvl`qKQ(`Rf%(s@Ao#wbCTQwPVQ^OI5WXxLN5&ElmNcGh3S)4~)|D zy3E?fC{tBsrJAynJ|vYW&~6e4t)8jpjmnL#+sZ9{l;RUzB<7JS7yM*hnbamsBfYEa zyaoLQ?Mko9c@3FD!KrTaiBQrIdGW_HMVzp0fRiQmDwM7tDm5BiH0GwcrB}Q0*;H*O z>I-!NQD5Gk;D3u|yZcf8uP!pPXvXSnd^1re>U4agE|%tJT5(a4u}psaM5Q;=(Fpr) zRtA#Z(rH9wQ?(JLumJq3i&UNQ+o#XYh}k2I&eXSy2J}7w$rZclf&pX}5{uA9|-e*350ZAsci0=&J)XT;AT`r^ms<#{0Gf zqncrdhuG(%kG`)F8knI(UmM-jAALj$f0cs`FvA4Zte3fIU@BtvodYk+5=&R5g=j!F zgbj(WVHR*o#FcehK^E+YaB5pzmlgSfMIx~eOvYT?FtF%_IgUW!(+}2)Tu(p|clGWu@rC<{~+vZVCrff4^MN;U8RWY{= zTxAqJn2K77JA#L-Y*bnF*Gb^P#~W5~?R93-DIVEmMLIUDUgVOL&V^5`$)Mc!SR3q} zP3ULNT#fZsL*JHFN_xe&xn9Dz+Y~flmu4n;<%6$CrXla%lU!`=y_#rgHK-*M^wK|& zB}`G1)C{pJpEt*%FejiR?izHvBX_0W7US(?s-gftT7(dFfPh6?uzJ~Jw})cc{oy1K z!S*#bA_S4a-Nddhw|!diAfmQ&jo>sC^4bOm;5|jl*um7sle=>SM+ipJ5+BQAdteEYEQO{FP^y;5Q>6CVrY*u%VV05#BN>rdo zSaQr44&17YXjN7WN21CC5tKF;4QZRz_rQ;7umIS^NR6RL7Vw0qPcnp2BK@CDRnF88 zD}$XX_=PIUCRlZ9YFlYaJmR!CvJgYn>ve{REZP5g;hh2pXNSy!HiczS{%4;&Zc)x} z$em0cutQ}aDe`W8z>1lGVc4U1I3$F7$!DYM zo?)_Bl7{r|Y?Q2qLm0&kEXxtn=O&V>h++XPAwoszZOU)V4Pyb)xmI85EtY9v6z&B4 zZ;LC+sgw`0*;Ji=_Sx?zq72wIn5&%ubm0(Z5hmZumx!j0ww&Yy<$~nOA~F$UBD`)F z6cgasiRJFp>ci>3llzP}M=F-%;kAq3Sxf=TAA*3dEKMD&oL6W{=?TDIO&b+$J}1*& zgi~vA=U&iC$@@t7se=Qm_Qi@CBT7PnKw1re5r0>B2na&~=B4C9Clg{j;pZ(7XVv9T z38dH*6tPN$60y)Agkj`-{_65VXX}*)CSyo8Zcnm_`q&}mPjKtjH(tUDDuz&BBH;nrNA>nw~_zeCi@nA!+nsl@#gPQ~PrL;xRPN%k@sGvW18$KPq z7#;em=56Va9PeOQqkME+#RTJ=M-befvdXEnk1C-TP}$?zr(R7`;B77+yWih#A({Lw z>6O9y71o~ae!P9LQ=7{x_Ythf%T2JoD+c2_aD>lC;)g<$L?BK^oeaLO5#2IM+O+!@ zJ)K-$MG7MdzaYX#cjx3g>VhYdVJJ+JI|BLc2ZHJU732S7i%j8CO_wP)maC8u-$L0> z#?eEzoG}rWxfMb|Mkf_&G?llA1r)OzO5j8wLMwxJO0R|p=U`LjRK?*G!Z7lwRGL&3 zkahO(u!~ndMA?p^5TmBYLk}Znq66G8P-lJOed8fBFRIxEb60hxfb7V@6?Ua8JW~?L z-~E;d8FrOUUfmtV=@~~>j@z76E0a;obV(#Gtnc+^qE=#x6R79Qxz-+Fj+!WGT}Sn; zbW6mDx7=B15f_osbBc?hWAd^3B)A{2gnZC*p0k4(J0_Zyn^T7i!Lt=Fp)OKNMNDe0 zt>B!IY(6^JT^Q5_cRCi^OD#op&ykYQcN9Si5Q}mvJ>8S&6RM56jtSm#ktXj$d!sgp zuC{4#l1{JTtV9~nOUro}7RU{al98sEp2G+4e0_c-`BS$8At-9c5ob~3@=)3oUCT$W zLdD@j>Nh(VEDIMb@nF2TEU1xmS@feK9wUIW@<#mmR9(Oj;BMb^Su2#N4kKn!vB?3Z ztiUQTLrSbkjZdi&M_xcWv6b{j7C5OpCInGzeiMxMw04)OzrSK7yeix@tkDV+x!qv zV8y;JDevdHvULXuyVKcQQoE4)Kzd)}Qgl#(X936UfI)+Y`Vlg_i^Ge5wXD^M;)OUvM1Kxc=>?%g;>!)Vn1;06KF#RF^DtPsW8q}-nN#)RO(mV6;!T7W|#$v%MHyK zXt%5^uY6POQ?GUUA&^-c6`h_^AXTaA>!dEai{`Ph4%kGTE9B1l(U1 z)Kl5GR*G%Bn5E1BwNiN)Uu_(UiN5oaXi1usQ8JSw@119*V56V+1gWL3l$f=PMhx2{ z*`7at4P;q2<+9SbPY{3hZ?6Y!jU(f`sQ^B8`q>}1S0e!eHZ_{XTt+naW^Ue~+hpb1zst%nrVVuU zu@4>0k(4|5l#gL!95)f$Wm}?WGwhg|j)LykMM*{pLA8aI*2~_=k8~5z>U_)bvqeNa zQ`E|h!AKjy!T6-gG8|yYc*ecHd%!3O``5Id34!uFWON3CzVcHIB19Ogk8$>}KZx=T zF;{0fmJ4#zf?oSQ2pbW%J)Kmd=8P}~yXh3CLI6}mQBgs=IS|p!7fFKWkOjc|wQq6o z?XFAh z0OTxfrG3IGy>@kZC>Ol6g6889Cp9O$*3NllIP=%}R@b}<%BDa)IDDqSVO{#t4&)VO zaS~~T`sO~dIF(aE;`&>B%r)`aEDf#UXnLG&_fyEXg7lC&|8!my zhPZlhwYuLi(d3yaPJj!5(e6Lm7ul9NeA zjh>uvf^E7fICW@1PAHn+`;Ty)#>b^-lnm9REo3iNI>>PxPZmZuZ58lj_XRg-I2D$r zW`f+vwu~dAc4Ofo5snoZow%MmSz$tWk=D?Bq=If!9Z!t@iu4q>8a<#=$uXfqmz z38+dkqsi1Z719TXi6@ldm#9!&PwlKNuH0ZmjX!yO`q_jFBv=1Nt$m3}_q0R;I;})V%O5&oSfBfi>-E^uB&=)XF^2#k~8DL9TCX-sZdy zfH?om3~&d6DpATE1i6Z>Ad5BCaD1lbY1JBa4DqxkPDWuazCM7J2iGuj1ac3MyR9Wx zAM9K?NuEcN96KCJxbtJ+&Vek?V=8_nu=H^H7_^M0T}O6g#=`B7y^3CBbBAY5kBdM; zn^PK6JJ~(0rCY8XZ{;3OzpKC)Gj2Kv7mLB@aPPAyjNWaAAx!l*LomKJzkU}Nq`d#| zI%kr1bJ~}1kS8N+igTG^H@iv*z8`7p05%rF^u9qj`Rz`Y9}$LC+& zh~6o8c>91sg^YoHPvGQ~w~~LOmd7v@M|Tn!1kVM;1K6x$bp{xqb1s0W;ePz8!1Jb$ zrL=Uv;J!4rbcuml7}6pm1Z=t<+L}6&*n&0RyUa4YxzjZ<3brZNV-vt8u)D`^k428z zedfaDy`#JZ?E0^-eFr+a@$Ih&$8esmVme6g;V0D@;DQ{11WiwHx6^GfbMCau5BU?`uPdSQgCG%Ti|7gny|cNeo1+ zkL*!8t8`_Qf`-EOkO5YjE34u2U_LM1~|HXvb(&sKeI*<3{_> z{ctTjQ(w3X01kYHfDv?h=Semg+JZBaK;JrpxP21YCx0}VW^mmFC@z4 zAmr0(ZV@~x_jeZi@umA)gj)8MK9n!TBg1e$;BRG7c+|GqP#Gg8Y*F*o9&>~dh107W zUTQZwqP7L!CTR8;s{>;9cfo1zVp(^W_aZQm9l7f`3#so67|2W%o(G^Y;gCPyhbqt_S&Edg#!l zW5@5EBmIH&xk#Y#1C_?E{T^g5^g9jwKUfG!NU~PX_q@KSdnL(>`>XgrCy~%lvobd+ zx=5dO1MS|_OJ8f7$@JTW`q0my`+jA?@0>W0ebbCDve&)9C{41?k?1kQJBkRt?!tO_ zWt&i|sjgd(RrWS>;|C^r(1T*`IO&9Xjoa;+A+}aJTkJWt=W5Jw$rOHEFExm}WRrnE zV%&%Q;5u^d#6w2;yeN-+H$CK2RMOhS60`&eu!y`kf5kHlF{%xBY=>O2wNNA2%f~S? zQwSXNCvfk$Kj-?O&?sEoUva(b9{%Pfd#L}rv_IXfGcAARGFbg#Rc>Z}hrgs9{ssU1 m^2;y(Jv=7KP~o^^{KJP-!p)Gfa)aOg=YIh;ey9JqHvj;MWy?eW From 29787bc17688cb62bb8760b24db2c0695eef2359 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Mon, 20 Nov 2023 17:34:22 +0100 Subject: [PATCH 06/40] Fix duplications in metrics data --- data/metrics.rda | Bin 1707 -> 1653 bytes inst/create-metric-tables.R | 8 ++++---- man/metrics.Rd | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/data/metrics.rda b/data/metrics.rda index 64a94e6cf0849927367c1d706e2c1e82cd9d4e32..d6c5b0949f436d2e3f4ceee74757b7c261d45b00 100644 GIT binary patch literal 1653 zcmV-*28#JYT4*^jL0KkKS=D&MRsaNP|NZ~_d!@(ka2~xU{=vWR|KUIa5D)+aKmY&` z00IC3;0GLi5`!BGPyq-414xrkNib;CdYeN@gu(+xX*9r186Js|lhiN~>M_GZMok(t zX`zvj!T`ttkZGU*20+LM4GkGIXwjyIMneb#AO=CEfB+c-APEWyRR2`{CaLWN6HTDh z$jtyI(J>eSF*0BQqd~(%Mok(tX`zvj!T`ttkZGU*20+LHL_%l~CR4;CRPvroq3SdM z(9p28$6O|Ja`E_;Hz|%u9m629F;CAqF$iJ~{yi%+1}Y ze=HCoYp~JEIdUl+8yZcqrZzEyO)<8kfN6mtmi}WuaSG;#5;-Y7rV3_~OFLZ;X1i$Y zSEc7Fxb9(kf~TI|?u5emQjc4h?A!e2YEnk^(Wb;MB1s5nK(-nI5ez9HkOK0s>NJ3} z5U>4e>a70Bt4SueR;4uSC~u7uZ!%jbr^XWY2F-E2=fAWmyJxTFyNqQRAg!Quc(! zOqmPJJ!S0e;FT9G`e^hUBW6Psz4)9h1VK|Rx8a0R+XZbuws*Z!zKVu>=0S!mAf|XL znf5qX)nkE?i8=o(RLr)7)!gn1`CyG0jNXz&Fb3+eVfdO4ICat>XuAk&;?`NMIIy!w z-Zb}oUX+sSzTj0fvoV7v*;ZG0a3rbOk^r!8+W=%)vl8cqVvoTBGad63N(fqd(L&}= z1&7Jft{OoHj#me?{5BHLeM+H0j6E>490NHj7@)>HZPr~s;tGqag}j&n z-|1`tP0x@Sg_e##hY(qiBp9e3d-S_$f+Gf+2RVmSp38w}e1-fUFkubNvIqN!ilW=T z2C|l;#*Vc_izy6|@z1)FYk=ZTpsigW)=;>pGxL$D$0cD z`5BrSCbyBxFuy;LSq5!zX?>P_AphhBWVP&zhx#h&iZm4TooKTRIf=~%y`DW3rPtmF zRzQxiSM}YW_m&2sre|oV&^+{ydgG2>D45vBi#u$RQSb&4kTE@82t%;ar}2v(F6{>g zVcSPWG(P$6@3)R(dCKe=YF^F}7|}|owmKBNsT>8C7B(cfF^pi^O^Jt_c%@;k4ZJU# zI{BF}LF16}<+}=O$$#5HnC*o!Giwx=;5S9i7yOx2^fJa2Qgi*C+g=JIfLjDY&NMOM z5}=j68g;rZjMdo2+o1rxRf@XYvZyMdY8(nKG@xwluSqbZRA-2wY)dkxho$CrcmS=x zXAwv&N!PoLaWF~LKIP0MOfF|3oEOI z^WDi%S_eSYZ!5H^IaF=tl<-63!|gCLMhSyp#+w^R?Kz@?=x|18%nlT7H?*NBziFKe z#JQH^t%3^#qXB`%`r0G4cED7Vltd=YNMRusS4E*}qI9Av0|QgHv*TszF{aVXBaTCc zOY>W?6SC{iQTz}?PD{%FJ5ZI*iCv^vu$r|Qc&Wu9y+*@YL8@R6gb)Qc8vPNj7yWiR@gp(= z7Md=iY;EAE5{V$Hcp+*A9{msmg8_xlsE?sdoR!2%GHNzwq=(h6h+5o89Us-mx0SF zV-v_hG&D3aGy@CpS00D%=27*L_Xwd;QG9yzoQ^`Dp$R4Mu zwFZp=i1bIOJvBW|sDxqyM!V@F29R!{0!P-wKXdXVOBknY@l={w95FSD!PRoW2(gzD&BRM)A>x(3V*{pxYi}G`mGFTm)8#gQovRyY?0} zYS*m#<(!2Pp`$qK>@;eqftWo>;+n;!yDTY;idni!n+hUoS^r`no);@!aU)hpt_3tl zG^nQp4St5T4?}auN*k*}2*nY0L0B;U5q4)H2pEBk zqQ!@yi0}s-;s%Q}LkII|4AN&X(N+s+(Dr=ZAt2&lB58q;%v)lXz}!fgzQRFNY}E-G zm0N3`Ate%ZAV_Y`+bh8#&XWuyq+_tMX|Y1VSx6r{U|Qqx!F+uz))_FGNi7bb#)gL9 zG{k2j3Lu{;fY3Pfx0d`;C%qhr)Yv`!F|eraR1)ENPrzgy6pTWRgog6dpRdPIKqRkS zNg>+j>^O@%_W_=ML5dtYXKs#@SVAu%|M$|?aY;O+=P^iEkX;|HUh1)>Z?wS7+6EqU zdjl3Y&^LaIp|Vvn)a`Y(aF*lKA^X%AGP^SER< ztSC%uV?`Z1NfUd4gk%f{qro9MjRgLvsn6Az;NEhZn;IWny1MJ!loIHY-ZHbbTqOoG zM3n&4=1$U590e8>JFx{pjBH~Dz-=ZUMoK3Of@~RubJQNjr34+x2YW4;Ox2S6s|Ckg zCy{oHQV%(@EN-{y%7=xOFq)Ck{Cch`2#2!(je$8-`Oi5!x+ZKOn_cG zMI9^^R0TmT193IYIfkxO>5+uugE`5Dq^?lR@jILQ0V?g)%qawf;tvwHSMel{Ck&3NZ7TYfr_aIUGc+sOscZ5TKhODi!kX87Sfu^63T;a z=q0%)!H3~!E{qcsVq;B>q;fhDF?5@ebX*P7Z8xl$B(2CVGTFahzB5}g{Q z5`uU&NEr!Lf}ue*Ni|qf?+|mNy4Hcbz#niR5|3+KlHe-!_k8dRG$R(8DxhsP@Ck&G zA_^yhOFI6(f+9dbOp^nxCK5K-VL%8l zL|3`xFiL{r#R3`;;9$7+^kpIC!B`Fr^uQ%f5E-2*23XYR(N`kalQRWz{p|{%u!a@T z^F2@mGkAC?W;BBXS|CB9G6?8()^(YbsdJE16k-tHjGp3Xut66sI2JgJZ9!f@PI?dEVPRMZ z!T^5{A;BS&aDj>oF2{MnFWf;QuoUpj*Fc@NTw<@^#+`_Sl-Z#)BHi(uf`}?>7|bF< z4cMs1L{^Cr6u5<&8!x4BsDw&cuX6V$j?{bM<2p@9zx5@OKIaN+K!nK=h`LgRVJ$@z z8n!f)45D7)i_q2u;sTIy=rPF25}pK%)W~t43CQ~adn}j!V8{Q(+>uTcBoT)_bATs+ B_}Kse diff --git a/inst/create-metric-tables.R b/inst/create-metric-tables.R index 7ae4d8f3e..c5d1e8de9 100644 --- a/inst/create-metric-tables.R +++ b/inst/create-metric-tables.R @@ -6,7 +6,7 @@ library(data.table) ae <- list( `Metric` = "Absolute error", - `Name` = list("ae_point", "ae_median"), + `Name` = "ae_point, ae_median", `Functions` = r"(score(), ae_point()), ae_median_sample()", `D` = r"($\checkmark$)", `C` = r"($\checkmark$)", @@ -18,7 +18,7 @@ ae <- list( se <- list( `Metric` = "Squared error", - `Name` = list("se_point", "se_mean"), + `Name` = "se_point, se_mean", `Functions` = r"(score(), se_point(), se_mean_sample())", `D` = r"($\checkmark$)", `C` = r"($\checkmark$)", @@ -164,7 +164,7 @@ bias <- list( under_overprediction <- list( `Metric` = "Under-, Over-prediction", - `Name` = list("underprediction", "overprediction"), + `Name` = "underprediction, overprediction", `Functions` = r"(score(), interval_score())", `D` = r"($-$)", `C` = r"($-$)", @@ -212,7 +212,7 @@ relative_skill <- list( scaled_relative_skill <- list( `Metric` = "Scaled relative skill", - `Name` = list("scaled_rel_skill"), + `Name` = "scaled_rel_skill", `Functions` = r"(score(), pairwise_comparison())", `D` = r"($\sim$)", `C` = r"($\sim$)", diff --git a/man/metrics.Rd b/man/metrics.Rd index 85b3633a0..d026647cf 100644 --- a/man/metrics.Rd +++ b/man/metrics.Rd @@ -5,7 +5,7 @@ \alias{metrics} \title{Summary information for selected metrics} \format{ -An object of class \code{data.table} (inherits from \code{data.frame}) with 22 rows and 8 columns. +An object of class \code{data.table} (inherits from \code{data.frame}) with 19 rows and 8 columns. } \usage{ metrics From b4f505237c256fc5aea1f7ac98c2a51a767d85ba Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Mon, 20 Nov 2023 17:58:22 +0100 Subject: [PATCH 07/40] Update metrics again and fix actual error, which was not taking (unique()) on the data in the vignette --- data/metrics.rda | Bin 1653 -> 1707 bytes inst/create-metric-tables.R | 6 +++--- man/metrics.Rd | 2 +- vignettes/metric-details.Rmd | 5 ++++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/data/metrics.rda b/data/metrics.rda index d6c5b0949f436d2e3f4ceee74757b7c261d45b00..64a94e6cf0849927367c1d706e2c1e82cd9d4e32 100644 GIT binary patch literal 1707 zcmV;c22}Y%T4*^jL0KkKSwhG&D3aGy@CpS00D%=27*L_Xwd;QG9yzoQ^`Dp$R4Mu zwFZp=i1bIOJvBW|sDxqyM!V@F29R!{0!P-wKXdXVOBknY@l={w95FSD!PRoW2(gzD&BRM)A>x(3V*{pxYi}G`mGFTm)8#gQovRyY?0} zYS*m#<(!2Pp`$qK>@;eqftWo>;+n;!yDTY;idni!n+hUoS^r`no);@!aU)hpt_3tl zG^nQp4St5T4?}auN*k*}2*nY0L0B;U5q4)H2pEBk zqQ!@yi0}s-;s%Q}LkII|4AN&X(N+s+(Dr=ZAt2&lB58q;%v)lXz}!fgzQRFNY}E-G zm0N3`Ate%ZAV_Y`+bh8#&XWuyq+_tMX|Y1VSx6r{U|Qqx!F+uz))_FGNi7bb#)gL9 zG{k2j3Lu{;fY3Pfx0d`;C%qhr)Yv`!F|eraR1)ENPrzgy6pTWRgog6dpRdPIKqRkS zNg>+j>^O@%_W_=ML5dtYXKs#@SVAu%|M$|?aY;O+=P^iEkX;|HUh1)>Z?wS7+6EqU zdjl3Y&^LaIp|Vvn)a`Y(aF*lKA^X%AGP^SER< ztSC%uV?`Z1NfUd4gk%f{qro9MjRgLvsn6Az;NEhZn;IWny1MJ!loIHY-ZHbbTqOoG zM3n&4=1$U590e8>JFx{pjBH~Dz-=ZUMoK3Of@~RubJQNjr34+x2YW4;Ox2S6s|Ckg zCy{oHQV%(@EN-{y%7=xOFq)Ck{Cch`2#2!(je$8-`Oi5!x+ZKOn_cG zMI9^^R0TmT193IYIfkxO>5+uugE`5Dq^?lR@jILQ0V?g)%qawf;tvwHSMel{Ck&3NZ7TYfr_aIUGc+sOscZ5TKhODi!kX87Sfu^63T;a z=q0%)!H3~!E{qcsVq;B>q;fhDF?5@ebX*P7Z8xl$B(2CVGTFahzB5}g{Q z5`uU&NEr!Lf}ue*Ni|qf?+|mNy4Hcbz#niR5|3+KlHe-!_k8dRG$R(8DxhsP@Ck&G zA_^yhOFI6(f+9dbOp^nxCK5K-VL%8l zL|3`xFiL{r#R3`;;9$7+^kpIC!B`Fr^uQ%f5E-2*23XYR(N`kalQRWz{p|{%u!a@T z^F2@mGkAC?W;BBXS|CB9G6?8()^(YbsdJE16k-tHjGp3Xut66sI2JgJZ9!f@PI?dEVPRMZ z!T^5{A;BS&aDj>oF2{MnFWf;QuoUpj*Fc@NTw<@^#+`_Sl-Z#)BHi(uf`}?>7|bF< z4cMs1L{^Cr6u5<&8!x4BsDw&cuX6V$j?{bM<2p@9zx5@OKIaN+K!nK=h`LgRVJ$@z z8n!f)45D7)i_q2u;sTIy=rPF25}pK%)W~t43CQ~adn}j!V8{Q(+>uTcBoT)_bATs+ B_}Kse literal 1653 zcmV-*28#JYT4*^jL0KkKS=D&MRsaNP|NZ~_d!@(ka2~xU{=vWR|KUIa5D)+aKmY&` z00IC3;0GLi5`!BGPyq-414xrkNib;CdYeN@gu(+xX*9r186Js|lhiN~>M_GZMok(t zX`zvj!T`ttkZGU*20+LM4GkGIXwjyIMneb#AO=CEfB+c-APEWyRR2`{CaLWN6HTDh z$jtyI(J>eSF*0BQqd~(%Mok(tX`zvj!T`ttkZGU*20+LHL_%l~CR4;CRPvroq3SdM z(9p28$6O|Ja`E_;Hz|%u9m629F;CAqF$iJ~{yi%+1}Y ze=HCoYp~JEIdUl+8yZcqrZzEyO)<8kfN6mtmi}WuaSG;#5;-Y7rV3_~OFLZ;X1i$Y zSEc7Fxb9(kf~TI|?u5emQjc4h?A!e2YEnk^(Wb;MB1s5nK(-nI5ez9HkOK0s>NJ3} z5U>4e>a70Bt4SueR;4uSC~u7uZ!%jbr^XWY2F-E2=fAWmyJxTFyNqQRAg!Quc(! zOqmPJJ!S0e;FT9G`e^hUBW6Psz4)9h1VK|Rx8a0R+XZbuws*Z!zKVu>=0S!mAf|XL znf5qX)nkE?i8=o(RLr)7)!gn1`CyG0jNXz&Fb3+eVfdO4ICat>XuAk&;?`NMIIy!w z-Zb}oUX+sSzTj0fvoV7v*;ZG0a3rbOk^r!8+W=%)vl8cqVvoTBGad63N(fqd(L&}= z1&7Jft{OoHj#me?{5BHLeM+H0j6E>490NHj7@)>HZPr~s;tGqag}j&n z-|1`tP0x@Sg_e##hY(qiBp9e3d-S_$f+Gf+2RVmSp38w}e1-fUFkubNvIqN!ilW=T z2C|l;#*Vc_izy6|@z1)FYk=ZTpsigW)=;>pGxL$D$0cD z`5BrSCbyBxFuy;LSq5!zX?>P_AphhBWVP&zhx#h&iZm4TooKTRIf=~%y`DW3rPtmF zRzQxiSM}YW_m&2sre|oV&^+{ydgG2>D45vBi#u$RQSb&4kTE@82t%;ar}2v(F6{>g zVcSPWG(P$6@3)R(dCKe=YF^F}7|}|owmKBNsT>8C7B(cfF^pi^O^Jt_c%@;k4ZJU# zI{BF}LF16}<+}=O$$#5HnC*o!Giwx=;5S9i7yOx2^fJa2Qgi*C+g=JIfLjDY&NMOM z5}=j68g;rZjMdo2+o1rxRf@XYvZyMdY8(nKG@xwluSqbZRA-2wY)dkxho$CrcmS=x zXAwv&N!PoLaWF~LKIP0MOfF|3oEOI z^WDi%S_eSYZ!5H^IaF=tl<-63!|gCLMhSyp#+w^R?Kz@?=x|18%nlT7H?*NBziFKe z#JQH^t%3^#qXB`%`r0G4cED7Vltd=YNMRusS4E*}qI9Av0|QgHv*TszF{aVXBaTCc zOY>W?6SC{iQTz}?PD{%FJ5ZI*iCv^vu$r|Qc&Wu9y+*@YL8@R6gb)Qc8vPNj7yWiR@gp(= z7Md=iY;EAE5{V$Hcp+*A9{msmg8_xlsE?sdoR!2%GHNzwq=(h6h+5o89Us-mx0SF zV-v_% +data <- data[, 1:6] %>% + unique() + +data %>% kbl(format = "html", escape = FALSE, align = c("lccccl"), From 5075c47116b15acc3d03169cbdbe6b795c64d43b Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Mon, 20 Nov 2023 18:17:21 +0100 Subject: [PATCH 08/40] Add startup message --- R/zzz.R | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 R/zzz.R diff --git a/R/zzz.R b/R/zzz.R new file mode 100644 index 000000000..ec51225d5 --- /dev/null +++ b/R/zzz.R @@ -0,0 +1,7 @@ +.onAttach <- function(libname, pkgname) { + packageStartupMessage( + "Note: scoringutils is currently undergoing major development changes ", + "(with an update planned for February 1st, 2024). We would very much ", + "appreciate your opinions and feedback on what should be included in this ", + "major update: https://github.com/epiforecasts/scoringutils/discussions/333") +} From 51ea97ad23ca1c66d53a79ba63fd7d57f67a2269 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Mon, 20 Nov 2023 18:34:58 +0100 Subject: [PATCH 09/40] Create a PR template --- .github/PULL_REQUEST_TEMPLATE.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..42e8017c2 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,22 @@ + + +## Description + +This PR closes #. + +[Describe the changes that you made in this pull request.] + +## Checklist + +- [ ] My PR is based on a package issue and I have explicitly linked it. +- [ ] I have included the target issue or issues in the PR title as follows: *issue-number*: PR title +- [ ] I have tested my changes locally. +- [ ] I have added or updated unit tests where necessary. +- [ ] I have updated the documentation if required. +- [ ] I have built the package locally and run rebuilt docs using roxygen2. +- [ ] My code follows the established coding standards and I have run `lintr::lint_package()` to check for style issues introduced by my changes. +- [ ] I have added a news item linked to this PR. +- [ ] I have updated the package development version by one increment in both `NEWS.md` and the `DESCRIPTION`. +- [ ] I have reviewed CI checks for this PR and addressed them as far as I am able. + + From 5b4d8333158231ac84b67d7845f6aab1074ab879 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Mon, 20 Nov 2023 18:39:17 +0100 Subject: [PATCH 10/40] replace epinowcast with scoringutils, making it slightly less obvious where we stole this template from --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 42e8017c2..ad2022e2c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -19,4 +19,4 @@ This PR closes #. - [ ] I have updated the package development version by one increment in both `NEWS.md` and the `DESCRIPTION`. - [ ] I have reviewed CI checks for this PR and addressed them as far as I am able. - + From 586b2d0690bb6248543cfed584ebc86d3644ba3f Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Mon, 20 Nov 2023 20:59:37 +0100 Subject: [PATCH 11/40] Fix linting issue --- R/zzz.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/R/zzz.R b/R/zzz.R index ec51225d5..e89e0dfbb 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -3,5 +3,6 @@ "Note: scoringutils is currently undergoing major development changes ", "(with an update planned for February 1st, 2024). We would very much ", "appreciate your opinions and feedback on what should be included in this ", - "major update: https://github.com/epiforecasts/scoringutils/discussions/333") + "major update: https://github.com/epiforecasts/scoringutils/discussions/333" + ) } From 65e13f5cebc8ba7bf1efd55e06d0e48f881ad5e9 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 22 Nov 2023 11:03:52 +0100 Subject: [PATCH 12/40] Update News file --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 5b8d2810e..ffdb9b190 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,7 @@ ## Package updates - `scoringutils` now depends on R 3.6. The change was made since packages `testthat` and `lifecycle`, which are used in `scoringutils` now require R 3.6. We also updated the Github action CI check to work with R 3.6 now. +- Added a new PR template with a checklist of things to be included in PRs to facilitate the development and review process # scoringutils 1.2.1 From 4a8fb923c4d94de2483eb614faa1b75b30b3e808 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Fri, 24 Nov 2023 10:17:11 +0100 Subject: [PATCH 13/40] Remove specific date from the package startup message --- R/zzz.R | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/R/zzz.R b/R/zzz.R index e89e0dfbb..1c6f431e7 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -1,8 +1,9 @@ .onAttach <- function(libname, pkgname) { packageStartupMessage( "Note: scoringutils is currently undergoing major development changes ", - "(with an update planned for February 1st, 2024). We would very much ", - "appreciate your opinions and feedback on what should be included in this ", - "major update: https://github.com/epiforecasts/scoringutils/discussions/333" + "(with an update planned for the first quarter of 2024). We would very ", + "much appreciate your opinions and feedback on what should be included in ", + "this major update: ", + "https://github.com/epiforecasts/scoringutils/discussions/333" ) } From ce29ee5177e8bc71f38796329125db2b34b8dbd4 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Sat, 25 Nov 2023 10:22:14 +0100 Subject: [PATCH 14/40] Update news item (and capitalise a word in a recent news entry) --- NEWS.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 5b8d2810e..0acb4ed63 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,11 +3,14 @@ ## Package updates - `scoringutils` now depends on R 3.6. The change was made since packages `testthat` and `lifecycle`, which are used in `scoringutils` now require R 3.6. We also updated the Github action CI check to work with R 3.6 now. +## Bug fixes +- The metrics table in the vignette [Details on the metrics implemented in `scoringutils`](https://epiforecasts.io/scoringutils/articles/metric-details.html) had duplicated entries. This was fixed by removing the duplicated rows. + # scoringutils 1.2.1 ## Package updates - This minor update fixes a few issues related to gh actions and the vignettes displayed at epiforecasts.io/scoringutils. It - - gets rid of the preferably package in _pkgdown.yml. The theme had a toggle between light and dark theme that didn't work properly + - Gets rid of the preferably package in _pkgdown.yml. The theme had a toggle between light and dark theme that didn't work properly - updates the gh pages deploy action to v4 and also cleans up files when triggered - introduces a gh action to automatically render the Readme from Readme.Rmd - removes links to vignettes that have been renamed From a1bd8a93c09f9f13fbe7bd4d916b2f593aac9f37 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Sat, 25 Nov 2023 13:05:29 +0100 Subject: [PATCH 15/40] Add a unit test for an input that's not data.table --- tests/testthat/test-convenience-functions.R | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/testthat/test-convenience-functions.R b/tests/testthat/test-convenience-functions.R index ad7a40550..4fea72fa9 100644 --- a/tests/testthat/test-convenience-functions.R +++ b/tests/testthat/test-convenience-functions.R @@ -82,3 +82,20 @@ test_that("function get_forecast_unit() and set_forecast_unit() work together", expect_equal(fu_set, fu_get) }) + +test_that("set_forecast_unit() works on input that's not a data.table", { + df <- data.frame( + a = 1:2, + b = 2:3, + c = 3:4 + ) + expect_equal( + colnames(set_forecast_unit(df, c("a", "b"))), + c("a", "b") + ) + # apparently it also works on a matrix... good to know :) + expect_equal( + names(set_forecast_unit(as.matrix(df), "a")), + "a" + ) +}) From f4bd44d8f45e998c36a6ab459d3f9d32871ed4e4 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Sat, 25 Nov 2023 15:29:30 +0100 Subject: [PATCH 16/40] Fix changed URL in data that is linking to the European Forecast Hub Github Repo --- R/data.R | 12 ++++++------ man/example_binary.Rd | 2 +- man/example_continuous.Rd | 2 +- man/example_point.Rd | 2 +- man/example_quantile.Rd | 2 +- man/example_quantile_forecasts_only.Rd | 2 +- man/example_truth_only.Rd | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/R/data.R b/R/data.R index b0f305421..056581ada 100644 --- a/R/data.R +++ b/R/data.R @@ -19,7 +19,7 @@ #' \item{model}{name of the model that generated the forecasts} #' \item{horizon}{forecast horizon in weeks} #' } -#' @source \url{https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint +#' @source \url{https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint "example_quantile" @@ -45,7 +45,7 @@ #' \item{model}{name of the model that generated the forecasts} #' \item{horizon}{forecast horizon in weeks} #' } -#' @source \url{https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint +#' @source \url{https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint "example_point" @@ -70,7 +70,7 @@ #' \item{prediction}{predicted value} #' \item{sample}{id for the corresponding sample} #' } -#' @source \url{https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint +#' @source \url{https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint "example_continuous" @@ -125,7 +125,7 @@ #' \item{horizon}{forecast horizon in weeks} #' \item{prediction}{predicted value} #' } -#' @source \url{https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint +#' @source \url{https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint "example_binary" @@ -148,7 +148,7 @@ #' \item{model}{name of the model that generated the forecasts} #' \item{horizon}{forecast horizon in weeks} #' } -#' @source \url{https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint +#' @source \url{https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint "example_quantile_forecasts_only" @@ -168,7 +168,7 @@ #' \item{true_value}{true observed values} #' \item{location_name}{name of the country for which a prediction was made} #' } -#' @source \url{https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint +#' @source \url{https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint "example_truth_only" #' Summary information for selected metrics diff --git a/man/example_binary.Rd b/man/example_binary.Rd index 188361c0a..7cdfcfb3c 100644 --- a/man/example_binary.Rd +++ b/man/example_binary.Rd @@ -19,7 +19,7 @@ A data frame with 346 rows and 10 columns: } } \source{ -\url{https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint +\url{https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint } \usage{ example_binary diff --git a/man/example_continuous.Rd b/man/example_continuous.Rd index a82df3bb3..7c5de286d 100644 --- a/man/example_continuous.Rd +++ b/man/example_continuous.Rd @@ -20,7 +20,7 @@ A data frame with 13,429 rows and 10 columns: } } \source{ -\url{https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint +\url{https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint } \usage{ example_continuous diff --git a/man/example_point.Rd b/man/example_point.Rd index 77a893df0..1a3036617 100644 --- a/man/example_point.Rd +++ b/man/example_point.Rd @@ -20,7 +20,7 @@ A data frame with } } \source{ -\url{https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint +\url{https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint } \usage{ example_point diff --git a/man/example_quantile.Rd b/man/example_quantile.Rd index 5e3ac4c9f..0e996eef3 100644 --- a/man/example_quantile.Rd +++ b/man/example_quantile.Rd @@ -20,7 +20,7 @@ A data frame with } } \source{ -\url{https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint +\url{https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint } \usage{ example_quantile diff --git a/man/example_quantile_forecasts_only.Rd b/man/example_quantile_forecasts_only.Rd index 4ca76a5c8..00520c8cd 100644 --- a/man/example_quantile_forecasts_only.Rd +++ b/man/example_quantile_forecasts_only.Rd @@ -18,7 +18,7 @@ A data frame with 7,581 rows and 9 columns: } } \source{ -\url{https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint +\url{https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint } \usage{ example_quantile_forecasts_only diff --git a/man/example_truth_only.Rd b/man/example_truth_only.Rd index 9958a0e24..8cfa41498 100644 --- a/man/example_truth_only.Rd +++ b/man/example_truth_only.Rd @@ -15,7 +15,7 @@ A data frame with 140 rows and 5 columns: } } \source{ -\url{https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint +\url{https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/} # nolint } \usage{ example_truth_only From 2e9212fde9740f3dd0eb57134e402c76ba296562 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Sat, 25 Nov 2023 15:29:59 +0100 Subject: [PATCH 17/40] Fix broken link in package documentation for `check_quantiles()` --- R/input-check-helpers.R | 2 +- man/check_quantiles.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/input-check-helpers.R b/R/input-check-helpers.R index 07e80550d..2d7e0e51b 100644 --- a/R/input-check-helpers.R +++ b/R/input-check-helpers.R @@ -219,7 +219,7 @@ check_metrics <- function(metrics) { #' Quantiles must be in the range specified, increase monotonically, #' and contain no duplicates. #' -#' This is used in [bias_range()]() and [bias_quantile()]() to +#' This is used in [bias_range()] and [bias_quantile()] to #' provide informative errors to users. #' #' @param quantiles Numeric vector of quantiles to check diff --git a/man/check_quantiles.Rd b/man/check_quantiles.Rd index e702cd139..b0a2edb0b 100644 --- a/man/check_quantiles.Rd +++ b/man/check_quantiles.Rd @@ -21,7 +21,7 @@ Helper function to check that input quantiles are valid. Quantiles must be in the range specified, increase monotonically, and contain no duplicates. -This is used in \url{bias_range()} and \url{bias_quantile()} to +This is used in \code{\link[=bias_range]{bias_range()}} and \code{\link[=bias_quantile]{bias_quantile()}} to provide informative errors to users. } \keyword{internal} From 090e1d01416746ee692668e7ed247f9cb850febc Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Sat, 25 Nov 2023 15:33:39 +0100 Subject: [PATCH 18/40] update wordlist for spellchecking --- inst/WORDLIST | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/inst/WORDLIST b/inst/WORDLIST index bf7cca877..61d2d80b9 100644 --- a/inst/WORDLIST +++ b/inst/WORDLIST @@ -1,16 +1,18 @@ AJ +al +Bosse Bracher CMD COVID CRPS Camacho -Comput Cori DSS Dawid ECDC Eggo EpiNow +et EuroCOVIDhub Gneiting Höhle @@ -44,9 +46,7 @@ facetted facetting frac ggplot -implict jss -matriced medRxiv metacran miscalibrated @@ -58,6 +58,7 @@ pval pvalues rel scoringRules +scoringutils standalone u underprediction From 3fa5741f67185c16ca7a2a05f20e2779f6b37af2 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Sat, 25 Nov 2023 16:38:00 +0100 Subject: [PATCH 19/40] Rewriting DESCRIPTION to avoid CRAN warning about misspelled words --- DESCRIPTION | 4 ++-- man/scoringutils-package.Rd | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 4bd770f5c..cbf8e4e1c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -41,8 +41,8 @@ Description: Scoring metrics can be used either through a convenient data.frame format, or can be applied as individual functions in a vector / matrix format. All functionality has been implemented with a focus on performance and is - robustly tested. Find more information about scoringutils in the - accompanying paper (Bosse et al., 2022) . + robustly tested. Find more information about the package in the + accompanying paper (). License: MIT + file LICENSE Encoding: UTF-8 LazyData: true diff --git a/man/scoringutils-package.Rd b/man/scoringutils-package.Rd index c6efc6415..a44ad90ef 100644 --- a/man/scoringutils-package.Rd +++ b/man/scoringutils-package.Rd @@ -6,9 +6,9 @@ \alias{scoringutils-package} \title{scoringutils: Utilities for Scoring and Assessing Predictions} \description{ -Provides a collection of metrics and proper scoring rules (Tilmann Gneiting & Adrian E Raftery (2007) \doi{10.1198/016214506000001437}, Jordan, A., Krüger, F., & Lerch, S. (2019) \doi{10.18637/jss.v090.i12}) within a consistent framework for evaluation, comparison and visualisation of forecasts. In addition to proper scoring rules, functions are provided to assess bias, sharpness and calibration (Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) \doi{10.1371/journal.pcbi.1006785}) of forecasts. Several types of predictions (e.g. binary, discrete, continuous) which may come in different formats (e.g. forecasts represented by predictive samples or by quantiles of the predictive distribution) can be evaluated. Scoring metrics can be used either through a convenient data.frame format, or can be applied as individual functions in a vector / matrix format. All functionality has been implemented with a focus on performance and is robustly tested. Find more information about scoringutils in the accompanying paper (Bosse et al., 2022) \href{https://arxiv.org/abs/2205.07090v1}{arXiv:2205.07090v1}. +Provides a collection of metrics and proper scoring rules (Tilmann Gneiting & Adrian E Raftery (2007) \doi{10.1198/016214506000001437}, Jordan, A., Krüger, F., & Lerch, S. (2019) \doi{10.18637/jss.v090.i12}) within a consistent framework for evaluation, comparison and visualisation of forecasts. In addition to proper scoring rules, functions are provided to assess bias, sharpness and calibration (Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) \doi{10.1371/journal.pcbi.1006785}) of forecasts. Several types of predictions (e.g. binary, discrete, continuous) which may come in different formats (e.g. forecasts represented by predictive samples or by quantiles of the predictive distribution) can be evaluated. Scoring metrics can be used either through a convenient data.frame format, or can be applied as individual functions in a vector / matrix format. All functionality has been implemented with a focus on performance and is robustly tested. Find more information about the package in the accompanying paper (\doi{10.48550/arXiv.2205.07090}). -Provides a collection of metrics and proper scoring rules (Tilmann Gneiting & Adrian E Raftery (2007) \doi{10.1198/016214506000001437}, Jordan, A., Krüger, F., & Lerch, S. (2019) \doi{10.18637/jss.v090.i12}) within a consistent framework for evaluation, comparison and visualisation of forecasts. In addition to proper scoring rules, functions are provided to assess bias, sharpness and calibration (Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) \doi{10.1371/journal.pcbi.1006785}) of forecasts. Several types of predictions (e.g. binary, discrete, continuous) which may come in different formats (e.g. forecasts represented by predictive samples or by quantiles of the predictive distribution) can be evaluated. Scoring metrics can be used either through a convenient data.frame format, or can be applied as individual functions in a vector / matrix format. All functionality has been implemented with a focus on performance and is robustly tested. Find more information about scoringutils in the accompanying paper (Bosse et al., 2022) \href{https://arxiv.org/abs/2205.07090v1}{arXiv:2205.07090v1}. +Provides a collection of metrics and proper scoring rules (Tilmann Gneiting & Adrian E Raftery (2007) \doi{10.1198/016214506000001437}, Jordan, A., Krüger, F., & Lerch, S. (2019) \doi{10.18637/jss.v090.i12}) within a consistent framework for evaluation, comparison and visualisation of forecasts. In addition to proper scoring rules, functions are provided to assess bias, sharpness and calibration (Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) \doi{10.1371/journal.pcbi.1006785}) of forecasts. Several types of predictions (e.g. binary, discrete, continuous) which may come in different formats (e.g. forecasts represented by predictive samples or by quantiles of the predictive distribution) can be evaluated. Scoring metrics can be used either through a convenient data.frame format, or can be applied as individual functions in a vector / matrix format. All functionality has been implemented with a focus on performance and is robustly tested. Find more information about the package in the accompanying paper (\doi{10.48550/arXiv.2205.07090}). } \seealso{ Useful links: From bf70109bbafe5b66dba93732fa48f750cb2f32f7 Mon Sep 17 00:00:00 2001 From: Sam Abbott Date: Mon, 27 Nov 2023 11:47:20 +0000 Subject: [PATCH 20/40] Apply suggestions from code review --- .github/PULL_REQUEST_TEMPLATE.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index ad2022e2c..e42ea84ee 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -16,7 +16,6 @@ This PR closes #. - [ ] I have built the package locally and run rebuilt docs using roxygen2. - [ ] My code follows the established coding standards and I have run `lintr::lint_package()` to check for style issues introduced by my changes. - [ ] I have added a news item linked to this PR. -- [ ] I have updated the package development version by one increment in both `NEWS.md` and the `DESCRIPTION`. - [ ] I have reviewed CI checks for this PR and addressed them as far as I am able. From cf7ef28c1284ad006eb9b1f6c18e54e8a66271a4 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 29 Nov 2023 09:56:22 +0100 Subject: [PATCH 21/40] fix Note on CRAN about too many threads used by scoringutils --- .Rbuildignore | 1 + R/summarise_scores.R | 1 + man/add_coverage.Rd | 1 + tests/testthat/setup.R | 3 ++- vignettes/scoringutils.Rmd | 1 + 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.Rbuildignore b/.Rbuildignore index 6cc6c4991..ebdc8b742 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -17,3 +17,4 @@ ^\.devcontainer$ ^CODE_OF_CONDUCT\.md$ ^inst/manuscript/output$ +^CRAN-SUBMISSION$ diff --git a/R/summarise_scores.R b/R/summarise_scores.R index 5283a8061..615c41e28 100644 --- a/R/summarise_scores.R +++ b/R/summarise_scores.R @@ -279,6 +279,7 @@ check_summary_params <- function(scores, #' summary is present according to the value specified in `by`. #' @examples #' library(magrittr) # pipe operator +#' data.table::setDTthreads(1) # only needed to avoid issues on CRAN #' score(example_quantile) %>% #' add_coverage(by = c("model", "target_type")) %>% #' summarise_scores(by = c("model", "target_type")) %>% diff --git a/man/add_coverage.Rd b/man/add_coverage.Rd index ad658432e..45ac422e3 100644 --- a/man/add_coverage.Rd +++ b/man/add_coverage.Rd @@ -33,6 +33,7 @@ the unit of a single forecast. } \examples{ library(magrittr) # pipe operator +data.table::setDTthreads(1) # only needed to avoid issues on CRAN score(example_quantile) \%>\% add_coverage(by = c("model", "target_type")) \%>\% summarise_scores(by = c("model", "target_type")) \%>\% diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R index 5f2b22a2c..37aff0af0 100644 --- a/tests/testthat/setup.R +++ b/tests/testthat/setup.R @@ -1,6 +1,7 @@ # load common required test packages library(ggplot2, quietly = TRUE) suppressMessages(library(magrittr)) +data.table::setDTthreads(1) # only needed to avoid issues on CRAN # compute quantile scores -scores <- suppressMessages(score(example_quantile)) \ No newline at end of file +scores <- suppressMessages(score(example_quantile)) diff --git a/vignettes/scoringutils.Rmd b/vignettes/scoringutils.Rmd index 72d1f159f..10061aed2 100644 --- a/vignettes/scoringutils.Rmd +++ b/vignettes/scoringutils.Rmd @@ -19,6 +19,7 @@ library(magrittr) library(data.table) library(ggplot2) library(knitr) +data.table::setDTthreads(1) # only needed to avoid issues on CRAN ``` The `scoringutils` package provides a collection of metrics and proper scoring rules that make it simple to score probabilistic forecasts against the true observed values. You can find more information in the paper [Evaluating Forecasts with scoringutils in R](https://arxiv.org/abs/2205.07090) as well as the [Metrics-Vignette](https://epiforecasts.io/scoringutils/articles/metric-details.html) and the [Scoring forecasts directly Vignette](https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html). From 5495432695518b11301b408bfcfb9529d569e446 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 29 Nov 2023 11:25:58 +0100 Subject: [PATCH 22/40] change threads for data.table from 1 to 2 --- R/avail_forecasts.R | 2 +- man/avail_forecasts.Rd | 2 +- tests/testthat/setup.R | 2 +- vignettes/scoringutils.Rmd | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/R/avail_forecasts.R b/R/avail_forecasts.R index 7f183e2aa..19a175b03 100644 --- a/R/avail_forecasts.R +++ b/R/avail_forecasts.R @@ -27,7 +27,7 @@ #' @export #' @keywords check-forecasts #' @examples -#' data.table::setDTthreads(1) # only needed to avoid issues on CRAN +#' \dontshow{data.table::setDTthreads(2) # only needed to avoid issues on CRAN} #' #' avail_forecasts(example_quantile, #' collapse = c("quantile"), diff --git a/man/avail_forecasts.Rd b/man/avail_forecasts.Rd index 5eef64b30..d8bf8afa9 100644 --- a/man/avail_forecasts.Rd +++ b/man/avail_forecasts.Rd @@ -57,7 +57,7 @@ number of forecasts per model and location). This is useful to determine whether there are any missing forecasts. } \examples{ -data.table::setDTthreads(1) # only needed to avoid issues on CRAN +\dontshow{data.table::setDTthreads(2) # only needed to avoid issues on CRAN} avail_forecasts(example_quantile, collapse = c("quantile"), diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R index 37aff0af0..b1bf295e3 100644 --- a/tests/testthat/setup.R +++ b/tests/testthat/setup.R @@ -1,7 +1,7 @@ # load common required test packages library(ggplot2, quietly = TRUE) suppressMessages(library(magrittr)) -data.table::setDTthreads(1) # only needed to avoid issues on CRAN +data.table::setDTthreads(2) # only needed to avoid issues on CRAN # compute quantile scores scores <- suppressMessages(score(example_quantile)) diff --git a/vignettes/scoringutils.Rmd b/vignettes/scoringutils.Rmd index 10061aed2..3d6390afd 100644 --- a/vignettes/scoringutils.Rmd +++ b/vignettes/scoringutils.Rmd @@ -19,7 +19,7 @@ library(magrittr) library(data.table) library(ggplot2) library(knitr) -data.table::setDTthreads(1) # only needed to avoid issues on CRAN +data.table::setDTthreads(2) # only needed to avoid issues on CRAN ``` The `scoringutils` package provides a collection of metrics and proper scoring rules that make it simple to score probabilistic forecasts against the true observed values. You can find more information in the paper [Evaluating Forecasts with scoringutils in R](https://arxiv.org/abs/2205.07090) as well as the [Metrics-Vignette](https://epiforecasts.io/scoringutils/articles/metric-details.html) and the [Scoring forecasts directly Vignette](https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html). @@ -36,7 +36,7 @@ Most of the time, the `score()` function will be able to do the entire evaluatio ```{r, echo=FALSE} requirements <- data.table( - "Format" = c( + Format = c( "quantile-based", "sample-based", "binary", "pairwise-comparisons" ), `Required columns` = c( From 169bad9b60de0f9fe348846c219aff8dd8e6fc01 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 29 Nov 2023 14:20:51 +0100 Subject: [PATCH 23/40] Set cores for CRAN checks consistently --- R/avail_forecasts.R | 4 +++- R/pairwise-comparisons.R | 4 +++- R/pit.R | 4 +++- R/plot.R | 12 +++++++++--- R/score.R | 4 +++- R/summarise_scores.R | 8 ++++++-- man/add_coverage.Rd | 4 +++- man/avail_forecasts.Rd | 4 +++- man/pairwise_comparison.Rd | 4 +++- man/pit_sample.Rd | 4 +++- man/plot_interval_coverage.Rd | 4 +++- man/plot_pit.Rd | 4 +++- man/plot_score_table.Rd | 4 +++- man/score.Rd | 4 +++- man/summarise_scores.Rd | 4 +++- tests/testthat/setup.R | 2 +- vignettes/scoringutils.Rmd | 2 +- 17 files changed, 56 insertions(+), 20 deletions(-) diff --git a/R/avail_forecasts.R b/R/avail_forecasts.R index 19a175b03..5fd6160f8 100644 --- a/R/avail_forecasts.R +++ b/R/avail_forecasts.R @@ -27,7 +27,9 @@ #' @export #' @keywords check-forecasts #' @examples -#' \dontshow{data.table::setDTthreads(2) # only needed to avoid issues on CRAN} +#' \dontshow{ +#' data.table::setDTthreads(2) # restricts number of cores used on CRAN +#' } #' #' avail_forecasts(example_quantile, #' collapse = c("quantile"), diff --git a/R/pairwise-comparisons.R b/R/pairwise-comparisons.R index efce335eb..b6ffbb977 100644 --- a/R/pairwise-comparisons.R +++ b/R/pairwise-comparisons.R @@ -52,7 +52,9 @@ #' @author Johannes Bracher, \email{johannes.bracher@@kit.edu} #' @keywords scoring #' @examples -#' data.table::setDTthreads(1) # only needed to avoid issues on CRAN +#' \dontshow{ +#' data.table::setDTthreads(2) # restricts number of cores used on CRAN +#' } #' #' scores <- score(example_quantile) #' pairwise <- pairwise_comparison(scores, by = "target_type") diff --git a/R/pit.R b/R/pit.R index fa91dc82d..b6d781da2 100644 --- a/R/pit.R +++ b/R/pit.R @@ -62,7 +62,9 @@ #' @seealso [pit()] #' @importFrom stats runif #' @examples -#' data.table::setDTthreads(1) # only needed to avoid issues on CRAN +#' \dontshow{ +#' data.table::setDTthreads(2) # restricts number of cores used on CRAN +#' } #' #' ## continuous predictions #' true_values <- rnorm(20, mean = 1:20) diff --git a/R/plot.R b/R/plot.R index 8af22d507..af93e3c13 100644 --- a/R/plot.R +++ b/R/plot.R @@ -24,7 +24,9 @@ #' @examples #' library(ggplot2) #' library(magrittr) # pipe operator -#' data.table::setDTthreads(1) # only needed to avoid issues on CRAN +#' \dontshow{ +#' data.table::setDTthreads(2) # restricts number of cores used on CRAN +#' } #' #' scores <- score(example_quantile) %>% #' summarise_scores(by = c("model", "target_type")) %>% @@ -582,7 +584,9 @@ make_na <- make_NA #' @importFrom data.table dcast #' @export #' @examples -#' data.table::setDTthreads(1) # only needed to avoid issues on CRAN +#' \dontshow{ +#' data.table::setDTthreads(2) # restricts number of cores used on CRAN +#' } #' scores <- score(example_quantile) #' scores <- summarise_scores(scores, by = c("model", "range")) #' plot_interval_coverage(scores) @@ -835,7 +839,9 @@ plot_pairwise_comparison <- function(comparison_result, #' @importFrom stats density #' @return vector with the scoring values #' @examples -#' data.table::setDTthreads(1) # only needed to avoid issues on CRAN +#' \dontshow{ +#' data.table::setDTthreads(2) # restricts number of cores used on CRAN +#' } #' #' # PIT histogram in vector based format #' true_values <- rnorm(30, mean = 1:30) diff --git a/R/score.R b/R/score.R index ba7dcb913..0b653a8f4 100644 --- a/R/score.R +++ b/R/score.R @@ -75,7 +75,9 @@ #' #' @examples #' library(magrittr) # pipe operator -#' data.table::setDTthreads(1) # only needed to avoid issues on CRAN +#' \dontshow{ +#' data.table::setDTthreads(2) # restricts number of cores used on CRAN +#' } #' #' check_forecasts(example_quantile) #' score(example_quantile) %>% diff --git a/R/summarise_scores.R b/R/summarise_scores.R index 615c41e28..11fda06a6 100644 --- a/R/summarise_scores.R +++ b/R/summarise_scores.R @@ -42,7 +42,9 @@ #' provided to `fun`. For more information see the documentation of the #' respective function. #' @examples -#' data.table::setDTthreads(1) # only needed to avoid issues on CRAN +#' \dontshow{ +#' data.table::setDTthreads(2) # restricts number of cores used on CRAN +#' } #' library(magrittr) # pipe operator #' #' scores <- score(example_continuous) @@ -279,7 +281,9 @@ check_summary_params <- function(scores, #' summary is present according to the value specified in `by`. #' @examples #' library(magrittr) # pipe operator -#' data.table::setDTthreads(1) # only needed to avoid issues on CRAN +#' \dontshow{ +#' data.table::setDTthreads(2) # restricts number of cores used on CRAN +#' } #' score(example_quantile) %>% #' add_coverage(by = c("model", "target_type")) %>% #' summarise_scores(by = c("model", "target_type")) %>% diff --git a/man/add_coverage.Rd b/man/add_coverage.Rd index 45ac422e3..56f2a2e3f 100644 --- a/man/add_coverage.Rd +++ b/man/add_coverage.Rd @@ -33,7 +33,9 @@ the unit of a single forecast. } \examples{ library(magrittr) # pipe operator -data.table::setDTthreads(1) # only needed to avoid issues on CRAN +\dontshow{ + data.table::setDTthreads(2) # restricts number of cores used on CRAN +} score(example_quantile) \%>\% add_coverage(by = c("model", "target_type")) \%>\% summarise_scores(by = c("model", "target_type")) \%>\% diff --git a/man/avail_forecasts.Rd b/man/avail_forecasts.Rd index d8bf8afa9..60fed26fb 100644 --- a/man/avail_forecasts.Rd +++ b/man/avail_forecasts.Rd @@ -57,7 +57,9 @@ number of forecasts per model and location). This is useful to determine whether there are any missing forecasts. } \examples{ -\dontshow{data.table::setDTthreads(2) # only needed to avoid issues on CRAN} +\dontshow{ + data.table::setDTthreads(2) # restricts number of cores used on CRAN +} avail_forecasts(example_quantile, collapse = c("quantile"), diff --git a/man/pairwise_comparison.Rd b/man/pairwise_comparison.Rd index 4c14b975e..d45442b7e 100644 --- a/man/pairwise_comparison.Rd +++ b/man/pairwise_comparison.Rd @@ -64,7 +64,9 @@ The implementation of the permutation test follows the function Andrea Riebler and Michaela Paul. } \examples{ -data.table::setDTthreads(1) # only needed to avoid issues on CRAN +\dontshow{ + data.table::setDTthreads(2) # restricts number of cores used on CRAN +} scores <- score(example_quantile) pairwise <- pairwise_comparison(scores, by = "target_type") diff --git a/man/pit_sample.Rd b/man/pit_sample.Rd index 8e896ce8f..335b66574 100644 --- a/man/pit_sample.Rd +++ b/man/pit_sample.Rd @@ -76,7 +76,9 @@ In this context it should be noted, though, that uniformity of the PIT is a necessary but not sufficient condition of calibration. } \examples{ -data.table::setDTthreads(1) # only needed to avoid issues on CRAN +\dontshow{ + data.table::setDTthreads(2) # restricts number of cores used on CRAN +} ## continuous predictions true_values <- rnorm(20, mean = 1:20) diff --git a/man/plot_interval_coverage.Rd b/man/plot_interval_coverage.Rd index 9c7da16fe..8c95634ac 100644 --- a/man/plot_interval_coverage.Rd +++ b/man/plot_interval_coverage.Rd @@ -21,7 +21,9 @@ ggplot object with a plot of interval coverage Plot interval coverage } \examples{ -data.table::setDTthreads(1) # only needed to avoid issues on CRAN +\dontshow{ + data.table::setDTthreads(2) # restricts number of cores used on CRAN +} scores <- score(example_quantile) scores <- summarise_scores(scores, by = c("model", "range")) plot_interval_coverage(scores) diff --git a/man/plot_pit.Rd b/man/plot_pit.Rd index d0eceabd3..af95f421a 100644 --- a/man/plot_pit.Rd +++ b/man/plot_pit.Rd @@ -32,7 +32,9 @@ Make a simple histogram of the probability integral transformed values to visually check whether a uniform distribution seems likely. } \examples{ -data.table::setDTthreads(1) # only needed to avoid issues on CRAN +\dontshow{ + data.table::setDTthreads(2) # restricts number of cores used on CRAN +} # PIT histogram in vector based format true_values <- rnorm(30, mean = 1:30) diff --git a/man/plot_score_table.Rd b/man/plot_score_table.Rd index 9984e8000..acf07cd7b 100644 --- a/man/plot_score_table.Rd +++ b/man/plot_score_table.Rd @@ -31,7 +31,9 @@ Plots a coloured table of summarised scores obtained using \examples{ library(ggplot2) library(magrittr) # pipe operator -data.table::setDTthreads(1) # only needed to avoid issues on CRAN +\dontshow{ + data.table::setDTthreads(2) # restricts number of cores used on CRAN +} scores <- score(example_quantile) \%>\% summarise_scores(by = c("model", "target_type")) \%>\% diff --git a/man/score.Rd b/man/score.Rd index 2cbeed348..4804efcd0 100644 --- a/man/score.Rd +++ b/man/score.Rd @@ -81,7 +81,9 @@ as well as the paper \href{https://arxiv.org/abs/2205.07090}{Evaluating Forecast } \examples{ library(magrittr) # pipe operator -data.table::setDTthreads(1) # only needed to avoid issues on CRAN +\dontshow{ + data.table::setDTthreads(2) # restricts number of cores used on CRAN +} check_forecasts(example_quantile) score(example_quantile) \%>\% diff --git a/man/summarise_scores.Rd b/man/summarise_scores.Rd index ed63cf1af..d41d5a6ae 100644 --- a/man/summarise_scores.Rd +++ b/man/summarise_scores.Rd @@ -81,7 +81,9 @@ respective function.} Summarise scores as produced by \code{\link[=score]{score()}} } \examples{ -data.table::setDTthreads(1) # only needed to avoid issues on CRAN +\dontshow{ + data.table::setDTthreads(2) # restricts number of cores used on CRAN +} library(magrittr) # pipe operator scores <- score(example_continuous) diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R index b1bf295e3..11342633b 100644 --- a/tests/testthat/setup.R +++ b/tests/testthat/setup.R @@ -1,7 +1,7 @@ # load common required test packages library(ggplot2, quietly = TRUE) suppressMessages(library(magrittr)) -data.table::setDTthreads(2) # only needed to avoid issues on CRAN +data.table::setDTthreads(2) # restricts number of cores used on CRAN # compute quantile scores scores <- suppressMessages(score(example_quantile)) diff --git a/vignettes/scoringutils.Rmd b/vignettes/scoringutils.Rmd index 3d6390afd..ecf2c1734 100644 --- a/vignettes/scoringutils.Rmd +++ b/vignettes/scoringutils.Rmd @@ -19,7 +19,7 @@ library(magrittr) library(data.table) library(ggplot2) library(knitr) -data.table::setDTthreads(2) # only needed to avoid issues on CRAN +data.table::setDTthreads(2) # restricts number of cores used on CRAN ``` The `scoringutils` package provides a collection of metrics and proper scoring rules that make it simple to score probabilistic forecasts against the true observed values. You can find more information in the paper [Evaluating Forecasts with scoringutils in R](https://arxiv.org/abs/2205.07090) as well as the [Metrics-Vignette](https://epiforecasts.io/scoringutils/articles/metric-details.html) and the [Scoring forecasts directly Vignette](https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html). From 787e99e301ee227b6dfb6cb018cc54e906b3ea6c Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 29 Nov 2023 14:23:12 +0100 Subject: [PATCH 24/40] Fix linting issues --- R/pairwise-comparisons.R | 4 ++-- R/pit.R | 8 ++++---- R/plot.R | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/R/pairwise-comparisons.R b/R/pairwise-comparisons.R index b6ffbb977..f18be4831 100644 --- a/R/pairwise-comparisons.R +++ b/R/pairwise-comparisons.R @@ -229,8 +229,8 @@ pairwise_comparison_one_group <- function(scores, # make result character instead of factor result[, `:=`( - "model" = as.character(model), - "compare_against" = as.character(compare_against) + model = as.character(model), + compare_against = as.character(compare_against) )] # calculate relative skill as geometric mean diff --git a/R/pit.R b/R/pit.R index b6d781da2..de5361bfa 100644 --- a/R/pit.R +++ b/R/pit.R @@ -127,10 +127,10 @@ pit_sample <- function(true_values, # check data type ------------------------------------------------------------ # check whether continuous or integer - if (!isTRUE(all.equal(as.vector(predictions), as.integer(predictions)))) { - continuous_predictions <- TRUE - } else { + if (isTRUE(all.equal(as.vector(predictions), as.integer(predictions)))) { continuous_predictions <- FALSE + } else { + continuous_predictions <- TRUE } # calculate PIT-values ------------------------------------------------------- @@ -219,7 +219,7 @@ pit <- function(data, value.var = "prediction" ) - pit <- data_wide[, .("pit_value" = pit_sample( + pit <- data_wide[, .(pit_value = pit_sample( true_values = true_value, predictions = as.matrix(.SD) )), diff --git a/R/plot.R b/R/plot.R index af93e3c13..0aa7d4185 100644 --- a/R/plot.R +++ b/R/plot.R @@ -472,7 +472,7 @@ plot_predictions <- function(data, # it separately here to deal with the case when only the median is provided # (in which case ggdist::geom_lineribbon() will fail) if (0 %in% range) { - select_median <- (forecasts$range %in% 0 & forecasts$boundary == "lower") + select_median <- (forecasts$range == 0 & forecasts$boundary == "lower") median <- forecasts[select_median] if (nrow(median) > 0) { From 394735911f6556fed52930c11b47ceb6be466ce2 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 29 Nov 2023 14:31:22 +0100 Subject: [PATCH 25/40] update comment in vignette --- vignettes/scoringutils.Rmd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vignettes/scoringutils.Rmd b/vignettes/scoringutils.Rmd index ecf2c1734..d7995e0b9 100644 --- a/vignettes/scoringutils.Rmd +++ b/vignettes/scoringutils.Rmd @@ -19,7 +19,8 @@ library(magrittr) library(data.table) library(ggplot2) library(knitr) -data.table::setDTthreads(2) # restricts number of cores used on CRAN +# number of threads used for data.table computation, update as needed +data.table::setDTthreads(2) ``` The `scoringutils` package provides a collection of metrics and proper scoring rules that make it simple to score probabilistic forecasts against the true observed values. You can find more information in the paper [Evaluating Forecasts with scoringutils in R](https://arxiv.org/abs/2205.07090) as well as the [Metrics-Vignette](https://epiforecasts.io/scoringutils/articles/metric-details.html) and the [Scoring forecasts directly Vignette](https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html). From e17be3086777d283620fad872804a41f45b620c3 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 29 Nov 2023 14:34:22 +0100 Subject: [PATCH 26/40] How did I manage to get a linting error in here?!? --- vignettes/scoringutils.Rmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vignettes/scoringutils.Rmd b/vignettes/scoringutils.Rmd index d7995e0b9..022e0a2ad 100644 --- a/vignettes/scoringutils.Rmd +++ b/vignettes/scoringutils.Rmd @@ -19,8 +19,8 @@ library(magrittr) library(data.table) library(ggplot2) library(knitr) -# number of threads used for data.table computation, update as needed -data.table::setDTthreads(2) +# number of threads used for data.table computations, update as needed +data.table::setDTthreads(2) ``` The `scoringutils` package provides a collection of metrics and proper scoring rules that make it simple to score probabilistic forecasts against the true observed values. You can find more information in the paper [Evaluating Forecasts with scoringutils in R](https://arxiv.org/abs/2205.07090) as well as the [Metrics-Vignette](https://epiforecasts.io/scoringutils/articles/metric-details.html) and the [Scoring forecasts directly Vignette](https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html). From e2cb5d2c410e7f322643adc406eb001a2be4596c Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 6 Dec 2023 23:42:47 +0100 Subject: [PATCH 27/40] gitignore Readme.html file --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0b003a9e6..0884bab49 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ inst/manuscript/manuscript_files/ docs ..bfg-report/ .DS_Store -.vscode \ No newline at end of file +.vscode +README.html From e3897cdc685d6acf0ffe1a897370113a2387f2c4 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 6 Dec 2023 23:46:59 +0100 Subject: [PATCH 28/40] increment version number and add News item --- DESCRIPTION | 2 +- NEWS.md | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index cbf8e4e1c..06302fca2 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: scoringutils Title: Utilities for Scoring and Assessing Predictions -Version: 1.2.2 +Version: 1.2.3 Language: en-GB Authors@R: c( person(given = "Nikos", diff --git a/NEWS.md b/NEWS.md index 208129252..583df6130 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +# scoringutils 1.2.3 + +## Package updates +- fixed automatic rendering of the Readme using Github Actions by adding the file `README.html` to .gitignore + # scoringutils 1.2.2 ## Package updates From 35c2ee6f5e1ae7736bef609cb833ce3a5841205e Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 7 Dec 2023 14:16:34 +0100 Subject: [PATCH 29/40] Undo version number increase --- DESCRIPTION | 2 +- NEWS.md | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 06302fca2..cbf8e4e1c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: scoringutils Title: Utilities for Scoring and Assessing Predictions -Version: 1.2.3 +Version: 1.2.2 Language: en-GB Authors@R: c( person(given = "Nikos", diff --git a/NEWS.md b/NEWS.md index 583df6130..208129252 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,8 +1,3 @@ -# scoringutils 1.2.3 - -## Package updates -- fixed automatic rendering of the Readme using Github Actions by adding the file `README.html` to .gitignore - # scoringutils 1.2.2 ## Package updates From 518af316980e19d0590058601bbfc9f214c3ec2c Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 7 Dec 2023 14:53:58 +0100 Subject: [PATCH 30/40] Correct link to the paper in Readme --- README.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.Rmd b/README.Rmd index c0181275c..59055342b 100644 --- a/README.Rmd +++ b/README.Rmd @@ -30,7 +30,7 @@ library(knitr) The `scoringutils` package provides a collection of metrics and proper scoring rules and aims to make it simple to score probabilistic forecasts against the true observed values. -You can find additional information and examples in the papers [Evaluating Forecasts with scoringutils in R](https://arxiv.org/abs/2205.07090) [Scoring epidemiological forecasts on transformed scales](https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1) as well as the Vignettes ([Getting started](https://epiforecasts.io/scoringutils/articles/scoringutils.html), [Details on the metrics implemented](https://epiforecasts.io/scoringutils/articles/metric-details.html) and [Scoring forecasts directly](https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html)). +You can find additional information and examples in the papers [Evaluating Forecasts with scoringutils in R](https://arxiv.org/abs/2205.07090) [Scoring epidemiological forecasts on transformed scales](https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1011393) as well as the Vignettes ([Getting started](https://epiforecasts.io/scoringutils/articles/scoringutils.html), [Details on the metrics implemented](https://epiforecasts.io/scoringutils/articles/metric-details.html) and [Scoring forecasts directly](https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html)). The `scoringutils` package offers convenient automated forecast evaluation through the function `score()`. The function operates on data.frames (it uses `data.table` internally for speed and efficiency) and can easily be integrated in a workflow based on `dplyr` or `data.table`. It also provides experienced users with a set of reliable lower-level scoring metrics operating on vectors/matrices they can build upon in other applications. In addition it implements a wide range of flexible plots designed to cover many use cases. From d3fb96ee7444b00d5d210553f7154a3328276293 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 7 Dec 2023 13:55:09 +0000 Subject: [PATCH 31/40] Automatic readme update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index de5214c86..96d12d06c 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ You can find additional information and examples in the papers [Evaluating Forecasts with scoringutils in R](https://arxiv.org/abs/2205.07090) [Scoring epidemiological forecasts on transformed -scales](https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1) +scales](https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1011393) as well as the Vignettes ([Getting started](https://epiforecasts.io/scoringutils/articles/scoringutils.html), [Details on the metrics From e43e008791c34fb1ff7a945c41f2172c6cf92a63 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Fri, 5 Jan 2024 11:33:29 +0000 Subject: [PATCH 32/40] fix warning message --- R/metrics-quantile.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/R/metrics-quantile.R b/R/metrics-quantile.R index ce14a37b5..4897812f0 100644 --- a/R/metrics-quantile.R +++ b/R/metrics-quantile.R @@ -241,7 +241,8 @@ interval_coverage_quantile <- function(observed, predicted, quantile, range = 50 if (!all(necessary_quantiles %in% quantile)) { warning( "To compute the interval coverage for a range of ", range, - "%, the quantiles ", necessary_quantiles, " are required. Returning `NA`." + "%, the quantiles `", toString(necessary_quantiles), + "` are required. Returning `NA`." ) return(NA) } From 61bab0cb428c4afad35a19561cf3ee85e083ea95 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jan 2024 12:02:36 +0000 Subject: [PATCH 33/40] add a test for interval_coverage_quantile to catch insufficent quantiles for requested range --- tests/testthat/test-metrics-quantile.R | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/testthat/test-metrics-quantile.R b/tests/testthat/test-metrics-quantile.R index b941a95aa..cb4b23c19 100644 --- a/tests/testthat/test-metrics-quantile.R +++ b/tests/testthat/test-metrics-quantile.R @@ -601,6 +601,17 @@ test_that("interval_coverage_quantile rejects wrong inputs", { ) }) +test_that("interval_coverage_quantile throws a warning when a required quantile is not available", { + dropped_quantile_pred <- predicted[, -4] + dropped_quantiles <- quantile[-4] + expect_warning( + interval_coverage_quantile( + observed, dropped_quantile_pred, dropped_quantiles, range = 50 + ), + "Median not available, computing coverage as mean of the two innermost quantiles in order to compute coverage." + ) +}) + # ============================================================================ # # `interval_coverage_dev_quantile` ===================================== # From d4f779e3f113e02a1e19e973f06c0a28d8eb3651 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jan 2024 12:04:09 +0000 Subject: [PATCH 34/40] update interval_coverage_quantile to test the new warning message --- tests/testthat/test-metrics-quantile.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-metrics-quantile.R b/tests/testthat/test-metrics-quantile.R index cb4b23c19..f0274484c 100644 --- a/tests/testthat/test-metrics-quantile.R +++ b/tests/testthat/test-metrics-quantile.R @@ -608,7 +608,7 @@ test_that("interval_coverage_quantile throws a warning when a required quantile interval_coverage_quantile( observed, dropped_quantile_pred, dropped_quantiles, range = 50 ), - "Median not available, computing coverage as mean of the two innermost quantiles in order to compute coverage." + "To compute the interval coverage for a range of 50%, the quantiles `0.25, 0.75` are required. Returning `NA`" ) }) From 8a9915ac24697dc2bad4be4ecd8495286867a6fc Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jan 2024 12:06:39 +0000 Subject: [PATCH 35/40] make interval_coverage_dev_quantile part of mainline tests (i.e. drop _dev) --- tests/testthat/test-metrics-quantile.R | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/testthat/test-metrics-quantile.R b/tests/testthat/test-metrics-quantile.R index f0274484c..1eb8810de 100644 --- a/tests/testthat/test-metrics-quantile.R +++ b/tests/testthat/test-metrics-quantile.R @@ -612,16 +612,16 @@ test_that("interval_coverage_quantile throws a warning when a required quantile ) }) - -# ============================================================================ # -# `interval_coverage_dev_quantile` ===================================== # -# ============================================================================ # -test_that("interval_coverage_dev_quantile works", { +test_that("interval_coverage_quantile works", { existing_ranges <- unique(get_range_from_quantile(quantile)) expect_equal(existing_ranges, c(80, 50, 0)) - cov_50 <- interval_coverage_quantile(observed, predicted, quantile, range = c(50)) - cov_80 <- interval_coverage_quantile(observed, predicted, quantile, range = c(80)) + cov_50 <- interval_coverage_quantile( + observed, predicted, quantile, range = 50 + ) + cov_80 <- interval_coverage_quantile( + observed, predicted, quantile, range = 80 + ) manual <- 0.5 * (cov_50 - 0.5) + 0.5 * (cov_80 - 0.8) expect_equal( From 866c9d05b85cf4ed8461f44847ed0bdbe4012ce1 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jan 2024 12:12:08 +0000 Subject: [PATCH 36/40] Revert "make interval_coverage_dev_quantile part of mainline tests (i.e. drop _dev)" This reverts commit 8a9915ac24697dc2bad4be4ecd8495286867a6fc. --- tests/testthat/test-metrics-quantile.R | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/testthat/test-metrics-quantile.R b/tests/testthat/test-metrics-quantile.R index 1eb8810de..f0274484c 100644 --- a/tests/testthat/test-metrics-quantile.R +++ b/tests/testthat/test-metrics-quantile.R @@ -612,16 +612,16 @@ test_that("interval_coverage_quantile throws a warning when a required quantile ) }) -test_that("interval_coverage_quantile works", { + +# ============================================================================ # +# `interval_coverage_dev_quantile` ===================================== # +# ============================================================================ # +test_that("interval_coverage_dev_quantile works", { existing_ranges <- unique(get_range_from_quantile(quantile)) expect_equal(existing_ranges, c(80, 50, 0)) - cov_50 <- interval_coverage_quantile( - observed, predicted, quantile, range = 50 - ) - cov_80 <- interval_coverage_quantile( - observed, predicted, quantile, range = 80 - ) + cov_50 <- interval_coverage_quantile(observed, predicted, quantile, range = c(50)) + cov_80 <- interval_coverage_quantile(observed, predicted, quantile, range = c(80)) manual <- 0.5 * (cov_50 - 0.5) + 0.5 * (cov_80 - 0.8) expect_equal( From 060eaee6ac3b447d0fef52133d828c2228018b66 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jan 2024 12:15:30 +0000 Subject: [PATCH 37/40] add a test for interval_coverage_dev_quantile --- tests/testthat/test-metrics-quantile.R | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/testthat/test-metrics-quantile.R b/tests/testthat/test-metrics-quantile.R index f0274484c..a6faf051c 100644 --- a/tests/testthat/test-metrics-quantile.R +++ b/tests/testthat/test-metrics-quantile.R @@ -628,6 +628,12 @@ test_that("interval_coverage_dev_quantile works", { interval_coverage_dev_quantile(observed, predicted, quantile), manual ) + expect_warning( + interval_coverage_dev_quantile( + observed, predicted, c(quantile[-4], 0.76) + ), + "To compute inteval coverage deviation, all quantiles must form central symmetric prediction intervals. Missing quantiles: 0.24, 0.75. Returning `NA`." + ) }) From 4fe7a4a1ebcd13842a9df0146f58869ed8d4ab82 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jan 2024 12:17:58 +0000 Subject: [PATCH 38/40] add news item --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 24bf6832f..7bfcf0743 100644 --- a/NEWS.md +++ b/NEWS.md @@ -38,7 +38,7 @@ The update introduces breaking changes. If you want to keep using the older vers - Files ending in ".Rda" were renamed to ".rds" where appropriate when used together with `saveRDS()` or `readRDS()`. - `score()` now calls `na.omit()` on the data, instead of only removing rows with missing values in the columns `observed` and `predicted`. This is because `NA` values in other columns can also mess up e.g. grouping of forecasts according to the unit of a single forecast. - added documentation for the return value of `summarise_scores()`. - +- Added unit tests for `interval_coverage_quantile()` and `interval_coverage_dev_quantile()` in order to make sure that the functions provide the correct warnings when insufficient quantiles are provided. # scoringutils 1.2.1 From 85733bf61c0f5517b8ecaf5bc8bcc3e1c131e4de Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jan 2024 12:23:37 +0000 Subject: [PATCH 39/40] ping CI From 6cab635a831210208dfeafb86581e42dfc2f1d9e Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 5 Jan 2024 12:46:49 +0000 Subject: [PATCH 40/40] Automatic readme update --- README.md | 50 +++++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index fe3428f1d..b1d3dcd4e 100644 --- a/README.md +++ b/README.md @@ -143,17 +143,20 @@ example_quantile %>% digits = 2 ) %>% kable() +#> Some rows containing NA values may be removed. This is fine if not unexpected. +#> Some rows containing NA values may be removed. This is fine if not unexpected. +#> Some rows containing NA values may be removed. This is fine if not unexpected. ``` -| model | target_type | wis | overprediction | underprediction | dispersion | bias | coverage_50 | coverage_90 | coverage_deviation | ae_median | relative_skill | scaled_rel_skill | -|:----------------------|:------------|------:|---------------:|----------------:|-----------:|--------:|------------:|------------:|-------------------:|----------:|---------------:|-----------------:| -| EuroCOVIDhub-baseline | Cases | 28000 | 14000.0 | 10000.0 | 4100 | 0.0980 | 0.33 | 0.82 | -0.120 | 38000 | 1.30 | 1.6 | -| EuroCOVIDhub-baseline | Deaths | 160 | 66.0 | 2.1 | 91 | 0.3400 | 0.66 | 1.00 | 0.120 | 230 | 2.30 | 3.8 | -| EuroCOVIDhub-ensemble | Cases | 18000 | 10000.0 | 4200.0 | 3700 | -0.0560 | 0.39 | 0.80 | -0.100 | 24000 | 0.82 | 1.0 | -| EuroCOVIDhub-ensemble | Deaths | 41 | 7.1 | 4.1 | 30 | 0.0730 | 0.88 | 1.00 | 0.200 | 53 | 0.60 | 1.0 | -| UMass-MechBayes | Deaths | 53 | 9.0 | 17.0 | 27 | -0.0220 | 0.46 | 0.88 | -0.025 | 78 | 0.75 | 1.3 | -| epiforecasts-EpiNow2 | Cases | 21000 | 12000.0 | 3300.0 | 5700 | -0.0790 | 0.47 | 0.79 | -0.070 | 28000 | 0.95 | 1.2 | -| epiforecasts-EpiNow2 | Deaths | 67 | 19.0 | 16.0 | 32 | -0.0051 | 0.42 | 0.91 | -0.045 | 100 | 0.98 | 1.6 | +| model | target_type | wis | overprediction | underprediction | dispersion | bias | interval_coverage_50 | interval_coverage_90 | interval_coverage_deviation | ae_median | relative_skill | scaled_rel_skill | +|:----------------------|:------------|------:|---------------:|----------------:|-----------:|--------:|---------------------:|---------------------:|----------------------------:|----------:|---------------:|-----------------:| +| EuroCOVIDhub-baseline | Cases | 28000 | 14000.0 | 10000.0 | 4100 | 0.0980 | 0.33 | 0.82 | -0.120 | 38000 | 1.30 | 1.6 | +| EuroCOVIDhub-baseline | Deaths | 160 | 66.0 | 2.1 | 91 | 0.3400 | 0.66 | 1.00 | 0.120 | 230 | 2.30 | 3.8 | +| EuroCOVIDhub-ensemble | Cases | 18000 | 10000.0 | 4200.0 | 3700 | -0.0560 | 0.39 | 0.80 | -0.100 | 24000 | 0.82 | 1.0 | +| EuroCOVIDhub-ensemble | Deaths | 41 | 7.1 | 4.1 | 30 | 0.0730 | 0.88 | 1.00 | 0.200 | 53 | 0.60 | 1.0 | +| UMass-MechBayes | Deaths | 53 | 9.0 | 17.0 | 27 | -0.0220 | 0.46 | 0.88 | -0.025 | 78 | 0.75 | 1.3 | +| epiforecasts-EpiNow2 | Cases | 21000 | 12000.0 | 3300.0 | 5700 | -0.0790 | 0.47 | 0.79 | -0.070 | 28000 | 0.95 | 1.2 | +| epiforecasts-EpiNow2 | Deaths | 67 | 19.0 | 16.0 | 32 | -0.0051 | 0.42 | 0.91 | -0.045 | 100 | 0.98 | 1.6 | `scoringutils` contains additional functionality to transform forecasts, to summarise scores at different levels, to visualise them, and to @@ -175,6 +178,7 @@ example_quantile %>% score %>% summarise_scores(by = c("model", "target_type", "scale")) %>% head() +#> Some rows containing NA values may be removed. This is fine if not unexpected. #> model target_type scale wis overprediction #> 1: EuroCOVIDhub-ensemble Cases natural 11550.70664 3650.004755 #> 2: EuroCOVIDhub-baseline Cases natural 22090.45747 7702.983696 @@ -182,20 +186,20 @@ example_quantile %>% #> 4: EuroCOVIDhub-ensemble Deaths natural 41.42249 7.138247 #> 5: EuroCOVIDhub-baseline Deaths natural 159.40387 65.899117 #> 6: UMass-MechBayes Deaths natural 52.65195 8.978601 -#> underprediction dispersion bias coverage_50 coverage_90 -#> 1: 4237.177310 3663.52458 -0.05640625 0.3906250 0.8046875 -#> 2: 10284.972826 4102.50094 0.09726562 0.3281250 0.8203125 -#> 3: 3260.355639 5664.37795 -0.07890625 0.4687500 0.7890625 -#> 4: 4.103261 30.18099 0.07265625 0.8750000 1.0000000 -#> 5: 2.098505 91.40625 0.33906250 0.6640625 1.0000000 -#> 6: 16.800951 26.87239 -0.02234375 0.4609375 0.8750000 -#> coverage_deviation ae_median -#> 1: -0.10230114 17707.95312 -#> 2: -0.11437500 32080.48438 -#> 3: -0.06963068 21530.69531 -#> 4: 0.20380682 53.13281 -#> 5: 0.12142045 233.25781 -#> 6: -0.02488636 78.47656 +#> underprediction dispersion bias interval_coverage_50 +#> 1: 4237.177310 3663.52458 -0.05640625 0.3906250 +#> 2: 10284.972826 4102.50094 0.09726562 0.3281250 +#> 3: 3260.355639 5664.37795 -0.07890625 0.4687500 +#> 4: 4.103261 30.18099 0.07265625 0.8750000 +#> 5: 2.098505 91.40625 0.33906250 0.6640625 +#> 6: 16.800951 26.87239 -0.02234375 0.4609375 +#> interval_coverage_90 interval_coverage_deviation ae_median +#> 1: 0.8046875 -0.10230114 17707.95312 +#> 2: 0.8203125 -0.11437500 32080.48438 +#> 3: 0.7890625 -0.06963068 21530.69531 +#> 4: 1.0000000 0.20380682 53.13281 +#> 5: 1.0000000 0.12142045 233.25781 +#> 6: 0.8750000 -0.02488636 78.47656 ``` ## Citation