From de9573da0d889de79d1a0d102f3d18f82032b98a Mon Sep 17 00:00:00 2001
From: nikosbosse <37978797+nikosbosse@users.noreply.github.com>
Date: Mon, 20 Nov 2023 14:37:43 +0000
Subject: [PATCH] =?UTF-8?q?Deploying=20to=20gh-pages=20from=20@=20epiforec?=
 =?UTF-8?q?asts/scoringutils@e849d60b19c9ffd1a9b36f08723c9ddb7afaf0d5=20?=
 =?UTF-8?q?=F0=9F=9A=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 404.html                                      |   2 +-
 CODE_OF_CONDUCT.html                          |   2 +-
 CONTRIBUTING.html                             |   2 +-
 LICENSE-text.html                             |   2 +-
 LICENSE.html                                  |   2 +-
 articles/index.html                           |   2 +-
 articles/metric-details.html                  |   2 +-
 articles/scoring-forecasts-directly.html      |  68 +++++++++---------
 articles/scoringutils.html                    |   2 +-
 authors.html                                  |   2 +-
 index.html                                    |   2 +-
 news/index.html                               |  10 ++-
 pkgdown.yml                                   |   2 +-
 reference/Rplot003.png                        | Bin 13482 -> 13454 bytes
 reference/abs_error.html                      |  12 ++--
 reference/add_coverage.html                   |   2 +-
 reference/ae_median_quantile.html             |  12 ++--
 reference/ae_median_sample.html               |  12 ++--
 reference/avail_forecasts.html                |   2 +-
 reference/available_metrics.html              |   2 +-
 reference/bias_quantile.html                  |   2 +-
 reference/bias_range.html                     |   2 +-
 reference/bias_sample.html                    |  14 ++--
 reference/brier_score.html                    |  14 ++--
 reference/check_equal_length.html             |   2 +-
 reference/check_forecasts.html                |   2 +-
 reference/check_metrics.html                  |   2 +-
 reference/check_not_null.html                 |   2 +-
 reference/check_predictions.html              |   2 +-
 reference/check_quantiles.html                |   2 +-
 reference/check_summary_params.html           |   2 +-
 reference/check_true_values.html              |   2 +-
 reference/collapse_messages.html              |   2 +-
 reference/compare_two_models.html             |   2 +-
 reference/correlation.html                    |   2 +-
 reference/crps_sample.html                    |  10 +--
 reference/delete_columns.html                 |   2 +-
 reference/dss_sample.html                     |  12 ++--
 reference/example_binary.html                 |   2 +-
 reference/example_continuous.html             |   2 +-
 reference/example_integer.html                |   2 +-
 reference/example_point.html                  |   2 +-
 reference/example_quantile.html               |   2 +-
 .../example_quantile_forecasts_only.html      |   2 +-
 reference/example_truth_only.html             |   2 +-
 reference/find_duplicates.html                |   2 +-
 reference/geom_mean_helper.html               |   2 +-
 reference/get_forecast_unit.html              |   2 +-
 reference/get_prediction_type.html            |   2 +-
 reference/get_protected_columns.html          |   2 +-
 reference/get_target_type.html                |   2 +-
 reference/index.html                          |   2 +-
 reference/infer_rel_skill_metric.html         |   2 +-
 reference/interval_score.html                 |  57 +++++++--------
 reference/is_scoringutils_check.html          |   2 +-
 reference/log_shift.html                      |   2 +-
 reference/logs_binary.html                    |  12 ++--
 reference/logs_sample.html                    |  11 +--
 reference/mad_sample.html                     |   8 +--
 reference/make_NA.html                        |   2 +-
 reference/merge_pred_and_obs.html             |   2 +-
 reference/metrics.html                        |   2 +-
 reference/pairwise_comparison.html            |   2 +-
 reference/pairwise_comparison_one_group.html  |   2 +-
 reference/permutation_test.html               |   2 +-
 reference/pit.html                            |   2 +-
 reference/pit_sample-1.png                    | Bin 18100 -> 15675 bytes
 reference/pit_sample-2.png                    | Bin 17190 -> 18257 bytes
 reference/pit_sample.html                     |   2 +-
 reference/plot_avail_forecasts.html           |   2 +-
 reference/plot_correlation.html               |   2 +-
 reference/plot_heatmap.html                   |   2 +-
 reference/plot_interval_coverage.html         |   2 +-
 reference/plot_pairwise_comparison.html       |   2 +-
 reference/plot_pit-1.png                      | Bin 18027 -> 17063 bytes
 reference/plot_pit-3.png                      | Bin 38034 -> 37900 bytes
 reference/plot_pit.html                       |   2 +-
 reference/plot_predictions.html               |   2 +-
 reference/plot_quantile_coverage.html         |   2 +-
 reference/plot_ranges.html                    |   2 +-
 reference/plot_score_table.html               |   2 +-
 reference/plot_wis.html                       |   2 +-
 reference/prediction_is_quantile.html         |   2 +-
 reference/print.scoringutils_check.html       |   2 +-
 reference/quantile_score.html                 |   2 +-
 reference/quantile_to_range_long.html         |   2 +-
 reference/range_long_to_quantile.html         |   2 +-
 reference/sample_to_quantile.html             |   2 +-
 reference/sample_to_range_long.html           |   2 +-
 reference/score.html                          |   2 +-
 reference/score_binary.html                   |   2 +-
 reference/score_quantile.html                 |   2 +-
 reference/score_sample.html                   |   2 +-
 reference/scoringutils-package.html           |   2 +-
 reference/se_mean_sample.html                 |  14 ++--
 reference/set_forecast_unit.html              |   2 +-
 reference/squared_error.html                  |  13 ++--
 reference/summarise_scores.html               |   2 +-
 reference/theme_scoringutils.html             |   2 +-
 reference/transform_forecasts.html            |   2 +-
 search.json                                   |   2 +-
 101 files changed, 225 insertions(+), 216 deletions(-)

diff --git a/404.html b/404.html
index cb2b64019..8b75706b0 100644
--- a/404.html
+++ b/404.html
@@ -24,7 +24,7 @@
     
     <a class="navbar-brand me-2" href="https://epiforecasts.io/scoringutils/index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/CODE_OF_CONDUCT.html b/CODE_OF_CONDUCT.html
index a1ea6bc5c..f72bfba5b 100644
--- a/CODE_OF_CONDUCT.html
+++ b/CODE_OF_CONDUCT.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/CONTRIBUTING.html b/CONTRIBUTING.html
index 56ce90e05..ea9201461 100644
--- a/CONTRIBUTING.html
+++ b/CONTRIBUTING.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/LICENSE-text.html b/LICENSE-text.html
index ab7ac53e4..555bc915e 100644
--- a/LICENSE-text.html
+++ b/LICENSE-text.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/LICENSE.html b/LICENSE.html
index bb1b3d205..c0130ba08 100644
--- a/LICENSE.html
+++ b/LICENSE.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/articles/index.html b/articles/index.html
index 5c0221da2..3398b9d2a 100644
--- a/articles/index.html
+++ b/articles/index.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/articles/metric-details.html b/articles/metric-details.html
index 0cb81e54a..d9e8aaa34 100644
--- a/articles/metric-details.html
+++ b/articles/metric-details.html
@@ -26,7 +26,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/articles/scoring-forecasts-directly.html b/articles/scoring-forecasts-directly.html
index 49980141d..517a6ef52 100644
--- a/articles/scoring-forecasts-directly.html
+++ b/articles/scoring-forecasts-directly.html
@@ -26,7 +26,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
@@ -113,17 +113,17 @@ <h2 id="bias">Bias<a class="anchor" aria-label="anchor" href="#bias"></a>
 <span><span class="va">true_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Poisson.html" class="external-link">rpois</a></span><span class="op">(</span><span class="fl">30</span>, lambda <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span>
 <span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/lapply.html" class="external-link">replicate</a></span><span class="op">(</span><span class="fl">200</span>, <span class="fu"><a href="https://rdrr.io/r/stats/Poisson.html" class="external-link">rpois</a></span><span class="op">(</span>n <span class="op">=</span> <span class="fl">30</span>, lambda <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/bias_sample.html">bias_sample</a></span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predictions</span><span class="op">)</span></span>
-<span><span class="co">#&gt;  [1] -0.115 -0.845  0.735  1.000 -0.800 -0.810 -0.105  0.855  0.585 -0.005</span></span>
-<span><span class="co">#&gt; [11] -0.795 -0.355  0.200  0.950 -0.500 -0.745  0.190  0.190 -0.435  0.045</span></span>
-<span><span class="co">#&gt; [21]  0.955 -0.455 -0.400 -0.320 -0.470 -0.240 -0.660  0.040 -0.470 -0.090</span></span>
+<span><span class="co">#&gt;  [1] -0.150 -0.925 -0.775 -0.700 -0.030  0.795  0.485 -0.720 -0.805  0.910</span></span>
+<span><span class="co">#&gt; [11]  0.790 -0.855  0.375 -0.850 -0.560 -0.815 -0.625  0.110 -0.605  0.775</span></span>
+<span><span class="co">#&gt; [21] -0.690  0.105  0.940 -0.975  0.520  0.600 -0.535 -0.590 -0.745 -0.965</span></span>
 <span></span>
 <span><span class="co">## continuous forecasts</span></span>
 <span><span class="va">true_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">30</span>, mean <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span>
 <span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/lapply.html" class="external-link">replicate</a></span><span class="op">(</span><span class="fl">200</span>, <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">30</span>, mean <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/bias_sample.html">bias_sample</a></span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predictions</span><span class="op">)</span></span>
-<span><span class="co">#&gt;  [1]  0.64  0.18 -0.08 -0.06 -1.00  0.72 -0.14  0.86 -0.25  0.14  1.00 -0.24</span></span>
-<span><span class="co">#&gt; [13]  0.85  0.70  0.08 -0.37 -0.01 -0.58  0.59  0.06  0.65 -0.20  0.44 -0.01</span></span>
-<span><span class="co">#&gt; [25]  0.14 -0.26  0.92 -0.37  0.36  0.70</span></span></code></pre></div>
+<span><span class="co">#&gt;  [1] -0.33 -0.61  0.89  0.60 -0.48  0.16 -0.10 -0.16 -0.34  0.10 -0.37  0.68</span></span>
+<span><span class="co">#&gt; [13]  0.61 -0.44  0.15  0.52 -0.92  0.34  0.58  0.95 -1.00  0.51  0.73  0.28</span></span>
+<span><span class="co">#&gt; [25]  0.10  0.61 -0.55  0.37  0.39  0.28</span></span></code></pre></div>
 </div>
 <div class="section level2">
 <h2 id="sharpness">Sharpness<a class="anchor" aria-label="anchor" href="#sharpness"></a>
@@ -138,9 +138,9 @@ <h2 id="sharpness">Sharpness<a class="anchor" aria-label="anchor" href="#sharpne
 <div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/lapply.html" class="external-link">replicate</a></span><span class="op">(</span><span class="fl">200</span>, <span class="fu"><a href="https://rdrr.io/r/stats/Poisson.html" class="external-link">rpois</a></span><span class="op">(</span>n <span class="op">=</span> <span class="fl">30</span>, lambda <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/mad_sample.html">mad_sample</a></span><span class="op">(</span><span class="va">predictions</span><span class="op">)</span></span>
-<span><span class="co">#&gt;  [1] 1.4826 1.4826 1.4826 2.2239 1.4826 2.9652 2.9652 2.9652 2.9652 2.9652</span></span>
-<span><span class="co">#&gt; [11] 2.9652 2.9652 4.4478 2.9652 4.4478 4.4478 4.4478 4.4478 4.4478 4.4478</span></span>
-<span><span class="co">#&gt; [21] 4.4478 4.4478 4.4478 5.1891 5.9304 5.9304 5.1891 5.9304 5.9304 5.9304</span></span></code></pre></div>
+<span><span class="co">#&gt;  [1] 1.4826 1.4826 1.4826 1.4826 2.9652 2.9652 2.9652 2.9652 2.9652 2.9652</span></span>
+<span><span class="co">#&gt; [11] 2.9652 2.9652 4.4478 4.4478 3.7065 4.4478 4.4478 4.4478 4.4478 4.4478</span></span>
+<span><span class="co">#&gt; [21] 4.4478 4.4478 4.4478 4.4478 4.4478 5.9304 5.1891 5.9304 5.1891 5.9304</span></span></code></pre></div>
 </div>
 <div class="section level2">
 <h2 id="calibration">Calibration<a class="anchor" aria-label="anchor" href="#calibration"></a>
@@ -197,10 +197,10 @@ <h2 id="continuous-ranked-probability-score-crps">Continuous Ranked Probability
 <code class="sourceCode R"><span><span class="va">true_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Poisson.html" class="external-link">rpois</a></span><span class="op">(</span><span class="fl">30</span>, lambda <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span>
 <span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/lapply.html" class="external-link">replicate</a></span><span class="op">(</span><span class="fl">200</span>, <span class="fu"><a href="https://rdrr.io/r/stats/Poisson.html" class="external-link">rpois</a></span><span class="op">(</span>n <span class="op">=</span> <span class="fl">30</span>, lambda <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/crps_sample.html">crps_sample</a></span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predictions</span><span class="op">)</span></span>
-<span><span class="co">#&gt;  [1] 0.701250 0.516400 1.278100 0.522450 0.621800 1.001350 2.554350 5.215725</span></span>
-<span><span class="co">#&gt;  [9] 2.457975 1.875300 1.109525 2.505550 1.681925 0.973200 1.333575 3.204275</span></span>
-<span><span class="co">#&gt; [17] 1.325825 2.290900 3.644650 3.761125 3.134750 3.563725 4.126400 1.799000</span></span>
-<span><span class="co">#&gt; [25] 1.304850 1.328775 1.551425 1.240325 2.940300 6.601350</span></span></code></pre></div>
+<span><span class="co">#&gt;  [1] 0.437750 0.526425 2.028150 0.411950 1.418025 0.819125 1.008050 2.439350</span></span>
+<span><span class="co">#&gt;  [9] 2.645050 4.083900 1.774925 2.544100 5.016000 2.483075 4.007525 1.068925</span></span>
+<span><span class="co">#&gt; [17] 1.160800 1.071375 1.087625 1.020400 1.285150 1.290725 1.248575 1.311500</span></span>
+<span><span class="co">#&gt; [25] 1.587425 3.729225 1.389350 3.174075 1.651250 1.446800</span></span></code></pre></div>
 </div>
 <div class="section level2">
 <h2 id="dawid-sebastiani-score-dss">Dawid-Sebastiani Score (DSS)<a class="anchor" aria-label="anchor" href="#dawid-sebastiani-score-dss"></a>
@@ -214,11 +214,10 @@ <h2 id="dawid-sebastiani-score-dss">Dawid-Sebastiani Score (DSS)<a class="anchor
 <code class="sourceCode R"><span><span class="va">true_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Poisson.html" class="external-link">rpois</a></span><span class="op">(</span><span class="fl">30</span>, lambda <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span>
 <span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/lapply.html" class="external-link">replicate</a></span><span class="op">(</span><span class="fl">200</span>, <span class="fu"><a href="https://rdrr.io/r/stats/Poisson.html" class="external-link">rpois</a></span><span class="op">(</span>n <span class="op">=</span> <span class="fl">30</span>, lambda <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/dss_sample.html">dss_sample</a></span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predictions</span><span class="op">)</span></span>
-<span><span class="co">#&gt;  [1] 0.08618786 2.98559786 1.46976495 2.34099849 3.62407591 9.73895158</span></span>
-<span><span class="co">#&gt;  [7] 1.98313057 3.90895141 2.96015505 2.11907238 4.68271144 4.47945581</span></span>
-<span><span class="co">#&gt; [13] 2.59996394 2.77948300 3.66267936 2.74522043 3.80306668 4.97143605</span></span>
-<span><span class="co">#&gt; [19] 4.31862922 3.89337389 3.26161140 6.10044027 3.19136140 3.46318559</span></span>
-<span><span class="co">#&gt; [25] 3.38871634 3.84638962 3.99732263 3.34833904 3.85810706 4.17401075</span></span></code></pre></div>
+<span><span class="co">#&gt;  [1] 1.201671 1.106247 1.393634 1.788146 8.381764 3.481015 2.065385 2.506923</span></span>
+<span><span class="co">#&gt;  [9] 2.791431 2.442296 3.159900 3.216813 2.617877 2.976601 4.631072 9.775002</span></span>
+<span><span class="co">#&gt; [17] 2.751900 3.458061 2.986383 5.149527 3.007225 3.173718 4.868428 4.623816</span></span>
+<span><span class="co">#&gt; [25] 3.797667 4.396779 3.288402 4.326138 4.169233 4.985792</span></span></code></pre></div>
 </div>
 <div class="section level2">
 <h2 id="log-score">Log Score<a class="anchor" aria-label="anchor" href="#log-score"></a>
@@ -234,11 +233,11 @@ <h2 id="log-score">Log Score<a class="anchor" aria-label="anchor" href="#log-sco
 <code class="sourceCode R"><span><span class="va">true_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">30</span>, mean <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span>
 <span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/lapply.html" class="external-link">replicate</a></span><span class="op">(</span><span class="fl">200</span>, <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span>n <span class="op">=</span> <span class="fl">30</span>, mean <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/logs_sample.html">logs_sample</a></span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predictions</span><span class="op">)</span></span>
-<span><span class="co">#&gt;  [1] 1.2493238 1.4910234 1.0449596 1.5655875 1.0975545 1.7011817 1.5305268</span></span>
-<span><span class="co">#&gt;  [8] 1.0834896 0.9961612 0.9489941 3.6769227 0.9405285 1.0693227 1.0154917</span></span>
-<span><span class="co">#&gt; [15] 1.0614725 1.3418976 1.4577452 1.7541831 1.6804492 2.8461110 1.0636520</span></span>
-<span><span class="co">#&gt; [22] 1.4800348 1.0343293 1.0278244 1.7132284 1.5020562 1.0706665 1.0368028</span></span>
-<span><span class="co">#&gt; [29] 1.0395886 1.2623404</span></span></code></pre></div>
+<span><span class="co">#&gt;  [1] 1.9280996 0.9180784 0.9872729 1.3282525 1.1176960 1.0086863 1.2985107</span></span>
+<span><span class="co">#&gt;  [8] 1.6362565 1.1220392 0.9922387 1.1432151 1.3851892 1.9439170 1.3101964</span></span>
+<span><span class="co">#&gt; [15] 3.2849860 1.2641101 0.9449585 1.4246469 1.2998487 1.2791942 1.5384806</span></span>
+<span><span class="co">#&gt; [22] 0.8795043 0.9449376 1.6840263 2.0839656 2.9628825 1.8347612 1.5779114</span></span>
+<span><span class="co">#&gt; [29] 1.3147235 4.9312163</span></span></code></pre></div>
 </div>
 <div class="section level2">
 <h2 id="brier-score">Brier Score<a class="anchor" aria-label="anchor" href="#brier-score"></a>
@@ -255,12 +254,11 @@ <h2 id="brier-score">Brier Score<a class="anchor" aria-label="anchor" href="#bri
 <span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Uniform.html" class="external-link">runif</a></span><span class="op">(</span>n <span class="op">=</span> <span class="fl">30</span>, min <span class="op">=</span> <span class="fl">0</span>, max <span class="op">=</span> <span class="fl">1</span><span class="op">)</span></span>
 <span></span>
 <span><span class="fu"><a href="../reference/brier_score.html">brier_score</a></span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predictions</span><span class="op">)</span></span>
-<span><span class="co">#&gt;  [1] 9.975442e-01 7.032452e-01 8.554911e-01 3.057581e-02 1.032670e-01</span></span>
-<span><span class="co">#&gt;  [6] 3.699259e-02 4.383884e-01 1.552730e-02 4.501522e-02 2.735159e-05</span></span>
-<span><span class="co">#&gt; [11] 1.333007e-01 5.782685e-02 2.755121e-01 7.752817e-02 5.448333e-01</span></span>
-<span><span class="co">#&gt; [16] 6.876504e-01 6.058707e-01 1.770059e-01 7.403249e-01 1.341089e-01</span></span>
-<span><span class="co">#&gt; [21] 3.608961e-02 1.156040e-02 6.170601e-02 7.775925e-01 6.875119e-02</span></span>
-<span><span class="co">#&gt; [26] 8.468132e-02 8.020570e-01 5.994507e-01 4.433694e-01 7.353284e-01</span></span></code></pre></div>
+<span><span class="co">#&gt;  [1] 0.603223037 0.029673510 0.407015505 0.506307687 0.005642171 0.140049312</span></span>
+<span><span class="co">#&gt;  [7] 0.041655720 0.712188179 0.962009950 0.082081483 0.324530596 0.094023639</span></span>
+<span><span class="co">#&gt; [13] 0.076481431 0.061101161 0.358976157 0.553464690 0.360497195 0.002520284</span></span>
+<span><span class="co">#&gt; [19] 0.169877178 0.100306497 0.602803465 0.129698620 0.005134905 0.002643595</span></span>
+<span><span class="co">#&gt; [25] 0.601742141 0.023382466 0.255938699 0.420297424 0.290987315 0.876199227</span></span></code></pre></div>
 <div class="section level3">
 <h3 id="interval-score">Interval Score<a class="anchor" aria-label="anchor" href="#interval-score"></a>
 </h3>
@@ -300,11 +298,11 @@ <h3 id="interval-score">Interval Score<a class="anchor" aria-label="anchor" href
 <span>  upper <span class="op">=</span> <span class="va">upper</span>,</span>
 <span>  interval_range <span class="op">=</span> <span class="va">interval_range</span></span>
 <span><span class="op">)</span></span>
-<span><span class="co">#&gt;  [1] 0.14146594 0.19799671 0.16409937 0.16358175 0.16151246 0.37690919</span></span>
-<span><span class="co">#&gt;  [7] 0.55464477 0.12684004 0.15252842 0.24688999 0.31001926 0.25944660</span></span>
-<span><span class="co">#&gt; [13] 0.14553859 0.01066308 0.21402588 0.12998395 0.17358867 0.22675278</span></span>
-<span><span class="co">#&gt; [19] 1.48765122 0.08471481 0.15074407 0.63991832 0.25178951 0.26549263</span></span>
-<span><span class="co">#&gt; [25] 0.19100429 0.15244351 0.22823608 0.13784232 0.27805907 0.11023177</span></span></code></pre></div>
+<span><span class="co">#&gt;  [1] 0.11903087 0.57353725 0.15732685 0.10296708 0.29979639 0.14926685</span></span>
+<span><span class="co">#&gt;  [7] 0.92400696 0.07768853 0.79109767 0.18002388 1.42849397 0.21166370</span></span>
+<span><span class="co">#&gt; [13] 0.66613211 0.24871027 0.20020090 1.25006751 0.23292766 0.17809847</span></span>
+<span><span class="co">#&gt; [19] 0.22016549 0.08261552 0.19754827 0.21248512 0.22215709 0.12071060</span></span>
+<span><span class="co">#&gt; [25] 0.02870520 0.22496183 0.15415855 0.14431259 0.22507579 0.26498555</span></span></code></pre></div>
 </div>
 </div>
   </main><aside class="col-md-3"><nav id="toc"><h2>On this page</h2>
diff --git a/articles/scoringutils.html b/articles/scoringutils.html
index 4e0a5891d..64821fa1a 100644
--- a/articles/scoringutils.html
+++ b/articles/scoringutils.html
@@ -26,7 +26,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/authors.html b/authors.html
index a4bd1e2ad..e973e5c6c 100644
--- a/authors.html
+++ b/authors.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/index.html b/index.html
index 00e8edbd6..74fc67371 100644
--- a/index.html
+++ b/index.html
@@ -26,7 +26,7 @@
     
     <a class="navbar-brand me-2" href="index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/news/index.html b/news/index.html
index 0ef3bcf96..96f74efa3 100644
--- a/news/index.html
+++ b/news/index.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
@@ -54,6 +54,14 @@
       <small>Source: <a href="https://github.com/epiforecasts/scoringutils/blob/HEAD/NEWS.md" class="external-link"><code>NEWS.md</code></a></small>
     </div>
 
+    <div class="section level2">
+<h2 class="pkg-version" data-toc-text="1.2.2" id="scoringutils-122">scoringutils 1.2.2<a class="anchor" aria-label="anchor" href="#scoringutils-122"></a></h2>
+<div class="section level3">
+<h3 id="package-updates-1-2-2">Package updates<a class="anchor" aria-label="anchor" href="#package-updates-1-2-2"></a></h3>
+<ul><li>
+<code>scoringutils</code> now depends on R 3.6. The change was made since packages <code>testthat</code> and <code>lifecycle</code>, which are used in <code>scoringutils</code> now require R 3.6. We also updated the Github action CI check to work with R 3.6 now.</li>
+</ul></div>
+</div>
     <div class="section level2">
 <h2 class="pkg-version" data-toc-text="1.2.1" id="scoringutils-121">scoringutils 1.2.1<a class="anchor" aria-label="anchor" href="#scoringutils-121"></a></h2>
 <div class="section level3">
diff --git a/pkgdown.yml b/pkgdown.yml
index fa50a883b..464b0f1e3 100644
--- a/pkgdown.yml
+++ b/pkgdown.yml
@@ -5,7 +5,7 @@ articles:
   metric-details: metric-details.html
   scoring-forecasts-directly: scoring-forecasts-directly.html
   scoringutils: scoringutils.html
-last_built: 2023-11-20T13:44Z
+last_built: 2023-11-20T14:36Z
 urls:
   reference: https://epiforecasts.io/scoringutils/reference
   article: https://epiforecasts.io/scoringutils/articles
diff --git a/reference/Rplot003.png b/reference/Rplot003.png
index 69dfe56bce09004ce7273179080ee88d4950a0c1..579927dfd3b862b8721244916297a7fbca231bb3 100644
GIT binary patch
literal 13454
zcmdU$cUV)|`tLVbP(iT`q6jz|uz>U?RcsKdf>IL@qqHDB^Z+^|4g-h=F?4hcB9efV
zfOHfIgOp%^AYcf}KnyVi3=kj!ccJJ0erM*HbMAfaJ@=2xW2ifOueJAD@B4i}-`B+R
zHfCG5?AroC&{nwl89N9P8C`pA+6aDw?Q2X2Up8K`G&=*nf{(@1b#dU!=0J1j5C{_A
zx%LvFF=D?$&;ba3#`I!D4wn%fgP6mzRt192vxnLht4<!s-4QWxOAE1cQ^LtpDmv$H
zz3TjQ?|hk0c}-~VXB`=uAN=&IC(2hVjyKT^b*!`8--Y&0*y$AH`S0w4HlhOeiEfVY
z6NTN#NI2=2a@{Yc<ySLW4xeG7b?~RVS}37Xw|SJ2yrdB^b1~LrX@BmCC}C%2$T`t(
z<Nf<j@saI<W&5b;Xy%mf=X-r<2X}XO{ahRO59Nx}7J&`evmQ?xrG#O7Q)@-c59dXE
zgwI684F7Q*)oGA$`!-MSK^7K^4cOE&U8>M9B+X}k`0%0B+%6!1=bEhUj_X4uFRp^)
zR$Xv&*wujO=xCGjNd80+{0nw<(wCftC!9Grb7y8|hSwJ*0b718&39~RX+bNryP#p2
zo=d)-Ya^mx?P_dntR&Cppq7`f#AZki)2FAWSD$f0x~of%boMzG&kHSPe)ES^mm+Eq
zHV-<+|0r9zVP4qVcH3iN@Tx{+WDUaj0<9@5JX{qHek+4oA=(wy7a&v$6(SoN8hH0%
zs7@yDeGQ*P-bs<Jn5bdp4J*LuLr%z4*>sEXooH@>k&zKg-ZKBfyGE=~?L*zC&fC|~
z<<!Uq?1$gCH<Jz#n?49v0)&mLp72RF`$SV|DGx&y+JdXaO$k@hlpKh948d~1Vq@Iz
zg5{>S0d&?rG`BE6aw2mX_X^yy=y2W$_G#E^9HTCrXA18JzfzS(jVQsv-H>!eub$?z
za{WomxA#U>Mhm8loJbC(0z+Q-Vfkr8-tk6Wz#KO2C*M+HXQc!sLz^ACw5O5B923U2
zg*GL#DGQBpH+e!-BTs%7yBb0k+RS5DH!}roH1@0z-8<IEi>sNBt`Yvc%f?CHxq3yR
zbg1J>elAUr9HJg@%bur~z(to-k=l*3hZ;R6-F<z9QG+2d3D-Spx#%Y~tB-XAJS?*b
z%nJ!FytE`g^sw}74P!?$Y0Q!wxR;Mzox@pi$20NI=~-$L!yV&aGRMZoN~`Fxj}a>J
zhuQb?^71Z`Jt)%raufaD_W^U+__$MW6W-*cez!E=l**sVBRQjzFERLSE;35^j?yyv
z2+pREnzLVrq2c{ea=-^?e3Z?=-M1+$ox#QttE&ZXGAYR^DJd39_Xq_z<B|S;Ob+1(
zN=CL_(M17RIl8`Nm9$w8crI>fSK^ttWnMsVCW%auPJ7CWuByqXe&X*+dmG>$(s;f9
zS8Xuop(?`B;qJcc=vvl7=<q2L-Lk6mT!_N;2cf|=h$%9CMW$g^jnz@bh9_a_4#m5+
z$=fzsIjG>}H@7^$l6>M4ZpM(8GPjOI7I=Sjx;JAOV6{soLP0K72EOIyXf$pmIy_vk
z6HXB<>xP3BTI5i?dXTM2Ud2W=@>UPSna_%aDHo5;$s18!%dx`w7ESF*VXT?5RNaVl
zNI2s?559#&wzZhNOK^VRfxvCOdAZYt1E;O+5M_KvyR1RjS7s&!wlmR^M2-Wa+J$SZ
z^&8Z%SHZ`r>FY1$w%nG?&jt6cXzn9?-w}L=TMa@7qwVhL%QVy2!>3qQ9rvNLCWylW
zhyYi+kl_KpW%QdWP0Vl>`-Cw=#}FgTx;8}_;;~t#+YJ%Il|*}6BW)dfcl%gYQt-FC
zvFGqAKF_9**%ZR+c8Ax}1j`=q!+ny(WuB9)tSkqH<O)|j;KkJeOg9`M*C2eViawHK
z4r^oyS7PC;sgVX@PYJRvPm}VMOt5a`nW*~s)F8TT>5MW|TLo@p&-#~+UY}+x8+obV
zfvbD=f|*NtxjxK%@Y1F3wP4x3^-7tlV{w0gwQhK=I(<k!B2S>r6jYn<OM@{5t7@56
z#sSIgVy4qS!_qs(4N}w{``o(salkG{RT*<+I}nvbyq_zrlWOfV&qZ&mKF;1$RXVs7
zP>Gdo4~dza$0!fC`C-c#B7X|6QYCgpvrW1IBlXW_%pu5y8Kb||Ctcmsxz8GcMqkBj
zxeP&45~9*j{HE>A5OgLU+6WyGU9Sc~I}XH)K);-<pV{vB{)KwU(R{s+r^qfwUk#f0
z24aYT4N#@A=!?z$zH_Bl%HFjR$-UU^f(vEzeN_H4-H`~CFhzId6Hx`a9D5)>`*O0g
z-Ldq}L#c*4(T24r63?8HUuYJ?dF2Ep1YOz%sl^3K{L2wQA`;Na^_##r501_nj#?>(
zq^M79gjerQYURKNm)a@@KQH3s?uNdT<}aRl6o}H~Ly(hq4p+oWBePQ7^X#$oNnB{)
z<;$bQBGj*~AD>()%OIxAs5X85lIJ-g0zL216CZU_4yz@&@)u*aCw#c>A)Tk$`!2><
z-U*%0`TY64(QrK(JwUN2pzSKKV{e8RzL@+0ihp`S{4$!GguggzNToCLUi}eVQ;aHm
z$MQ8b9IJo%@}-(Yi)4bwf+?I@^Zjsxjeg0Ib~H1EbQqEyIkHrQawv6^b*~L&r+_%5
zTk2Xpb@_25MOuk%3O|-^@q=T7D70PME4umg>C?k7Zii>ZZP?<iQy+rhlRO@;b39nr
z($33^C8L1VmrLcfdsd)85a`QXo6_AA*gc_-&cA@utmgzI_omo0G%_x<O*Z_!D*KwM
zwXLos6n`t3zxI6p>F%C{)<N4v{`FG$erS%{uNuDLBay72GN;c!qPaWqm=-m}RJj~b
z9mnE#CBD``b^25@W1~B-G5U)gZ>w;?fF6pzP<&5D<9nfPH=D1WfFQk%;GypR_E7)#
zF8d!m^p`s22klo&)G6-|upeA0i^zq!t}cv@UXCuXJ8=MBbn%%{K2>aac{%s`_3J|4
z)91jV3C0_VKs~?y{i6D^`1J!=R9cjnhLu>5O@D2@f}7p|DNA*W>PaQ#s<7!`E&c2f
z;VPZL>eMiN+T@X3W1QVJ)ioYG(9diN;|@6$U95e4@fgE{G>4qq0-fy3X;^=qq@Bi$
z&hz2~P(7EI<__T=NN(FkIcH{8O1-L4$cPnkZQ-=h(T9xkYaJ%^gvOyurI1Unk>EYw
z(RBM)*8$AfOvDP!Qg!+80OkhjgK~U|x~;|1Qv!~l1Oj**j1N|OMwh2j9;@Mnqkr;d
z1;#NrSl;`<vh%j4WTYCeER0=8p-|nv^Ru%@a&6iab~Jk*?G}yj_V#vF!4KoRJSE{v
z7LU$HAaF}Up@Dr--&U|XUEpuPN~8~+DX?2SuYxyVy2#*X;|VRe#`8yZSJh5#ZvI+<
z;W)_ZhyLztgeKA0l@(F(N;b?5$RoS`aiR9hSTa7NG`Ak9ES1)4l{;k&!jp3Mx<3b{
zzhAU^Mv~Hyi?yR8{^Lx9XG_A?+p>5$YuL_G?C|hNRaX#FdFOhy_zffy$=V?7bI-DX
znIG&dkmW_(h8@SBfySPK04P;IGZWM>{Yu;h?SejCxS{ORzl%(~n|GWX9S1hc==w__
z=<ge%1&zfewQqc^p?KHzfoDF>B)T`SA`M!47=scNX-s%{+&NlG_phQ*&Y}22SCa3+
zuye)k>14VygHJVa_sYl4@ATRr0)^Z}{UsESN~=NpH&FRF5GmZ9tNj^2;xQ@er}4<J
zK|F$f27NGIWb{Zx-k#{sWo;^kp}!+nfIv=ksR_Pju~VdeEwTsRk6r9=_vMW=bWd(D
zeVDG~s9&tho~q-yp_3zvIkg_rC3T<6J64)8jMkgb#`p~?5-qxhU#D9R8?Aojcl+jr
z(%>#+-~Pq!Eh?u#dQcV>3U~ukjJn)@bU;A31G)^1;o;fWV?iLz?b7vszYvSZn3<Uw
zAn8BJte+OGh*=CyNllIOX4GPuSmh<GNMTz0AGgqSV$lnt_^9q3la+`Krh!8s!{#(@
zF0QM~hWt&z`_e25Dz6V$jm6Ga<jb<k{dKaDJJy>P!bGJrP6GwHmG@$7TOmFT%M_06
zvCvRKcc-Vw=r$@zw1^`Hv?+$MGp&188#uLXHzBw9bLY-ghOlf3wUKV6`yfxC!dB2V
zh*<8*vM{$FQGPYif#_6by;CH4@u);g$yOO1J<{H!)@?-RiYJqmN8%&7MTL2^3aT-O
z=w@r13J(52%u`ygmUV@6$xx0tS?j-j{c01T7&)*Z-oXrdd^C^6&Srjn0`_cfzoGDR
z1!k&-rElbr;z(%_j0k^3bvhyjwzdSi?Jr!90Lga0I+btpz7~U?Gc#Gb@ah^vP|q?_
z=@2=d0&NVuc=N#df>+reZ#-^o+W``TNMUqyrq#qgROeQl|52%=0eDn<jnYp){lo=@
zLOiU%u4o=a9WBkZNKYA*3sg*l%b*+xc10?1F|mysyR-Ba^QrPGQb}-B=RoS!wdkEv
z0o`f6QfOJL4dS1N^oUf#hZ7YNl3AHvq&blM_E>_K0;cYJHjCao<RbjYUYExjTpDiE
zpi1*&t|QbiDiFj^uYhzYhDX_taTd6eS;K(kvTjYv&p6{lAinrCgD?f1=^C```Il+U
z_72E(AcUAayLjv|$Z2z$UF)FBKZY!+d;<ABjgURbb8U)U{)K=(9Iz*;_3F2T|GOU8
z{QpA3d=D23dZ*4P%B6NlhrC(4{oC3Kl=isC%pfV!#(nEfb{ve~k-5u8&lK#ifXENG
z_H<(q)qEjyb)^Uw%Pv=eUYvmrX;7rWTI`DDMKfI|%ctbOzLZb?wM7YX*&j!MKrW4L
zKKZkb-@EA@Xztz;4&_Q&T+PDAq=DKJ@lLb$t9E&;<4wY;x~8e}5_Lr@*v_~z&#tUE
zqpZdCYTF+j*t9aN8atD;%k12_oPI=OObw`E9N=JA!7U#sUyx47HCdTSW%A0W&~E&!
zvI0xI|NS%2I{w>TwuX<4qUZIBtAoca=H}*36pJaZ2Z1|k_N`ei{WFIn<61lWiTUc4
z`kI9LPiEVEF$qZTN^Wkh2IVUq8P1s7g~yP{N=Q2T8i)|1zhFW`pV}3{<YT5^Y9i@t
zQCO;$IPeCf^5zSB<>d!TW9O6#wMk?f4q4-l$iEaHf#AXaS@ku1=K9&U%?bqJm1Huv
z*vQaMSqfkGc{3F6@Q0tIURpaLMO_JOZ#?0rb?er7X&X>nghe1l!(v3Bq+5Tp%)9QW
zQxN%Go`Q+s8qQ>!E4l%w<}&b7hj$>pXXFjf$@F@sKEB^FcaTB^nu_40tYTYW$G{C~
zWOn+@g(RyNgMTpcKjfbHWmIPxCBqbcj+UOsT3#@x{J~%BadDk7q!!D_m$;B8vBeIg
zsC)vlBx(b6!1sG*%_d+6{pRn&n$90FRDrLF5U>FzGAUOp2KhkfI_YFHREr*;KjMe!
z-vvRWH|)BH;<sK|<NTjB1$<KpCAmIUWlry5N7?0)K<>-3zTH>jt((Pdq!SX=BzSe4
zk~#{qf7Y;+;WF$=V(^}MTI0u}poz_*(&9GCQv9i=)fIaXDs%b`Svev1&0(*svvp*7
z#HWOnm6cZhAR2PJ2~loVq6buD4I_G-8yuUK`hC&pit*q<XKN@)rtSYoSN;bD-nYW#
zpEN?xk@zkp+6D+pvU&`yWD*2$-?$A?BGBcx-n*N<EQb#JJSH^Vzn{my11g90`!-Bx
z5Pl<wD`3ykm=?8lVKNZ3zrb_k=WDPAH=37kwTTitTx$mflut)q%NVYh!t?LWrjp~r
z=~Hnh?T)Zb=#t%6H3p<n{&zzy%be<MaJeplwd!4+YeCGXQmo#UhSn|<)KyHyD*3>d
zo2?YnbSOmg0y`vb+cpV_9BuNuP*hf?7}R{K*IRO7FMP~?Dms_Y221H<f^=jf*)qK*
zC=o%eKUBkO&1qvPl-F0vWHq}g=vl{ItFHPo!*yz`MvfNd(EjL|fbHB|0jlg8VzxyA
z3=a6udFh1i$tp6L{bKA+$PkB+W^3%cCi5oA)zwOJso6E3`UO~Uo=)cm1<MDW&ZPiM
zO@8_m=ulc$H|$IrR7Tc@+QiDULtSo5%7;0CKO^&4U4wkTcAP?rn2rydRsa@5$Vjm|
zWflm>Oe_838ZZEbG&m~`ovc1NHT8fz<+h1CIcaAo7e0bMmj0`3$JcWr2%Hus#X3Bn
zNLJ0${2ZA_YBCzm7@QdAM+S_)NzW%T5F-yATgPS^z+g>W8lvWcQYdnpsr(1|F<;vI
z!7LUxIBKV;d&a~VF4L<UF?GdRI8Y+q;e7g}2Bsi}pv%h3^XfkRmXtthAsv)TI=;C@
ztt@J85?kVHVJDrivSAP6d9MrSbDT~d3uEr<4lcnj@ukn^L{HU36UmqGXH(R}Ylym+
zq!V0g>+0gFqUL-H2ev$QC}xYRytsT@aa+)d50P$#WS#LS#x0L`b-w-27%Bs~tVk|>
zs2h|*yr4@JPgtKTsIen12AJEEPxqpRlGRJs<`M$NfMZ1-&I_XK73<a;OTMr+m(B3D
zQn!XEua=i)4!QuaB|gq&9*J+$o>3$&nbh{Kh<s18S(`qsNqT0dG^(>Pa{`I!zjT8D
zio>FdLo{y>I?*}AJJ>M3CZt^fPfV*+nKX`HB)(N2jaJCD4;O~Q1y;I&K-2%L$kV9>
zBt)y<Eya;v=9;vF<fi@MN{_t>7mr;Z1eJ%vg|B`Pgh0>koUOn`DdZL<HjJm52S*XN
zfHyt809o$3A^e2U6@{9w&^}2|@5Ra83w5GBs#S%cK#6@^k<xUnBkZS4&#e&TR`OPE
zS_5tpr?3xfO?98$&5o9}cDq=jW^bhV_<mmux(^FM(-zWiU*FOtDjH`6ei4VjNBpfe
z^QcQ}IOBOnjv~7-ianQKeYOw$Es0`;i<+g4HSP!X!{s}&wAYqBX^+raK^L}tyL@iK
ztPvcEsir=d%{{>ZY~37Z^I#Thy?@<`0X!O(yVJ)Ef|Q{y#N|Uz5^YcaYPpoSR<}0i
z3@iBXwA`E3XRxdHM@B}f%@|^Vl`+iKpLxQsMz1fRXFVj0FwAIN32t{M(!iSlC~Tja
z1QlSg)3`r`D~H=Cl*W25YTh0j!<pA<A7>UcO}FPfM-P<TYaa%S?>e)Q(VophEK8tC
zZ?#IB`zEdPCLJ^=S8w0Goge<zAW#`d0~&oOWDR`xyk@q<fr2bGwYK!!6F++mO8w(1
z303>1WOAgd5tTCtK(PnlOXq3f0D0ye;yx-YWW`YVay_rJBTU>?@BxNH<Id*$8a8*(
zu^VvyX|uO#{I^s@!2_Mqg(WS+hE&csIKB<d9Xj#Jqa=>f)-fKeA<<%Tf7a{{dY#!d
z;XJn`?PdSKz<_6Rq=jn3xmQY0N=sGOSf1X^qw=Z%{sq_M<Q$DDz^golIis7<q-83Z
zCo|rf95(g7rf8mD2U}z?7)DzU9~VuTk1w3uQTQXdRvvKYBWwc?=_e`bO?Lqzv%P!q
z!Nq&C)#Jg7wrD}VvQGnnsX>YP-6h(6nF~O5fR<3x_amUl7idD5k(!Rqe022u=M(RT
zhlRS741vWEfR46!7?nR_CnGElu8GMc_>#^YS@RJLC0h1)PMYM}G&Q-9N`?XM32Ef9
zZXQ@@2lYs-b{93yV=}n=Il?LCNpxf5sBq;d&e$)x=8cQ6k{|+NknNQ0M$>63C8Gln
z=)lD<<2(kyhyx1aFf0k764xnMHb!;EmDAnWk#3&geo?h5kX8c2{HYq5gbyqQBwM&5
z2WMGhW3t923P;|0?^S4*=BwgjK^+v`{zttg)ll;UCQ{)R0I>$B&W)(fFKQC2PY^1r
z{`$F%u@Mmqip!qE+lXW;cLWrn>col-kKe9(=iQyX$mRIJB`u?<2;Ezmdr_xQR^?M+
zG-><Qra&dO2vZ;i*c~@5{LY^#HG=!-ADBA)dP{DUE8yw-0H+7uOlc0+tB>NQ$Zo;u
zE6g3<nHb9y#0^M?IMyKC^!GO6u98v!yc}>s2LQ{#i=MuS;|crKB;+gT+q(MV4&GZR
zj4)hzG?a4#Sr=wx#GvMToVekWUrI~5ZLVc#7?VY?wTKE|_DEBcC<TwU9?Y-~pyuTg
z^1KXq?0WE2kW+}|{{`@es{>XCJgbxTR`s&;>f(;EbHPyrwp_kv_A<CKid;INTalbL
zG&JOIqnH+_5gA=Op1x-FV9{JxA{j4@O0dyq6Kv4j#3kzmFvn7VG+LOWcmHTn5Y*&A
zW0OlA(H|=b^4%Ew4Cn3Z4TxkOZnwc)pZzclSq|#Ydq-QQ?#=ddhpX|5JagDiybF4j
zh)_8U!(u>qJcoNT{zVehe@!6^(;q5&*|XaA0hWJ8;)AgXvmZMyt^?``m<msH+QeG|
zt?b=OD(JDccHShcot+uubR0qjKL!dr;1B6~O|E8o<rSIRxw|vR=|iQ|FMRi^9J2*<
zo2Yg#zrgbwdiR^`C`~+bP@*MJ<4iIu5|#C?g9vJK`(kXWuW9e+Fm(n$rj`Y&@Ee3M
zpC#73*6?Rw7OY1Cv%tr5?_s0m*<LZ&ElP&Cr|Ssd5@ELqNea2=Ne)f2scoy4*Jbd8
z2w<&wDW$*B{bn9jl-xf{xBxj8#{75%Jc0q-mqs2wZkgk5;z3%je6rrP2C;$xQr3a7
zszD-~fMkXSZqWrs(N(adJv1_7Mx@guZ;mDwReNzzlhmh-xI*=bElh#rVxtMYw>RV*
zxLPhHBOLrVDO^d`=ES@eEc+fyU#d}`2=L^%(^j{%Om_gR6LI-Tz6m2(=j}SF8<}~k
zU_l~33Rl(;HNLoivfA_6+qZAaY-s(!gnCdJ^~I!MuGC7q?jM_RZLJ}3#$Y#=DCQ5l
z=e%EMQ`pNhp<N?76vrKBGpSYe<R~#BIh8v^=A1xt#Yki(o&Yj{?mm=pu0_H80vPSU
zK=`DUiFi|)?JoS|9Jcr`eelD%mB4LS-HwrWasr#kadv**O$s9);-jvNDn`#L5+!{;
zrZ~D6F;?OU?v_g?A@6qe>}Cq6>9f>4R_<V+8q}#dtR194grD2Vb6W?EN^9$H{8|rd
znhHx*ED;0AbE1a><KCS;0oNU9bpPR7-r5O=m*PW92vtwB-c|ba7Ni<4eM-Er90Ss|
z?*#42Sk{Y8ZBp>ZA;aCB$?CemcNlsa3uv2)_Lk9Jt(hG^&mFc2$o5c%Aa6UMn#&26
zT4j3-*@POj7)b4pE<jBecI9t2Z2tuvAqoX{0@W;lVfq=yuC(w8B9P2diRT<&Ej?Eh
z)%}spgKhQx$mjuzSz2q$sIIxoA$I4`J4sH#o>yWJWcKK-WsiH$>%&~bCyBjxH@O47
z4TSlCd6eU>F$ikj;k}#a-uwEn{D!P|!rZUGLvDKr^h>uR0pxh8%*^@t)%q!+rd#4U
zAm6P#w~X5MVOPf-f7=26x@!cn0k{XA1{=Q~gFjL{74=03SM9uJ5Au4b=7wn%xGlYr
zi4-goMow@xLGfoVKY@%Ff8dIOZ=byNF*k#ywIfi;HKw%VtygCBtr2HAXF+`y1D7<{
z_e)-k0gSyYGZLIkt9#_<^!BSXO2!WWAY0O2-nen2;>jHVX0ibeF4v&|yP+fcyQ&{k
zjt52o8|UA};+sk*RNwN5=(a)u9DA9BEvcvo5%}?m<dKQ4>~9uzs`tkfP`E2^>K1+H
zf_77$jI3Ij->r!6K*}by8as-AbY?lDRY7XGnhgI}PJ>>!`dJ9teQ>Q8;O||BjPtZQ
zg(4v8)IFbAi2>6ls7lOGORLhF*661bZH{w4vRe+nRseP*RKhD@n6R+M3$((qnT0$*
z9Rpx=+vNHHgyby(2v5g@<E|=+wV?V$jN0A*8%qiLZ$sxlNhAKBob}}UYh{0C55RE}
z6B8Q-kxv9}QDDbij4}QM{1`tpde~7$<KKe_YuPj^OA;_Eg8Bq!PT-0*xdbrx&#KVn
zvOuY3;HQMbt0(x22P9gCU#S^-J@{a{ycN|sH|Q4@pn|U}0qz26us)K1En7FB(aP?2
zu^TXOvMTToNo3D2(bR8fVvS6kp?v`c)VoigKFt9{SUDWfw0so)hQ%LwldfcP2u{gR
zacsEaQ70IQHu)Sma<qiA(KL`*fOm^Z?Z^Ts`)fRb8Rl${Ba%((0lpG1;fMydIYTn2
zy!*2k;0kcZsJ!YG^9dGzQ7ya6!H^mqRQmN<amf}K^1vG5OH#MKrY}`<V3V<cpTAm<
z^7HF-?ix!qaRU<1%QARhGbrHV7CSYmIS~uf9$wn`<E~GD^?n<+qxqzGh}1W`E4b$U
zw{=Hz+;Z#H7_N`WWM*0v_#fH5Wx&EtCtJ3Hu6Qi{&24blB7NuuK@2AQVD{FjN6-5&
z**ewBVP`LVCldqp6WXBQ1Xb#YZS0p_+`)=A=LUejpA{CiIU7MCE&qg1O6Pa-$!XO|
ztyhAxih3EK{Jz;5+59)lH@;Wb<1BYY$a6OawskV$K4*xMVN&}C&a!c$7Wkb(F6TuM
zMHe6JL+Sh1adiqi<`Q)4A4LuM6fUZuMIpDnlEw>{g&$gOGZor0S&@0bdtRQuhDTNj
zyeCwjLM3H~4jr-)cPPFz9I+dE_(RH_J0<1x*fo_sRnmQkO-Qq`02udIf@`Yh$|(sD
z8r-g2y?S+a4ds6<pZLKM5%d(a9z0cjYYwgR-V2`R=Q4WZ8w?WFIi01A-d0m0zF%Ba
z`eetZo1-ib=_+S*uI&7bp{m?^Q#V>2vi}>3uEbTv7&F5@-353bh);V~Zg?I)RTFU2
zXrTfr5IMlFHv@&GeK2wl={9f+9&ca&vU>jWV6VE4j#TZlnyc-&1zLe+9L$GYoaK!K
zG;qHxzj>WxC65t((d(fVr_bKD|Lh**?huFV$5WI%ME<?;Soq^{H31`>SqBV-)mIoK
zl|O$ttzCQoj35Xp14$hcjrec^4P@~B{FT{4Gwh9NHUMTnm~IcX0Q|IBL^MKV-`>4%
zyK@b4>7fxmZczL;{}J|X-NwMzasW@x169BbL+Wl4Exc``wn_wEtRfs=M?9gULqA&7
zb<KJYz*A<RSqCAz{-t|zYHBI~DgfN5a6H^<KPG~9ByI@Rh(#O))a5tOBlN)F&N*nE
z-W5v5AD|848n0<Jwoslgcqtj%bL~W@J=I8%lZTym_L|c-FaU)U3-}0YHR?Yb@Bc1J
ziU%FgkkRpr=vY`-*jmEbhstpe_6Ph07rIi5;pn^|qDjb!a=G}$pP+MB!3G_>u}*Ac
z{`D?fJ*-^gVpl+zvkmPGB(ER(lrz&N-}L$#L$`r#eLOtU5Gs`@j9UOeQ=5_@17m=2
zR+=`8Gg0{)V?lfX_$EJ*T&t^MAWPb#)3-m<Nv^r{nBMG>KPthsTL#sES^pQdg|RZu
zH$i})N69FaZ#M)PdY_lben7SGp=jfL$yaLi$0b@qJon0JX;NL#a#5ciIhxG&oGyKE
zKd@g3f-ZpgKp)y;($>p2bd3{wvj84al}a+&p42L*MyVVza|VU|{hSTY)hB?j@ep>1
zZ9(P7#t``>wUf_A>-pUrV_rSKmPBp}{eHUn3H+l+kFM7s{M~i3jbLo0|1#LE3hZ=u
zZ@Gn5IGWvc#~Hl~sPEH2z<vdXn?1=EgH)7VtE?y))lAF)(f~ovwJp5M0N<{{*rc^5
zG8H2nW_I7!lUFF@86V%hMHgOMw9+`o8Avs+4f_&S=47P<ycvXdK)^trQ8Z4{X!d++
zM{TB+=SU98gVb=j_lY%t`k#XwDhdnDSb5P?8`g}s95IQ@FJ)w9$)G#yPHl=gs5@*`
z;>5UIZny7ohc)Q_R2o$e#O{X|p7sHw$W5C@6=7-cL+qQtci@@Stt|}y<i&d(S-6!H
z0DY%5I`FEF2k+?_-7`;qK{PNno+?LJ9*WO?4v<~I>dG>nH(X5}ytZj<<^^C5`rG#F
zP2VI>ZBsaotgU=JXx|Ktdbf@RaT#^GU#G&GIJ2j*v8#eOlfHwC4-bIu5Q?-BsFljh
z<RV`WS{?_HE{YmG@;Ju0sqY%Z3O?aQh92HKuxX#<t4gWXRSh5@w80$y(RuRkHo#w+
zRQ}DO2jnp8p!m&5B=Xm`v4ElH9xYEhYE8Gp5No!i8dUkqnS+m5`Eeuu@1Fn2E}t@B
zF4}O)<=z8?{P5O#(}$;TZ>v#3E9@OO8|3#bW_)i%`M=OD&FsP-uW4ig6sl`pTi~SZ
zR+wlznt`#3i;FXJbscUTYB|(;C8(w&58REj-ShF9nN~y66$jHTFmQc6HMKy83KWr?
ze@rT=z32m)qz#}^3JF=K<4GESTnS9Pg#Wd>!TOF=`#~af0BB|WnIi=KHlZBP5u7)o
zR`hbqY;?bM9Drd&g65eAeV|*yCVRi8o}RQralhjw@Y;*Y3;Un@k&<yZMg1pS@>*Yk
zaOPfeJB$B_T=g+B&rx^8E#kv1G1dC%S3l8*dcHe<&XDBRt=H2kRkOR~aDVLCa;r_D
zsHv$*6W_rw?9BjTGZy4`U;qjgFkt(UN}@!ehnN1QpG7leO<cnikn<wY$>NZG%}y^~
zsa+(yG=u<$d2Ac{i9#!)e_j0I&(Jx0VEx1{w8%Gl#JqQ2nZ%Sv=O(M)sJIEKrEYKb
z9_qC<A-#Qj!E4UOXzn6dl9wAl{C}rg4@{LiDW=s>q(gopxPpci49-vUA11=bzjMSi
zpL{P{D3>0AO2x88&G5Xklr9o@3g9W$J#|uEIsI5YkHyXK85ts@QA&;_U(a4&3d&<`
z1)-=$(B0QUNs$3m+yKb1Uy8?ZfT_IoCz1Po$seB+feKt}R~-m`(dN#%1Ukf?0~X&$
zr_)d9>qj}+eR}43@SZtb130ZV(bkG-x!_$p6Ff$rdu9{ffsayT|9wJ);mF>E58l!V
zmmIZGz|e9Lwsfe}`K1(R*#jYf%?FLXRDe+_2>RL?z^Q&flPWNHa~MGC-2=#j_dbD|
z{W0Cei=!f!S{<3E>qxCAQ<G?urCU)zXACzWLo@NrZ-+MB^aXxMo@*XUVQ1o*;Ly-M
zA06N+k|Xm#cLULxZ)>K>1YR}BIW6zkQciT0dPSGa{{0fb3^(viWeh#Nt&OICe*Yq9
zj17!XZd<vsJz)Qt=CEg`+mmaFGV4tpL6epnUc0#8w@#e`&V7P$WUV7WFd6BlvFYYj
zjZDKo%0cG<5(M#*xKiL~=mSw8saH~Gm~O$Xrt2GhzLGpRRX%dD=;EX$n5fLo#GiXX
z2nz-Fs-Al|m<0&l`$L^|H{C3$s;a`O3?QWw)Zvy06DsUDU@9k-r9$TAy{Y_qAacgQ
zxdnC4jyAlo-oL4($GOyWdjzOEE&&Fbzh;;uJOcI(+cq_<@h>JmRuh%1F7(XuN-hHo
z9|%w_U#l_Mc`$iH_T?a`^P_b@RIf*Wl{IA61`_{TQpL?*s~9|u`+Dlp^b6P=Fl&Li
zn(ehuvlpcO9#POy01Cw_#7>Mn1ifhfd#PO^9@I&;D+<k5w8;*Rx-+`}(_O7&?Sr@0
zyWAj$B@*rp6?ur?RHvl%`I9hF!Lj1MSX!88&50hSWUw4bE^{x71OY>pY};h@fb8N4
zgwtn^e`6WFDl+ddn$(LvS{WY(zXY#v8wKs&)HO49kgS~JCLW~%Y$m}*(;j(@e2No#
zwT`d~y2m5?8uaGFamJp>K~Zm$e*txetu$%{HhB)ALCFaqVY3NTZE1aOs|(1!p5XLJ
zN-sQ<Pn?iwN!^QD(VBGIU)SRzJcsIJJ@$WH#uRVq;Tr}V0q-uSuk3+K%An-@)MgVM
zf6G5xdhwXi{B(<C_3TK?+au`~A5Kdeg~s$@UsPV}c%TVx+nuMv!0Y^pF30O5^^18=
zg&CHvRemSxE*@JxHff9)1_ffYPQXZv7kG=qv2_0-0DrnJ2tK%gCNnIM6+EM%MZTwG
zgq(A>Iqb~^U3D>k=klxVH=`DhNVH&ajXMCC8r?%5y3%LeSx{D1#saXyL`&{O)6X6Y
z_gRtQXBch!BYx-CM~3{)-}K%md2k;FSkd64vg+0Xt~uw;pUh8;yaMPlqkiiFqhwG)
zyq76R<akt*$=0!3+RV$G5Gxh}#_xU|+ODe=UFZ!JbNG&NH!sq0>yddwUJh7C`Q$H4
z`}Xhm1NTJ?b~N3B3<^&l=5+x1s+VfBe1G&ur_f(oErTD>eaz6|Ay8T6UmwTxcUI~2
z2J6-}>Paqa0=w+bt)NS7N&jdCHJWmdT^u(`OLhKUjg#hKY5P?1TG-XD&Lq3XE1&>H
zk*`@;EYW&;dXjh-{rcz9|GS9SOv&`Gam6C7$=j>Ro&zuYR$Q?R?OvwUuuE}?Qqj!X
z806cR4#uZ~ZD@d`kpM0Lp{okGL7Dytpm+hZ2QXzSubwI$f(7n=cE@;0bn8BUJIxMl
z<Z9Q7@ozN0IH0~QiR9r+f~GzPH+OB#JTK$_o_&`}@3*VU!sG0#$qq<Id#gH9SLJ)N
zFTw-ysqy6bR{Up>*XOD^8Gzkjl%<64|FD$e43x@)1lnO=bg`OH1rW!OQh$i%zT*Q{
z%esrgU#u#YLE-6^X{ERt^d+pg6j)2HQx5Kq_QzjeCuLsCDp_@K4>zN^FIUmuaGvee
zJTAER)p*nBq07mUG-=JJi6|DJ1cI@pHb>CoO7#T2Tk`U&6X8ul1|IqQfqd$1`p~be
z$m4U-0hNffC-cnD^-r}1^ULUNE8KuM;WgH}>d7VFtF`0)a~SRZQ}5|_uFCcan3J5T
zO-tEfuIN@we{$vnp*ek^-evUZ*1rgff<#>#(2kpJU|_&J!tMpJ&2~*wj0YE5><W-x
z>y!mVrHOFo)T16dR*H2sBCP|~^?uc=Db?YvcMYs4EM`KU!y+#&eQ#y_uRbJNK;BQZ
z*$&1e$M@Mej=wQjc`LEzy=&tt$sEVIHIm3*YO}P4*B*NH($Zi4vr9&J?2kTHw5J4~
zWz_QhHn&UBF}0;lac2!SK$k@#9qjOm-L^5ggqEc)Gn{d8B9s?;Q}NW^=KUWGFJBJY
ztQEAkxeaD?xF|Xy$b&21y#L4e8?9_@TojD!ueH7T{V%$5qTn=*^E?#J!SdO;fi2Hp
z>}YQO82lTd;exJ6JpSR0R!>T*`ugcNpd|jbU;pwcp3Yx|+(p*~G{mNYUI_?-pS3wt
Ja_aKU{{fR7feHWs

literal 13482
zcmcJ02Ut_vy6pl{+zR4Wq$;3>B2uMFJ=h?CARR&{QbSRC2ODgu5`@rE46rEy0RsY3
z6bZ6v0wy#eAW}jV38BNAai4SUIs5E$?|tvxo9~M;%ayg}nsff+AO9G?k8kN|vL4|%
z0znWfT<e-41Tl=$--i!@BVnH!Q^3a|2OZ68;1j$yuhvI^kDu;nS@=NE(G&DHL*Hol
z4g?86@M{{zfmw^AA)#5$VKckSemxq<j$_phf_<vmZ(jc-s3&k%2nK@<!jHSAaW@E<
zREu{!*BKH&`qV=t`l%B#wJ@{rg8qf4(PGw!V_ki+{o*b{?=@EL2{9e!Q$Bm>gzh~i
zOQf>@!eGVAV>0Xb2p4vt7;Ms_PUDTfncZNDW7lk5_|~XT53byKsYUC@aR1F8Qo@hy
z1O1(yXLzmSRda%6h(uzFUSYqNx5Y`tQ&wKKYP!UMD`bhuM&3wY0=IJ(cVb&WJXxOb
zCdkfioLAVWs-$FI&VYRvkHv5!)#Rn5z6HVO78c(0ajjaw&k)k~1HgIZf<sa)`_)xd
zs|_R79=s`7DHgI>k@0u*Q!Ut-ijvY69+7AQSC=L1Of@$*kD_nm<j|eETN&+xlSK+f
zlXCTSb$z7aSUuWG>t`v>YTUhh_x3FYW=eky&#a0TQq0uI+q|k~E`dp&^1d8Y86nU|
zp;8Rk!_KAa+Sp29dY(|d78e(F=tl`@>byKWb(K~Ug@OmOv$MUDV8~AL@@V~fG2$m=
zoW(k|Uw=6nhVm#aYiqrF_3D5*s(f*F_Vw4>TU7h?rO^Wqm+#AfL9D3FP-%}QuR;om
zzwX`OifZDHnwW<by?8N+E-NcDBQ2>bXY1_;aYgBuTGf7kBgN9h6E)Eb=HW(@P40Xt
zo<HP!yKrs_J1Yx|`*2gle(&&ZMDVxqx98EK2M!#_1yg(zedy4kMlalMBAy(OgQxKq
z%=YrB1b)-XH>?#re4os-uW_8oEXw}13`<LVaYNN5;w3I5DZihR3)VD(D=JnvL`sPF
zWYk2A6pQ9NDU}mZ6T+zV>{C$_llEK**nUdDr-HT?sU54uyPMN7r!J-@4B#R<N@&%C
z!a`KlDXUJ4-uKFS4%-<x57e6;ZVy|dHOVfQ?ciRY&=?Q<%0*Oh?Y_Rp5;@vF@H$u2
zHke%TwWe1Zwnx3|&qrpqIin^nfm@cq(uYT6B<*lun}~~II%X0R5*j-Z+f&}Pp&3|B
zB_$=&rzmb@XYAgru6Gl$oV@oKwu{B#7G;s0_4frfrhU73kfn1u>ZD=wTC4C{JT1Xj
zDQs`!WqYayGU9c23iW4rlQ5j(yHx-lQ%PCbE6uI*!-o$qvCD9TDCL2NheshC;Tl&f
zo}!IsV_^xcHJw+2FDHExEmBU>@*PkQpX_kQ$q~}7g^~NaTC?Qtj-d5Iw=>o}N=!?8
zX>!DB!JC8rLY|h%s%^p}9Bz1<mtk@l7hN;4RliMn7vgDT%A@+rwy$m`TK+n6hT*fy
zp39C$rFZR)7W?(9gft&my1-!@OZAV4kR}4{6!>yOw)!4h>*U?@aH89qC3-I;BxL&p
zT!^J*0AEuhS!67_FG`uj?}r_-=h}0I+u};Lr;J6{a_<Y6SB9)FA&um~KKf!V<{v9P
zo$2S(BzU2cq;}S|0(P^C<ptUuk2q67I)s)5k!S{Dq1L%4XJ3YxcJ16<9G=z#Mtr4@
zGgaKu>9?uxHPO-0O>_`B+E1b0#M>~!RlzSrFdYSD)3Le>+kw*#-GOhx<8Ob?^54!_
z^w7x|qRCnBXS`h0{ajrToS<LLs{gsMAiX}Hrnrkm;KG||yOT3nK6>uOjY2BZf7=Iy
zeN2vCA-N)8dCXS3?kr0S1N`C~GifiNv^AE~>j||{iX{<_s0R;+hAocP2Q=Ynv}@;9
za8fL}DZVRvyW5FNBQ?ve$WB|GVesrS=+xAXshz-4)exGBAY~EE;!z{&27s7o!LS~#
zDnEcY_5I=>jXUgc)?folA}a=GL{ok<6V6l~;zBLfJP_D$;EEa%jMp;cii&umE0ilu
z_*z1on?xH+V3y?wZ;ELCBrQ$NwmHgHX}~3)-d8p+J>W*_3i*_pa*NN^*vY}hg88Oi
znx9I~%HxrNc6a{%wN`)Te}dSwChh#b{o!hlkqp9l2FNAcha!swyUejM6M~u_)v`B3
zP$w^w6a?`d2Y(Ix3v>V)XJBA~BAI_2e9z1F7Oc~kjBk(6OBjo$AiV-xc#qYW9R-(S
z`o;o|G{v<#@V_MK(KJ2_zek|%3;bGY+Z5irwKktkJP38Ng0r}gsZX^9S3jIje9x1Z
zlYZ^o^MJ<H>~>;_8Lycf1LVWJ9mxQFi-ZnA%?B8{Ac*a-9R#Te{NZ3$4s%xfZO==D
z&!UtBhl;=|nC1SeyL`xFY&7hQ3?a1mmPl!Cy9xwN^E#JD%B1TKb5&ge!3J0A>+kn@
z`(C=$;V!DeT$e<6IA9wQ)}}kn07;P*kB);)oP$+f-3*nB$*a#3sdFi=*bHrS!-du1
z@qYWht4odBOXfw!!EcO3RV66yW(|zccxDw;O;U@1w+b9~-(CDkR{|4aE5@;XTl|;8
ztbvgc-s=X`GLe`uL5k~NPY-J*qzS_sYz^UktQyCu`lqY>7l$|c@^uCP5REn3w?v(f
zpXNrEVwbfHi;Q`^i5DTL@p8z)&jSNBI(^T<a~d0(i?I7^83qIdpd>K$)@X#KrR_xu
zawTx*FY=jZ+NZo{bmiEf$jwfQ!N*5eCgkBYMDhTctS&-nK~v+sLMJuglEJ9p${?`h
z&RNWVU*kVr_&+_U*ub9ewLTWI>{?;IV2@fnf$@7Gf>^5LgP>K>$dixXUqW`)u7CE?
z?{pl+qXuqB%!4T|XM8O*T;rA750#mgNOUkj&1ztIL;twEf4b)Xvj-o%?r=!gofblN
z;?PSX9I*;Im22wDi;LE>va-JBXv3mUd7)~yi%UzM9<HvJjd?Gbi*aD;7k+{woxU!d
zx+9bf=S7+p7#7LHHG_kLCDIgya(nnMa>T-sotU}h8dCTxb1|WxY@y~W6Jn7d+=I|d
zoinRMk(i~JyxBb0D6@GlE~IC>Qx&sM?dPtwb#mDHXoX=BH6FZKQ696!W?)`?XdJKb
zq__xvrhB+1)$F~!Jq>sV0B2-NOH0k|OX)t#wKF-n+J;_U<)e*Z+v=pK#!i!RD5HOO
zoxG0$D3o6|PncZU!4KGM>zB<l3b$B5yvUp<)Nr^UT!BgBV}!~o*y~TboTBdw1n!yC
zN(-TV3t=hR`I%O0N1k2gMvgTh+Y(Zfl74oUQ&v)Xs#iFPK5Qp@%9(+_7cXKu&iv+7
zr70NJcuQpA33h2|iDWG5hkAZ%<0Z_oYUpQ_hl7jD>}ajk(g~D@Ro_)~0({H&`K>`p
zu`uM*)19sCN1ZirUMt0G7xs70F_bB)9;v$}@>z&dy|wM;B^?pC;}3#V?~-5THUy>B
zD6;)qK=_Z%(M$<yhCaFv4-c~^w#o1+c&bdlrrtT16xcd%yT4_fFz;{=Y`0F7xVZRm
zm2)Cr=(o2|JL<f`S2G1&YhjFB!VjTCQp@oP33Fg>eGvT!-pk^@9E=Y8GerCm7|jMB
zbMSaPZm`l(hv?X~yS>g|w?g2}CNV*;uR|dx5))Oa8pjvn&jmQka3Md9jqTlq&u5=H
z2q|hpAr5g#u&{}vwomXRi%}hk+H0IPffFa34>CYi_y2gyg#(8oPabE|@b>kkuJX#e
zE6!DxNf#{F&6jPLc4WBd0g%p$6gcdVs7f1j)J&}FAj@<sUM(@jX~VsFnn{S&mBfh8
zuS+k$YsP|0<Or{oV|?$Ojbs>S+Wo@W^zrQj=4@h#X$b=FmQocwxwXCB6hbsHvsOCl
zr=T7DV7~6?L68bot=O-hSsx9^r7e!klscK2Vanco|4q;NAg2I8YmUg?UW53!IK>jc
zrL%~LU%|4toX|M_fvd{5zu+dOz7%Yt3||X2&*ROjo!_TA<N(uNV&#?JsqU!LIIfbZ
zD<o-I<2m}IqYeu9XpK1))9O%u`t<3Yf{wcO+vt)>xwDLcIq-bLVSp&E>U%4R`aTv4
znO$Y2rX@JAG3`>hkW@gHHCr8eUF{lo7rhc?HOsbNI(DRzALo78WIz(q&Y0PTt&!`?
zLM|O14|2J4=N3rveu)05wPFT4Q9!OE5E2;Mm4&JvrIOm)?I&<nwcZ%8EqsjFuX}*q
z<cpBuh&^Ld<9SBY2MQe1kJvI&UHfFzJ?H1^>oWCQO7~pt6$p8w6d%(zscb!fGcDO~
z!mihXB-xb!5#pFMo|T7AUF6~8^EX|6cU1CIE2r&(udSS3{LM`Hh7qpSpaB4*-r;zm
zXBIy<LkX70nVc6s*H(RRSPqk%s9drnZ<%7IA`5On2Ncf|-At}tJ<ZOR_3P!k=1Kr}
z<NM8;{~6SR-_~AiXlRIyii$d>V<?(}c-k{lB9qPxHHS|AS1^{~Ovm#0Dk!1VVFq3&
zliOqVLZt3uOkQjOw@r4t3d-Y_Xo{K<c%LsC1+W8cOfQJLpcYpP*_br!=Dl(|Rgj%B
z_`Af%J`7M=N!G!a($hb|modg@->b)&c7>IDxK6^iUen=*@2{D$=GpevmOoqxAX1|C
zk)w~^9fQvS2=1Elu1QSnaH={_8dg8e!^6`(<z1fPMju{!|BGkfJP)*INmF@KQ^c*l
z=r%9T*jCOkiK*MG05y94E7TaDJUnqzg%2Y=y+B`sp}UmlCFPA<YDgCTIfiGhKR17K
z0y~lC%$bJo01@g3?~am~`2xpJ!<+6{a6(q+A}8XdZ`AuQR+6_!HK}!_AaM4#TO8Jp
zL<+U>UjXD~{FKyqFCm_njKWzj)_DSIFhIY7=YZe6T9I^J(Zlzr$UD;tO<zGc3u(hn
z>?23({Pz9Nt^Q=!{K5^tMGmJ`<z`DOD$bTpgO~yJF#g5~1{N(u<?Y9LB5q;SUQIIC
z8XdNK-;?rnFrG=*paVX=lZonO2bhm3mejZrMg9EzE|)NOGM_(xe(Nn;`1mUjxdR{i
z@`b|>AN_9;uK&Q-|B9MzsMtZxcTpZ5DbmvlOZCf*{B@<^!KW&1WX=N?*XFVEr4_88
zcKD1k`7+sYbf9T(EvX~8D-KY;tS10JcE4kqd;#vxzp{}*^8Z^l3UY-~a{K4oakVCW
zXIdRroZym5Ssq`~KoV*N*dFbs&L_qZ+hI*(g8m}oO*^8P+>VWxzGzfroQXZJs;bJ_
z=^KZqy=ZlihHo%J97miPPRF)(C;K|9<(A(t@HJz<&e`P~Cq11C=#~_wW7JB048i+U
zJD-jI)<R7Go}fACxEk6k@N%uDkhdQYEILgnG%WImi-SP?ymgt71{k&+8UgYt6zSE?
zILNj7P`bbjokWj;M82N1P&IxF6iB~^=Lni8Qfbh3h8${4!Zo+V2Jql63ut|9Z4vD&
zlG(+V!p?Ez(HZoo(b3V2jw#PvH(mM623TfS!tJUdXF+iZ_9M#HYv5_Cyo1?6mVJG*
z&tq+UZGD|hd+VDqGN$#5DIAUogP@NKzlX$zEUZ)6fL$<8WH=PikNCP`Zl<^AP#&>(
znuVDhYWVcP0V&Pn&1nG+73SZ@AF$=9{@8-oIjynR&i(8$TDRuwd|l8lAV4y!$L7E{
zMkcB0{IG2r<wMf>?zx5qM@b_comg>9N4FcJ`QYIH3&;FZ;`u*u@S{j%*98hA6nU_~
z0NOY28xa&&I@ka~FYRQHj2{&|gMM|r&sIyjCs-4L#GrV_z-xvEeU4}~Ns5<XSd~97
z1Z6!-KmE`V)}Y<jhN@9NOJU_XiiDs$j9*y8_axx_Xlf!pVx6=Up=Ai8s1w<`Z(ma2
zMtY|BYF3!Pu|Q{=->H=T9ftd6PkjrAmwy~AAvgG~Tf%kbn4K=Azmj%x6isnU_Qe(!
z&fI_8D#R<3Oy}wf@iq+V7hHYcg_S?0reWW1Dbu;70sCk=z;&<S{>&w`?<4G)c*1*!
z3RR=7DpJPcNR6`_xu*M0`u(aNwUDT*&O%9AX&@uEJB}KQ3c?Y!-uZbV-ljGA`PU1x
z{WJP)+Oqvf7ChYCcTK%|^R9zD<|H4?+H$KSRoutas~s&^MM60{7jj72ER@=Wgw$6d
z`Yq~wDWxiW6kkMJ!WnQ9_}P*BcBC9<om4Sbl{-<X64u43sT?y6JV;$l&1iHxwkiW$
z&gfFQ?_Ldo>jjPGZ!y0V|FODS9;gsA4S-h<ws(55Hw(f~4h-I2Z%1tJTyD`gKDaf~
zAD`}Nqn68NibJd=4R=kk2AinxsViQ(^t}Oaf*g{|!ph<McAFN?*(R6gORI8xO!~Qy
zH#9Z-&8+>1o@R1_7dT?Gs{>aY`iF*s1|3Ixdwa_+mc4dpR{AXJP%(%Wi8+B9C{z7f
zTmZXnz-|*v$q&KdyZ4+=GQD+x$CApdx8q7O<j!!~hORK6JX#$Mkey)?^EcVA_tL<N
zzudZ>M8PwPrf6^V=}3;gsIa!u13cReceB<ez<}Ma%%aMvsl$)+N1)9E8pHOw*rNaL
z#3N2MBrbd7{@g=uBqB~)(gGxu{xUahLy-SVZ5p<|%@IqYt%<S(7%aOHRErI)%>1^a
zZU91ZXLoKes1#Q`csaK}tGC5W2N>)+O6-}};rq`bgphI4(FGX+lkp0sf(?3Tpa9Tr
z9If&kt$Q&!$<UdDDywyRfMspmhNdfTY)2}<>waCLfm$Wk-OkpQECG+M99l4d+XK#i
z77lO+2dF%)i14GfeB_=_IQD+amPeT_unB9T#&aP>d!)vT93G!YW`!bA@niSHwG9ig
zt8z5A%?5PVQVAnuch-%)xzBuj&#s!60eo|P#TJOiYQoMW`@UK_3Zm-ajq=Fl)K3>F
z580}K$Sp|%(Qyf`e)&{92+k)UIM?BI8ps*O5XZR9Snd4E{s4aPt0ND1>c2%12Q^P7
zL@+?$CAH55ms}ZT0t$G0Ejk>_X|o@MDYStgbEzfFr{|nY>yGryQnm0aVeCF1oArx|
zSZ@f5O#V14i}ixLmDGnl$Ch^?6d>p?Xx~&Uvurin#xm^86W{8A1i&enoxJbGrxDax
zFGx_L9e!NNGZ*_HIg~+Ikfo(cSy`DpThBIr?UDV)b19b1kY7%cW?Q(F<{H^xqasC0
zA&bwFfFSFWC!*A3C9w_%nh!rWH}~e}dk2_vlqch4y5rg=^-W8Nr?zQ}yr-MaFlme|
zjn-$MR+*U#Fh|7;1u++xdhHn3prwVBfvOf4J9aO*7s%w*BpbTQ#u~q?8tPhWr8XL1
z8?QT9a62x@$AjAj8_tbd^*^NXv4Sqz0V(euwj9$jb)aqJDo~aL0sqEUJP1@bk-!Y(
z&VKI{jz9LU($Cfe9su8>x+^?#Z4h`}ia$9pVXbl8SH}=kR8@G|lEtijbwnF;$CUS_
zm^}3-)aV4%0pZ(pQF`(%U@An1xXmJ)!1#wo1tldHvw`O3d)nx4s`zD-4Q)xQ8}Ld}
z_$XrA){U(2TkHk8+=g0QuYn&K85vRL(L+Ey4U9RzUd@X7w9x1g7lgx>UpIIvPuS5#
zzmu{$TO{VH)kg;En|je`L@G8M-eeyaRD<{T*<A#hykrkG;M02`D92%Ibb?)O-~L*|
zc3SaHDbW4B;-tNjUzJ!%<O;(zj$2vqZg&9rHyf*sy8(l*1cu7S<joY!sb}joy)Z0t
z9|Rk_-=1r_m^jC_zBsbp0T18zIf?O!nR@q5W@vS-R*I!3mCA?gj3*7}>dgHFAZVkc
z1(3HMLfU+b)EoEB?^9mNzOx@Ouqjwrg_T5v085JmvNNKZGzLmKCop09-M8!07J}4<
zd=z5x^x<?>@&MXKb!8@5%vp9N;YNoDoRAigLgl5~285aHh`r7B#~l|h9BKJR=0(c$
zAX7CBs0vfwn>BRV8wmC_lz|a{B4V2g^vd<z_ez-xJiCe0cj4-|uwARZb-@O|>dbea
z;-r(Lg)9P;UE_-OZC|HQZ*fJb0!1fhMOpL4jVQq|{XRoDzdH)0jHA6sA*77}MJmLj
zB|FgaALZ?(fKLP)VTvjmA?PQ3;_;FcOGF2kk8Q1$jS^oYFi+xBI;Jp|sAa9f{+WT4
zO?R%S6p<LdDiZA`G2&d(Mq!}J-qLVZ4qy+vp_OFyR&T~+HQW4|fFu{7oLa&8lc*tU
z^*4bY*UPo~76{qPY-g0HmZV`xuz&kus(rn^6$At-PXqzv(jIa{$a+(kZ+)7Ba`r;)
zwlI3k5FE`~m`Qf8%4Kc2gQcw%r;AO#PEAc6NpAnjpGeMe4-&kPq@`kqYv@CS#Dk)`
zQ>MoqPs(|28(scW%eOE<D0A8CFA2&W8#mKZDyOYoiqB4^jt+SqF#;uXNGZPQ{00B+
z%QLUE>YVoXb|0LK8kM9Rtq)k1h$km$!OHNYJ&Rf^J*};Gm5v>8FfbclY;0^1i-Q$N
z&VqhQrqEa_7HrohU`Dh94S6S*5*|(688QBw`YkW~OP<TMlVfF7q;MU>+qkeReP8PM
zcMWQ-64+Q-vt8phe>2`y-~5_<75#}w^lQM=bozJpIK^k~sx71y76dnbiqozny7}Gy
zGE)Bd+PRoK7nN(e$C=Lagrr;sGRaLGZMXPfmu^jLd(KR<YQO`5U$qgin|q&gZfEYk
zSgCo|G4=htj0NGY`u=7Qn@&ueG)AxHjXo%R`QT<c2!kTyJwd+2{efY53HVBPceg$Z
zD{I&k@RCCPhf^ve0c`*FP%_X|0uv#4fj`QA)2wNWI7u2#biMKtOoc6Y`xAF1`w`oV
z0*4#9q9*)_+1eaY6a28`cIP<hO%O&K???LAq*nqyEj0~rtpZymG0I+%zU*}%lx>im
z%Ma}pPe)BWm75O*wuQE6%GP7)X|IBL+rIs{yjh<R+Cq>^F<E`cyZafl7-Xbs3u{Nz
z!&q8u2@VxKCZZ|jdL~FvnDD|HBw1RJXi~y{;j6vuqI6HrWYr$wkfo)uhLBur;zRq_
zUQs=ZJ#kHpz>-{69CIu+7L5?6sO#3e!3Rg<@QHwKExV!?>r3%$T86;HM8!+<>*O0U
zzmqymnVVB`E+#*+RmN6y80f7X4D_2$cP<0*tG=~#U{wd4{TrCP`OtOgOpbS#<m+!R
z+c#JunQg)v{kOBx;ewQi9tZvAxQF(E71)Sl_s4gl9x&J?4B;v++Q-YF6@x+6P5$J<
z|4jz{u5RQEPd%S3$W_<NpT5%U6BZn`*$$7VXD@RBG}=ax;s6(K%owcmo5!nfyk+bD
zwzkax-EkGdaPSfbuPavT@Z^m87dXY$HXv|~?*^8EVvv7>A+(I3cK_wxdy2JD+(8JE
z`6LplE==KxZ#cpiFuj0305!h@)Y`VBqKApk`MsP91ElC0&$wZU9-i~G6``Q2NUR_w
zUwd)UzCjatFY^gC039srFAK)0_!Br!4o!k9HObRdgC`z+1LE*cE>sR`p&SjaoankB
z3wy^2LCib{zuiQKJ0F^aK$+>nUTXVkP?+CDjy4yt^E-jqj<nQ=gEPgXj2amCR_m6{
za$|0Q$JQ!1GR{T}+n+6RVS*wbJ$z6CLEr4^!kQ(M*j>f2ZB6{^pWWpT0%b(O1X@m}
zUR&JW^&=<FXW{9S27zB!>AC*6ZoJ-AC5eC1Sx^h2h#sI%V6=j@`lOMag-0H}+e+^|
z0NFL(^*?U<uWE6DxtOOoM{N6W+PAIw^5wL@dKLd#gWi;r2f7OmuU}JLy_D;jgO?u!
z22Mzg#|X_P{tRdYXnr#;7Ky`8Z|2c87kxYL+Rxj7$O@x0R2EfGi>FA#7vrQa{I0KX
z@^^?q5W5<^7oh6D=|Y^NTbcQWuJzJJjzDAbiLEkg4hsvbhy5=7K6-S;J{ks!z?y0A
z+7ZDbJbn$2R^v!)V*^!bl9fb2w=qy*{&G^}Z{%=U4U|dubPRxT@(<q9|3p#uN5ATS
z?V85}EX4j<dZ^x;`4^RqPM3cO5myhzwtn%(s4_qw<FXnUK`97~!{3ZWi9!@Def^$;
zC)>WRZX$sY0bty4cNgFyM{Fx>;wWS;l6CNo7>BZcUp6qIc&_(a8?al;c71c(Wkq(1
z54qFqfcI0o7mmUWfj!{|8X4;K`W*f)rQz^SJKH}zDGHUY0wBhK8O6)T*C?3Wo)yxg
zy1rwK{=OOd(v4W0sDqX+=rSo!Bg=aXQ*#6kcl%q{AaI_Iv|-e8fQ1hz)|Q`AGbK>k
zXgsYRD5!pK{j8xS#iMq}&Xedd$s!mfO5pHTqI0-fZXzfdZXIXpX%cM$GI$AEA_Z$$
z^lCJX+fj+*HGuN=+?}t&v4C7;5mk1n{lu}c@aM2X1{F}kaq6pWe&zmAL;4I=I?m0`
z2ATt<OIG^--)|7JqQji;*AG8Cui5N<$CNZQj9N4VtM5#(`HeR-IIn?%GCv{VnDZLc
zd8|-VgY(Fv5!k3{gzcjEG+QEB0>mo3K+hjk8><{M4F7D5{@7^aQ~@zE-A$MR=8V3!
zc5RtQ+-3`?ucl*x^8#@c1f6OI^0H36bS#kUB?>#%G6vohcrVq?L`k3+A-#AYDn8o<
zgdoBZ=NC+f{~-m=1D$X#H|xU=__3V-ojV_0-)iSO@Fp@svUFuIlQcXA5>CzQ()!@F
zxrsoP@NytkLy_$)Y;1{3gB2~VP;MA)=!FhOLUm%3Nh-3S!nMC(oflbWH6Ta$+yu(N
zc)Bh006O$hwiF++`TnH-tJiN5)<Nq9-Kfa?VfHX`{lzE<h>X9RHQnm_I~KJG>jgJ6
z>jZJW)(VI3Ym#u5X5QAEUC|J5C~o0Qn}Wxi@3jK4F{va2NC1x|t?Q^EXi{DA<LT?r
zf#%23Y^v0cc@LP`lr9DW)-)S)vFw-27k?=rq`U-}{GTX{dRH@?<-NvsLckpFxA=_;
z9L~8Zl#Iav)8K=j2W0-s;qlFx`+!MRMNnVxb7QVo4K4iUh94Gv4Ae#De-iu=YDqss
zRzHe2zHdjkppUZP!l4iFv=bOP9NwCcR?tyjfI9q_)sZ(rKAwSr8~ZW@%JW0$r@q$5
zeLB8<`-Y>Fiq?si=xm}@g~KD8?`VwdXt*pRSMyyN!kaIY!rZ+8&=7TJOW`Z9XnZV=
z`FsM77w`)7FPGE=uen(VPCWYI762vx!Gj0>c6QAHyv<py8s>lEz5fx*fMx3?@A)}!
zYIb%OS2oXsL|6@20X-OIT?y>tc!_xg3R7FkY_}v3IdStiQ@LvB#<zL?h@HEIJuL$b
zOMVg^aGs;%FEkUY0SBGr!T3ow?R+n&SK*5^YN7Bq>I(TJOHZ4^1K49*GX)#Q6Lq}2
zmh+xJf8K%y@+ZgC>gJs8wEbQ%ZOFMqIbg|s3rfX;iw$Qh2Pol!Hm5MkIB@Sf)olNw
zy6T=z?wLtE72ArHp4MrTp?lN>3#iSZViop7!va#)sq4Mh`&r7nZ|C4iDm&j=BtteP
zqHTpPO#EGP9iCCgh#h9MjM%<51I8l25vvLY%6Ez2TRV3Og2=yH4M^Q`*m*iG@u?04
zDU1ao*NElee%mFsyL0}iMajP^vd6+H^YbHsf*u2o<qNc$?ca@c(Udy;Es;=jGyt?;
z@bc2KveYGz(BMEAIiK8h0HQao0RNbG8Hn7f67${pMjz;UcLdruWxkG(7DC=2C^UBX
zjXruu=`H+&2|rmA+v?ygv<mtzuE%mvn(rNGkLC}5-QqZ^D<tFGO?pYP0o06?@I%2L
z>g|0VE7`bVNVnO#4*+|Tm5pr#6m{F4EhSf@kF`%BU%K?3bH;OsG6o*HFYp9Ui|fi%
z$X%}V#3RC%SV5nLq&0~t(x{c|B$%hx1XTAdG`1ND<=pGnS64MHG9EL-nUWWa3#19K
z6OZ46evRb$f$~Hv;=xEGdr-LoQ({K&#A6MNpQfg!bt=jYG;#nZOOroLHt=hbW>&)t
zT32W-A>hqby2`c%?yWq60m^)zi8z#|t-?0|%*7Yw5xetx>Yru;BGv=Wi$7sIaxPN!
zaa$YT7XSW6TZ9{~9(mbjs}Do4A{~gN7?%(U52P+I4puFP&M500@BtP~w;KcG)N?;D
zoegN*z_|QRk>cO?j{MQePy?=V#x+aJ%r7e&Q^4Un?Qa26pO6*l0lgPS7L-fRopXW+
zo}XV!Q-A?C_MKzClJ7I{ke@v^E-^Wy8vM!6;$r)2iJ0UTdNaen^s4;ve*eYD?rgbg
z{efGaR4h2tKRVj*IzNBv)?(WO(7aOHkpn?4H@a;Yoc|~mrzceW-hCh(RgNcpruP+q
zU8(GZn|kpIWUr_{@$;ON2wXlo@|mh>z*i{a&+VCcU{Ydg0#c%hG+`&|Z&t)W(-qeH
z&3d#Lze?%d-Cg=Pf`CYk`Gju2-F=3?AJ7X>N?|}F5QYmGzVx?n1<fvT(hcZ1EH7v>
zpav2E+xdV{W@pm4r7&jWKjf$>QNK_LoV^dfdx&C@|JDQ4mlMkDvO#w_X@ZcfLUVv5
zon02p2tBj7{12T2z`AA@3=1r#H~Excu^oVLy7=oI;93QqPO-7*y650a`%m^=udrz)
z!H^UNI2=jlVg<|L>-FW_XZ%u|f~PLpUO?&+Ei|CCtB3762YVv+d_cr0mjr-hrC4$h
z@?pi~G5x0vO8=LO{=eG*^C!J7GTguW3BrH6gW}SEuSX_}ZvldeH();l_>U!-TA+YC
za(_I=p<;tAp1`E>(z!S&fE*U!PAoQ2FS)5Me)#?vz^TA~V^#N|%KR2Yy|asp%SWO&
z9H@onw{iFsZO0Gx7Rexk0M%kw`<U&@K(I-^;pJ(dw&P#DdUcGAjX#;awm4l2W{B>b
zo&B}nCb|7KD&LShChym|mrk+RLC{87Zbf{G4OAHeg!m5o^;2eY$O`k%i0#q&vU<<#
zl8locV{bOEf(GDCaPBYk#(_%U&HH{p%)5^Q5aU3`t@>fKV2Fz;zrDnhhU#M5CdD<5
zcam1!VKI3I(Q?TvfhsoJywkd1%vLzliw5kMLp%>^Jkf%kXrGv{FC6;V*VlJ~>1~8L
z=u}Y8{t)WyrD2K6Wn2B7Y7=x7%4=#AfCJ!Fw!6@ML$wr;lVy-9x`8KZ?R%-?k^%d5
z+~aF`hB(t`YZ7P>@WA14%+@5JB*ex^7v$%+Qf^4As4To05IBq`y2+*M3MJN+TH;Jc
zIK&ZOx9;8q6aN~NSuZE+I*f`wRgn;d_11CH_12_JP-KPqgJ(saOS*#qAXY7I^JZ|f
zP6u`(7Qp(?7{3yto28CU$63t0L`+^9R<JDB8J%iYLTsrp?;MJF_CVlpRCKgH7-$&D
z<R)d);|rMJ?qD!R4Ka=oI51j%_eE}FB&@eyoXlPpi;S1n*9vaJqUpgso7>KX?0jDI
zqjyac2Kq)>dHXjGZi7sgbLXWZRB#tC1AZfq5vDs1S&IjjK;9Lk+yby8dO-}Bm$0Jx
z`g%EFwW<-*ia<kNq!8#YsP-Z^?(&d<Z8^^P=e&F2%IPYcv8V@W(A7w<=jW`0v4X5R
z-u`FrIe})&bFMG_-3H&-#r*^muOum{UT*s7y`UiMX2`vhpf`c;w+_7D2J5)hMo1gz
zG4+uUqxva;{%T8l76ncYdq(p)_$he5&G>t+)gJo!y?C)Ko1?+Ns9LEpDIt22!(<(B
zyUXak8tG}_#*n)U%doI!u1RVhcg`cyFdH{=Z!qAz1UFKmg4Af!C(O%|O?42jFn7-c
zmv22~B}2#pe1|zsI^s(`XPZP7sgcCBdVw6TNE<O=KM1-5H$C4Aq85Sf8;0x*;9&{d
zF{W0HgO-W5#{J!8g|(9ClLn$Gd4N1@_>#k3zXB2}5mf7m%W77j9vq3M7lq^i{wh~s
zC2*!{=i6>AwctHQsvGF#;yV3{VUh0$An;!6qtPJJC^>*kH!fbA=}9+h_W3XhHtItC
zV~aa&XhEH6;)o`|0yNc;9M*Y1*8&yWn3;AH(EJ|YHb%IpczZt1n=RBh9@+wKI6fh8
zc)C-fXYx+Dv{T`-5Z#u6Ts&7-#P3<8{8R7doSL>2fc0~fv+%Y!>0D9vBlGzc^Y%5b
zUd2TmX|t;^w+b)b_Pkv;l~J}&nU;G%*#d4u`72T?!HZerjsc)!iJ3uRYc!e!=qpeX
zgBKaQ8K~vvQeC#&Q|pHYh}Tx{TB2SZE0}8<8nAw`Cp1DzIHOE6mMJeucSjkCuj>wQ
z&M@#t>k1{y6H0aS4L#5aR4Uc#*8ZwP#cMoh<3p*ll{f8eC9Hr)R!4|&BzMtZ$vl!6
z*`R%y^ckZUdqC|$;vKcMx7@*CRy9TC=eK^FS#1N|OW;2p<O9E~`g2!fnNUXJ$gu9f
z+xL5!VWVBxrlM-RleL-Qq=!fGlv>W(b%;OuU)oPA@BTyk=|&hK%@Sn3w}6GNUO)v#
zIaOLEuc`O9wP4a|pwqPeJ)+*yZ(qk_dd);;Drh=cJfNz)<ni=mwS2#0#$ZHMdHp2C
z>q|AMbl&Ygyar#Cx*f>J#<-zqj>?VbP@@y-`UKEBx(v8!?&*k*JkU?`(p*eJKi_Z>
zjkuKFSzA~<lkW1$%4ki!rL841ou;6o5_<Jg`cB)*NLOPcPr7?;W|;L&S^gS)x>*11
zU4`vB<($2_8oZN@nT#~io4;pL1L9`}ngJorCVo-5=`1K7=OgHbW}tlL?t4(om^JRL
zl{d{^IkXw7R!KT`P~)R<V#tdr9)*hi^zK#1s9kQ<;@bXq$^GwntM@Yf!sIRud4t<z
zxTtxe+NzGtZF|xD0#5JVti#Do)@3}VKO$)$3t>mz`a@%p>q7HmFpnW9)I3TsOpGs)
z{{!eR0{-S{*M-O=cX?$%Gjda2y}l(POAJCLiw|sEf*Y-?6DNDcc*HR<BCTf!x%<Y+
z)^^C<|2<+m8r1eJ;MXR7<+%^~zIKc*%)8~?nD><MM8)rgYbCFf7v?>MIs_X6Em6~c
za-MbcRrjQ=0TuNhUs5wWx_!T3?cr=%!NiBT4OrT0d*j~TQ~x7Sv&#3USo!$u+zoAb
z8+ASr7N(74Q1N_3xWe5mHlbo?XJ6#Da49ko9*i*DW}nO(=8Uws0v$No-y!IyHB5F9
zh#dOaON9wsMt70i2E(NouM8Y{`SR!H=6Bvv*ax5qyxH<&^A(w?HZDd*Je|P&$?MtX
Zj_tGzEBrlPO`ZPF@auZl%C6Ym|3BY`OM3tS

diff --git a/reference/abs_error.html b/reference/abs_error.html
index dd637ed90..f37f75e3a 100644
--- a/reference/abs_error.html
+++ b/reference/abs_error.html
@@ -16,7 +16,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
@@ -100,11 +100,11 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
     <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="va">true_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">30</span>, mean <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">predicted_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">30</span>, mean <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="fu">abs_error</span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predicted_values</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 2.01879202 0.69207765 0.91175498 1.71395928 0.74646413 1.51484910</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [7] 0.15624959 0.35110899 1.47857662 3.29085515 1.07546222 0.22773463</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [13] 1.33546528 0.28994391 0.25914987 0.26462259 0.41897401 2.14605889</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [19] 0.82280576 1.75390591 1.22319941 0.99206998 1.69741206 0.50146093</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [25] 2.80167576 1.37368546 0.54469432 0.59079704 0.06504385 0.07585973</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 0.04335247 0.78169750 1.72566866 1.65897287 0.92520395 0.57798934</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [7] 1.40227305 1.53346288 0.04651595 0.72696856 0.61086690 1.16778933</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [13] 2.44073495 1.64170045 1.83422664 0.53999974 1.18998257 0.37440980</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [19] 0.95626491 1.46325686 1.21805693 0.56912591 1.17229418 1.80345311</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [25] 2.66248107 0.84947828 0.24647184 1.04848255 1.28635859 1.81194099</span>
 </code></pre></div>
     </div>
   </main><aside class="col-md-3"><nav id="toc"><h2>On this page</h2>
diff --git a/reference/add_coverage.html b/reference/add_coverage.html
index f9b78ff50..e9a6436a8 100644
--- a/reference/add_coverage.html
+++ b/reference/add_coverage.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/ae_median_quantile.html b/reference/ae_median_quantile.html
index 11f54ae9a..c94f657ee 100644
--- a/reference/ae_median_quantile.html
+++ b/reference/ae_median_quantile.html
@@ -20,7 +20,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
@@ -113,11 +113,11 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
     <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="va">true_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">30</span>, mean <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">predicted_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">30</span>, mean <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="fu">ae_median_quantile</span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predicted_values</span>, quantiles <span class="op">=</span> <span class="fl">0.5</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 1.812709892 0.414207758 0.093280028 3.162829089 3.906720909 0.850222619</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [7] 0.304749051 0.001270822 1.193929248 0.919874521 1.182473820 1.831074557</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [13] 0.264693752 1.968146319 0.809231613 0.713629639 1.924207600 0.594296163</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [19] 0.373904488 2.058168624 0.688137071 1.124096332 3.128342150 0.056254366</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [25] 1.070399093 0.278518244 1.339064990 1.052953923 1.033862842 2.321194807</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 0.009167113 0.872604126 0.515186522 0.626420244 2.133607786 0.580075604</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [7] 2.085917221 0.728393181 0.641991758 0.679059189 1.520541579 0.182937000</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [13] 0.071280772 3.176288601 1.428557684 0.907705662 0.062685133 0.426616895</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [19] 0.744986362 2.283214165 2.277201261 2.651972106 1.856070426 3.913091404</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [25] 1.790528259 1.671142980 0.050199270 0.433765063 1.154432409 0.070320886</span>
 </code></pre></div>
     </div>
   </main><aside class="col-md-3"><nav id="toc"><h2>On this page</h2>
diff --git a/reference/ae_median_sample.html b/reference/ae_median_sample.html
index 1015cd1b8..2f75b60fc 100644
--- a/reference/ae_median_sample.html
+++ b/reference/ae_median_sample.html
@@ -16,7 +16,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
@@ -101,11 +101,11 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
     <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="va">true_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">30</span>, mean <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">predicted_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">30</span>, mean <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="fu">ae_median_sample</span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predicted_values</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 0.06347216 1.53636267 1.26550790 0.49051927 2.07635562 0.23897194</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [7] 1.28540707 0.11143626 0.01528276 0.57545113 2.99596646 1.30559601</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [13] 0.46210542 0.65158364 0.21668173 1.12933887 1.42363120 2.18281301</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [19] 2.08457395 0.74101704 1.31789897 1.57850901 2.44268252 0.24034947</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [25] 0.51951770 0.31845583 0.08615476 0.68660076 0.30681817 1.44981216</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 2.0964594 1.7387376 0.8076599 0.1504663 0.5778161 0.9841827 0.4550418</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [8] 0.6935833 2.6801398 0.9680780 0.4389062 0.1298220 2.6308444 1.9791294</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [15] 1.2582431 0.3692456 1.0107132 0.3056360 0.8101142 0.1494411 0.6472083</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [22] 0.4068241 1.2441764 2.6587995 1.2021722 0.7939360 2.0019957 1.2597231</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [29] 1.8978191 1.5750900</span>
 </code></pre></div>
     </div>
   </main><aside class="col-md-3"><nav id="toc"><h2>On this page</h2>
diff --git a/reference/avail_forecasts.html b/reference/avail_forecasts.html
index 31acd983d..d5327f363 100644
--- a/reference/avail_forecasts.html
+++ b/reference/avail_forecasts.html
@@ -16,7 +16,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/available_metrics.html b/reference/available_metrics.html
index e226370fb..c4682c028 100644
--- a/reference/available_metrics.html
+++ b/reference/available_metrics.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/bias_quantile.html b/reference/bias_quantile.html
index 8a5696008..766939482 100644
--- a/reference/bias_quantile.html
+++ b/reference/bias_quantile.html
@@ -14,7 +14,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/bias_range.html b/reference/bias_range.html
index eb56c5f27..af45e8036 100644
--- a/reference/bias_range.html
+++ b/reference/bias_range.html
@@ -16,7 +16,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/bias_sample.html b/reference/bias_sample.html
index 032e54378..b89fa241f 100644
--- a/reference/bias_sample.html
+++ b/reference/bias_sample.html
@@ -14,7 +14,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
@@ -129,17 +129,17 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span><span class="va">true_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Poisson.html" class="external-link">rpois</a></span><span class="op">(</span><span class="fl">30</span>, lambda <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/lapply.html" class="external-link">replicate</a></span><span class="op">(</span><span class="fl">200</span>, <span class="fu"><a href="https://rdrr.io/r/stats/Poisson.html" class="external-link">rpois</a></span><span class="op">(</span>n <span class="op">=</span> <span class="fl">30</span>, lambda <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="fu">bias_sample</span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predictions</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] -0.065 -0.930 -0.100 -0.815  0.275  0.585  0.510  0.340  0.130  0.165</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [11] -0.785 -0.820 -0.160  0.670 -0.970 -0.460 -0.890 -0.915  0.960 -0.090</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [21] -0.255  0.395 -0.785 -0.270 -0.245 -0.590  1.000 -0.955 -0.180  0.730</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1]  0.665  0.520  0.465  0.895 -0.985 -0.465 -0.550 -0.370  0.670 -0.890</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [11] -0.565 -0.880  0.755  0.190  0.030 -0.415 -0.725  0.940 -0.650  0.535</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [21]  0.630  0.575 -0.275 -0.240 -0.170 -0.470  0.495  0.530 -0.095  0.305</span>
 <span class="r-in"><span></span></span>
 <span class="r-in"><span><span class="co">## continuous forecasts</span></span></span>
 <span class="r-in"><span><span class="va">true_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">30</span>, mean <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/lapply.html" class="external-link">replicate</a></span><span class="op">(</span><span class="fl">200</span>, <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">30</span>, mean <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="fu">bias_sample</span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predictions</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1]  0.06 -0.92 -0.96 -0.22 -0.41  0.97 -0.23 -0.43 -0.60  0.21 -0.45  0.21</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [13] -0.87  0.97  0.05  0.99  0.68 -0.50  0.96 -0.82 -0.30 -0.74 -0.86 -0.45</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [25] -0.66 -0.49  0.30 -0.31  1.00  0.62</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] -0.02 -0.76  0.63  0.92  0.49 -0.23 -0.67 -0.84  0.21 -0.19  0.34  0.85</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [13]  0.67 -0.98  0.81  0.52 -0.54 -0.47  0.34 -0.19  0.67  0.73 -0.45  0.53</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [25] -0.97 -0.52 -0.40 -0.50 -0.72 -0.97</span>
 </code></pre></div>
     </div>
   </main><aside class="col-md-3"><nav id="toc"><h2>On this page</h2>
diff --git a/reference/brier_score.html b/reference/brier_score.html
index e94f87bd9..9b38038af 100644
--- a/reference/brier_score.html
+++ b/reference/brier_score.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
@@ -102,12 +102,12 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Uniform.html" class="external-link">runif</a></span><span class="op">(</span>n <span class="op">=</span> <span class="fl">30</span>, min <span class="op">=</span> <span class="fl">0</span>, max <span class="op">=</span> <span class="fl">1</span><span class="op">)</span></span></span>
 <span class="r-in"><span></span></span>
 <span class="r-in"><span><span class="fu">brier_score</span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predictions</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 0.3523912288 0.1652343826 0.7890216269 0.0009722258 0.0762192058</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [6] 0.5930475789 0.2965580809 0.1823641411 0.0383435004 0.9586519512</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [11] 0.0313536327 0.0001852220 0.9999462993 0.0055385235 0.2591137215</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [16] 0.3501003067 0.4612365644 0.2210000000 0.6576505469 0.1554480141</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [21] 0.6381849505 0.0464214581 0.0014798787 0.0853600302 0.9918920937</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [26] 0.7998561744 0.1744389547 0.0871353702 0.3980443052 0.1664929109</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 0.0112829959 0.0752915165 0.2847032519 0.0199814359 0.5776206858</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [6] 0.9898446829 0.0994000260 0.0004272894 0.9905717892 0.1014083421</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [11] 0.0454723878 0.6849139082 0.0140767038 0.0888843918 0.6781037162</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [16] 0.8549518110 0.3778401233 0.0602922087 0.6247052148 0.4125147605</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [21] 0.6405651209 0.0049223738 0.3352601066 0.8346801463 0.4579342835</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [26] 0.0143894841 0.2619389016 0.1100379825 0.5566204308 0.5573266538</span>
 </code></pre></div>
     </div>
   </main><aside class="col-md-3"><nav id="toc"><h2>On this page</h2>
diff --git a/reference/check_equal_length.html b/reference/check_equal_length.html
index d9c43903f..e7c9fa3b2 100644
--- a/reference/check_equal_length.html
+++ b/reference/check_equal_length.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/check_forecasts.html b/reference/check_forecasts.html
index e16f90604..2d8761ec4 100644
--- a/reference/check_forecasts.html
+++ b/reference/check_forecasts.html
@@ -26,7 +26,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/check_metrics.html b/reference/check_metrics.html
index f073c6204..5a0488949 100644
--- a/reference/check_metrics.html
+++ b/reference/check_metrics.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/check_not_null.html b/reference/check_not_null.html
index 2308dfe6b..fec2d60b9 100644
--- a/reference/check_not_null.html
+++ b/reference/check_not_null.html
@@ -14,7 +14,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/check_predictions.html b/reference/check_predictions.html
index 6758e773f..e88493023 100644
--- a/reference/check_predictions.html
+++ b/reference/check_predictions.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/check_quantiles.html b/reference/check_quantiles.html
index 420ef2b82..7aa2a3eb3 100644
--- a/reference/check_quantiles.html
+++ b/reference/check_quantiles.html
@@ -18,7 +18,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/check_summary_params.html b/reference/check_summary_params.html
index 108cbc8ca..759e319f6 100644
--- a/reference/check_summary_params.html
+++ b/reference/check_summary_params.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/check_true_values.html b/reference/check_true_values.html
index 4c0588f2a..50bb13f56 100644
--- a/reference/check_true_values.html
+++ b/reference/check_true_values.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/collapse_messages.html b/reference/collapse_messages.html
index 152590629..a82b20ddc 100644
--- a/reference/collapse_messages.html
+++ b/reference/collapse_messages.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/compare_two_models.html b/reference/compare_two_models.html
index fa1dc4670..8448e300c 100644
--- a/reference/compare_two_models.html
+++ b/reference/compare_two_models.html
@@ -26,7 +26,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/correlation.html b/reference/correlation.html
index 048ac603a..347a4f975 100644
--- a/reference/correlation.html
+++ b/reference/correlation.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/crps_sample.html b/reference/crps_sample.html
index feb980415..50671a36f 100644
--- a/reference/crps_sample.html
+++ b/reference/crps_sample.html
@@ -16,7 +16,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
@@ -102,10 +102,10 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
     <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="va">true_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Poisson.html" class="external-link">rpois</a></span><span class="op">(</span><span class="fl">30</span>, lambda <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/lapply.html" class="external-link">replicate</a></span><span class="op">(</span><span class="fl">200</span>, <span class="fu"><a href="https://rdrr.io/r/stats/Poisson.html" class="external-link">rpois</a></span><span class="op">(</span>n <span class="op">=</span> <span class="fl">30</span>, lambda <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="fu">crps_sample</span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predictions</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 0.493375 0.289675 1.255600 0.650175 0.590175 1.180775 0.634925 0.707700</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [9] 0.771975 1.169275 0.812675 0.900500 3.160875 2.941025 1.229875 0.919850</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [17] 2.975750 1.115300 1.921425 2.843275 2.219425 5.387575 1.101775 1.375850</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [25] 1.481175 1.413175 1.464050 4.910000 1.536100 1.311625</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 0.506600 0.599175 0.532050 0.693600 0.898050 0.740175 0.695800 2.560775</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [9] 0.698925 1.744675 2.023825 6.090100 1.026500 0.795300 3.337050 1.323525</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [17] 3.239475 1.667400 4.251350 1.313575 9.930200 4.993325 2.040375 2.070175</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [25] 2.055775 1.302100 8.186025 1.244550 1.396025 3.978250</span>
 </code></pre></div>
     </div>
   </main><aside class="col-md-3"><nav id="toc"><h2>On this page</h2>
diff --git a/reference/delete_columns.html b/reference/delete_columns.html
index f2e192ef9..b63323a2d 100644
--- a/reference/delete_columns.html
+++ b/reference/delete_columns.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/dss_sample.html b/reference/dss_sample.html
index d900f6411..280c64f3e 100644
--- a/reference/dss_sample.html
+++ b/reference/dss_sample.html
@@ -14,7 +14,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
@@ -99,11 +99,11 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
     <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="va">true_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Poisson.html" class="external-link">rpois</a></span><span class="op">(</span><span class="fl">30</span>, lambda <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/lapply.html" class="external-link">replicate</a></span><span class="op">(</span><span class="fl">200</span>, <span class="fu"><a href="https://rdrr.io/r/stats/Poisson.html" class="external-link">rpois</a></span><span class="op">(</span>n <span class="op">=</span> <span class="fl">30</span>, lambda <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="fu">dss_sample</span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predictions</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 0.9390834 0.7371880 2.6558613 2.4016219 1.4872945 3.4290742 3.0316810</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [8] 4.0471224 2.3517762 3.2996424 2.4827979 3.0100034 4.9783075 3.2694590</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [15] 3.9500410 3.6489525 3.3176212 3.4205666 3.0294219 3.3162573 3.2252580</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [22] 3.0564948 3.4780737 4.0662328 4.7203740 4.2505824 3.9260245 4.0784721</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [29] 3.3347969 4.6343248</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 0.7529455 1.1681697 1.4945488 1.4134274 3.6974837 4.6144495 4.0673072</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [8] 4.0583601 2.9199115 2.6286791 4.9543650 3.3044757 5.5078382 4.1154664</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [15] 2.5331940 4.4050080 3.4696983 3.2481334 6.4119310 5.3720219 3.4350516</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [22] 4.8295404 3.0192077 3.4335919 4.0517961 5.3975668 3.4011170 4.0102301</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [29] 3.2499874 4.7312445</span>
 </code></pre></div>
     </div>
   </main><aside class="col-md-3"><nav id="toc"><h2>On this page</h2>
diff --git a/reference/example_binary.html b/reference/example_binary.html
index 73a2d1763..c1f62b877 100644
--- a/reference/example_binary.html
+++ b/reference/example_binary.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/example_continuous.html b/reference/example_continuous.html
index d7e8452f4..89487e6ee 100644
--- a/reference/example_continuous.html
+++ b/reference/example_continuous.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/example_integer.html b/reference/example_integer.html
index 56f9a8989..431ec0b65 100644
--- a/reference/example_integer.html
+++ b/reference/example_integer.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/example_point.html b/reference/example_point.html
index 2ca1568ac..aff066aa6 100644
--- a/reference/example_point.html
+++ b/reference/example_point.html
@@ -14,7 +14,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/example_quantile.html b/reference/example_quantile.html
index f3b15a22d..03810bacf 100644
--- a/reference/example_quantile.html
+++ b/reference/example_quantile.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/example_quantile_forecasts_only.html b/reference/example_quantile_forecasts_only.html
index 2d633ad29..4878a6f36 100644
--- a/reference/example_quantile_forecasts_only.html
+++ b/reference/example_quantile_forecasts_only.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/example_truth_only.html b/reference/example_truth_only.html
index 97526b58f..e59039f8a 100644
--- a/reference/example_truth_only.html
+++ b/reference/example_truth_only.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/find_duplicates.html b/reference/find_duplicates.html
index a5b231970..c2560b5be 100644
--- a/reference/find_duplicates.html
+++ b/reference/find_duplicates.html
@@ -14,7 +14,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/geom_mean_helper.html b/reference/geom_mean_helper.html
index 6d3f0ac0e..4595b3449 100644
--- a/reference/geom_mean_helper.html
+++ b/reference/geom_mean_helper.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/get_forecast_unit.html b/reference/get_forecast_unit.html
index eee7adb49..049ec8d56 100644
--- a/reference/get_forecast_unit.html
+++ b/reference/get_forecast_unit.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/get_prediction_type.html b/reference/get_prediction_type.html
index d5b96be30..5cca97e98 100644
--- a/reference/get_prediction_type.html
+++ b/reference/get_prediction_type.html
@@ -14,7 +14,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/get_protected_columns.html b/reference/get_protected_columns.html
index 5c4556cf0..478540c12 100644
--- a/reference/get_protected_columns.html
+++ b/reference/get_protected_columns.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/get_target_type.html b/reference/get_target_type.html
index 434f545d9..37c874cc1 100644
--- a/reference/get_target_type.html
+++ b/reference/get_target_type.html
@@ -14,7 +14,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/index.html b/reference/index.html
index e585fbf96..ee24b0c00 100644
--- a/reference/index.html
+++ b/reference/index.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/infer_rel_skill_metric.html b/reference/infer_rel_skill_metric.html
index 48f2f5746..49bdcd00b 100644
--- a/reference/infer_rel_skill_metric.html
+++ b/reference/infer_rel_skill_metric.html
@@ -14,7 +14,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/interval_score.html b/reference/interval_score.html
index db1440869..ae3244206 100644
--- a/reference/interval_score.html
+++ b/reference/interval_score.html
@@ -52,7 +52,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
@@ -205,11 +205,11 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span>  upper <span class="op">=</span> <span class="va">upper</span>,</span></span>
 <span class="r-in"><span>  interval_range <span class="op">=</span> <span class="va">interval_range</span></span></span>
 <span class="r-in"><span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 0.27293349 0.18546570 0.43039104 1.04487964 0.09440684 2.16892458</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [7] 0.15922210 0.15676948 0.23634107 0.17408376 0.64563459 0.14810861</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [13] 0.23310837 0.18772632 0.15344188 0.17462532 0.26430655 0.14241786</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [19] 0.22042963 0.15942879 0.19539303 0.24145151 0.10768440 0.32457468</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [25] 0.18910149 0.53891555 0.08214458 0.10659996 0.20495251 0.20553410</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 0.1822568 0.1970791 0.2332590 0.1173598 0.2386262 0.1792888 0.1862348</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [8] 0.7123322 0.2117145 0.1722823 0.4906667 0.1277423 0.2021887 0.2265155</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [15] 0.1313495 0.1780059 0.1348840 0.1519159 0.1521136 0.1632998 1.5697556</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [22] 0.2974954 0.1727772 0.1965859 0.3131081 0.2503699 0.9567075 0.2227665</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [29] 1.0525656 0.2037366</span>
 <span class="r-in"><span></span></span>
 <span class="r-in"><span><span class="co"># gives a warning, as the interval_range should likely be 50 instead of 0.5</span></span></span>
 <span class="r-in"><span><span class="fu">interval_score</span><span class="op">(</span>true_value <span class="op">=</span> <span class="fl">4</span>, upper <span class="op">=</span> <span class="fl">2</span>, lower <span class="op">=</span> <span class="fl">8</span>, interval_range <span class="op">=</span> <span class="fl">0.5</span><span class="op">)</span></span></span>
@@ -226,35 +226,36 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span>  interval_range <span class="op">=</span> <span class="fl">90</span></span></span>
 <span class="r-in"><span><span class="op">)</span></span></span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> $interval_score</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1]         NA 1.46364084 0.33481206 0.93192886 0.18168710 2.08230042</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [7] 0.08550798 0.07930845 0.61179960 0.15602139 2.59530642 0.15752202</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [13] 0.19052900 0.11386365 0.08943640 0.14771378 0.25058509 1.20513877</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [19] 0.20827582 0.11900102 0.08399175 0.20612850 1.56251164 1.52590516</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [25] 0.14197488 1.27746481 0.04878732 0.51578138 0.39172486 0.19794848</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1]         NA 0.13799207 0.18007754 0.05725046 0.18893065 0.14852296</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [7] 0.16965832 1.05394105 0.33253174 0.11388981 0.43369921 0.10145401</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [13] 0.12607438 0.12456986 0.18931956 0.08312870 0.15224190 0.93500917</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [19] 0.18240163 0.34947402 1.54103261 0.13225473 0.19374337 0.76565650</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [25] 0.18129546 0.30883740 0.13999485 1.40240250 1.03164032 0.13596727</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [31]         NA</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> </span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> $dispersion</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1]           NA  0.117563356  0.008662780  0.015151200  0.181687097</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [6] -0.061200790  0.085507980  0.079308453  0.156692099  0.156021392</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [11]  0.138894171  0.157522019  0.190528997  0.113863654  0.089436398</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [16]  0.147713782  0.250585090  0.010752790  0.208275816  0.119001016</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [21]  0.083991752  0.206128504  0.003812329  0.096152881  0.141974876</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [26]  0.081811218  0.048787325  0.022467765  0.113553512  0.197948479</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [31]           NA</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1]          NA  0.13799207  0.18007754  0.03178123  0.18893065  0.14852296</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [7]  0.16965832  0.07206667  0.12086215  0.11388981  0.06023895  0.10145401</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [13]  0.12607438  0.12456986  0.18931956  0.08312870  0.15224190  0.01614648</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [19]  0.18240163  0.08154878 -0.05042919  0.13225473  0.19374337  0.10115010</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [25]  0.18129546  0.30883740  0.13999485 -0.07381066  0.11466466  0.13596727</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [31]          NA</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> </span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> $underprediction</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1]        NA 1.3460775 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [8] 0.0000000 0.4551075 0.0000000 2.4564123 0.0000000 0.0000000 0.0000000</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [15] 0.0000000 0.0000000 0.0000000 1.1943860 0.0000000 0.0000000 0.0000000</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [22] 0.0000000 1.5586993 1.4297523 0.0000000 1.1956536 0.0000000 0.4933136</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [29] 0.2781713 0.0000000        NA</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1]         NA 0.00000000 0.00000000 0.02546923 0.00000000 0.00000000</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [7] 0.00000000 0.98187438 0.21166959 0.00000000 0.00000000 0.00000000</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [13] 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.91886269</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [19] 0.00000000 0.26792524 0.00000000 0.00000000 0.00000000 0.66450640</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [25] 0.00000000 0.00000000 0.00000000 1.43564123 0.00000000 0.00000000</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [31]         NA</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> </span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> $overprediction</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 0.0000000 0.0000000 0.3261493 0.9167777 0.0000000 2.1435012 0.0000000</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [8] 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [15] 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [22] 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [29] 0.0000000 0.0000000        NA</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [7] 0.00000000 0.00000000 0.00000000 0.00000000 0.37346026 0.00000000</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [13] 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [19] 0.00000000 0.00000000 1.59146180 0.00000000 0.00000000 0.00000000</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [25] 0.00000000 0.00000000 0.00000000 0.04057193 0.91697566 0.00000000</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [31]         NA</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> </span>
 </code></pre></div>
     </div>
diff --git a/reference/is_scoringutils_check.html b/reference/is_scoringutils_check.html
index 9fb6f6604..794c74b5a 100644
--- a/reference/is_scoringutils_check.html
+++ b/reference/is_scoringutils_check.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/log_shift.html b/reference/log_shift.html
index 4fd6adbf9..6a262af0c 100644
--- a/reference/log_shift.html
+++ b/reference/log_shift.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/logs_binary.html b/reference/logs_binary.html
index 4cf7790ba..768222bc8 100644
--- a/reference/logs_binary.html
+++ b/reference/logs_binary.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
@@ -98,11 +98,11 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
     <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="va">true_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html" class="external-link">sample</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">0</span>, <span class="fl">1</span><span class="op">)</span>, size <span class="op">=</span> <span class="fl">30</span>, replace <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Uniform.html" class="external-link">runif</a></span><span class="op">(</span>n <span class="op">=</span> <span class="fl">30</span>, min <span class="op">=</span> <span class="fl">0</span>, max <span class="op">=</span> <span class="fl">1</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="fu">logs_binary</span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predictions</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 1.99297273 0.76611191 0.76338707 0.17870758 0.73110535 0.79260410</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [7] 1.13429992 0.93289482 0.42156539 0.22229161 0.99467703 0.33112339</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [13] 3.99983158 1.31923934 0.03721416 0.33524961 0.48426520 0.10720729</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [19] 1.62850953 2.89970529 0.53315687 0.69754142 0.62716305 1.99952220</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [25] 0.27048504 0.02102428 0.57074522 0.62028884 0.03238712 0.27333754</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 1.20906030 0.02033411 0.05340961 0.76117242 0.17654643 0.71277523</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [7] 0.28237863 0.63359548 0.37188147 2.42112348 0.51437877 0.67967567</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [13] 0.34696851 0.08778954 0.30022236 0.68583582 0.74310165 0.47061757</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [19] 0.31336920 0.05072400 0.02918284 0.40280624 1.50419322 0.30777101</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [25] 0.95504696 1.46372405 2.76565118 0.95719771 1.52262728 1.69986720</span>
 </code></pre></div>
     </div>
   </main><aside class="col-md-3"><nav id="toc"><h2>On this page</h2>
diff --git a/reference/logs_sample.html b/reference/logs_sample.html
index ca60c160d..7cf558e59 100644
--- a/reference/logs_sample.html
+++ b/reference/logs_sample.html
@@ -26,7 +26,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
@@ -117,10 +117,11 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
     <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="va">true_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Poisson.html" class="external-link">rpois</a></span><span class="op">(</span><span class="fl">30</span>, lambda <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/lapply.html" class="external-link">replicate</a></span><span class="op">(</span><span class="fl">200</span>, <span class="fu"><a href="https://rdrr.io/r/stats/Poisson.html" class="external-link">rpois</a></span><span class="op">(</span>n <span class="op">=</span> <span class="fl">30</span>, lambda <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="fu">logs_sample</span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predictions</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 1.485173 1.614623 2.004157 1.706181 1.921952 2.173025 2.016472 1.934295</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [9] 2.151720 3.131112 2.865392 2.238183 2.602272 2.888706 5.822223 2.711567</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [17] 2.434798 2.523354 2.591544 2.425989 2.645081 2.617121 2.386618 2.613739</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [25] 2.511558 5.206675 2.752121 2.737813 2.629800 3.482330</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 0.6171748 2.0233417 1.8573057 2.3973257 1.7390020 1.8890290 1.9355473</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [8] 1.9645747 3.8215958 3.0070751 2.5518583 2.6304564 2.5590551 2.9976456</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [15] 2.6427894 2.8578589 2.2871401 2.4836108 2.7957332 3.5362551 3.1394826</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [22] 2.6714621 3.2088482 3.6395443 3.1061173 2.6878009 2.5935118 3.3132310</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [29] 2.6611293 3.8780586</span>
 </code></pre></div>
     </div>
   </main><aside class="col-md-3"><nav id="toc"><h2>On this page</h2>
diff --git a/reference/mad_sample.html b/reference/mad_sample.html
index fcc656161..4224eca46 100644
--- a/reference/mad_sample.html
+++ b/reference/mad_sample.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
@@ -101,9 +101,9 @@ <h2 id="references">References<a class="anchor" aria-label="anchor" href="#refer
     <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-examples"></a></h2>
     <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/lapply.html" class="external-link">replicate</a></span><span class="op">(</span><span class="fl">200</span>, <span class="fu"><a href="https://rdrr.io/r/stats/Poisson.html" class="external-link">rpois</a></span><span class="op">(</span>n <span class="op">=</span> <span class="fl">30</span>, lambda <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="fu">mad_sample</span><span class="op">(</span><span class="va">predictions</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 1.4826 1.4826 1.4826 1.4826 1.4826 2.9652 2.9652 2.9652 2.9652 2.9652</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [11] 2.9652 2.9652 2.9652 4.4478 4.4478 4.4478 4.4478 4.4478 4.4478 4.4478</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [21] 4.4478 4.4478 4.4478 5.9304 4.4478 4.4478 5.1891 5.9304 4.4478 4.4478</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 1.4826 1.4826 1.4826 1.4826 1.4826 1.4826 2.9652 2.9652 2.9652 2.9652</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [11] 2.9652 2.9652 2.9652 2.9652 4.4478 4.4478 4.4478 2.9652 4.4478 4.4478</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [21] 4.4478 4.4478 4.4478 5.9304 4.4478 4.4478 4.4478 5.9304 4.4478 5.9304</span>
 </code></pre></div>
     </div>
   </main><aside class="col-md-3"><nav id="toc"><h2>On this page</h2>
diff --git a/reference/make_NA.html b/reference/make_NA.html
index 175994da8..9c2b0a2cf 100644
--- a/reference/make_NA.html
+++ b/reference/make_NA.html
@@ -16,7 +16,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/merge_pred_and_obs.html b/reference/merge_pred_and_obs.html
index e6db9f44e..282e0dba0 100644
--- a/reference/merge_pred_and_obs.html
+++ b/reference/merge_pred_and_obs.html
@@ -16,7 +16,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/metrics.html b/reference/metrics.html
index dd5fc8bbf..4d7250a74 100644
--- a/reference/metrics.html
+++ b/reference/metrics.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/pairwise_comparison.html b/reference/pairwise_comparison.html
index 4a4936391..c282b0ea1 100644
--- a/reference/pairwise_comparison.html
+++ b/reference/pairwise_comparison.html
@@ -52,7 +52,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/pairwise_comparison_one_group.html b/reference/pairwise_comparison_one_group.html
index 8203e8b63..871df3f79 100644
--- a/reference/pairwise_comparison_one_group.html
+++ b/reference/pairwise_comparison_one_group.html
@@ -24,7 +24,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/permutation_test.html b/reference/permutation_test.html
index e0edcbcf5..7550b4a2a 100644
--- a/reference/permutation_test.html
+++ b/reference/permutation_test.html
@@ -16,7 +16,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/pit.html b/reference/pit.html
index 04b46d115..fa6a5ceb8 100644
--- a/reference/pit.html
+++ b/reference/pit.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/pit_sample-1.png b/reference/pit_sample-1.png
index acf2ecb22ab4f6f2d9da87e145642fb04e83f5c4..338b602e141d92896dea0ccf144ef10be8c01bb2 100644
GIT binary patch
literal 15675
zcmeHud03Oj-gaEERir5@3IZ)eQdd-D39GG93<y{>tOAM%2m+EA0)zn8QwUl>gR(Cw
z0ul(Ih><mT05Jzllr5|YA`&2kAX|hG@=Zd0Pp|iV&mZUdzJE@?a$Wk!Gc&)L`K|Zd
z&lq>w)>83n)vqBCh$8AF(jEd?7Yc!_@!YT;yy;7RFa;j#J*_Q~;0b;fj#Wp2$5&TR
zI%6Ra^;p@TRf!)zU4lTqgP@Q<o(;{M7>Ftxz!7<$*dk8Guuf{(p2sTt_7p#JbyJ8h
zzDa-h;`9eI)NyU!yegOccAU>8+?8ez{l3Di$w?#E;DB}1B7WinMKvqTny=U8Qtx?O
zcH38M{F-$Chd>qQ)s<f{7A7@C=Md+x3>!?HXn5w$heKGM90HRU$uij_{XE;k%aAYh
z|IuHTDSq8+ZclVS`M|Ckn2bX`C5R^J5e#eh>H0FC)v`$vA}%&)L_aFwZ!Dr1o1hX9
zBdPvL8){@t^{e7H5oG^>KsLwMZ<IYx|8QMJ_Pl?3wXW=W^ubzp*)#g5ujE4@zkVCK
zO7`OGT~l)LTdW4x$$tOg?tiWTzboBqz!vKD+2^nmqobpFsBDV~=k0Y>C8bfL4`P{u
zYsN~7gl*B<S+YIWzm1QNuYF#^_fVVPxpN1Fl5RSZdQm57>0_DCDx)^Jo;JDB`rMq+
zXtI}(Q`<ue;@*aPZa-mZsb^r2w1+}*K@0~&Zhj}z;8kSh`xrG$UjR*!uqkE`!3wi=
zpZY~*6Jb;I!_e2NGS&WnVS_(`8~wEf$e$q_{AI?!%m}jXpUIUwy2}){Kg036%qmOx
zB3M`ex!^3iBDYTv>BnPcdGyy2#hiv0BWWo2tl=h<4!PeI-8m0Sxzh9WBmYLarmP5^
zf8c^s5DpIy-x{x4zCgyq5eUZ3suB+ubf=Fa)dfc^Xm<`SV^3T&ZtlcMKEOOqPLIG?
zCl6;2KdLH;Dlg&FK^1lFDt2zKa@s5txU<JaIh18Co`9c4&vn9bQOltb5fLA|sIHa(
z$?Qf27V^V!WS05_uKCrgaSG)?5yg+_#<t5!O?9dQTe$;k1_NVYO7OAbbCuH~g+&!P
zATf&W>c>@G8*AH`w6BoA*j89rC}P&p1Nxqu7+mR|dO9{XmZO945%gBrRZst8ukhR`
zPM5wk9cA_O!lc-h7D)Vok#Ad2uE)5C;?TX{q+1wuT>MV9V8~_%*U3TZVeGzRKm}9%
zN`?Ww*P_UFFb04OVK<hB;^N}yllv_ls4jDjKf0^Qh73M=rZts~Wg_fQPqk1Q5mPUN
zgh6(0Y{%K<bB3>lb^Ma*Md=RnBd`5s00d&1?68(IP>s*I4MU>`%(&36E?&II<qN_%
z-_Z#S@-Cm*WG?6FrIs}}Y7Auk>n~nz+EJ66o7>FBWmS~~<pkNG(dD0T5pUA<FWxZP
zi*JZgY$+%xpx=h!+~|U%V*EsRaG)Dox0hu}$Gj+)$$(hqfJv)LQs$CpH0~TUGm}hp
zcei@r7p6bvqBifH3hXYsa7{EhkOEITiFw!2(P6mlATJGdz$jBF2Y#uYH?>vC6JwG}
z&pO3{KoRc{fnDMz-pp$;2<$gH4CFOcm9R=E<4-$_9Hp@hl^U{1>mT?wsn>w+oI7{!
zzz|_|W?&b42`y%^Sks1A@oJVK6WyOGVL^}i;Vqv(e-2YK44gPHjOSO?`FeSI%|vpf
z4X`35oQ_P^;AqFq`fA{;nr2H=4Jb@o1dpE(AD?xOjUz8LJczXO=qH{b@5N8{FH`|<
zSjSb2SR5;mUL;3likIIQawFb9V6K%3+F9!ojBFTrlM$8aWsFKNo9{j&q7{_zZOLcR
zxTXgY&6zI<b8D1L%Obx~@{=3P=qz<Z4Qd>wX+sXAi8+;;CBTC&k?}AF2&v_v^SYc{
z)j6zF{k0*Uo}Tvoa)rfE_g^;zFkNzz97r~_v$c)%m6<8YOR_jeKg*uOp^|kfF@4Vi
z@U4;o;OMn^?PC*T*Ni=~#m-<S7s&!;W#!VkVY0PG;CzsL8CTC!jcypCN~*>wIkvX8
z+0N)rM+&9a$jq$vN&DENKFPuit4`nCJaW>MZl3Oi63+c#5;9?=uNQ2Hat~pdiA=l4
zuFVsGDeknFycoLKw^<k@52mt%)Jh;YKR^FD8z<?2si4s~Jei0JJc)VP*w{F3lVKn*
zZ7to-(Ek3jiu*ty&A62>U(r4%|H;8DeSQ782AFW7cAHmA^8Os#=`jiMIoMJV92zJ)
zMMP`c7EP|4AB@sh&Ln@z1IC{jQrN8J<SCw0mc>sr0vJ&WXT&CDcGp+PkBOs~wWbo_
zVIJh_q;o05mlTOgvX1pXQ{$!<@4yJ*d`H-VA2|mWW=TdXM#w8o{m+GQ^<oZeZ_+h#
ze>Xw+0w&$hkf4?cWRHMBf?R6C9vk(nGC1QbSp|mJ&Ct{ddo1d4P3rF@)Ea2~L&gk1
zuAbcX;YxosIpQGW35sbuZCERc(r1jc_IHXEP&(T#Y?F10?s964%@NpOaNZhKA)aI<
zSlD-ZN|(gv$(p+sFys{24!6ciSx3mtXQ|g|+(dGO5u*?l$R|jPTl?P=JiM9HbEoC0
zI67>T_LKuEJ7@S7YLbzw`C{G>Rh1TYvQP7p&g^oRr_u(QnA5*BZK{#(W6<BU4pbZN
z3cDzjL|IJ>lT6L^nhi}4Fk%-+#9rizTSjnM@1xN4%{AI6_tdo5!hw|G52)-m@kB37
z*KYa|Ow)S0oFlG2JoC0<j>uk+#PH-P><6aPF*ebY%sn-|%n9$hMwZ?k9%|jmnsO=`
zcg-<>J0_D0ak(E?PdHA-VL8>i7z%pI0h;cExJ8j7|H4&yyJfM?YGIMMD=K-|jvOKS
zrgSc@K~q82bUzE27=LMG(?AQ^G&GgHnDxg7sVQop9=Brn_Qsy)LkDDI>a~1_HgjOk
z92!TzzE<`IGJ7OgK^TY97_T)ikCsIB(xjIaTjh!4trc*nIU@g-N|}@;?&56<{oPW2
zT2poTL6SG?Cg^!VwITH@#p{`a@^w)qkuL__uh-38UXv})vG*&$G82Wac$G4GHV-df
zQRpThnIP_J*;W>X;x799u`63dwR5XqKC{>rWjN?8U!qG<Tzz$AkmTjMi6{^5;>}eo
zc}9He0;Et;`2;>2zJNzj!sgv?4=-^R-@`IgQ&&`U56rE!E9?xHrD=`1Namh=YRs4+
z*BsKbM4W4*3AwO^sQ=pRMMXE!G)zTjA&J9c*-bArcT1~Lb#LG9ldpl=(WqWiZk|4z
z7B-t!gAK5qZb2<u-vG=O_<3f320z8DGGp+`zrhiQ7)WpC2T3rwdiu5VijsFe%3mWa
z9)ofDE5C1#)=Bi055526e`j~}!`2IZ$@(arBvai))ANF(**$}&<bA2{&JLRCGv-^7
zZt~w%-y#Ri3>aKEB41dmUdx_bUnh_0!N(oQL87HbTs0mR7EIPOoHCR5x2(|7cp4JO
zU7Uli*g^Fj5Ks{$p{_5H4?LRwRSjN573p6k$_KQa8mui2s(pHWrIOtCUM<X-r?LVE
z7v;^5(&`D0WDg-#7lnyf!<C!%YKNNtV7f9Y5O_UI({}m>Oo+k+ki|(I3ZEqe)zq(K
zBk$v6uB+qRCYlK)C#p9J9dD?A@oA%nayik*-vWi}1`ejnr)mW&tu4InFE{+bSZz5Q
z^H1iB5~r;??-NG;K;AbdiD=d$t}tf)`a434V5Qbi4>8K;XL^e9I%%U+i`!m7H}vF_
zJUembI^LAgiE^)suH8#sat|UplQY!PR~Cs*H@+ciGyN7%Cl7x_Rn1J)J`bwRe-I{L
zkV^sZNn^$$XVD+!-YLd%O~v#+ntOY_sl1sgvw$mi(Jf8N9B?)fcd;~<jN~l>S<5Bv
zVpko`jJ3QROHsG@94v3?l_lHRy1B_4O9PkIWXfakekahB1>ZGsp2{<!T)50#WWG#X
ziB4PK{Xw%-70mG8+r?cTniD^)$nQiB4)oK^v8%*U)j`CuXdn4jeW@Uie3S*tE%MGj
zSD}y;qereOqSfg#D*h`3fJB9)Ot!0qIYWTrkiAgcY&~+szC?5K`<k6g$9m)wB)G^T
z<!0;7+N;4p;St_U9+uTx`yHcG@y$wHTG{l!G~ttB@LdZ_7bki$hws92{I1H68qp4Z
z<C{6o_+1DKMqg6q?cqjn+=wI(E1*L3mn3re)Xr;zSru@Gh#)s2Wc_`{@8<Ts{X4Pw
zSSP{v1x$}Op+TW3`F$fB7H!}UvXVZ7-0+#W!keC71ym>HL)hvl%unPGi=68HdP<%^
zcL|39x%lP@!1IpaeDwDVtks*?<XwsVx(@6)cxm-ymEGpxoiUFnln$KRTx4;q@g~b(
zN>AC@mAX}j2qrHmVIF}5m|a&-w6mK21S_?}Sd(XxCIZd_G(EV%z`&pjoOO;jWUtv?
z2RD8HmBhkfMvyx&zWB2Zo32oUiHKNgg*k2}+}1`bU&lX~w3&{9Il>XT3<@mileAS<
zk@NZl2TCbvQb`@areI>FFs+yiO*gkDRuPhN+h4!dX8n?roz0va2<$ddRmZj-+M$+#
zasZ3_N4D8=t}yrA=g*<Z`&|vhAEqxbF^|-|2W`APCry(2)4c-7w&eZCvOopgo*ooA
zlTrCG*)_&&{jn*EW9=SR=qo+&i{)*#pqfM61twnBQWhfU%?uriLy`HVbaNUvQY<4-
zZ7n3TnIXNG8#Hn^Xe)5qUI^!@T;t;J<rcKU!q#$@v3Qnh?-u!sUlHG~7KU<rU#@u2
z#%rM6mb)m7F_4#tqZiUcU-$m>@Y6|hGu%06c&JXK%aMl2hmHVU+rphO8|b1rScZbB
zoEkssul|+gk!wjy&9Kr@s@%VwqID(*ffJ6dnxC-7c#wtg1ujY>@ci#y2b?4F`Iq$3
zeCH}s9z&4WqyLIBw=wk+V1b{<{|>0`JK6lL8=JK6@?h2P2|EV_6kAU@G-X#<Bv-5i
zuhd6n4LhTf=ZVj?8Q0eQp(t)mi>r?66feAj1+laN&ZoenS`2tn=AB_5l#ceaJZSbg
z0dF|M;Chct4ydEtGrfeFW;k*mxn>`OP0mb9`=-WN55FkydKIFrcy0z4HtZY2{fF+W
zfk-j=+*?;^i;<7~d%^ja!0+krXmz+%VtP9*vi*-4AiZwx;y|s5KPaOgZ>X;n=vvTU
z;xk&}Qor0hM-~0C%)9r|IAzfix;aZIpL%LL=o{5VTSk(3=C3$D@P%ujv<AssAB3O-
zuvMf!BbgHpHUeMpB$p>GS&=<#r%RKT5@B4cr;lhNYgCE<m+q>dQsBH3ykD3ydO3uk
ziQ17Fb{c~6An;2EKLA#|MlM`VLb=}=j%n`3W4$Lw%=R)SW4u?AS{^WZ78>4@J^U+5
zBW3t+FuO@&NDzRGVfSAO<rT%ZLT617JquOWw}Mv>LTuFE$vx|6!#4hA&PP;&_0vb9
z8TDJUwer{0POm{Kx#j@l6^zx(@+s$}c|D{pA0^i+uAEHYs@kmMd;%dGHnw`c<m5D=
zcJ2}?Fwj?i{*89HJiduzJ#(^OVnt4Yk1@+dy>)ZxF(WI>%>phn4&xK}rMtq=LKbmm
zLK?^ow^=hIpZS?R08jx$KK|0O3EoFA;c7vc`moY+2DIEG@?FJUbEaakCM9HPuz5vf
z0T2n&f3?yyK1K0Mjs@PnTpVQ9OUo8Mj>#kPO%yETad6Z`9~uf=!~ON0jdnuUC}LUe
zihM`ld|2*c>$(Z4pl&Yjk9Z7_4U<$|E6kOkTDrC>^W=01XR!{(HaNI~81@2*nx`=C
zL6Xlf8a&LJyo*Qxot~I3o{X8nOetNG2bz&zy#BR;y@#>0e{TW`pH4&Mu+02LzrY*~
zkE}%WUVQ*Vi>=9-cZX*ZpZH+OXUGwhmk9atTnY<KLVX7A4KKlh>_9G8PWnw}0Y%n?
z^>j{&<nra%Ga2>)gcfU}miEWu0cvV%>&llih-ydUjvD+dwdfCcC#sX?-dR5<*XBvR
z>mq=3OD}%`IF;I}9FA+$VFpSuLherWKbOYE7TdrU?8z2Z({(Vs&H@Xy>_aAyC(P=F
z-1hq4ky<-|D|!qvLMRCoR(zw;DnAI7L2P!e_}&T*S24V5Bj4bNuhtZ&qh4RWh|~tC
zqWtZz#{DLtnyuPOXI;ai`9ibSl^DFwdcU5tD-1R@$vQ^Nag1GXllPttUdd_y{wJ!(
zPgc3}={jS~upq`Nb8Ze^>57(2Wb`&Quff`>gBCfze_-4f{RMa7Uf`!REN<_X&uruy
zs}?HMe7%zEcOJ1kB7gm##M;Iy5~6<ue9=G>zbev>%+?9BLZeLp`B@MGsH!GGA5Cc1
z*y7;NE+I~LcX!jR4m-2w5GaRTmErs@YSN8#b=j5&&p!j;hQVN{09*mc7F!I4ycMv0
z&nOgDiA%-!A|Nw|PM7qJU31qnHV&7Oj~zJTk7T%xvppKkUoH3c(+OQ`Ipu(j{R)jH
z>N9-F78;tG9+fhhHIPQIBD;=P0csc<ixB{t>nhm>jn2v$9_Fa8d=oy_nQn4A<$zoG
z1@bPOn};!AeK<WLT0mb&bBv;zy8++;+hM$7x9a)Jj_zR}vlOrbK*tghj1-?JcY35`
z_dp>2e7QWPcHVs6|I+o+h*L}lPV&A|GqCsB*fK2VB>CODcQf1YawA83|7%i}M#`1@
z^O5|mmPaNywc@`4P7APiqIp4|XflY^eZBY66TtMbWOo6&yJfFaD3okk_spBBvHXl|
zKzz5)&%8_8M%*q_?)^}}@BKG^RcSW)gMU}?jtt$khJ_Rg7lnE086`)JZ*8eY4<D59
zFV%iLHX}&O&W;QXsf}2ee)lR?UdVi%_D&AkJS`CMZY~&Hs~*Uay5Rv2UHU0*K9@q#
z1@?;)&Ad}1n_hgvlkegmT{r9d1R+B89Xn9uBwimdVAWGvzus>jc96a1#yc~Tu4J!J
zm@X^HX7Ia;AyUkxmSo9en(1Z00*m9j0OTzHB+}gGvq!e0XzvQGsX$K6LNh8qX}J4h
zq8ToVFkUqYL!r@*`-WD|hw+z>58Bwn7ktTJl_I9(>Od84ZxWy)WLi|YWwP>FTCexp
z&~m(xlL#Br37Y|*24VrhUS^8wD)o?B13yUdx=h|H_kw<|8*p=1<1A*jZNbdPrYyOD
z!M7Dd-}|F5ooGkNH4J9Dl|rerM|Zvg41Ii-EgU#SNDJ*%b2BT8$F@w`H&z?@(%B~q
z*9r*m$qY$lpCCa8<;M2G8)uo73)wj0a!1EqlfTIn7k&I)x=Cn~m!F>>0emvB!TgLy
z<KaCi!!a<GQ>RW9bDmvpXEwZ8BG<8BFLO#M6tvV-K2F=!|9_<|7tHC4<3*aM?A4dA
z$+QT$@gd=i?D?m@7x5oTTP{ZtM4D=%Fs{PVOuzartL#U4Hs{+>Oa94O%<BbCHxHTM
z@?YFQqfEnpE;3LgZ~ehL_xLyae!N|BnPwZ+db00<-j(yN_w&}+7o0d8f@#g2fOY@W
zbh*px=@r<II}Iz)@Pwz0ZYjMF15jIpJ({MKTYj|Pk~o~AB%E8lS=2uu$`)tNHpMD&
zMIS}LL%!bi1?@+Fxmok&M$MO-H2<oDKIK3(=dgjnd^v(=wma`EZvR<$-nHY^1IuFQ
zC&Aq|ug<yCX+;=jhStz-@sBL=e7;SFdG*ESggDB^2dW_MZ$LjKQ26(a5xf928?cXi
zR_X5bzH(*aY37U*@(6~BPC52#zC<c~qWcY$AW*28ZL=1TLd(XDox;Fq6AY6|;R7GH
zM;u?XcdU2n!D>kPYlO6c;(uRN5F+-Wlz^bI(z%lRE!;MfRFEA`89)5E9_{G16;!o&
z5{Z`+Wu6_mICgXI!>`%ygn)CzxlSnbQ>~DwTPGe6I-G-|Uw|n;yiyA)uG-{q$0vtq
zkf3Sdj@xl|kQzkv-?VAdS-16&qj%QUob`-|FwG`RR#IB-uXcTYB`jd6&04odDZGR4
zAzfs@(yjaaP;%e0({K4UX+|;(w|EDdz)H|KK&v!OXKa-PD?lk<E-`(%y!7SL(yA5i
z=C2Na=>a&(=^y^`2U)=UrH8-rgUk=MP=3a(vJUX|9c{~;vD6+KfCf8TjX8lrGz>n@
z->^*BiKuqr$&Z^aoczZ(a9+NA>55^(wT85z!8XV%tq5+i{bC3BvQF>1VqjSEs!|Sb
zw_SX*wgxbWi-mXFQnVjxK<R8QmtNTE*V!wWw;+*%MlxxZjcbYm0%ePcZgSCEB?MPL
z%(p;^2p;R13pG(uNh#41X&&);>eKS_GX6v(5;@UV?!&rQbotAey{m$57M_74nG4xz
zN}H9G=-Z(=B+_FxU+8=sD?fAW>rLMfDiZYJosn05BZV$}Ov%X($jF>&$l^cfJZvbN
z`>3HRTBU4@ZH_rHjOuXfZkrO=GA87CfNK1!@}#7sJN8wDluG)i_@HxSdl$o*Wj!C}
ztlKFY{~tl`IX>IbZT(i?BdWZ3fkJJedufj0bIsZOG0AsYi}dMxr8OlKm&eRBJ)Z*z
z!+Goje5YR&zCTWKA8*4|T3lS5P%!gbG=3NPwtDH@cSB2&^-EpIde>4m_ck^mb4Cvv
zkYsP%vtwS^Us-<0(WyEUYj^5YzZw+3e;CKEqPo&vfjc!t2X5SSi;~O*OXlQ@sRox$
zhho$MA6ISK15EQ8e9X6`l-#(NVjmt*0}f~1&jVb4?JQHh!mTrV9>kTE<cX1tpoEN|
zx=K*wyZ6)jzoqqKtUE8Ae$p3My+5#8@SM^TjSjHMeE2;tVlp?=ain*oqwtI}@0FUt
z6{jgnNDBgA%0NhQo%%A3p5B($w$#-Kt$_yTV{3y;6oJY8g>#*b9$Ce{lk}UUQWi$)
zs^!^oyyNVnz970p_tmRck$ZVdEYI;4{-|5fZ=_|?=SbwKbLU;VN(_%TR;BhTZJTx6
zTI>v(G-D<A;U6>4SdG1b25V^@@Q#0_+Sa2~LL%L@UyNi{y5Ve!YcjF>0h+SfP3sfz
z&REUg(70<)`IuSy8!6-zRT<l~>*x4z<T)rb3Bg-wPPSJiOgw2`E*>{_DA$KhJ?|5!
z81sB8y-Nbd=N9vsQ>{PQvC2BILU0@V1N76UG?B&f=MOoFB)5&!LN8%ByE@WBE0R8L
zEZhJ2{Z7gSzE0rut7Ou!7aaHz{=NggN0{rL*_9-(gh$mAoz<LUv7047XxSb|$}VG7
z^*m=F47IiFt&~dt*eTT*&LTpg0FzzR)7+^F7;va2iIkd|xiqmSU49Yi{JSozdNm(b
zwxl7|_$#q^y@G;Nuq+W9B9Z-JysRK5B5HSj-D}>RhFB%doQ3`e4<2{~7-aaQzZp?U
z4qu6-k59c<b1XH9v!z(3g$neJz46uYjzY)uH{OA?K5imO9r}q1qR=Dz!`K{Ylz;Ba
zxJysKI1&k*;I$DvU@6J&JWXVy7c#<3Yf^Rl_HJ`iQ`0)tmGt5C+gaQyr0MT^gF@%R
zCF8VOK>!Qw3YELfFZ#H(D;Px=`kx5m9-ru3ceU<r8_qIOLqb5a<%zKPH*j}MlM~LT
zN`nruwoa8CmRag2tHSZavQ0;CZDX@Vx>`et+h)!Ir&(ujMc)CXyW!*n&{5}|FRQ->
zfhx(cGyT+9l{qs{@n7Z8n1|ry2!6{!?Iz~7`9UbS<IDv?)+gE-D?3LQ#iPGsn6BDG
zub6M(XRMlwiWTl#A$V$9L;JHbJgj$TJ${<~(V?g_+jRn)wGnduS9Ne*h=*k+h6HkG
z;C!KkcKPP$uL&Avi_3T0H1UDU8KA>bwbjPekKew_wY12zUsTG<pg#d!?auz*HQdf>
zjHa!HVbVX()94bqqg%V*$1T5;j^F+(Q-ZV$dFzj1vhwau3Vzcps?vzw)5)I`4rdDE
zNxrl`k5bv2kC{1-JN-l_kf*?Hs`Bx0$e#I5EZv0dSK&iBATLWYHU7_^=yg^=ep#=g
zqM}<mcG(wMW1@AydbheituJS%re@UC>pR*$DhMv?c;h4~)DpP~*|>3c9;j(oy}X8t
zux9wHSEo|WSSg(w+{NkftEdZkf?Kt((O%S;H|lTgrd<Ei{!7xHT7qctL~bzVdWKZ&
zP&8-KE-bTMyXWrwL=X0I?z<NVt@T)@Sj!WOjCj)Nrx;=qW?1{&u`8~EvJv9+2E=I)
z0|5$FBC=5*`s5HV!qsZ*Oon05vP*eSf9RnZ{fswkUS`x5Xw<h*;Vn{;_CpJ;5^zYH
z$(+dnZFjOv@CM$5NUOqaP~wamhG|~2Cb<c$kTfao#dNGK`=+HMYD5s*en3hXQKl+i
zryR+6QwyR_I-IG=L~yr4E!3gJ-FePYrM24!W_#UT;2{zv#WDxLTk#9*tvnJ);m9TT
zs3@sfLqmhEaG+W=?n0eE%?u+TO}~NC%ibc?;!k9QOw>*ay+07BS-mvf;$ajzHQ3`_
zUip3AGeh1I9Qv*w5Hi9OjY6ir9dp4D{6`6kcOPBXI+cAKjcebZU%DS}1Hn2cEYFPQ
zCT1suE&n5gm2KwO#h>G8JtR&KGjv-kvT#~uZJ>rAsSawkyW@v5>FHQbvb}MKphmWA
zSGe&nD(5!?-c)@0l=|3I8nDEZ_wV0-BE1F)&z!^MN|~fo5{XEdpQyw<z?#p#Jr<>R
z`0&#xaNW@B+O^rnmp2sPT3R$0s^f_hCt|dRn9O{-9{cASD73<bI?&|0=7n#|JLLk`
zQy#99>5vio<I+NaF>aAg?p8O&48Ja{d|qz5)}i-(LoEHRUq$3);6d$F2Yx9i9Y5I_
z>XinD2b5Bk*VRDEtq;xl?7V}m^BK7z)9HyuFfZsuNFG0~uzvlrrH@lzP1uZ`Zf6a|
zjq(V-%}M#KnnR6rr_t5>+o1HG%7vU1Pupm8NrrnLmXBPeTfYy`cK;i=$qOF<NA$>W
zkH|}fBkM*_pv|5<OONK9J}wQLX27Mvr-&Ec#R2{_Xn#{Cq{tcgr3yQa`EujITkQpQ
z9_~e*T6wcKD(jrf-}#!FAP_rt=Vc_1<HU=Lfo~9&NK>=JhYxdeD*6Ym7(|(AMe1lp
z?t~_O%#`xMec2RkEnT!31iH6NS(j3Z?kreE8MQ$0l<*}M8Y=F5XZY$Sem;AC$oL%;
zXau0OAQI^{!a|R2usaXC>WVai*W&hAA{GUWMlf84Gou3P0!-k4i*iNIS~lK;egbL9
zsk~>YjhSU4uBNU#Uv+OZPNk1c8Epqsa-%}P|H?t|mTp0zL?SV#^6h+AT)oR%n^Hj&
z?qRgb`E3Z^T-N){0+bPMyrZ%56Ckn22WmK=T=EO-aw31wTCX*<49b~BPHu^c_E5>)
zHZ`$WtVVSxawuhc00<R@%)Cj;0#O=Smf-^Nc420}=&pO-MpPZlyf<k9<`hnDqJMsS
zOiA*_6bJU|e_}<qP4Es#%kU{)_=j_9&-w)H@X?~uI^)W272~UeM+MzaE-Ia91heIE
zuC?1}23EL~2bvfgFTQRSn~N%0k^p2xRnHZ34Jem^pcB9r**gI;{_N?$I`u+NxTgv+
zplR_){ieOWQ5dGQ3M)~V3Gm6<P<#l)NA^m5Yy$}5SLRfD<#XV>8O9BI*eo{hC*${%
zYoMR35xhG9$<lfC*VhP|T08wJvJ*&_#uWoqsR)R7xxHC%{=%XQT7Olsm$U~!a2zx^
z0GX+(sVSZI<UO%nYt{Zw{qotR(hEMJP3L$YE(25?>10}&%1+%jS!bxFu>*|w6+~u<
zIpWlAGbr?3AWfkvO+ju#Y+6w%+FAK+pTkpW2Y;^Gq(NMz_+7E`b@H`#e?oB94}<Gc
zn;smCjofmEx98S=*XQR9`wS2~ZzR%c?ES5*s1q;5O-XP&PUV&y)sal;&pw|&1oBoJ
z`BV~yB1Xpj(Oqk2pWhp|z;lUs4`f7{9t*Bw!g`htCh+{{1a1i=v6dD{HO=1UMn{f3
z+pT5(4fGHKvCwlX?S=AWq;dY=2c(FD2M@m3@_v$2ODK&adGkh!J=)fnoowk+=3%eb
zIiPGHPkxf{8sn6~0jG-e`4)&W1pTYX@4-~7abPcjS)<lnkddpPN?2SuqN*8B5-D!p
zY+in$A;x06hDPKZ(?l47tsXGna%SVCrMH-TQ-;{yAfzdz?fw-m)yx5l{r!@AB%{Lz
z50;4sU*1sG%dyA+SIg-S-uW&SfvU~R%(-0QQVQ^2aI<<!u`6vd;ci=jaYke$$acEb
z5n>3>IU&0{z=NN!6CJqE(k`>mp-AH36UzE^TxL#v54#Bvp}oe3&~A`=KLaD97kYU|
z44NCb<3b@F7aAHGF|y{^v19aP>~e1E;`calr8wiwh^y9+Nmhn{lsc1HUEvbY*Pvvv
z)bR|V3zj~*dY3c|xzvum6&{`UtivpL_hK(_-q9{luEvb73pPTbyYt>?Yiq-KMNP)g
z$8GBP{$R%996#!?pd@^0z<G_k$^G5DC0S_BB&61!wwv&#vaR{Z0$)gKGe|mjGG_o)
z3UY&z=bqb=aAdzeP)==&=K2cKXa>J6kj4^!6A1EhGpK*Pq|d%qV|$^W><hB#zme?n
z?=GW?I>Td+o0OePu7V#p?%_IOu6VHE<Agk(1)HROzy5t-10Q)Fow_W(67XDn#!4mY
zSV+4+RauVl{wMnQ_un`ffc3ZTOlgM(n`(g?<f!G@u$os%!+H?2D|JUOzPmZ{`&jiZ
z<-4_}SL^CpHnMcjXNyflow}Zr3kS5VXWdspobvnt>VZJa&R;~DszOaD2jo{rqh}Ip
zB#C+&)`zx^b2OkVo4(1OjjdSezC_l@jrMG;<an1Nq+WGs1E8-*Im5s8A++4%NV>-x
z;}79(#*j$<T0?KFr@{q@SKxl<b{T=TuQWw#sO~b2=70uwfBX|=1b8wRpC9kx=nM$i
zV#I17$|;g<qb3cwgoZ%MMm7LS{Ct?(w@@jjSKve0vlbFvVJYKaS}qci{i@JrfRlK2
z!sCsiy+s=!(fS2|mjL6|M;z3OB=!ozTO$uP5wx{RPaX$rSSw#cd|jAE2C49>D2jS|
zKX|)A{x(sn22HeVyvoBo?Ztpa+>$ZFZ)>IC;Pki(EH*q**Le*jdeX*0_My8L^>6Y(
i|GsJeFGkFgq)O<+7gm+Ux4n^xK^?b6mi%=7m;VE((@ZM>

literal 18100
zcmeHv30RZY)@}raTBg&|iptd12wDYHWG2KG6~owqfPf4NiVOlW1qdWiTKO$1`hg%*
zKoL-eM8JRyVGso)LQo))DS;9|2qYK?kpSWDgr0MH{`>#;p8MbX-20q!56`1t_`bcr
zz4zK{?e(sAt#IuJdn<)ccYg|l!4y!Zjyu9&+hSla>GPk+fIGvP_ZGlK=Ddy7ac~7c
zE0&EoaM>Pq$}Iv0Q%@29+mbf<;Q|cy6%2L!yVEgwbECM7QMBju2RfQ#du0E--y53Y
zHjjgjE?pWKPxd{2vITDPAVT}??tQ1Wy&#m&eYV4%Usz~Dh;l;@etG80Pw%hYlGY9$
zx~TQ&Mn=h3_Yd9nG(Psi(cK;UrFrq?x3-HDSV3BSALFX+5!94fIyJv@A`dgAFTipI
ztM-be+U8V2eDik6T>sDf_xQ=_R0K?Ws-|Fl?kc=g8|CcGZCQ9GC(z@a5|7X-wd?HM
z7URRt&SVqHEZnW?{0l4SlP2+_$H(OlE{i|FU}~C4D&lM54^n#K>#=)Vy~NkV@3%{u
z!hZQYW{dd7r+XKq#MdvZ$F_;D`!ZVP#MdA8{o}r%f9$*xSb|mfVz*ICaaoybM~i1x
zbux;vu{Mp|sM~@06D%>^j`U7C?7Wt?wsv}jJ@n-Pm%Qk<Z7OqKtlKcyRQFAFvmOmK
zwY84()Hce!fx!F-%HFII@;jh}dA1|lw!m6ZK|VfX{n=5aeH799s!D{~j<(+3A7o@S
z2HiOQd(`D%kG~L$`ipTqhC{J48h!H4_i-zY;F%jiMWCxXVKC!>*q6J-qW*{8o8(kP
zYrLwes#y(zU7p@#OmTB|o}Z00<qncDf|O{^NbK0wo|T{Ade*!=csVrU;XKd-r-zI-
z@SRc*OH}mn6aIjDQ&CY7wTvxPMKNx%n&74VQR^dRWelSuM|eu}(W@E02>-@W3o?Nn
zs=dO76HoDcQCHLRoalVLfv9!#d^CogV-R%l#-!bAi5B-7C80+fqL8PZ2P#qSD8><U
z^Z22UA3q)|?TZbEsPrhZDMj`szbQF6Im+2tv$8DE(v$9RmJ~Qp{o%uhBzRhR+3a4_
zvUWg0^iV)c-K_|Ttt@r#Q0U6k!FWRV8VE@9!-qd~WDDvp4@Y-)+)&lS=i~@Ope`EG
z8^>9mALiz;V~LHN2Ao~>hh9NvXQ!rhb^T}v*S;D-;S$_xrj!&%L+(AQ7i$1aIH}0^
zwr^PT0f8N)xE{7R)h*DD^@pCgS00#Jd{*vKDLjKVIRu2!uHAK3+~L>;JGB)vcp)qQ
z4l4sbSmRH<oMKi&V5>GDJEs<h(98rhmpN|vht@i~HLxGYYh-0)9GwRcDIvEH`G$op
zl*=k<*g88??^1)56ciAR1U9_>GMiSZWN1KVu~@yY?i|2<^%A+zD(x%~iU)4}yiWza
zQ0rc)4}r-J`<s=Nm1Tc4ZMxW<6T5v_)hOibk958_nz(*Ify%$V%5A~T_PTn~7k5{P
zMVJ`AbYiTM9T$sU=?(6*wX>Tqa3h|nx*v6)KJxU6FMB*ivssr?1`jQ`d-vlK<<O>r
zM1oFs)x_1sj6ucsk)~-Q>WyeFBUwAV$}ZPX0lX#QA%P9{F(HyaJFrndQ$O3#e1M{&
zxHyEVSd<pedi|c>^4NT&sX$R)9!`m^QSS>R*|T*8-L;yWv%y180(|HMPt8+2sDV(Y
zsHEgw70_2=-xuK-`Q}8*jhE@*2jxOt5g#fr*wT&Wr^)CQ-XwEQmzsT9bGWUoE&HXg
zeu8Z8Fp>}p&4bA`$Ac}bVMI0lu&FsVPTpG?Iv%z0W|Y%V#o2n^TD!UvJ<1GKR@N{c
zL%4EkWc0&WvQmigP=2f2?`9Fn2dj3}j`e^dP4kU3gyZkty$e8jrNhCNt-Y+mH>skL
zcW0hr7R#K=PfkDWwk-O6ERUA03^n7hu}YLF%ImPHsc==iEw|yShQS}-0L8`oxk2;8
zwU-+wdbu}m+#pv2tqiIgHc}V5fd#lh`9R;|-jL{CjkvF;B+kJYYL0p74voZ2++eD!
zt0#Qt>|A%+nG_Hh80geEYG1nea*^76$lQGGty?q0`bd8QqJ!Ys*h7sRv4c`u_?4ma
z0d>PE2uf>ftHWqmyMlL4=ce9Y956yJXi=U&f8NtTD7ed-cXl34XZb)l+wcLfESFG5
zQ><XS*ZHr~u8wKOM49FX-rk&1W?{Pw>eXQr6O(MeD5&C0)o)Mabe|qOC<7CAbXqJ|
zEdo6knV_0QeByLTF4o6NM>sM>3X0&=PivvvD`^g1;gr4JlEdEq+k_}tiIZ$~{N|s}
z8U7Ra^M5Vjp8}Ts18w7O0I_{^L+U3pYoN1?nMjv|%LflO*_S$z+$&3IQDid;A{C*O
z>I)!X<LdJK>idQ$_A+P#JD68irs1Y(9<z`Y+PpG_QMsbF10Z;B)m^&=<Hmnc;dsxv
z!N&adEI9rQT+QCtlU(G2o2h%H3)CsnG|$905cH28UD8KkeXIPp_>2QrQ7xdCuVZ{>
z;>MC>y>eOesR&tEUU2Z6WBajjRkASqPemTE$Is;z6m*t9$%D@}EOP<0lt(W(W&i+o
zF+4oHs?#S4u|DHKefjbf#Om@Eg*OYxTpix=gAah`GH1I{-RBO&jJMm`+h61|$f*b!
zxl4Y2a0i^QPX!|nGk&za>5B^+qV@Xu+Tk4-`5o_^3WWVuQe1>!B;AbL6mdoN{C7Iu
z&1c89eT532&PPu*uW~%*4xc&ZkB=<@!GzVhRB>$g|0MMIJJ6}IgGZ%zi|93wg}>IW
zO17{Xz6TY3JuNLwV<r`#QoYKuS!pYijTaLW6B}W9H4-FfbJ}|w<IS5l{s4Va;FZ~e
z*#>L~fKlqZcCB<rHzs@lSktmIjsZRfKpz?inu}OxXRGzq^0L6MD809DK4LeR(z3-;
zfeE!12p)Bu6o3Q_`mP9c+S}Xd!NBcNi?h6hzS7dt^Z^N4h0bCtra6(OtH84vx;m3c
z1hxYh-O1VcvPotHz{_l9Wo528El2`*UYCSQajP$Md1d(lY^3C1Yuj{XYW?zX?P%4N
ziH_9Ll9J|JRyChZFKA3yU2eQNIU)uG`zdz%Eurrok5lsPr|(Xw?%ZiyEAfmlwG%Do
zewEg|tMEr`VU8toq$zfB9mU9oSGur&{Y70typqCQ37^I2HO^JnKzR@xvpiQnAJ*9)
zm0&5DekXNPq658mo8R3Z2Pn@1<Ky!caP~D3(>>BDW*7eeMbErnX%Ja9j*8My*l#Fk
zDE>ZyEwHz>9m#oM!E;TgP$>T3rBnB~0L53AnHNdM&b;6JNfVgM^#%e63g&Cnoty`f
zM<*9RR3PIo*Jd|k!Mfa0k0Ml*&Ay6mBrmqfYR13m$u}##8UCx0vorrU8LO@0HGJBq
z;&HS;irmx~f2r2Dq@-k-TH#u8=1pMyLqY?A>k#hj?Ch{pLqkJNQ&aO2$vuq-UI=$W
z(XL;=9)IcHltep(R^B6RxqjY)>HRh8y)%&-D>a<aMuE?Sa+IB|Z7_0)%qF&oR#6u3
zLh7{B<Le1*74Nv2;%Trw^EU^3zygj#jfx2^Wo22~E4Nqjc8O=U&k{gL`_c$*YwIy6
z&uI8CSnU%|IP?2Z>Y-c^v<wf;FE7K{9+kA3^w+y%^cvpZ7ie(Sfw-nmN<#a@y^r*2
zFP&+QUomRzil9}vMZCL&Fg7+;#v$ttId$7PJ8xvb^X>fy&Qm3zFQZsbikqq9(VRcC
z9$XVX`D#%KZR<&&hd(Nty}C_p{vpu$IP^;X33Z9&_Su8^!K0wMy(kc5P2AH2&@w-h
zWq=A_7;8&AICrV0ap|oM)#fcnB%FDXw@EfQc+a)v!etfJV%TG2V*?Vn8n^oEmj^^K
z06=4mg5NC^fhpNQ7f=AHe|OQ>*Oyqq7LKAqc4=yQ8+q%K-=EMk6$dhP<z;a=PkJ4&
zTVw`2h)zft0qRe4appI3?&0)7tfsdZ<tY}VPr&=!R?Nf=BP=>Ha%me-LJ$y@JbLuV
zZa&%!3^sL3PhLr>B{Gv(2gyfyBYozFNNFBGYzg`;Xw!Tb2&e}EK<3=o76SW#ww9Jg
zy=Ba$OP2}>Y$ITXSK$Ebnp}6#u8-&70KEkczJT@W6PA$Q`Sa)fL6m9BTP`c(pu!JP
zkXrSDVwh_LVD9Qxdu0FOZsU<vXJ^C|Hv+_9%m9)*63?0~E3?REC2?#wAQr;e+4G2n
zg(xrE3?ossn<tnfZDzjb&;O>1@)}O)rqC)qn7J%+O+vG8=S>8vY$AJF$R=(CLK|-q
zBt2f|Ha8?o;wqp0pZ)W4K>t8s<NrPrm-9(e*KT8Nv2wyvE`76r)uR}o@V%||5)FAD
z_|1Z%_DVin0AGoLQdJui#qC-l+VPl#AShv!zX*-wu*^Ux>h~atE$MO)Ww(`Kq-COm
zGNbr6aP#pjoOV5u;;pqpks(O<#K#YglA@k9!<A83Es6<BbckX(O7*(KN*$%APTUI?
zj{xhu@bz*Bz*f2G6}Hv76zxXth99Jp-CQnEf-2k--6V~K9p785%^ic(v$5}ol%b~m
zl;mvezTw>Tczuc;N|XT6q<SQW%MTBJk#8b5XR~tP)7mSJ)YTyUgcNQ<HlvGfIVgG6
z#j2f2b=KkMAf>%4E*h(?BnPb-KjGJxaTznctL1Wtuf(m6ZQpTZL6i$`xjoZ}M&`0E
zj0!dkD4|-h@g;Sk#s(Cw;I)7QXRCzSi(4mtb5uP>7Fr7~(JYDfhLCpQE`ple^s^Al
z3ToOz5j1kS-Ak5q!==1w!EU^=xJ#jNvO>NfC4J2iV$Ku%ZTGH(q8N8(=6Bs>b>R6Y
z7ERT~t&VAeePZ15o4X~jC}4GG->hAoj=?h~cCgQdiJ!x?E48G!sc|$Mp5*|=Pa0GC
zJ<6K0;wObjgU@2PJxdx5ns(LGTd91uwuNL%kKw>hi(;ULq(#Yz-&=HzeEEgskWAdx
z`&_*VRej%|jMhk#$g{BgnpsT2E!I5_n>ZD=R=`)og-9=o2Y@XVgR!sT`HkSd_fmyk
z?)2|_<)WM<c#G$6<hOq);JOhD{&>F>LlaQPKgwcgNCRrk=6l3zn848TP1L-pJjq00
zHw&+|VnV5LD{412XWei(y#J<>dH{{Xy>}}>rc4x!)QDDN@td;@1&gg)s$iO@OyElW
z6WYx;y;*M^Q2Gf}Ndw#a6t4g}U89C^h7L5c*HMN>ha|+#F>fFZiwxEzqNQaI#UENb
z4-KI3w^<jm8L!5N@Do@QinPLO3A*adl2J9~Mxng2SPl&>-gjBaR>q<_Xf4X}_jMF&
zamMmL)WRJoz5*7aKNId?$78DINjewm8AQo)uMel6#rqjkl%a}OOV}o+eK=ZUv|Ms7
zC;IkBj^!V|!zv$@{0>_zr07VPV%Viy|DK(<<FpN1S8Gu&LH?{avK8vxPP_9Y>4?J6
zqhH2Q9jf*Fe%ySk%+b$cT(wq8;c4vHP*(vzOqwp4&lw0vhDa&8BDe{sqImN%1j+5k
z4-7usB4sUC;J*G%SG+sN_JDTNf{K>j))mPXoYGGc;PL!6nVwDg<!o=#oOdgr{oye+
zj>g_?jFfC{fYt&dpEcdc4oc5^(`q2Om)Qh**MBRPb_f&%#Um&Ks1W;b93<Sw+;EZB
zmyENN546G)bu|vFY0taq`w?r<G9l@`RH{{bN%F~O%h4;c+=#z9(`u6<Bw8KziN{e%
zqrap$PAhvHRqOc`Bv*FLim;tho)Q5Sh8paODTili4`#4_YL7RZAfa5M=%Lo(g>BiA
z{=$<^NoZpVkWzkoRVjRBo+B(mX{qLHiko<pB+E@%yfi}IOWBJu1^N|1jkB}--9W@k
zn1!o0upQtl^&Dz)joQW?JU>Tf6EMEF0m|$><JMwP!>l*+MN@?Vi^#v4s6~ta`U`v%
zqU!@$$=S?T)VXN^4xp1Zze3ZZgR5K6X|g#-cVKa=2(}9~uC{V_CclE4vvFnfi4%Cz
zhhWzS*DpN4^Gm1DUx@D~iUy94jlQkHwyE9RAmRBb)3G}w_nX0evM%K`dVLuV?4<qj
zn}s<zkxSs(gh%|*?D25dNP3z>I0kan+BIu3HHc!|?hK)Xow&LF`>}3A^Rb*Jf8FPy
z*K~j~gA&c&qAHCB3CiGRyK^>eR?h(J-gTfC>0(Q|Xai|$$2ufONanp%Tmo$A3K}|)
z-egJ{MB#5exC5L|Gu|(>oh~`w-h0@|GIp&(BZHd==a5p)$bE^35@r6J?Fu}fdNoL&
zOyzut<Hsv(o_18R-di7ulwzuE;-D#T)U0mY3}+<4{k2WV(`3C1!tcbc=@!7eWdgcu
z!rFu;Zt{Hhv-?A?&I4w6LDRG+`CT$B?>cyuO81ASnkZf=3i~x>I{j=m>zG7LKNQ}+
zecOsQ`p%;}?$CO0tS3G6<8aIRFfx8z&O2t~ypnWSb-*_ZJp*Kv7vyIJ9e}RGciB1W
z{bmaW&V##GT4{2taT_A@rC!TaV};c5ekX_g73O!O_!Zs-Kti-qE*+K*J)$1roy$r^
z(X<1!2OAzLgXM?4T^XV`XL^K^qr?=zjT<*6U+YuPf>_@^P&yTH3?eR7c$3apN~D}I
z<@ZSwd6h%(e6nA1<I{*ETN&eclER`yl2h-&^Lad<sW&818`v)azkMAPajiodrBUE#
z38DE-8-#XEiO#m(1JZCpUA;MhDX^}-@W%t&cL2rnhR*hR7Tl_;s?ujSG@DXV)AO92
z2MB;g#*a3lahIlh@?~!>ue+UQ(}~Ck0PV9;Cqcu5lv9v8h{20ejV0##_-Am?6RqEx
zzB(+MK9a@ifqUCm<1BQ|0pVQ%Snfg0I-rz6x+%iy<VkGJXo$Bt0M5Nt_&V@0$FGUN
z?ridQ9>^1ka&jaH<;+k{%*&xws7bJoDA+rg!&=NPecA{zRhngGvou?){yGpA6c}1q
zSb&fYpT`2>1ks+CG+}}+?Herci*R;!)iNPsWV!dL60JUXS-X9Q!s7YC>I+1hBd6*7
z6c8&gj2A_<mxnp;o?e-F;pF7Rv<n{;^4ZkU=7d$x&Tx)6LeF58WHZH~56j*^q!D%0
zWOKbw#71)dA=LCW3gkDYdX+%f5RZe<TB!1YhG&~uA>{l2G%J)T+-!maO;#7$l?w9V
z|1SO>^~G29PfKh=XsL`@z!u(HY!0qf0(AMVD9_SAXM1EDyT%QIX!1Qf&QABm_mDuW
z<qC+1C++b|&sXk#Db}8S4$5CI7o)E3Hs)3&)e+c}qm!03>iPx-BwSdSFs<?Yi(g%!
z4ry+D&%o%j@fVATJ>p(J5Wpc>dPF@YOS}H@^!hrg`R)Vz@b91th=O_RBT-WH5&C-U
zt6*7>4zUtdU<4dQj1+w^^5<uNt_3jNq0Qc0q!=t&=m9!K$l(^Cc(dla(diGYdCDl@
zfd)~{1%5|6cg33J_mBa<f(mB@cY;h?Uqe@dTos}{o;olSDGDkd5O#qjA=wS056Za`
zrQi3Xa#GzL7WYVkfJ1;&DM}$`KLBJ2rKZP!Ly<e8{*$Y}(4OgA4J<R1!)!sKdVWt2
zEzUOBLsWJ4N*X=BhyU8&VzDL#g9{Jm38sBr#7i$UmsZGI<e4`Zv=gUZVn#J=!=2IV
z-0+_hqSO39%8_X=;4tqTaK+=E`p!0lh>HS@)Tjo+0EuTVL;}~2s{&5KL~VnLU}tGg
z8?fP_HW09>4ojpO$j<6ef*{sdTe7A}Pta1|Uxh@Naid-!4l`s^J;R$+5Mg7j9_f)#
z(|HgX3yeZWo_>Qe6vyCEo;dS9rKsU%pr2-n>l~1ZIxsxH+zj}wz7Jk4*XqTKfZcp>
z4H1J8^s1x6H-5Z{B>9uMF?w<T+mSZ%NQ5Wd6)-^NQ}!duc@kswaj_d|6MNVR;I>xW
znlZ1nw6vmWQR~m)p@5RX+cMSH(oh6<kkOs(&Nbp%B=srrn`9IPfQL(2Uk#3t2+rqP
z;BSfyqyrD)2=h_Vdsjvdc3S9AFc|DcpECI)NJ|+(O|`2z>6IRJ&4ODd8X%@t$3Um$
z5E7GYy)8x^m8jZidL?`#gJp@L?Omxu;d5BOpcn-#x!6dLpIcD84}bOYBW)i3aY0wv
zPzASHP|XQ4)a+VvQHc7hKjn<sm1t%f=CES389~v8>DkS>=*v)w>fbm@Q%VmUL?ZoS
zFIBTTNJfDmr%i?^L1Df091!(HYp+-fB&SfT`K7>3{uQ>n_6PI=;Z%4WbdSSDAMA~i
zoa{oEba4<A3)!J`jbHogd*%88)hmI^{t-u}q|cq=jhLumtRW_cfXwl$zV|yOvUUOo
zg>hQPs0VZwdZ6D`WhAJeA3eH>2${gK5$vn)HHwZy1#FSIan}cca)<#w8e(-xZ0<ik
zwhmwrTYojbQ9oKxQDC_77i1^JUBu7;TTZW0W3K){(x1YqNjix@rvyi;aNKu|!;AeL
zQWJqMXkpfb@GJa>%g(t`g@lzq#6HN;8hlr#oC63S0<l(-RloNJ;7JF;R(U=C(VOjt
zi}|CMVnen7_&HQpH(HGBP@vlLgdAZr+~|*S0^n~>pw~ab>$GDpK=<&z_<WXn>|Cb2
zE9;%+D+$&4*iXSQlA(Hp%zxGh<>6UG#g(;Qax~Gpy6B#pcqK5QqcVD-=cyqGhb>}$
z_N|43(geX#V+p1K!N{y4V;5Sx{s^T!JujcN9h5ONr?c3oWyAOci6F!PZ9N_uhdV>1
zk<4M>0XY)WNQ~_T60$Cl){~5)5O`jRVHQVomY$Zp`K#`&3^s<gM<GvB@|vT{F&p>&
z69WG77y!Ll4xre20>(Cj4jV6JB{z>dtWap}M~I0EcZHDlJRB5%fRa=czu}?qTEhSJ
zqJh^(@7-m}A~R>3r=BGbN`?oR>i5{?3=Ej;YMxAw1TVLK4Vo8R9?O%oQhT;TLgT`I
zF@0keBLI<1nU2}36{Wl}lH~dOEBBMo`_g0>?XeH17iYH;CAxh~>E9#!znk^ho!1$$
zPK4)MGE_}pO{Xkgjm`Z;JW^p=xBpne=yt4y<(OoQyV8FiWA-?A4B_ilBAL+G-TX^q
zWZYkV8{7n(|6i6{{oO*B5M5&%b!g&Y$cvxdTF~c==+p1H{^CW_75Rfx(DMqzYl!bv
z_q;sjVyNFYu;XI)Ua4<R$mM>r_v`b6Tlc;a%2^vW?vn~Y-?r=x-~P?8n|DLp+@Iz<
zCj8QU-YO0iZk6hg+nJmMAB&l*kIr8|WYFW+kEm<z&nsg(e9M1Swul$fU%y9|7GPI(
zV3#ryH^aUkr8X<$PHx|f0(S4+-1&bs_~(?vsF)8?8LJmCFkIq|hEEJV(T8?6J;>k*
ziZfEIJmquCX~Lx+>9E%K8IJhfN;1jb;-&>^=S;45Aj?UdQ#^Wc#^jGG?Nj9>4c+4S
zwL8T4$>bu!q$5zAF(_JCVnMk>qIvIj*16vE2q#OTeda0!3ocPNi#9jVke5LkJ#*Yi
zUSIwx9Sqdr&Ne9iV&fB+L(yBfBO@a<ft6fqUJjh9TWo%4r@TeX``&wL;&k%!i$sK(
z*{aa-l@+yW>40!uR7!JHZGD=9$7@7^6ZQJ3I<x4iZDCiiKK<LyzUnA*>gnkbU8>}#
zrAOK$T!Io}_fJlmp*d^yUPQsK*cRN6Ej?GNg0dV-pge>8hosCJ5K5xb5X)yQK(08g
zLEdnXPxmh9(rrA5s($h$gBw3>wukU<-AmW~&G*o~F!K+)U}K+$F%txn@2Y%@bd&9t
zo~bk+qT)o0>2U9rSFXHY^y1Gw|FUY(Yq>h>0!We<=}tgKhFiEdj2Iu69A71v>}nMW
zU%!5hnVOz1c-Mm4SOi%~&O;I))rwGFX1|hxDv>o$pMFeA#h_tu%9|pq#YsI}Z2yJF
z6ffc1EDHlshf(uwE6s$+p9dZf+UK3=)AQMK=%l#HP0b=c2DH0|UjbO#06SOJ8LxLJ
z$$rBtalr0ILn0q!hBDFcYyt^;pUG{(V6mbfpt!Ntq?C&nFM8c(#Xz}&`C&oRj}-_5
z1B0he(VGsv`pWhg8*#i<$w&BGm>`iGXCNiGINQ~}_lcp{Ewr9cO-&R`<%VLTp?MVV
zxw1n%w%CM3YrN_`t$T+gK_0a82MKMMrT1?RfaN+`uMD*%%eprG;88cc{+7oJx8AR^
zv|pY5ba^<A%y7_NX@fIuc~&Hvl8&w}BkO_JDLaY(q1@ft1ah>$x(5YTDiI7zGk71v
zj6oG3o)dU({$wR$?W<ZRd={&_IKxN~M@~*dias)uLE)H*hq{f84cCtM>Px&(v>2l*
znfA`zsWwOZ-Rl%n90-QfFK_SZbhrWP8fk+SU=z~z#s=-5Tm_<OSjXbZ`ew{4Gk8;Q
zD`;-8K4Q9)T9g6k_1j;D3{wSU$=oI`9PAmh3lIJnHSjp%&`xu{my)JMQ<Gbe9#}%c
z)EX!nSZqXD(AfM;D4swQ=B%#@yc5LY*Dc(H<<D-s*wvKWHs;YjK2Y906KQI8wcU3b
z6MOZBS0j^)v!IN&8xWAgBVBqeYd2839^=c|iwOwv;66>)L06BukagjQiOzIZeYvM}
z-SE7ZXONxlp+AkjB&7pGS0T|7lG}XfgHw|9VZ+-AAd_Ud1c_*U%*Ai0V`5i}Okxqk
zXR#}jdR_tcm82|qaYiH9#gtA_Z`j9fIfR5KyBqBL*@D(16e9}`COiuC!pNv@4`n#r
zFE(}st#^kc;4*Ph5MzN;;OQe_v|E2x1=5C7t|@CXhUcT1cUX2cH8n~hAo*(qOhJhm
z*=I-HB`;H_JqWQlcPw58hlFsngDP9@uo%E@C*mr}p!)GZ!zf2D)6v2)DB&@ai_C@_
zjfAyd@Td*0zn8%qAMa;+pFO+wn^1IZb4x<b_KbCdijBp2cXpaKYX2l{kJU}{K{kKZ
z=g`MMXcT2(rmw#!bXMfTvo#pB%kwicGmqfh3X+HkcUX!e4}q2-wWxH*F~p&Ik2ZBA
zVSb1qk@%r&9a>{{(2Ex@l1a33sHr7`M}l{uI1W6TXJfRd5fX{Jq)+!cbLQhkU@7My
z^c_c$F$F1~wt5~?%5=n501KS!&)!rWmGi;6Z^B;v_S<iJP*`k?nVy`SoTfGJ-i<S>
zJ!V{xA=xl*l2}niZk6fpSxo4YfB9{&$%+5MFaN@EKo1K4g=hbfa6Leyj~P=_Q+Vf~
z1XqX-kB#0xnbhX9&}NyUn6L&IJMELC<&!i?ENWW<4n1^q^s*K3%5d=gfb#TUBB{9v
z9QgTWXlxmseOdFxkgA(x{fFwoxlI1hp<3W&U9X4IrVy#GU(dZ4tU3WdwGAbzrp0+e
zX(7vhX+H?sj{;<52HDcxk#D#HCIXz@`%q$1&{AJtpLi<VoYD<|9ENd`K)$`n9;IJC
zE5(4)vyi9$dr+kHmsZLl2c;OfpdfM(RjW-=b};|2a%ZaFox2;GKU_gFs+M^DM9xFG
zGfGxz8szZ6z;BB)i0We4YaIa`T~=mRmXmdTbPdgm+8~-Gie@q3PAN5h4PeiFxmaZ7
zfnq;UF(ZbdF4nPu<_N?*v-M}dymvYIB47v~o~^^i#Ik>2;49|F)@r4@!J`}`wrgl6
za&6t=DX7c$JGApUfV^*J5QW3@vs;wkFD1ar0hX)ln>qVqkX<FeZD1<HQA1PFe|c`O
z&r>JNlS-o35ZY6eaBxacPwCW^T0jjpP3na1irRI(7mvW4^px5Y`Ql60_gj=X14JzU
zp^eYagofE~5kAjGfrBs7P5LaZP0s6<O4HajFzwIcM%i9Qv!Cz07euJnqMwcEe%O_0
z^Y+Zy`}b&ZvsT7`gUg~B{$#?1`5P5MQVE!TiE14B9A`EwrF+5micBcv+fE#I%?05D
zK8~28r0K<YAfBuqydL*U=SD2l)e}DQGzMTgbrQ{a<KV8KQO8#ivdL{99AC8;>0Z4Z
za}g@N<EZh7L=al)cQ{o>+3cS*L4k@8;Y0R0c(V~;AH8kD{hpH^uUrhQbXCAD&gY8z
zy>qr6#IwfT*;Nre7b^gq%yJ~y@<u<2(Hd|@)xuW-p+({4;EtYg$5*S?`=k1QIa}88
z&6v8p3^OMu#~Lbk&F!0sjFwO7>=E$fgFG<^_7Esx)vH|k`1b8vM~Ll`Anp5`Gcj)9
zd+3)R=@J-_1ywyg-VT6?Lw@(-2m-TQvC|#hIO_B@+utw<2xyZl2Ff=a+Oqye9_PvB
zb?M<gSVsg_lHvP{M@Vi#p?(rn2KyhuIHN0ne%IZwiO(0No?lbu+8sH?D;XRd+z9K^
zsSCD&azh$s0MtjmrpSdeC1|$CJX(z0Fl0s*h`A_yFN{v7B)gK={cdZMU?hTC2SOY?
zfR!F;Y9>5CBE1}#W;qIww{svaV!N6&rTfLjc+g4oGZK?-E&HECtuOw(pjOtmMY@S3
zH+Oe;>u^sf7}R%m+IaCkKX+J+2yOrE9~?iOM)z-fjK#<EC_$n%>c$%7ca6wvndnTz
zt&3~kR#Qz}TFp*c?-!&sNFungXSNOw8=#aY;c#ceamxwo{Tqf9oLZE+SR$=c5i+3@
z1n`Y5+04w$=uAg1<5~Aag_?3&+<MsB3&&U9H=pcIv^yQNj)xm5a<N4jywaiyS)ZB#
zcW*N_jc<$N!ymkW1y{SX6;RP|fUt2lpXMom_>n)Eim0vdRfh#9%4)<Ex(0dfWF&Aw
zLFmTlDLJWk#|2E>Mk>ffG3B|s90s}3=<QE1!$F|&>WVE-L6%D;?a9tWyV@sDmR8`*
z0yy0hJ$ib7F@ECKHP+{wr0I>%H@~)W)Oxczq<7yI3HN)fsG&#~mRRHA<%NCNH?wvT
zl9x$F?v6shasw`2oNCmz&->U7e?$`TAF{W+3og!ZLM#jy5w`hk6=C*IVyrwR#I9I4
z|G|$+;q?Ys;uU=(K=`|}8=)pGN{!8aB`|`LhH2#c2@q_WU%Aa*kAXm9JJZh<3ccf~
zs8xUY+_`ZF7qe*K#rVY;Y50k~Ucl8LqwYm*sqiCXe5#kdYCkg&e_riokTG@9jia5^
z2qA7(u-XzYlwSU+RcpW42f5S@gVO3jcXl@_T5Ba4+k$M=(0iOIGTn}8djuGsL-rT_
z{c%rd!c!n(uQ#PEyR*A&_G>ag$XW^_>t#7NBAtz>p$RVmN}E>#r#c(~(ISaf*YCdJ
zK%{%TtMPY07f;yicS4m=qsvLcbDEEJX^pb@>P)ZN9}TwT1xT-UyR9+<>KqDU3&~$S
z-UZ$&0tvSp?rcx+8m-uq(2V1&Uzw#|4@paO)Sw)pOY<m<@@Mt~d^Zc505$=!xVY#r
z$86bH0oh=5zdO-}_XbWxFbx_<Lp;5_L}a)oI3bnJ#!lX0Hh>Czdk~CWF3wmd(U_>W
z!&SbWM6H#90UFrE*C1+=rG4D7#f9`4V!>8rsLQZOE75O4D31Fgj{2{34}(%~1hH=>
z-h?vtp;L2%6}ULJE$PP5o$@yw?CiR;9Crj(;v9q4!2vzbu3B~%vf=aF^4sJd+t&sU
z@^0pt%l)Rjj@=jqQ%-^UrDYY(kAk4_S9oXoNK+`Yc{-n)gC4ok$<#S`aHRS%_yO$A
z%gc46(dMXm!K#Dq!Go{0>jRAWS*~WOMkr8REfW+I1e<>H$LU6{m-Njv?MGSfuM_+K
zmzz_J1nhvr^6EMrygD94j2kVW61q7n)1*`^+abA6W46$&C=~CN{{V)>k>7(aqlxHY
z#w1W|?q5M~fT>)9BObTqBz`ywVI$a;b@qE#1nrF*m&vqkPk*i`Zn5w8`k__sv0@1L
z3WvY;ARZ17%kyd19H$_1(8s6{vX?+y!cI;@)_bv~&i>nv{8n0^R7c-KVK+$aX5+Hc
zHP0`-eD>^_KRw7Z(dGpZ<b(Nbp2;dc>S~C%rB|TVIK#2IcBnKAXt$n@o?dB2_<S6h
zv$V9tC(+<5YoO~OR$=Ng*^676zUBjkYONHxvr~4csBmo@>E#+eM=qaNGUs|3mNm=U
z;(z4ly0i7<TSV?X<boQNtvNNo3<5CU`7cOSBm+5}fco7v&tA6QKY1Ev7E#Nd#1lP2
z>Msm0Skk~|KdR|g_c9k*L9gMlUXW3iBWIo;hkFXBq-+vR%C@$9Hc91e*U2|oNtC0%
zVru~PkWZrF^5K1beFet!c+7W&CxT{(>9s0fh=+4$)5gOFy5MRi;jE93K7djO)1X)~
z7_K-1P+n%xoHb7br~;%%F`*M`B4zLr0J0nf)(dj(cmUlZpgpMNxsm$CiJ(W@e2Oq*
zFP5siIbE%+-6vYI@px=^*hg83j^!5{f}!0uhYmK)mwEcL+Vi@zpMk#~U@rc<0IJy0
z=AoYfkUmXC#D84iy=d;OtYet+Tcwra%_%pEGnkFbLqWKEOfJzFedWprj=pB&Vvku%
zF3Xqf6}q1?9au>m1f=AExH$d&`5-AR#?|H4)VT|}w5+6Sbu;TFG~t1EV#}4<*yW#&
zP0-$YYX|DRm-kAH0=TQ$K+TIhlxJUB<7;eQJ7kd9b73b3dz8Wstds-XR6@IWDo^i{
z0E;`FK7y#27pA&Ne>K>o@e~?}8{7v5>$)h&1=dP7rpTqMnMA(tO;BmPN&YJAk%tW~
z{|UJ;5$IAd_!2d{eLTXoG1{SS7{S-<^4Fa*u$M#QzM+h!D5}TJudS`+U|*fFUP*-m
zLaC8ft$1kAut*l)ZA}jDH`tQyTeE5hNSWcVkIf{R&%xPR9IUe*^>+CuDX@;P)%iMP
zPCET?*JPHr=r=K7$Dkao>)GGak$XnglRGtWYbE@T+kGqfDOcM=2~j#e;g=(o%(L#q
z4L$@*=)Aa`TSuP1+=s$OKc&git7(sQG2e|rq)qMIqSq>X_@B{ce&gq!u2$vAqc;S=
zb}!QRjEsu+0u|TZcmR7!wKaC=u||xp|ICFw<=gKX=h8woVp+Fc_X5H!bjOYzF6(|D
z7tiT^a?}T`=Kk_e7Jdpo`FX}yej|^D2s}}Yqku>s1FDJj1<daP_-_7EUYk!C#y+Hj
z>op^N5JY3xYMRgk#Q~?55k7A1NtoaUN|6J;Kgt!KW^1n}T&w?X3T`*k;?TI^eOav_
z_JyEF+sD2SKCNWl*Wcf-r;x%makT^5BiAz^O$Q1Tpm*k^C#=;tJ3oIEggfXvrAWZa
zMXZBuyr*AqH#c|nkvltdnuQA_n(||a^wlJIfht2C?#9n)1l|J~53s>HKOo2gBo^MA
zSc_`l8?*gaHGOn|-%>2_aMEjhmjl!7Ke^P}ryxBXx)-(o!ZbK)Cj84gz5ev>;3k6z
zsMDSRr6^vSt3u4|)?#;du8NAv24%2%x=)&3U67kWF0?rct0?*#!Y@zWkNZ2}SJ$y&
zTmsn=qgCZ)HdraqfWc$VMVji_@(7NvG__Y!&LO;Qd2Nob()g?ES%baO><fG1#@sQ3
zGJ*?<rNNP6GZ$SOU%=%Z7_kHH$qp?PlsX+ifGGB%TDZ>ofY+b?12pnqx|L`LnA$4m
z5UX_a3_lBsYI?94H43}<q|?mbZR7ipIwJpu1g~GuRY%RnGVrH(vx{e@*GmzM`b0i8
z+T%=$;8R%ZCm^K&hQAv&oIaB7CwM3)iA=@{@Ug4jJuw89-^0~1xVOb+MpjQ%&#G$-
z)&Pkay28Y5>D&VA-3bn9SZjy7#RS%l=LaEed3xcz&lOPNQP6ZnUy!29E@{}Ii(8vu
zutTOBw^#|KqqsK@KivW=v=e77cUDsK4Y5Nn%K+_z+PtAsU?}4J;YNNJw6+!?&{T8&
z8U}lU5(C8C&>)DNZhp&edZCI4UzvURtrX}kNsGaJIsWNuO928ix0U0m1*aU1iob!u
zI<AUSrG|_pH@q1>eVkJe;-~{MPM?WUW^i+hU3G>ZHU<;_WD*AJ{Z?|bVk3tI5<XI}
z#1AY>8ldHUb<)H_>d5+EOuGJ;_VvHo>F%F4H07KVpS!u%NV5E>6ZXdm-~0UZ-vF>z
B@0I`n

diff --git a/reference/pit_sample-2.png b/reference/pit_sample-2.png
index bfe1982f0f2a1973927f8d7cfed127f8292f936f..fcd0bf9b0dd057b2c09cbded81a1dd9ca8f4dd83 100644
GIT binary patch
literal 18257
zcmeHv30Tu-x^EPfT0o{_D=G-B1}Gy6in2qj8wLbg70^Hc1wjcA_7Ju}ok|_6)PIyk
zHaDaMlOPzjY*-hJECGQ;wg3`92qbJFY$5kcn9iLuch0$Yo_nA3oHKXwJeoZJ<y+t7
z_x|4Z<Jz}wPFkOC{}cv;X(3M>Jq3fUkB7n3E_|{9T<OldKMB4zTySwZ3ckT-;Ru}o
zzBWdjI2R3r?MPStTaz&~brA;p5{5kb&6)VT@m|8u=-}j$1@5TdnB|ELSN4@}KJ0%%
z^0M<}-PQcM2PZ1)_da6n`T61VKgORwl09x}M-2TM`g!BEpg4y+jo~3i=b<&v&5wM!
z*Z9JE2vt6G_{1iS!0|Xp>*CdmH+H{X_mk$uLr_O%yKLMb`3LHqaQP+aXg7C)UpnTA
zNS4%$Boz_rum%X@HL3yszxmHIlf+kxh}4hoPisQ@X2U%^<gHS^AECT{m#Q0=z+aU2
zNu&fUB!U7%L&x2rPZ-L+%MNSKCaex(%klp8%3ojKuG3I{fBSWsj`I87w`<Lm--qt6
z^HqMQ9^R;W752mL<5jJHYB;%8dE;~E|Go_zY!|KB(*5$~%P(TlY&(uFkx@+|sdrDz
zrQg(dNEo?)B;5`D(gu5SdJ#@_Ew|(hkVu;hJ6VY`F?a65-qbrE>eAHMo*o?tm;LWV
z=RW4tQbcI0o`@HS5)w%)H3zHHaI6D#(KbIn@9ya#&4FJDP65wfiA7VpD}uSFb)uR*
zOQpZU-SY;^9Gq_Ib?*23CF}yF?J@X3K3VX9fPhjebFS;5hK+Vz6pf%*8PU|%CKxBr
zn;F(bcz7@`ytt}G=n&a}t}?H)7Se~j!)u1hLGKNrN97Z(t*v%(<L@Iw+a9|j?9I(5
zuhmkhF4L{>D&IHw^s86La&mGqj2AA^ds8Og?xxx^l9%RAlRYs{NF>)-v?3DXjdhd~
zEI8P`9X_z#$I$%JcME-`wUp=&FR$slzdWU@uiso(H%>WaVPR34Xuk;tt1EwJg`2EI
zkqLK%%vf~aK^vQeVKKKdyf<NVc6N3%@@jBUkZ&!vkL9E@RZDd*H$yJHxwWgbnQ^^p
zw5I2|#w<NiTr*OB>((u-F=yaKdIg40@y&Sod+(}nG^J+vE*wek^~JThS|vYLiMgq+
zeEMhFA2@XA5k+Lzyc|(~b4Q=+^a^~No*pA?pBW==qRTxtQw_s=f`1;BqmuEDgyhfS
zt3o@cKftr#oTmZxtONXgv1rNIa^BfS3}WtmKrbPk+vQZ16|E9=In`{fq|F_Xd=Wb1
zA*C(o$5<XXFngzYAJwN3qphvYeq6EqPTWh+FwLq_szmAvUq}0#E*JoVB5IG7PbSr$
z@rbo$-K6-jQ!cUk6TSlLVQcB&Ag@QB;!C*)ZTM>1_7Pql9=Rj)D<{x=jpfbCc3_60
z>vi4uUlXEUZ`s)UxVYGTlUCD8Wc};c8;Cw<`g?0*Mg3q{@@sGmad|~36prNPm6Axe
z;18pH27gUgS2nSD*e|s8XW`M25yP1sE#XycgNh6Tgk$tRo$ku8?mDn1`Cyw)d34lM
z>Ec=iZM0^rHf1NrYa~8DTVkhd;SfP|-7X#|#XwqG8XVu5RIj;d6Gd$5?$bD@2^!Ij
zpM0*4xGOZ^$1)RRVrD_Jm9*Rfkq0%Q!>m!d(%#;F1`;(dX)7Cp&E8;DU%_>dJp3f4
ze&s3hMVp7xmX?;$H;Y*sOgFC{c@BI@seGcFJM4y;qPUV_m6h>)%=(03hZ85+b7CN$
zgwY}|ZtO`kpA0V{h42qkB9HelWk8N{xv|ERry5lj?THWAWnw!r#SGasF_#EL<HCgt
zu@KIP)7aQp=<|_d7by$Kxj|RwQP6fN1>N=sY*m)8tX%5S-&AtCIK>KS$Kl@0$WZWj
zWbb+?aAvp(ah5MFCjJEPtinkIPR+|=U_89N$-HZxr8Gq&xAucLm#BxL$4!W;uze{}
zCaa^j&Sk~FK|wvqvR?O}GBWzkdUQM|kt$u?-LV)N>T(rZv0P#H)6JW#%LP<zqiteF
z!O(Mc1BPuyq!p5vJgD1e%?qIo+jHvb>fFb>Xrk-atyM#<V=NAD62Nov^6by>rP}ZX
zIf=x|3^d{pkkdD57`|^+K>f{!I{0g8_M5*~G?(^<dpX=ysleH^VNK|EKuAc)A2_kd
z{g#%Mi(MG%7uVBRQCtU^kLCXT(-Ux;r1^-riRqJX?*su^1ImYi;1h9)y{=ZhvpM=1
z>wqH(o$c7M>KArj`~T2^{1f=Ve@y}Q&j4lqwT=H7+jxYj@Bh^G<Vm0J_)<ENv5#}T
zl$21`8&3H6vGpNN-!|rB`&xJJu%7CvKi$87-^+L#Rg~ZY9Yl(I0ot4IWLY~pt~@4@
z920KNr~qK0S%jt1EBgO1R$6BVZW3eB1S3xWs0?^vw1)@PyCSFwfu*5DbK{*e?LQP$
zb%caDmlsn+!Aw7o4jaV^iM0L1{-nx)v0BRB7$ap{sjQ8Rs-CEZiyj`mS)J8O-47o=
z#ErjyI{5yvYi>%szkmPcShQc3@F6ms*%DpVJwa4cQ?nzHy3l-Su&8$B8i^F=`^ILh
zcBPm|BudyV7&T?Hjm6ltk~eSOm>F)Ph!!av4o8-kLXZfU8KJ;A^s2yJWhj_=`rJgf
z?Bfr4iQwK6aU3CE8g9z<6NFcpU19+Qp|PZL=yf);4M%t+xWpN%rAVx2UhJ@oOS0lD
z5d$M4CM%0}Xes-=x9#5tYQMHBpMY&|)A2rFFQ0Am+N7!Zh(x09P;P=xy{48{^2MMa
zBY=^kN=8KU#$-<s!FgInOtRn<cErjDzyy|{_FzRwltM1^vjc$cdy777RW}#C{GAf@
zrT%{QSpSZ{!Elc%^w8`Sc}vn))h7ArjQqJH$Z#rYY|gVZ_9K8FR7A|jmf21g0zhpx
zt$V`C(#}vh$ZsLm`v><aB9Hu`(=$6P92}CTXA?q0L(8V_+k$lthj0KNc6|K!v8BKv
zal;06^+K?@6UfKJCw^f6^IZ<Auu}_S@dNPo#<mEv97$>%T`ot6rUvVSw-^2r5ET{G
zmm^7@CKfKcK^Zg@xdg7DQAAeZUC%}@f_NoaXj;5@2EuZuR&*goMC4-?L~?$olUCIm
zZlJ$)>j@~U3D?-)-#^wo{qhu+YhLJ(IG0q?5i;RY9tpPOAaYoX0fI}pL9+_d?cS*h
z|LXqK4%Yt(iZ&+7A9CSQ{Sd*=^tW%f-9b=9U4Cqf05Rz?=vns!jajf7X1e&T6Xe_B
z91A+-q0ys7S-uWR=v@VYtwXz5ciDig(Y9^C&c*yrxp4Qa|04`%I&o>H``P8T2I!eB
zJB5{1p?n24G<w}3o+xZDIn7q?su+$?D%Rxt_wU)A0P9aR<ytBH>AixSKsbk0qaSFw
zZ{HNJmV(F@_Jh?AKldcZSFy<Va|G{}g5iIM=4(qjD@ID-H4DSJg%<nwM@4&h7>w#A
zZ~@|0RGsCkCgyn+s2#2^S+;p_aL~%rBesl03Tta?>qe110sI_2!ZNqDU3`pZIc5h|
z&n()7vlfO&l<UT6V*?$04lfZcW&$z#FuMfsxFZOg<aU2dPfu3|hQJHW2WsM3ddDWo
zkwYF)7^`~B?yyR;LK+zxYu^A6^^k`L8=&~iS_<DN$9eibKz}KLOLG&(W*=swU-yY+
zz)<lrb?7+B%CN3r@+#ZP!eZt%iDYEEG~sB>*@YCBlExk?Z=VdR$_C5mzo~Gsz1Y`$
z|NeJo-kMQqK|w)o;qS`^?3e{={xzfo2bfOq-@r$QuEATdEgcTBp6E_sDXNIV$$=N0
zG|V}N-9*E(@ZMUV=%#~6D2~n)Kh6zbJelFN*8ke_XyM9e|4^1+N*P7OmnLjU5bsjP
z(A^0R>*}I}eW}1)l$oMa7KW(5!;Qv^o?lTf<cq$7Wj<teJphWDy3!h6<wOdZL)bf{
zEXT8COo=ppXJ*&MEOh3;Kxh7E>gwtv-L{|kAKR=XF3!&BCyqbu0XxOizhVgX{?=|v
z^#gtY%oJ03NhLr3Uos*8f)D>0@Zle&)Xr>9!#zriRyoXwt43=jRZ!0@A!$Ri$~zu%
z0=ExvsKyH@$ldrHza>^Bzn;&GeiccKY+BD@Pel1aSR;HebWqDj<wE?Ah3uSU8FR+q
zWVj3`NIVY_3Wap!%4x_TtiM+E`flgb-o|++_U8!MF4GWPkq8yr7Bcs6j^y)o2k(cf
zp3x}(JaN!5Q^-W_b&Ya?xG@4nE8NZ^(M!U1E|1$7q+(4z>bfjUR-8tPo5qjuDbCZn
z$eV$4_d_MF(^B{naao4kkQK34*<9+*Js0~Y=A>{OyvCNJLl>}ZDsc0TIi1WThuf-|
z`s*`4VR%3yVo#1x7eA-QcO*W_?@WIt7d2iL6#9*=QFZ8Cb~{amN3M)C0^o57vcd@z
zaS)+ExRBG!8mK|0lww%gMxE7vTwTx4?~m5m{VX&HnmZrjZnSU(ZYEBO`;y~si2n|f
z87{bu$Om(hKXwgMW|o6q{P~sOU+&MazT_x=7W&2Tid~Ht^5F(=#^*GHqjCI2kL$KO
zl+!t6aB1hH*m-~C>qJ7nkePF%(wIAA-&-#oJ0GFE4{Pw;oF<c^MWC5J!-eU!T<OYn
z0~J|w4{`9bc!P#<{aOR}sPUa#sk>Q<p7NE{mG-0k@<im=hGr1HKQS*{>KS<r2%##Y
zg*0F8VRyzOz8b79ihs2K3)~u5>al%*Tf!hyD)@j8H}!H|CxCAgp$z<927sG)h76SC
zZ|cyKYjuep5H*%ZxGQuWv0l+8GRPGvmmw$J?2UjB?vcy-3X2O8Nc@1ZKT;UeZ+()b
z9ZY>SSSAff;xI+3-aI(3$PQjB$%HH7o~i>WeJksrrW|y4TnMDr`)}0J(gK`u#3nVZ
zyy(;XMJjllEk<E|gy)n~f-NeS;S01#QcyqTRLes=eY5JnXUU-X89Hxb#+p?yLRUST
zcMLom!B6PG>mhn*E6qE(DxGMly88)(AfP!G_D=c<Xe&K`ecg`ElQ?7y5||h!=2q~k
zNP2^%a6mq`6BW8mv*9JfHgh|W;gTorMxCv7_NtzeORm)|oq-BR7v;%FvHNr;oNtJ~
z2w4Tk?M|OS*UK}IEdFngUIyX-xH2=t1zdg~)qVPDg`Q$Ja;5tIpC@=~Gj6hAE)kgm
z=L(3jlXgTOqEK-u0??#j`C0AlI~c_dc|soiA(G-;ZdtK3<yatmYG2`{BHm#-zl_!;
zcEv!{7!?5zGyUGEYSlyC`{p-8k{NP+11xp@w(SYS3W}zt=JJv|VX>`GG+;g3+y@9}
zU0uu2_-g(YK+HzH3+&%<vfPmq4|xses6WYX$T(oV8TM`Vh=T;*%U+464&MV&hvQx0
zNTLIfMpI+g^@38O-rLaj^{}&LV!QZrony7E2Yd@<V{I=H`_OlVW{`cch^cNzE!M5p
z2Wf@zJ~qPLe(I=l`^mmc+gvNGSh`io69K{Q%L)D<-i7qd6;5rSr~xT9cFTJM0BoYe
z2A4~LfsgpRl>D(}OPx($J8fn3$JMRB0kp}tm$q_PZA7IbUpB21P-Dr#+pQ6*4ZCbK
znZm#@Ytv>_Vam@zYb6G8WbEbsW!3FRIdw~CA*&c(H_=B>cTjcBlW~7TbB!fOj$Ang
z*+EGMIdr1liRoUr)P|+Fu^~t$TOaL!tT+)hadNOE$vTtrkjVN~A#!&OU5!2Jet!u7
za=vARo2uF7mZSa2_kJ_i8@vywuKp!Ia%QvlfyRrb91YNB3L)?sMMcB`5!M8z&6^nN
z5;dV$m_K(N$&2H$Zwb#_u~2n4d*kt1$t5U}$PnA#WWqVrIG%y~{*IQDs@9W>N*v-P
zZ!IEa{!pRwyMmGTYG{#Et4XyMEjXYae8bM3bBc3Nt}P;X*k+|cdAo7-b{4pe;L}!y
z8VMuf%T9D2qH=yv)$D9B&_OM<D7}^^wXeq^&P+_d`YRLV*F{TNT+611d^Vzb{38ae
zl_Z9&PUp>pb(p`SrNm7AWwF)?xHvL5xZ#JaI5L7>{cyDdD(Ik@BE2yZxoj3gmZ1LH
zC~Q`0XrFu3<zNT?7A-1rx*=amL~c4@uJU1r{D2SBcZu>2mM_D#jqw<0fD+(K6ei>6
zGBvA)el>Wj%o4)NE@&N$5ukH=69&{kc1(yUoLb09oM7$ZFtwUitw+|~>MOEm6G&f3
z7!Yg|6bxk<FMN9im`w1QEqW~OcbIqzuQJ9bLc57RcLL2hRzx5xQ&*|0;aJt(O^o9F
z6Vq1tDl!&!-M?h51j@<Iu6{rf;f(PCkdBv!6oeWpEv*loB=#USELmSr*;O4m_{EVU
zN2qT5efiQ%I0ovW2XhOi^qW<%{)xY<_&Htpy1t%JP7$$PrzvnVmnbBfPXz#d7f53)
z%z!X~bGW%Xgl4I=X;aEWkaPm+t6dxoNYl$B);En&vYLgsL%wDgUlklz*!x4m(qUFi
z!=5T1kbFwc$0+7NtO)|{aqG&<ZI6}V-~3TDpKrKu3Gz0Z?U{&M5(29ECLjX%vGUGZ
z3gH2Xl+P@{7}~^7_FqEtL40$N2NGWSL}T|9@o+A7C4s7vgtL2rNMT*}Cw<oE_ig`<
zZQB-#;4l811}bQiNP6kATV(GlN^U^`Fr1k;bgfoeSjkKJ%%7vdJsbg&7Zdsz_?d?8
zi8$^E>inEx=ePV=>c~Bith(x^XCFVAZW;Q1GM=ik=Vv$Feg9xUft>aV#>-<NZ2nZf
zSzI#fcOF4i2#`g=<t+?=ECMT6C`azMwvO*!OKo+J{sNFEQf5Kg#q*DP!xe$lbmeY?
zaRX8C7{T{khnByA3Y$Y`nGT%q^(`weHe~7)+T#HdvH_X2tYO>7V7Wn9SW!zchL)QT
zQ`*d;?~K2^wi*BNetH71WaFe#%G>L{vk!bTl=@pXe&DlsE`I>7u;wt4;V$JyzyNEk
zU?&Sx7DsohZ`c5WHvXm^wg-aYAZ}iK+t_##By5LXY4&PI-w91fuswbHv?@|DH~&%<
zaW~%2O0&wjQNzv_-iPZFpF!iS79z;V`1<l=L97_h3Luee-<|s{*HoLmC7M!FQc_q#
z8gm+%|LDB*PAU9#Z`(z6xo|%*ioHkU69&jfj+qwTm1K{OjuMX?sUY+`efspdX@uR6
z`pwcdAYGA>#(?gJ!JK(7kMQ=M!L=~(X(QIH3wTbFItcG2_2Yx}8C=szTb+=aRXLtb
z8;VeuT;kWF-fRQ`@ZTnnOkd4uUU%>U%7G{))kfdUNGT?f@LZK<%~FSc7w?T0arX$<
zm3opF7db>@qR?<=B0w3?mpA*;2sQUi67Bh=e)2IJML0;7riASaQmN0|wwGJ9)JCSZ
zJ4fN6LiZ>yJ`PcqzTWZuZ?ynFYlwSlW~Sh6$f&=eA2e!?Bbro=%%%hD6a(c%Gu(`9
z@`P8=7D<+z=Jmc6Zqtd#ci~li(ZlJRH*YQ_GAxiFE?x;t<9{q(RXWEqJ^gZ?+q4-m
z^y3p96${b5u<{RNkqGzs*bA$;xZ?5!Z2=Cx&Axu01+w7ev=wp%6D0Y_{QBSJ86;MN
z|0a*Lax)jpTCf3&m@qvn0l6D<B=F%y9U=MF@)R_`%H7S)QRLQhJx}F}{C$BL&o~g#
zJhR`%27f6jX<3~_TD$`?Rqe4%OU@Zcn~bK<?dq<=va^4-4_=-~VUtKUk{`q4Rpa&l
z0T|}FcEF1C87;hO7BR*F4u64kwC@{{2@)kw;T@;As^yKaY&fsI3Tn0-`kU&3UySDZ
z<(H0(BuA@_C?(E6$2OwBE*Hp12|v2!bUh#_CvR8@=5?6*fi)pJU$RU%5o#k}sA>{C
zf4riuy>b?4{#?SQ1S*Mi14u4|M5^Uim^y)}af2);Db7*J{3T<e{q09?QDM+63RnN+
z{@yxMrEq`#3q_PAP1ulm!$2&BHJk(C)sdh<v9TSyuTVIFHxe5m?+7Kv3;qxvzTgH;
z{iw=(<qhoiTX1I0zcQOk(9C>)RZDleXT-6JA7GjY`g{&o;DRKAWOELNm}fhf4FVkE
z6PM{=WO!rfC#<~{xJ9Kp9)g7Lp9bKUr&M4lG$`mXf7j>d0hSQl4XzBDUs&#~?hZ5J
z*x8B{mUEq|MYP6()~jO32DTZ4iav^zYdWo(1T4AqS{<E*Nv@>h+(j(@V(}K-=gQfo
zu5=vjpAa0uHR1L8n5U<&9b2X3?qLBo(~com0UVCa{%gREh#`ZCe-UgmR}Upsi)4sP
z{`#txSRfi<nsLO<3FaI>c#Q>T4{{|J4i?1;x$-qf2hzmdWEyjdb+c87?Vg#|!H8zt
z3U7;~ot)RriC8gXc&4N=yyIJ|zj$<jqXOs7UmiqS!WYbfZNmRhftwNJ2}iH~IhJ&B
z?RlPv$`^hM5we9fNXp5mV<QzWkgNjXi7R|nF93sV5QIAslWJyE-rfJR1GHl*oCDMH
zCfUW;Lkx3H#E8r(^W$oun_SAN^ERwG$f<<$o2NQ2v$F%kg5|nMw}Su{gU_M9OMjTW
zX;fpy5j79KcY&5eg5^{2(^^d`L)hYVx9f_GV!s)uZM0AVpTOUAXrcM`wfs4XC<d_=
z8fway78=%+%tQwj46=K`?cX8?Wsnh#7Wp+V<HQAV&oJ_U>j+g?aEJhONYFw{`2~aK
zMkmRRbnNw8j#Lf@ws;!EKt*U#vDqZObW5b_@l+NM#D#ABgffgnpQqL?w8%+^pr?F#
zHDXNSFD6y`#9!twR$RAJmQ%qFDL@C}aa3Qte!b!_f06RnQh>M>SX;3ohiAp{qQhDz
z@50$V=MXu<Luxb1R{<shW)UEwSRU7XIs&Q!nQGhNgGfPEFqRYHy?6P~?7(i`w?{^L
z?{t!N9Df%6fLxi`R#;fW7u!KS-7S9>(Gq}6+`9{ehDgV_%z!u}JRZ_MXsy~u0Pn^&
z3EBX_ON6S7YxZ+IkQ<bwy%$jrJHMzJhf>0q_{ihLGsu)jGlFYyLK|L%mCHU+B3h_(
zH4}a4%H8)@WVa}>9d!$;+b1m2WJI*+pjMuNvYpfeo8uJ${Djkgi7ddJ=b9I<AbsEc
zB~mcbFR!%ZI3QPIAwT!&OArBsaL&^=;Kj{Ta_j-~KTFExos*7t1LuE|r4-HB8t6Bw
z5NL!xfG4VsQw>{Q?qE>E`twxd&$|!Ce`wXd!TL3S;~m>y37#jKa_zqj6Y7m1VsG(P
za(u7h)ds<U{pR?Q=`;M1j$^(mQ=Pi^x3~U}>$bL!NkQ)sj=&xYLEVy>M5<?6GqOqr
zgzi(5uwCCd4l)O#lN_^fd%BJSb4UhtM1I&`NMi%OO2pjIdSQdWul5bsUCE@LfPA7j
zhD?1h!OVloSf?yhOVfVl&5r;GjCl7gdTJNn*HO|20Gj~A2ur;vx}YW*y1pM+ue=wV
z-R*D5mt-=f#q<_TZ^1>W!SY^#nM(G)Raq;TkNIi<)$nd|(t;aan3K~<nCyRk+CeY9
zNXDq8Bs!>>CaU7#GSgUWE2yK{onZZGUHQQX@e$=JXBMQJuFcD}Q+kHfixLW5{C0LW
zMj4ngr|(c3GZga~p!c(Ls`57fPP^E#Gj0X;ouYkb4})qZFt0!PgV-8;{_9u1^((Lv
zaeyInY$=t%ZS3i#YtXmAH$4eB^t6O%?Lqty)?xiA`qtYZoTTTDtCrikqkjsy`^mAd
z9vt|l<dovd-FFW@c~tr4t6yHOyCryOh7)&v6>=Ww{C$J%vu`%K?c2LP*Z&0SNB+`>
zUb|w#_><01NpIEC{=^_hgV9!di|7Iic5s+{Zd~Rl+jj4}wJNusdVHhGA;Y#CuDbV2
zw^LWE3jcdOAbpL7)%_6KSw6HNKlN6Uv^;1)4KP)jss2BG)xVw+Z<aE3otuI>t_v>J
zwQrtW>%}<5${zvsO9zVG^0x;Z46Sr4ChD*ze;C}W^hJ~tYklvTC=)BwMP|vDdoGq;
z15w2-I|(DY=c2NWFF09GR3_tq^QsYR`Ff3t4m$L3oeFKhZv1|=?($y`{LMQ2BB<6l
zi=CO7F)N{gC_fD8i*m-tSnH+qgUqafjWYUfDo6|u7i$&WPBb%@ma1QQUd%tG+&Y+f
zveDS5f|{Bd0}9RYj&K4=aVcl=79SSf)}8>-LYq8=uoT2aJCDM1Z2CMgd$PP$;%v?|
z8vaz^jb%z-72S3>8Z}zD)7jaH+hb~KPY1P{lF=Aj86Smpj>1Cx8v$aJh-<oM4^uJS
zta9G{3afoyli@(R!fcWcjyr+z*y(K9auZ&nuC1SRAuevN1L<Z|libHjrf#k8$#)Bm
z#|N)+8|BE2@h)zCwqMbBxcsHzUw{3z3SuCTwY}neua+L=j9^(|JJ$-jv21w}veKAS
z!F61E?;H?9*1oAfLcLw1k}=+eo#N+-pq3M3oA2gq;;yP+a1Kad3UsovvQ9hWJF<sU
zT-bezQ8>VXqj0t#M&AGK-8*AqyGse6h(Sn|^K&Z5UizXju4L&{$Lrgt3bsu&i#scd
zt59d@-$iGzew_Hp3ywc+GUO^xQM6ca?JHz0_m9+bz3PJ<XL4{j+kA)J3~4g$ZMt$u
z{@Yz&?%5tf_I9E>dFe(k&rcD9=c>ZGdA(0+FyubxC@+&CdT?-X;+5z3+9u-s&N$;w
zLxae<aj;3dbVv_xPrpTp`n=C4K7Uo9@{{K0j6QrSfOBu?S5Ynft@RuU6i^`QEqGFs
zm6@q;i(3YTmy=C!1F>u=otVVpC3LUIm#B7t=GdI|?dl#6DO(tbn3uMSV@q$0#?kGL
z)4AyJiEWk%;^+R8XQA}t(LG6N16cdmv38G<a2ijM^0ZKeWlm;ZtNtD*;*tFv@Y8`1
zBZ-#SKbOE19hZ^h<9GJspxdWbs(H0oYdx`TUo{{=sPWf-e37p0_qOAWa(V04<@~ri
zUP70!s6(?LU*6&2)m!#18-+n2olUfTO`<by%AdE2+p-c=xXQfOXlK9cDI&uPdG*DB
z9XB~TB_+iP;55$k^fVT25-sUtD{A5L);2`#OBD!CHL?1v>e315bg>(=m@i`P;^>YH
z59{}V8cw=W6n9uh^j~~2Fv)^@L-x<!>EtwVxr#utln*6cFyGo!>#L+~WMjTmnMTp=
zzF%mLj$1Wv8gcei^-O=i$GfuUl=DK%rH_)md-f!sPgVd-0o*Cl%EW7Ocp%+i<v3Yd
z4?VgaB?BaI#@v$JR&NtKFAxWDo0^+h=kL6rBZH22*ETs=ShOLh=TIWWiRn;wA61^h
z2DxPE36mif+%9VIZ7-iZF|L%J)XN9<t-(S-$99ltfM!lEx@}A}HO9j^Xi$+^d-?qN
zfj2?T%<Ag-n5u42%!>od|0TyS<L1&~NE`ef@Hd~oW#Kia!d*Upx>j@S#Woig7k9&j
zn%21Qw&>`98i6ai`(#R)c?EJF&s19UB==hF_bO5Gzx&BQh1>h*aNhr#g-YH2Nub!j
zcJsgPZl1yRtbbCqu&|KTj_SK5tc08GDY6H~91$UWb1OqnS&&yfEO@i(aYhe(=QY{f
z)Fh7}OH1Hb_b4<pLE{<hDN4FD(wtui9C3I3KG|`eisU*;y|ytPz$>0k)+xuFN4Y|;
zpx|Jf=7uZH5!MVxPGP$eTuFks%v-T4Tih0QVXXi-9>yNdNwW0emFNBW8X6i-P&*(7
z9rxMUEsepAi9IZE{KPACxJUQgs)z42GVI@Dle}~V3_n(!3*P8+1}8{>tH*_Z_{&p6
zz_~KzbKQP-0}~<prSXak@`1;^#r}_xDHXrX4wf*u*WV$}I`YF^fcsCqC8h&vpY|>r
zILKE+LqoSu`O?lsQ-jfj#bUzZui#2fv>ySScd5;aawP9A3ItGLaPJjYPiK4zAt>mz
zLhW==L*^c?M6~APnzrDe;qHb)x8gyz;`8!N+2Kq<m4eeX5hs@wMtN6;aw4qtaN9y3
zR3sjv^4ruNw{(hNo>rf&0X2=xTHalhXt4B#Ns+xpS3W34b;r``VjV_&xwRleGN+3S
zr)S?a?gYQ5hppK7o{Q$k6X(Xa;Wqx%+TrC#RzF$8XU)+D_onTuo$JC#Q%!LFb!m}V
zbz9K{xwK|wQB;rK5HP#QXQ%WSnjg0k#0P{krd~&8t?_zvt@_vzva~TH;wHs%cz$Zo
zm^+T`uAF#4O+Yt*wdH@l__}(PeKuW*>RIo@`TKzFQ4PO>qR0{6`odSyjr5Hu7&Ll?
zF1lUSf%1}91U`^Do0u7FN;h^f@!N*m_tPt!5&k=qp%Rpcgg6kezXMeTmOfK>7NXvy
z>%aa$hsI9=JT@3W`Ju_m8^es7&BeNB(6_$6;AL_#5u&!E3hDiqjGW1_i6%pL;e9`)
z2BXlqfi94lam_D%uY*45?Ji<~ChO5}?t}oCtwe`lzu|nc<>T%N@zx1P+|#Pyv!Wi+
zIX*ZSM2IZnYa2KwS#w-V3+nzD--Pk4$PSF`QPu0z-3i<o-dYwb;#qjjFp{_h4!f`N
zn7)#eZx%b7?k4UMsnAIFkEt@&l6VPqoK3;~TNoJ5`%Ogaj;>=MLm-(E5QU_X8f-I{
zD>*UEgP-&DT_L;X{npmwo=58#HUO+9LYhQIB#4Oo1d;VSx*-)UR7y*+FUG!+w<rtk
ztYz<kS{IJXmblU-c?Mb5gR$A7G(`PLMn?U~^8@MZ?=bSGaFe2*WB}G50%*1={e_uv
zXTC7HqM`zOc4#sj%4w~nL{p!tas;U!t|vj>381R<5HjBg?`JZ^fj1!akx_gKnjVfi
zd)9jY%^It6)x-V?cOrG_y7U~uITR=t3IvCZ>{9yGQx2WgRF2{Fg$I;XdLi5Bza{U`
zp^IDdr0}6rhG0I2n>75fTBMv7Y_9<1yVwacO;(D+1{=>hyKkILGJc|~BESB3;PBEi
z5zbDrsjoeDWYF0}ae%|t$Ehlh>V~7&V+X@)M;|)@abr|~Xeg4uHZ&Ve<K;Rn`F&lq
z%sE@S=KWC`PsgYxc6Qdx*@R%-oB*@*CQD-<_U<aWopFP<HPzJ<!!+dF1e?0toZZm*
zZNA%xOIBj&xq4gJdsv3!;w3h();eLa0K_Ybil<xFm0i=v3vcOy;IlBJVDE&0Nk{O$
zhgo)l(|e*zG7V5u$~yn#6)dzIL^g;_VIolosz1kWpBh`gDaSc{wUpxDr#VPg;<Yeq
zHn5e(pa-f_wwL9Re6_M1;0oCIvGx)_R|q94yg}DbZSr)sX;WkG6Qf|S`w^L^SeTw;
zswMsV|3z$k3rtV;f&Y**XvmY)L*uT~yO42)3kfDe-$IGXNPyP$G&(vO`*l%IVp>ML
z6hscQMZEQ8kW`$;=}PQ8-o8%x+X|Fk_jZY-1vPI56hY=&m)~`ooz+;I`)ItoN@57K
zTw<J4N)r?W219+inj4};Q)@7z)V8uM<3G_sq_p44O7?=rD~1>j;FMM+A4Pa}d-^^o
zpH8XmlVI7m7Z&9!I39|fDYQ}^Mv%dCT&Jf|qFfAFLF<#4sp9Z=Q$@Gkpq!a7nkQmh
zjrS@+R%gL`*I-F9N{tTlXWL*Ec9<pXi2kKLpM+jbVYAuY6D!Ba(&s1<8VP8WlO`8m
z5sH0Dv|0Y;a~hNeg99)Qxk|A3;nZ7WVhnNl?fCdOean_D1-)#7LKb72zl0%M87h+|
zd*hmYsa}9cAlPx3Ackzkfb*LoP#erG0o{;qm$P|`xyVIOg)lz^XEVC&X}rR7C{%)C
zyZdMU9=fe++}ILks71)y6PA=d`{l_O#!&@*j9zfAZZ{BwEJ(8uKb(_6wrocI?)?eS
zeqeFixvz^%c_@SjMt=X)+hox`8ER2<fV>|4T%*jb5}fy}%wv7};XgzLKI3_^K7zt;
z&)&U)r`6!|?AdI)vY>fCa);&yCmryC^tNiri#jMDTp_n&3!fJyYkXRc&(=kwL(IB=
zx(kD*q`Bc$ORtqSG5TZ6ec*K~wyd*!2h1tLa;v5<8-$+&fHz=+NxTxEhQH^%^DDl*
z=j;CUXAKOC2}8U7S4u}Dy$|BoMef7i=H`4mb^vpDef80R(KIdIx|5He&gKGsfKC(w
z<y%RY-qq><v^;ag)&uj+8nZgv>c0<In2s`#w0zr{y!<htNh8AKb%&aF>Bj7ngJV{O
z^6_BBH*SP$t*5VFzh3A~#jl}!t`y43Eiu}@sj(6NlP{vo?a2c&e7&Ke!K8>0!gRz1
z1q~0QFpzy^b#*+;?+X6!lSfjLdbV5cdo{D#tQ)0fYHh(jasZ@e9@Z1?Xi!I<tWO($
zQp9%aMzwswfQjw_dMO`H<JEiVo_4m`T-RJ6NvY8h-Lw<Gw_Jp1?I^-nQh<|Hh61n!
zM8-}|uCqzuN7<4=A2@L{j>4>Y+#MvXa~Zuj__3ibX1f#H^3zA>!Cw#PRQ_FnPGFmw
zQ6dD;4{#9L_Y#BlWy=Glq<d?88o!434LMeH3D?mL*F&T%3v=Y)U~4RY8jtIVbdrIJ
z51Dc29bqzXWrSzW=>H8jg(+F@|1hzCXtumb+)y_Q#sPY)346Abs1mpQVoS_7_s*>@
zbEs|phU)L_trB`zU8w+CByk-8NjyE2($b6T>&4HuIw2q-@6=l?d(4BnxbWP+?8Ez_
zqN4QX<cTzO{rMnBTec(zd3QeyGfc4}=kEYzGuIl(8?zI1yVT4ovcgb)8y*WD%W_d7
z_ErbhQqmw;^4Dar<Q=O^4tTPZ<Y$S!Apt-}Z?=k~OMBHm9zH%GW+Hw4Fz6m|?l@mr
zf-o61yj^och9UmaneL`63-v?gc!vnXQfYZ45X`J5u@yLfK>v!fo2Z9~6237u!|7wb
z>36OI(iyfYo$o<QUQTo~zX^c@5Tr&w(?Vt**LrskIMS@W!zM}?3$Qq5(ImiRD1#UX
zSTa<DeO*lVPx|qWyFoAfbtf}GFTLTiJr8?jl!HH)ZCOVNTwTP^C)Yl?vLvQUFgD2s
z6+W>Y9#&5PT<>y+PQAU7Wvj3m5KA0p8ZFd$gT_CB_(N~Pk;$t`9i_E8Odj?eyEsVM
zn601Ni4{y|RyD#GI$M1V&NlcwtOZN8n+MFwGxGZNKsp~9;jxfr`U_ma((v-~@=Z9_
z6#}lRulDvervY(#v~7>v4CwA%?~DkXaxV-=GYTm4hG8(B%P$i8f5<D&x<NMb9}7hR
zPgYXwO9U7b;pgMyBh4K!y9%T+6-Z+VN@Ec^m>L@ee4T7Ifr*3T2YY&Y1_2yOlmY7Z
zshavGMyR!(Agk^ARJ!Nl+UrWp_Jp<q%ECdY<jrNIcvdeBA~#KR(J&IFF`Mog^eP7j
zm`f9x0<+tvQXcXa=l#vC@4}Vm#x#JXjB+s<I`Z&7n-0w14+jez&4(->s2gEK7cnIp
z`%i_Q(YE(0Zd-WODyHu}Lj=S8UPD@^)EksWW~2_x963BZJa_w)<I}g@g}M3pOuUkL
ztHCw`^6(5G52F{&X#6}pvrj_sgSZ(5n}|;mxV>)0Ge@)?=lcuu501Emkp1X~IE33m
zgZy)oYEP72t2Gfbb}`F)b6ru_kNWYI-}x^|fsGvpi<LB-BvpFVp-k^^{Bv+tTOm1T
z7CSRR-(XKjJ4KedDs^-)r1Sb{8s;3=wE@+#XFH773YhHsC=m@Q@N&lQFIMYxf_lKI
z8A>j1UdPFp(-_hXkcKYenK$U5wtIrLpJNB`($7MwlQiiojygbI5x}AUxZa`67c35i
zmK7G^w;^6|T$T$|yKw8oO0mWd^lP*)%SM}5WS4`B_%0?{caBhs*eDGZr2qSv<R80Q
zHW~-Vhu@hv3K>JD@P#RO_ui-iKL?Rij2~v-o?y`mzE(C^_o(yMBVXZMD8%=DX)mz@
z_YE5ftgy%(&0oM$_l*It0fS{nbrbFL0%g@2s!+Ng>x*BKG~=nlm|vFe!u@P9JvwIQ
z+$GO=!a+nLFS0LjWBnRf%T_OthN*jvavZ_CMjhbP6PNSzSjUclKVBicD4NZA;W4)c
zmTj(t=UWTeZqtH_0tGqpi?y&rxypQ&xo;r?qc|2&x)zpdq@;NK8R6C#Mebi+Nb>zz
zIMST3GGG1a8rVHoWlQgqyoo(|LU823i7J=C{q(&7$~#b^1d1^KU*(}tp`!?IwcGE~
z)Mn(^<1_1EbuWLX8fC5xH_=$BlP_FJ>tU%IcY^RCwMmwR++=)iYZ&*ZDm@qsrA%3-
z-jGU&j94XlG|x%mY=^-@p9}%XhSdrG(e&FtwxR!*PoMsvShMoHDL?fW-%Mpw$YX9t
JDTn>P{~u^3tzrNG

literal 17190
zcmeHv2~<<}x^En5tH`ySiptd1E!3lciiivm>V#ob6p<MP0SS{p2oNUgQI1tXcaTX2
ztwk%41i=^}j76mp2q=;;CLn<zfrKdp2!Z$C+xFaZ?z+8q-Sy6UZ=Lf_)@nm`_Wlpw
z{~Lba@B8BQ6ZTfhpKbjNgTW}{jvsNvVAe!qFssggvKCzFNq;Z_-qxPCu{r|Y!E637
zGZDOf8g|?z0)x>^M*droIy89!gZUDJJM!)6=$x^>#I!!5hj5Zh6x;5-h&xnJQ2Xra
z*CpRn9x5y<`O~wzGq>5cKYsED`i-4uoV&i*w4w5w;tmgY=Sj6)uML==?NV~+ZS&L7
z)9Zn*4q071$<Vm{^7*=FGrD=t;eBN7h+27L?^hYS$+qx3yE^|~<6sVHDvMMrlgH~O
zw#~AP^FvpnEB*WYFUP&mQ>E^D#rX4{{kkStTw!7C@>Mm@==+V^Rv<Gp`Id86g`EeN
zZD0bfG_a$v*81xSWCeA&^FMJ{p{odsTkVPbk#y+OD)hqD&!f?&KiiJpiur?e{~F|r
zZ_?`4A@3)?xvYk~-#@X^0C~53@DDsdL$wwLlQKMfFLX)8$w|S4cE)1YVNY=h@qI7V
zycX9vIdxQP0d2eQ^+^?GhOQo3HTkZvaOu>dq3B0vr;b)hH(ivATcUEyfFx$tEk9jv
z=~X0(y<oarRVXr*%OsM?BfA)P?%Y|g+#>#?k)dH(W#b@n>lc^RlFpkiPBc`AD@C!`
zwM{{a1!>H%VSblr5BPooPA*1I;lG#uH4+w6_a<)iRK6=clwadT6)nh-O*}0{PuQ~d
z^XSVp##n6p1(+d?-2etpO7OX{_P(Tp5dSUBAAH7)Sc&`Bd>`Hu%otjdOC-|GyB?l$
zz#3(^hvR6C$OARd#LA;FGjFV>`>&|X#;#9)Gulx^!Xm0Pqvy5{8RMT@PyPv1{qMd7
zWm#-e!TvpaF8O(ILv5#E$lu9HS`o!m)bufli2+=yhf?bFvyMl0_t+X*T1$5zAA*Nh
zp06EzPe)g`F=giE)2C0JPdiC3Kg>x~2m}KEl!*O^BK1Q?CoU6%g2oz0E0>PIoL&;a
zqxODbp<1y^*|77g#zwA#z5T(2VxgwCcI_^fuLqYuo8aVRrI;=%4E%=m%bk=bqDoUv
z7BUuQt<HpJ?1utAFl4=~Tg+NBQ_}^Gp4;=Wim}qBJkx#qX6{r{x!dP0;S4QNMbG%`
z<p&QQ=s7tB7l{_g1drttPEKyALmJ?(x9{G)W2Q*E(P7tZ$*g0ZfB8Lk`<5r7g@#k7
zPK9QZ66Lew#M-e725uHutRf6<G7j#JT#|CVmOFe*WV7Qvp|;_Jy{-3*Llaz?eFq=S
zU%PfKWgK;?7^^+ssJIltRbSyoD-%Qm+96$~Lr)=zHGHOs@Ny~z33!;co0mVS%J52<
zcfr}?U@U1i?qZG4Xk`s`TJ&P?ueW4g?)>2#zb0O|URk-Iu#iL??<R!R_*_d)mSksT
z;hdaWJ;d{-3FqSC7Wq)j8CZ!NS49=!A3b^$f^*cKFJe&`yoDd{?sQJkT3TvuZvJIJ
zB@qo2_8V4RQVK2VFx>`LSQGKUr?7Cr&)=l-T*I{sDm}T$RajkC06a~0cX8@qPfkv1
zs_ze)O9>;DpDB{^t|+PSNOMJ0k#<mQ6tb237RpEZDZo+s&(9rdv~_UM&UB->(hEE0
zPG}N5EYbyqg@tTQ@XE`{(FmlwK6Y{<T!247>YQSi=?6pEO2{CaHC8=6dKgyZdO5g^
zBmu>v@I}08lK<BHpL_xXM;99S%-L?dw%t@cSWgdmG2Vk~fE>GiXHC4k&+0&jkX(#i
zY^kIsmKPRw*cc;@7jxo!Ams5VIZ%TKP=)scA6E*;eRW@L+rjFl;PCJ%o58_Bu5Q=6
zKzd$=Z+#tU;HuivaGZpnARlMwx^Ub6j0|F_t^Q}dt!Jq2F<$;LyhC!)1{Iaom;73f
zyeFF;7Z%py;3#H%6wM!X@vry(+hwANLs9XKvgj%HV9sut);z_PM#HC_2z`4~qD}+l
zn`Ted=WRg2!yo*=Tp0?RvhpDuvoE22K-=Edw%C3u8_LScnlBpC3_h3P5!JN@_sYy7
zp)v!BOBl@1mOo>o$%6j+D~Y?!%@<!m7KZ!xM+pm^iV{Mb()CEjp}a-xcoYc_e}SXs
zOTP6}RaK=GiWaJIQTF!so|UH!tLI)C9Q28fp5p>Ik?6rCg6jwKAa&yqbHdB-zfdnK
zTxeoH(9X@yUPvye)JG-_3*VesM`}<uHa7>80o5}rDy)o+ja?W8$MLLZyyEad3t)}k
zuc_f#?|mB+pL|Q*IIl8%=(W{wLI|UW0s+f<3vN|JNI*Ou0_MjM|C`p9EnC3Clz~h5
z3I}|FVXA{}%!l`hkC7QMWos|oxCXJnt<*|ttr@Ea%xKKAh26;RgxM~CT2JYPCb8-P
z#c?h^phgDP{M@27xaT>xGmu_m%tAJC1Wp`no>9T^Ky+WHtE-!DLG(vqPTkYZnP$;K
zTRXdWEDLW&JKFU;xW@=bbs3qy7gBlA0UnJWt^n#Lv7UO)Ms3=gwf>#vM@*%w`cH(J
z{_cU~9}sw=xDk`$+nA!6m??0Ylu&icvMQ<kMAr6<Dk}F1z(wlI@Oq3TzjA)~UaQ@d
zgoIV920Deen0=b67`Wj%^8-y9y9r=11d+EfMk^1}3kwmGX3xw-rs|%rxRsLP7Z4ET
zU-8AxGdbOZmjEbC0SMf3qqthe>x~|~Rjqa9!t+amuW}NX=Bu?<VeW^<{IV6Xf`4d4
z{x9wNr%0^+pB>{*1D+}%_CS;~A~z9;;mf5yjVXy^0|$1MRZ{cYL|Xv($Oj(B(u?$7
zHNR6aI88EAsy^xDM6*H?9lslEG52Y=Zr+@UU5$Jm+Te+~zuAl^Zf;Pq=oRfbd(1zV
zaE3$k;v;$5{ZM42Jy8-~Zle~dnCnw=^9;?iPqYzyMeF(j4ex))<3v(Dki4G<!P8&d
z`PAnqP3S!jK)3XUe&vXIb>1q>!W|?@pP_*CP3k79q*}afI63m5lBzP+`E*cMzh>^e
zh=g4SoD-U~#TS`PggdI^!eeO3z!M3;Ucr_+2nvc6E9)h%Qra*Rwl%$HWF&xXBFuqs
z9$bo6>MBe;t{*T0--n4PjNv<0S|5;n(XiFr5P$~64=rTtp>7ECy9~llIx8jLuVsIV
zIIFsI*`HMLk56$R;r_4?<O;?VTzQB~z_I}RfL!=(1oWzXpxvLi7RO6#%WA%1>EX0&
z0p!7NEeH8K!e>0WUMGJ96_Ic1&Ibvr+sTUzD0Tb11Xn}5EJ9&Z&gtxK({qXN6KMHs
zRt#<_0&X<RR|LTZk)8(ElS|Q~(#RONQ2q*w<G}(3?4BcN!Fk^9Vn#D`vh1hixTRez
zg>LNA>({RviYgO){01jooSfpReId1>GI5`FHyzKdA2bi9Uu=J5n4FxP2(iw3a5Ma)
znAg#SC-wkJn}mJt#!k-Hv<*Myr(GEp;&)hLY$!kRoB>rBux{a&F2I~gckf@5pIF!!
zs`!$%<756)>t5#9;tw#7gGt_WN@}VXCy*|ti54m=sWj`UAE9kdPI`a5ZEW;$ns7IE
zUC($86lynh6Y?61P4s~spctr(G_Sw#ASeSQMMQw9ZTto&Bq=E=R1{}3dK_%Rtvt9Q
zIIchXS}>zW2@ZNvIQkk=ixl^d&%2|dW>M5GGpxhU!73M`Y_5rwsPSLp%*s?Q%5$5q
z@$$Wzan8kOCo7?C1dqgMfS2vuo(Fa1{>Vch_rr$|MP!ZR;qypNXu-M62<0aXg3Pap
z5!^Fd&tkEbxd4b&bV~M{{QD;4FQD3-E+#S5X6iLtzMLs=hI7)p#?|v~3yhh=q)^z-
z%7l2gYh1;VD|J&@4g+{Lb$g66{LGcI(_UbF>E;3fg`?h>eVWn9@M3_U_M_S3Fj2>(
zXu1oWaHfET<J}RwVarM6#VEMNLpYymRr$*hNVLwtr(oaKJgir?Bohc(v}k%sqc(N$
zEiD37mn-K^s(fm;!NGcp1}`4-$NHPH678nM%^|H}kZ~j>aqhU1xQ8Lh0bU3|c?s2n
zxq(e=o|u^MFf=p-a;>n+O}M^m$eq}Gsqtcad;3n|e1I?R8p^hP-~K|g3go57V2kDl
z<s>UV$uAyc!KemStW!p6XPdmw0S@ja0+HEQQ=SwChVXmeD68+^_r-Jdd-&xgMdHDI
z`y!VIXplwuvr}jSR}k=ZWKB6<;g4H7z{*ISc9*y|e@pOa?dw|jH3i$Om8Te|f-LN=
zFV8G6!?Bi)E@k~SV6ND#zQZyadCa6fLh<(N)E@r4H0|n!07eggp;`>`{oy0mOxN1%
zcXo2}3Ib}BEmR2oX6qwA8s{TmoYlC|wQS~nkfpV?wVi<7OWO8O&fp3D6^RcNVe%L%
zLBy$!sgZ@&DrS4g#b$R*5D_QsHF~NfDSg!kxm1J?#pe#itQ+?+GdDL^b6Jjxc{Iwe
zKqg44Z#&X&2RnJt3e2NC-54S9U$P?M|CmWlditLjMqR<$!>X5Mz0C(tk4-{}Pyal)
z{7f`AojRROfGv>3O!26}A}2HRY_7lvx7W~c+PN|2*0<Rxq~DCOTg_|n5D^XD&sj)Z
zy%MU_u+bC!+R#Pa`7A(Z@Z-9gJfHs&w;#!;2p*E-rC+n8J&e#=`Gf?dX;p!7adFFY
zb91*ENeZ2hi#Y)6Xq85}a2F(-E-h+$i0p7nBhmV?f7i6B?>gzbk1D(Cr|V|X`R(+C
zppA`*Wq8G5-SB}YPo8)HV<o~2XD31^oCPs-Dn`f3T0I#Tn$r7p!xrvPxovS0XD1^`
zzJJ5Vade;Pi2Jb=e4ppUPlrd>)R5c(gHV$vpyzUb2<mx0o(*w#&CJ^w_dV0a(R)8W
zqk&T;P8@tk_pJEyAd_)4I`aC*{CD$NAk@}KN<KJ>d=7o--RG_7G3_n^alnnq33Yif
zS<9o>g#7jWET}zK>Yy{vaEV^-E`r^=T(pmu7F>=&^Z~Qh5ghO&WYHm=HiP5#vmA6?
zG6WSk&#W$r2CtpaTWde%pxlgF#qQhJ)J-K;aFsqwo0w3uo28T*wn)>N*H&)%xQN@?
z<yVO8d?+TiCf<S95NtQq4At`cHPDGxJ$V9h!rm-F0qz(%=bm7!S==bMgh}B6G?Y}w
zq?~NOOk_vGOj9(8#H3r2f2B*?zWk3VVRr8o#$DgT@KdO@<$+SMW=p$z4jLl8z4v_)
zoqxyI+3z9h$uP<RUr$i&ri^j+<FS&6TyD);DoTiiIR<_`QKOUmUfhSv%NJZ;OGWn>
z=7@j)6`nTYy6pSH16g;{v3(n(^WV^y;#LXK2fu7t%_A{6hRQkG==b|nqn9i3)s>g0
z(E)Dzt(4?PQ4E6n7twA1m{mt|fb&O+`DDS(wN%s@hP_n}>Cf>8;TJ}uxtGn<E67+k
zW*06#O~U8b*`iObdJJ4a2ei$@B2_1*aZg;WYd1)2z-#R()n5fuvK~lWHm{tcid;`>
z080(U_vauVFdDBmDWiY<rAX27Vw3W;%U!_?v@JJZ;`s;(_PaN806H4Xs@D2$2n3W_
znpf@8PK3$NTXSFAXiIbvWMCIgfF=3DG839zc$B|l7*axIopfluvgU#fV4JETmQAd&
zay4)O<Xkgih)rn);P_`AK6)Oo*)H4$)|+RSlgdxQi>^ZHVdWMeCd+disz;A#<A}DZ
zwqnGpwupNexnJB5m_W5ij+r4zO~P*0P74*gh8_<m9X14FJ8`?vbSA?>Hr7QBjT=<Q
z=3MhUASx~^1tbW>+VME&?daqIOgz9gO#S4g?iY>Z#b`JR&T7_<uI;7sZVQ5yn*mC-
ztgl&;(uC=5AqW&J0{|r_W*F>#)qR@L)1wWo1d3OFS2+n{GC8fD3xARKW!ZbGRIQS;
z4m(~de-P-uOK|{|?G*$|rBS6dK8xbl+jCG2@V;-fF<N`vl^B{-({@IAXC=qW<F7|*
z_Yl1ds+a&z>1IUz0U0K#zT-%LUpKwRcVpx7dG11kjXCN<>*|1xCP1u6f5j|>Hxtc2
z4~X`JxkOGXR8jF)_DghtcbbaYVPITNxMP(mF`h6{(8KU8|GIJM1j-X9kvU<9h1L3H
zZU$9mtPi-Q_a&_=e7vj>GB(=xaW2ci_ZxU75?r_^tfDoLNx|i%3*wd8sKV~Pom~km
z8;hgbPK`oB$ix`8lXV4GlkOYZz(uVBI3^{YiJTl)Fpc^G?@#wt97V-}QU0w+hUlM!
zHFl8MDC)2=`UGtJ1ogp#!?FTR^XQ3Dpg9M54VOl84X~ipqa8ip<W*D4y<%wrWUv$j
z-%*{bL5pgFPaUEU$x>_Gxdhs=2aWa?iWyUu5dicHCxIth))=ZfjAP@%W4R=+=CLcC
zh<stFp`l3{6usa9INed{`ZikjoY!k@Pr)Sz`zbbLktIlhoOMh(>&(H;mIG1a-FM^V
zvr?pXL(X3mm<vr>fwr{CV&@aDV$%dexDe~`qp%U~6+9>31RkklRLFRwGNOd%nOeUW
z(6QWYmK(37BpiP@3em=kjkj-+4D3P{f;swu0HUt}k&^h_`7nAV?UDA%P+1LAowOfC
z5SHf_zM4phq6AY>IhGt({980r)~7h#`Ut)Mo93`pLU-GJTGrvNa0`ytCtYF+El?Mm
z)BqIkG5MxOqwHXoujntKVd313Yf&)YI{c(R>JjR`-gik!;)HXxGV#-4ZNt^T0ts0o
zz<UESLDFT;k$y$fUosBxFiNOnQm*`7O*1?Cmu{u4rjniN2Xv7kET3k7@|1Rb@7ubV
zHK=P((g#t?!EgTr)i(SbSFQxdeTB;v$jQYc3u6(uN9vjA&b__C@(o@G^;SQhp7tkK
znI^vvfluS|?h2}Lc^QH?kk7^#R2w%@uGjTgPt_WdWMUy~Zo#T=NO?kz+rwIbv>n26
zeupP2n|U8TYW2zWMq%uX;Wo~Ci+ssVW}hGI-r=%rIaY>#T@nBmmM?ZqJO>PqPnA08
zO@2J5RYky$Jiy2Q$RrAzBPm|x&mJN>!>p(3fYZ{d!~+1LTFH@TL?h=9ga!<_x&p5#
zJ*u&+x-Jey<Vo8==ve}2#RHcY@h6}BTtPN9pWlTl8K}$y&q1<*gL4TxS$t^z1Z+cI
zEQ4BYr+$DW4pZ~MMf+G}`&zA<M#jgi|8Za~+r>W$e=Qa2dKV+Po69}4<m|@?{7F3%
zwLkAKjX=p^ue{{hQ%G%lh4hogzPv7GjImA^x9!(0Jpph=_zoAfef|yPa6H@|HZ;oz
z+FS$igC-WxSyoY4bv?mEpr4+Y-Izl0Z@mD>m%A<lcaGk5|6CJ6Nm2+s*4^@9F;2GE
z1Klk+z5gm&V3K}*M^N0<G>fWw#*JuGB^4ERUwU&?%}y43JE7xrzouRIDOhdS`%WZG
z_$#=$kk9}fWV+9ug3^&#Ti4XU{t!TkK#6I>5of#cdij9H_e?W#)ovW=vJVm{0s_$d
z_Onu8zn-a<%bh2<1j0;%y}ey;JyV}G%w8D8C7738BI-;l4FMB0hGN`SA&~`h;-_S-
z*l%QgL@!#ScNSLU()#-K-JK-tF+LugpgVU#>N3afGx@8S@J-h$74zFYAxsM3F`Dxh
z-aY=T>P8#zZ)!wJZ5y4ccA(~W!UPoplnVw^avW%$6_Z1uN)NdiMOu75S!82@8ep9_
z;`9Nc$kbEIdrMyuyiNItuF5|`yo_D%=e)*T6Qlnc;-x}a;$nWDZ?Hd>ze%~(W@>@G
za0fCr-i=OHX9<QB<HC>n$GgHnWPfBG=k5VFrNqh8*E<V3wO^wq9`<%?$Ve6ULX_w#
z5%fr~yfkL#2T0?dcLk5G`JtA#`8bfJnGlO(+f3DO(<n1ywZm1kdJ|+v07;%APOvL<
zS$A=sxq=)>sB>|9%x)GJ9t)oPRlk5$k*wf1kQ4lv$m_`$?94@V2V-*|gwdJiwI6}t
z)^{(os)$5N(ONdTe9P=ltGH1V?!K@DbfUMm{fcR-^uk{d7~^O|k(o%QV?w)(NAJj)
zL3~md?qIfn2rtIM0Hjt!%Z4lkZiCj4TxPr*#Y=T{7lEXWSihW;C<2A_&05W<7)m~|
z3wnbNQ@1w&#6+QSm_7P&Lpv~X%H;_3IKnpnR?8nfBI)lVA67Z-b}4OI+q@o+imGks
z$STS44QqJw<~(T(;X5w+)cqU(9pCf29V5yH0~Jf(;N&H;V<}Vfal8^=Rdk~DB?arN
zXlC`TLeO@Byl!}Z^w>#<az`|){CBZXjDgW8Q&p}?)%)#};cv_Tzu&U5RV>AnY}S@{
zHnQ<_S5Vfb7jnx=7C|vL;kO=(l?F!x=&od3LJ3vmnTm=ozG0Jc+Iyu~gw{h!2E+-+
zv}%%UxIKR<5;0qk=z*&)Wqx*O7Y1g;3=Cls)^7<~MyN<sCe4=R>!Dpb&0rvOjK2Wm
z{~x+{{*RH&$Z6UHpFjV;8Ps$#1>14#!otxAP!*165+zG*2$Q2Pt7v@n%JZaChf+%T
zsFX}5=bPN=Le3zG+`1epS@Ib_C|O9V&5vNyX&z%gSq5mnJp*}cKDYIRTl*(JoW1Zp
zUY>{V@UZ?i_)go~%YQs_lKXArizx@AS7#oa={|GS@uX{EX;Y3MPxPlBD1>vCdo>DA
ze%p}kS8Qk9{xj!u-RD01=`j^;rNKUtm#lFzf-Tw}PpA>*2<17g-0KGu7Hjx&tSVuR
zq_?+M#&ONJo=SljP?slOvE=wVIZE6+A6Cof*K|c>IGW=8^ywY0O>_>5S!`+7E50Lw
z9s>SZ>-p1g2<(-an;S3t`RAX@lvK19pW8o<@S+fiu!?Ss;=nMKzVzbl+qZJs_!z2y
zONpuxYb5D<ZoIq*=Z1u!-J7fM=gtirfn#Sz-st-I`+MaIY~co_4I0re4-AVVO&joL
zhN#%?+F94Rg^M|#KB3ph-y5rK+5`v%1@7&56hB(C1y%2`7M-vAv2k+~Hsl}&*grph
z_UYYM?_9MvW7%E?5hye-etD$#ifraXyD`BuH_b86JZh}7>+Y+W*y>gT1B1tDQ-yn4
z6r6<G<w0D{HdZ?>;W1~SzM!DMVf&MeqQQY=ZC;m1`n-5%(m5Ory>)`DKktlY_K9!d
z9P#mS^gRV>{>PYQ0H#&HKM`3ytQEhpM+RltO??lA>KDY7@+FDqCA{jgjG~8Jks=VL
zQA^^vck@taw%Y|PpF$7O96!kD&OG(lKG&qS7Sx}`ek~q&MJ&v;*}*q8GwV@bzcx37
zA%E}q`W`vQx|Aqn=W18KjYZ|lzL(}DNlch2kEgF_7?`*kfsL6GO!>7`KYKRQ|8h`L
zySzl(K7YKHZ;>Y`qjR`O9+LjhD)r!{>!w;ZQzW%oaH>>Sk+J*mM}Yn5zKts+VFwni
zAGhORw=?SP<59$$QTbluo|X5oGVk5d#e9iHrofWXQ@xE<xI&VLB`v+QrfFLwfMm4K
zc_9l~R-GX8&u;sI`YNS15BoA`Q7XNjG~&VgiW%u%BAn(3eEhYB0csHzMV6T%sv9S?
zOu_|CCPa&#-mnH`24&wPPPYT>_<9w$srR8zYa*_@-I;`fyxo03Xm*a|Dq~~}rhY<1
zLPCNS%-GJFnwkoAH;B+ib_glnv|E(Fu0%Cyhw(-kNJ+~4TWcP3Mb(C^d&9%ST6|Ej
zH_+d;E5|%?q^${zbq8{T&CB1aWwo@nE(cJg8RWR|-J8eUXcaOEKhcNAm1jA=rggY#
zYp-A1x}B9v<jwy8T8DaFtmoSTMnd)U^x{L}m(k|aB%JXM=ZWD5$yy766sZ#V$F%U)
zC#N3giDYuZ0gZ&OSb0!>R&K81fzt+wPS%!S(SBKTv#l=6EReFija8;dsGXH7Dgg(o
zT9CF47bWPu%cObrbfFsE9&mmo4{~vJH9do`Het!FcR1*{*iRjYnV`9cP6EdD!jD~D
zUOpGe=mAZB#B|35W0qHHirgJmNpno_-n}`FKfjZN23G2;7rj3WQ~`Z|3@^S5V19k$
z#tm)Tsrp-)a?ts=90w;FccW^?X*uFi|50h}R)37aw?6;a5W3I$W75?2%?gy%vV8cj
z8xWcOOAi0i2SgA4Wf%YQ4~QQ8llaL(WARE(&*I{uO&~>LJ5>o8=oOfoYHY9wd;j8k
zs+L*Q*jV{6@0X8}_rt!E&J*mGmiSPL^a&(64w&+BI!9tq5ElSIjEYRI7$3|eZTnb~
zf5d&Ux(f8n%3nFYehf#F7k?fb8%wG8ZridUnV~a3=qeIJ+Ptv}<5GP@p5B#qwslNU
zlVa#*JpmQ0{|Z@xfOv<1M!fc5$CW%df?;u-BJESFttcs(kODRe2KefgUCv*Y(tBie
zRkDh9!bNx-HwB7c2cN&qbQP_7*f`&k{o73=x0p%Nu!Wfq_e^u`K@>*!ZtW9W%@by1
zWSp{&=>RY+kDy@XFS3kx85p#>FD^{CTJWp0r!RW!*bWMln#3(ZrH-%H$w2{lNjq~d
zsEx;*^HQ-?$vkLim{_*>A<}e7?(mf;=>tckXt8i%_EDFYPKAJyHW106{uR5(b1Ea!
z*x0&1REBBRjRIga9>p}~xSsl3WI9gSR8AH4id?uXvTmXtOGN!~8223X!Bh?Dg>E5k
zx%mdP$u@ed^obn2Wjwm#l$cJ7A1}A@z8O+8?}zn%t!Ww=(7S#2F$OOdA5tylT*|25
zTs@#1x|#Ut%{Su=eqK*#ZG7*AI>N=TL*AUgW0$w(2+ycTjJ|tfD3pO(e$-3nYU%~?
ztz!IV@q}J9so1?K(L}tG>aTifO(-n1O6_r0->tF1>j0Om>!2U{7Fp}E?bI)y(Uz+r
zgUT0|$nV9DuY2&6q9vETZ(<A5GE|Zq3=7f>Qka`xy}w7UDo>j_O(Aq>XYa`hq_|n{
zFt7>V%gT7=q2e>Chl|pQ*~@yehx_yJJqn#?zR<#W+~~31=V?(|DVC*%r?h%S?Ncq_
zR92?}f{u2y&X5vfU%H*82D`rr^)b9;pYF(WvwkwDFNihzPqA_B+aAMWf>>IN2h9N%
z%(tS(RBekqUs<8<?yo$jv$?x;Tr`N9h|Y##vwp?Jiyh_$e2_hU|A0W_nrW9N#FqVL
zH5eqR2^aj5VYoo3Ru3s3c#b4Ba-5@ejQ8K%^t&iH=zXuc3raif9@5#~-_lj)&nNq4
z3hW{xB9=#~OmTNQl)OPR)`cQfH6b1rE2U~-7hb$<WFx@0mY-MyvX;1Np=Z-mWYL(6
zy?a(j4y#~iI$T96I=CG}c_3H8awRem$P*61SlmHFGc)<&A`PD1Uc+t<L2UFfb1W&m
z{%JJX_u|D5XDQNQNV2oQJj>Svu<>FZXpRwIy?RykudLsB3voqp%j95Vs%}><)C17@
zWG;U`_viD-5;8vrxho1U>-KH@N5q{!@ygeEE6ioR5sc*l?WKW;!H>n*@^F84&iq?V
zV((-kb?EF0$?6>i=6l6kkxeHR0lzTq0586#Ylt=}-`ZKQ*bXiG^?q!Cy4U(e)vX>t
z4?P6x#&@xf87jxG*xw9tIn9TP*1L{N(ZwB@pdUbL2Xz_8E~pix@wNT2n0pk?!onNz
zb;s9@jjrqKbc*P_EMF@mDVxr0>N=5YFKnerJ4;gtn35G*@iT7{D1=KhGvL<+WApw#
zdY^c!VyAxD#vm~JEYz_c-DUm`iN$MH1eJng<Gv9KXUDoCdd}Wh(di2omAH$=&}D&k
z5VL}`*sQ~k;9vQ;-;ac&x0o1=esE08{B_`vZE$-vHYED@N-;r?y1Ke(xnLC;@Q<8@
z$AeTQ@3Om&Y*`&Gg}Etd&s>M&YsO6`Qg8dc7xfU(FAUi)Az^7dokP@#@qjbZ9knz!
zBx@hI^!|laD&XyBw+iL;5UB8Z!5+98M{ARG0q^C@mwdPJ&Wef(?LQX8&F$5m9If}B
znCRfFDS@Tp%dBov*Rgl2k-v?>@4NWjB57X5&jZU*yEiZPUNtzoZsolnH4cRJZ?E>0
zXhT^=bS?W*LsqED+A9^(xg`1w_u|=gQLj8Obx_v5dvczu=oHd@lnw!3$M*t>B|Fzl
zTP&0J(EMruNzp>*80gMRp-}6oV4NShs*1Ca?kY+IY2^*TK#Wr&<8~B~eTN<0C><*D
zB*If3cA|<tQ+liL)Bf}7ouT?&%le&5@|Q}kU|A3N678`lX5vYF{C!R=5e~K9d&=5n
zlK;dRucsima=sHyDJff)g5QvZ?kb4;BhHv*(9CAnAPHs{_yI?P+vD~M8mH2Xl}xT$
zvKEd>V+r8gjLC7)2Q&`Gw%ZxU5AjG6Y>!07xa+uq<C=)YC4&BFi+q95`5s_dRBdc*
zBwm9%Ru%_ti|v0&eV!3~%nrkPCA|kK6(f=QllK@1D|D36qnL&o{PSXl2a9f%ZCbtm
zZP)?x_79Pptasd?bNEH{1mcdNUt*i9)0a%`@ZUO5tSlyjw7d!{q<%No2?z@=psKDz
z(vde09gq!}va+CL6^#QE-|x(wu91u7Mc{~;o7B|QGSbuYDnXUA6?$@|zP|!oz6vED
zJDeCoFN!Ar*#2QXH!Kj7LEng-IPcukwEv+_l5d>i5$LMhfwSHrL&E>Nz`^1*mv2P^
zZYvcWvGFc_{RCg23A7LfseqAQQH&h8>$=aPS8%LY6<+iI2?&ucw0SV26d7+q`~gJb
zKl7Yk>0a=3rfw~MrB3_Vg%;3>*^ZlPwQ^V<jBQyLrs-mnm|IA{pQPrksc|rFt^oI3
zmj&>OywlnzkfL%^OUM;P)Pi4Ycy>1CyU}7+P+yhDdgF~&kT1F`5Mp`8qPfCgPi)|&
zOOu8oBZE5Wd2ssOobSZ->q<7*=O2te1&E*>L#&RPt7TkKd_qi&NkN)3g)3w?H}{^z
z+rf=MbOuuV?$B|W9FR<Auz9Y47)a4}cXeGj4Oe<fG3^&sN&R2QN-s$-crV;6;=glE
zjGdXIMa)Zc=wG(wATn`VSNR>-K_s(JoeT_oF3`t%hj3MAmrb96%({=j_2@a~RSHx+
z)gy3hWOx!pv;C}{a8*Ux6bqQeb>IcrRQe|G%HT69RoR~VQ)<@RKE5<_PXSD_n7%ZK
z{urin=#|^Lm3po~V9!4a?42tkCCa(l?q|Twuygo;5ppBy<rBFcM1lj@IpPI)3_zqQ
z6}^*6A!3GO4IwJ(x$)-prhOr@mj!9Yx+ZshCBQ^2D$`vt`zw_^Z47$Gz}z%2H~9Sx
z_f}M*991!!cYb=G>_ZF-h}Q0LJBEAb-%-y<g|0)MdksgWQkX|FO+C0UVnrSU&VIPa
zQ(Bm>wSMi+6c1O|*opUC@$b`l6r&ZVLJyPrjb=;jL``+?dS((%;T0JV8@;OD9l(Ek
z{}TQK?G$}e<QjuI)6E-_y|#?=O1m>d7)+mM^2I<IrZVSbxqJW${fVw00(iSyAX2qX
zxDIb!zZSGlhMn%w<nC3nERDg2Z7-6RUgE5_&1exD0|PDBW9=m8m;x92m`c#Xf7>6+
z0EQTh|8@nKs1tBl8ON9gUgJD0aDa<myYg$&Tg$)fRosD8@_^r!u3NVbzp%q+$51Nl
zdq6|%+GJiA*>h7XddE&M3wO0`DwqWboAR(vMm4%!vS~HdA8e%k`KCVrJHN8cDkaWD
zDE3MJTbR@zD6IXaS22z_jkeguKRx9vcs%TT%r#ZpU5%U)+qyca*Jwq%kD<K9jlaOj
zf-Fq3MXT^x_|qBmFT?u&T}9&GK@vN7AZ(6$gP)V1&%YxG#+6o8Evh>kPJ$VZv#&Cj
z3E-!V5u%M^t>1{pFKT~O?*A<KtV&gm%}nXjj@7ogb|PcYg8dE;2jQmt0m)XBV7Pb}
z>}G2T5SA1ETd&f!CHCw*tUS+?CSyaP+4t^|;s6G!EiWkvn`(P-*g$<lvQ>D<o~3?k
zFYi4T7RwLm954GRR=szLWKpE{C#FS?pc51#X89yZwZHzsgzZ4}Yo7wn*>(r$dZHli
zNC*qqW|Fu!fQ=UemmaWhhLN70evt91UFlKpv6(vG;OZbfqZRJAZjD}66GfU@n03d?
zh5}1Bt#%Eo&?-m$YyDpFN{kK*R1tWNuXUYc<c4KsWyD>8MlGMaH9tSET|{59W-AJ2
zv!`hwy{o+^D0}$00V?7}Fxo%F$v{bY;;~xuJnSmUFgP`(4b58#)fO>$5UIjuc-gUC
zbk6E9`u-o3JOlpFnXn!BRWsn2^D9ncIp5X!RZO$Zyq*DCbeU9d)3?~-vC6aD@(E{Q
z7!a7X1p3(BnL2v0@X-am5m5dT1^f;0jJQ1Z8}Kw*die&ox2$`=#)f5%$~f>}`QoOn
zJDM<<l~m;41&CtOY^T_cuO&M3qLbKso2ko=uT|Esy&@^Quf%Brf}+osDvO&i2Av}b
z2twQTux~lCBFy+H2u^)hmk*8?&&2K+$`1d9KI4r4_V)nE|M>d;z7(jHR+?)+bp*Z&
zS>*Wke8HZ!QLExrzgToG({vGCM~jHI*_yb~FG5;%V#dE{o~~KvDe-){n9;q7JK(+o
zv-u&r9MHH)I~Z8jVchIZ=D(rKSem`dSYG-;X3f4@NyAME)o#icFAT@<Ka<dgbqx-P
zduZ=LGuYO_&p`FM^K6qEmWIo-nYs=oS_nhl+y;fDm@d=!;jp{$7>v%%B#?Q$64J<3
zvhRe6{_a-+1*U^gfj7DS2AzqMorU061-O-9q|LM*{m$j?zq*jGwtfD|x;%@6)I%6d
z02vu+Kc52;p_qLv_8a`N8B6WenT;5X9SRgNnPH%>snH;s*YaA;b@4b{x#No!nB992
z{<&n*Pk)k^m9vhRm`Dy>f${$C6Y%dCcuy-Xz=OC=$9fD0^^`D4XFf$P`Y4j2P<0|0
z*H#A@A_k+5k3rJZ-4*{+Ci{PUrvDoc)TPZO)<&p?CC%>VBg5d1+8?1F^8W6B0ozXq
A#{d8T

diff --git a/reference/pit_sample.html b/reference/pit_sample.html
index dcc35588d..300cf0d96 100644
--- a/reference/pit_sample.html
+++ b/reference/pit_sample.html
@@ -18,7 +18,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/plot_avail_forecasts.html b/reference/plot_avail_forecasts.html
index 69c98942b..f1e75f0df 100644
--- a/reference/plot_avail_forecasts.html
+++ b/reference/plot_avail_forecasts.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/plot_correlation.html b/reference/plot_correlation.html
index 2b5e9e4dc..1a8d655b6 100644
--- a/reference/plot_correlation.html
+++ b/reference/plot_correlation.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/plot_heatmap.html b/reference/plot_heatmap.html
index 062265780..de5a207de 100644
--- a/reference/plot_heatmap.html
+++ b/reference/plot_heatmap.html
@@ -14,7 +14,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/plot_interval_coverage.html b/reference/plot_interval_coverage.html
index ba138be2f..7f3b8eb31 100644
--- a/reference/plot_interval_coverage.html
+++ b/reference/plot_interval_coverage.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/plot_pairwise_comparison.html b/reference/plot_pairwise_comparison.html
index 4aabd959a..40401c794 100644
--- a/reference/plot_pairwise_comparison.html
+++ b/reference/plot_pairwise_comparison.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/plot_pit-1.png b/reference/plot_pit-1.png
index 0bb1aa428d98605d47b9f48a8903bd63238a11dc..f669b5a68a5faaad6481cafdff22c574b09e21c4 100644
GIT binary patch
literal 17063
zcmeHvd011|-fk4dwurQcR#XI9*?<%wDk?MBYQ-?Bh#(Na%AkZvAcQbes~ksUhe2kk
zA|;rhV3>zk2O1F~0tpZZAYlv;41^Fu;I4$Wr{_E0{mwu4x!?WHJ$WA3Wbd`tUh6l$
zzjv+8wG$53il6NG1c5*(qK+Rug+OeKLLlV6|41I3VPzD~f`k0`w$?|%5qv~P>Jq@=
z<Iv-0!x0FzWa-bkl#vg<2*j5N)X{HFN99iSC#3ab<HkNPaWi)D>#pto{K)dgulvc-
zXFi_3_~E!*Ep1;`*6!=yyE7^?R$PUrQ^<mD2B|J)YN2^yJ{eQ`^G*`uyuIpizxC-?
zK8+wbQ)ga(Q)du%@A`YCE${B^MfuI(R_~gHT{EOXS-d_`AKtvJMyH@g7~UjQGvAHp
z3&NFUL;XMV%Nt)3ZGk8rU0{U_<Z~WDCB#W2vUdC@O$+sprB85$E;pzrIypIanGpi*
z;I2}aRqS7qRHT<Ot|fUYN&i4lX`e-{lm79E*6e!e@pGHOjnd=pG^&F1cw+b3E%#2W
zM@qjRF5KWLJtltpalQ12xbm0(xg97jL`$bq(-+MpsdQl_NsuE~uxGxrVLDQ*;<-?s
z8CS!+jJPpIC|LI4>N?_IHxr4`XUqHxi`7eslR{*}{5#=7^qnG^%rYKD)=3@`iJskD
zlku7gd3pKxgS0YX^rzk*%tbWE#s@^AUW~=21{bD)2c9+N?A+Cm@2e{nU!qnik!a}0
zu)e($Ro6c=V!Wq3DJf~QqM{aRvet6gn3ZH{xOeaD<tAS7!-xA#OnO=_jVFDfcLsC`
znKUpf_V`MbbK*yIUF676vc`;H+n%4o+e~!s-@o6WF6X*_rcpc-yE0e60~gAf^`i&R
zRt2NWnqB#^v>j3%{!1zKU%)Q^J%xXX@bcfUMiAf<uN*QpjdnmYLu~o4Ax9E%vJJ$k
zXVw~V^TXnW=^S%f%Nh5oU<?1y&{-XLGZPz!hbgGw9jdCTmJHS8<YX^*=Pv7%aEa}!
zSFcuvAb&Y&#K<UV{G|T*^XFF1Oo2cA)Gxo3yQJmp+~70aOKGy)zdwxD-QE36xnbtl
zUw^HH&STM5Ln0#awwPH)Bql*entPp?SnN-RveYwgKiFUH+d?QM*6Md$5(>|d+ypaf
zW|2F)D}z{gIXO9I7q(#^&6-d@d9<uQxGH7y=FL+-69aH)W&w%2bj8_u+B_B0iIrf6
zhlhD)?Q|(XK-{$**uVxcvfhMt0yNyFp^?!1W-ex7sz!KSk6i8Z{rB(dPzD3FSgv76
z_0UzZBc~T!6rF}0TbJwO#0&matZC)!5O43n7i?*Nh|rx9z81zAuF^)KnLNv-mtctI
zFeeW6E`UPz=xdiX%5GOl67pM+2?6jyRM;RjDIq(^5zVYKZMtpn{(`K@t%)0&JGQ8q
zM$AyDhl#|cSB-IEZGV=bvGF3Mq@?7uGj`~EI`cz|rlw|7Hm4bmHKP@CYa1hDW0xx-
z!IOdP*dYb4>S`h}JSD!7%rz{1je9DUBQ>m3C8??_AP>#NPS!U1lP!Q?czaxedY@hl
zC4btN)|Oe{8s!IrR8t$^<Kr{1A`yEvh72@bJKvB1rfH>(_q~5a1_Hj54so*9v*)aO
zE{6W%#f#J|!DLsd`z@uelT3joicBU3JSRGr$~V7yCGSRJWH~#d(|vZ)9#SyO_}C|-
zGPN#Qd-Xlb;e?Jc4~7R_sA5?yHdronpLe?D*tyTrGX8lx9i1Q%3^?+iq^72t!LdbT
zu1heha(dY!W}(lFi3Orz0ZUCk>$&ujNX)NI)RbA?mGK)?R&Z4lNQif!F*>Cf%((rm
zy64pJ_R;-^4lOtWxfww{y#W*hhyDJ8{neL-y;>KR)o^`Jw63jTnwAq@JKG<Jjl2$t
z%%FEaz1Z=nJvuZrRC{G*_`TghD`)4peW>2)g(XkAOd4rL$o0IoJTr|q^Y}`TIr}$O
zOqXz@Arz6=OZn@4H5W8<^<l+wq~j*pysvktSRa5&2+J4YDqur6sD>zh+1}o6I9VHi
z9=5Z$mtd7fbGm?Sw>j>AC;AZHkP8r(_cgQC4EJC>x-8lIgR?U>3fN~taDT19E80SV
zda4tdWJDkk5)J?>sf4VI%*|uzKqIT(&9~K3f(9BT6Vs6~{_q#5;Jec|-W;>G_VQv1
zHo<+ShYnTx7WzetTD;;H1_uXOrNpI{S~73Ds;b#tD90$oGU(;?FLJ=y<Nn-%TPZ24
zO;?sY049pxYrMFqaTfR_MAK@>x$5aw6E@-krr-$%SLVOYHDF15mK`;GYd}q1J^rGz
zbJZDV0r}dsYq(oCZ$9V`rU%wwx`@tYdBf8r=ty3(Qcpd8?xLOxIW}r^EV8cO2CgMn
z-Mo2oXFeVnap|k(=7iAX*|%@s9tM#Lcr^*NfB$~?Y^(vTbad4W2*_2PK}NH<pSkV=
zhVg_RJ8I~!B?|t1B-*sa#l^;<p`l`DXMrj(M^C0;3Bp05!6gCLDy;P&bEnCb7W$5a
zx`Hy|<j4Lyc<cr-t1RlJUcTYMgM!yY;xV(a>HG{18>#|5wD;HH60|$4?SNnHO-@#i
z-YpzRNGNOePDO`4+h`)0?})LGr@{=EBqJ0Uy2f$-!?yFDkvw)>5R4g%mORV}Hofz}
zGz(8|T3IU767ljjfu3}R3X|1e=i%5r+<mS-s)qtUjaqm6{NSRyEH1cS{})2m|NW`&
z-y`@phWZyNQ-3$*fL*iX3xnCRXtA^C8dcl?@f^wvlQ6Qx__qD%AUd66M2OX-v7yWZ
z^Cq%}EpEy~f)se?%p;_Im^&PzgFg>b6l2w;7pdh3x~T^TQaLy{(1G6uS)~FVVfFsp
zVkm57#~+cm%(0~4c<go}k?p}0OnTD4<#l4?T6{Cn3vEOq_zv>pfsoMI4}tqkOz^|(
zhM1K3(4AWcvKrJ8HgAs(+CStCs`1S8^EsJ>dM#QuWHo?~HKB#*^bnePi&tu5?~%D0
z6M|3Y&p10<;kl$L_FYazLq3T17uXkz^B9uhSDvlw9;fNQ>Z$gk&*#;1;Xo8Wt8H%8
zl-35NKq4fHf&67p8h6T5iTdu#c{v1?-yT4TwpdSvnVo9&6<Ky+PGgsXVXx!-1<2pW
z6wy>M*PCF^uYsIy7^`0CZc0cX*Ui*=raaqKH~Wji-13-J6Z1TfjKw_7F@C?YvT}p^
z<?rTyQfb(YjC!UJZ6Uu0e`9-8U7R7Uju5+>rk~ZAW0YlCg$suzs1<ius4dF$-2Cgi
z(@ZQtc%9NP@8q-+2e_6!xMU@bg|TP8eqmwdOwGfeZ5{sW)%(sU0~`JaXyEnmCD2V{
zVctYtbb=A>IV%3*lT90>D%$<*dwI5wmPnY@7$aPO%DCpMCtx58BTY}3GY>U&930pL
z2cln?tYMG05rb;S`Gl;M{gh<GIL<FlW@Y^_r<kyO92P>N_%TuQen}CARY#sH*pV+|
zqwX2o=RFgrL8}I|5G=z`8(xlwh5k!XSI^fx!AOP`Ox2{rIIMAD*X)eV{^-tdFB^WF
zoi$5yuHHe|=$ZS)X6g0E4-S47<;A9~+<+2bPN@!_B+OefmR><fN`R#b><S9SMyM^h
z?<w)qoz1erjyu|KV2?p1s?`RvhUjc#)|oyT`{cU89$-hPw2qOYKPcmy5T@DHmvKkn
z%X!VoV^3D;bqiJ%NqE_POQ)2Ra52m!$z1npAz3h@U@l{>4i5l-3Ff-3^6*our<l>)
z^=okMZB_P1^~gy#tNkewaUBEcF?e>Zn={Hu!Z&1WjJlX>!j^SG-FM)#sHP5zr%8(B
z2%5`7&hC*ljVlFHQbKbTMxU(qpt((6f9e9~&s@p_BgsK?(SIJvZLO^FM_zdxE2w0&
zI|d7vnsPN!kq7q5c(}t}V65tPd_%NVlMYP<UJitBp$zQ!wRRdKq4vy1V82MkH*0R5
zDX&^PgGI4(IgX}QUTK{3&E2c2s1;)2Z__xXjGJv+?11R4%5;uJFYxMYj*@Dw?B;vO
zgV;^Hn=x?x*EAxe3O}OH<Li~QMbYY3r7~tPQMof>FmGQf2O~o)2!z5tnx>2+LR`4>
z9~nXW{AVnia$Q;|tT4Bz@<@#yu9_>O9`A)5_#z5<yd2qaa;;Z+AV;Q_=zF-MHovZ7
z&bX60CNqIOw!j38Mn6=?gup5~_{T}~tgi9#aT6@MNuQ=iFsBfSW~}LjO`uq9PvNnu
zLiiBUJqV!U+-J|86&&9WqS6S8p<Qz^Ffgz>Ib7Cfi>tC%`9d#QP+IZVOSq`0#bSV3
zzofYnatnOy(M%-ZNi{J{vLqRl9S3#7`@tcIbcXAWMe~T<rAf#Uwg0qpSN(dt({KH`
z1H1TvEI=!IB%E=t48(^4u<N_j(jOlJ0IvVdTelRSvZnoYM5oY9RYL5%4_R8HLOmy)
zyPQ#NyuN3wxf-d}B#w}-Y3^uD<(T9NPRlV1x~_kr$IkWnnap*(nUd0UepDnad}3aH
z;o`A>-@Xt1P-Y02XqHi5=yQoq*fh8p`!%iDFP;tR#R}GRBz;U)on%yL&kRa($b#))
ze21JLrdS>SPs-B=Ss8o)>wk0P6+AQAR*t#)Ri;<G$cgX>1<P&PvfL3EGg-^-5@93u
z`MJGbo4lk@>by|d_@gpZSfpIRhYiUfuo;2U=kMy)wtR)0eE$-<%z5XNwc7^K1snxb
zydDUX%i(a9R*e=-6E!(pzv`~Dlmw(bRZDha3M|5>ONapp{xty|5A839Z)9G1dMCU;
zUOde_<4nP32(4<!DZ<RprQ;sA29_cwY{p7}>Bcw=O1iGad!w0pW-XV)r?c+fHQsM*
zEMcJ=qOp3Y;UwHwV<11FP;FTwh7NHjq#fKl4m$8O?h|2KJybz;rxah7VzJPDdRW=s
z{nwf;eoI@%@8{(|z@F-mS<!cxU9*kVNf>pfy8e|mP^rVYR?Pe}rPyQs_^5ri{awdv
z3;@|)kTWeZo2eeY4XHZ%I>?@52cE-HtsdP+@~GkGNGV;|{|)x`G^oBrdBdAmF4g=l
zCP2bJ{xou?Pvi|>gZ^V-*|(<Nr6XP&*bo-BvWG!2eEYuFPBk?PO`0mfCm>+5taVM`
zmtTHTUrI2y=<siE!>do1+|A8hz7I{y>cnL;7mrIXC!ro-HXYnrc8<??u<F88PdKlg
zr>3SBCjku25byy<{UVu}B5oI5GmP>-jq*(Api%ioEf_;~nopd+goSdLRCpU)|KSyH
zi;_~)QzFs5lo-tf>_FYIW5?bFSNT`=Q3AS%%(f2;XW+uN;p9Mez36GWo{OdVTzFl7
zuomD=+(FYVbaPOgIs>xCv07R9`Pw}1IyvEH$u<C?sELNEW1b#FvDx_g+0&L0v9YmU
zXr`kr-yam_nzYj}U8g2mvLsN&O?7`5*Y$}1nY(W<YtGrZb%|LqwB~BQ#1gY8fM&Gv
z1f;CS8$!k#^|X))w-B4K=91W1P%6r0HR>(e&&APa`s>Q<!=^pT1|mA_H}PN8-B$6H
zrK8027vFDT-$GU57zGae^0qRk@Tt`RX0(&soK($-`6BK0JY*zk0G>c@Gs~D6{y4vB
zc6jYlF}UQ5=GN6Q1UtobO{b@)u4qULK*T*OkR}VdD3%dF@X$4hV4(_p<|&}Lu#Q0@
z3p}h3J{or$_`8nAY8i7TVMAxpT)M&3ZrL3YkbmkbFp_s5v!#;-I$NW*O3x=Yb{!pT
zFrXdvm-wR?SVo0fd*~&&DuSV;Q&aS(WGOrh>zQ#434H{P0Clx?!6TPrfq$R6*=6)a
zQ<|c*C-0sef#bDx`{U++5fgILInS*d;^)uSx#-lC{9T%<)}|T4u~XY(BDL|6u+kx8
z*(^aeo7}?Q?&TQf3>!fvMzpIC2^w(V(^2Y}#W|UKEuHB|!o3=Tgjg96MeIq`PFbVJ
z?tSz7LQ}T!Pea*u1r;&n?8hfAONWJ+uK=Q=>ufi@B2-vgewqU?Q)1OzGjkXYC**bs
z`Z5hA-v!DVzi|SAyLqgJXARdI(Mk!ix-@5$w(8wAO&|q)TSg;+a}X8m5atV`QAG7N
zS&lrB<|)qx{7rpH1v}g4L2`wxv(Jl|pw@5frL2fN6=hR6kzMp@R2{v)zpp}9*5s3l
z&)4&aWQyV1g4pOq-3{!SaNKI6ztdV5evgwdZ8|%YAQMB{0>tA*uShCFWTP3p0D{If
z=!njTM!I%S4R!6A^Q5BCqeskW2@YWw{bhQgnjg5diPwd4Ywj^j=K%3(iELm^#W%76
zy=4MmqjB-~0>M{z=NczTMFE~cI)Z;#<2x<Ql<il^;#zsKEODQBr2Xv_hOqfyDEhZ$
zKU*NfimWa5dIXP7!E!2u(<vdCBWc&k%o)4=?bN^Fp_q#ghi3$N1x<H2ig@!GZB)9H
zyZn;&64hAXW4ueYcu1`W=I@SLad`;n7em?&f}J|0*nvL+WkR9|IMdH`L&%%?yhfR{
zb~YcDW5znGTV)Sr@}FO-Vx5Ld&A2KrOzLEp=NLfmJ!C;YfLaN446BMiWCkqcAzAQb
zhMl*si?T9tMNJkX?<Igo*U{gi%~B-AIFizzp<IkDbLl#MD&gf$o~-TQHXz7^c1pY#
ztfF?FH%IVb3Fcz{FdWcPh;4aYMX(gmr0Dd(94{?VJG6R;=A~VuM?2xq8_?=uQO#xJ
zjJpVw&d9RN9;$`h&FBr}xw|c`S(e1YU~ie_CpLDzP^+iSXBH{y)EuCVr99!?wi{@z
zGock;^^<9B+!kPAv-=L^NSdRzi3P0&)izPTkOvLel{Y=3%gkpSV=vDRPSF+xtidYk
z`U{hPm9gN69hI&it~!KCBQjlR@NO;I&Xcs|Lee?TD0VsrUr~Q(*dE2eF`Dvx%AaM)
zhP`ymDGBEc=fL$o>M=|RALB1ELd{&bpe?<c*!Zq<dws#L!<U2%$SUuFMKjyImXWg~
zd(E?W0PGoQ_pr|<n9^EIn;cr7td^p=_J814Al#;IO1$l8V>s^BeKdCrx%{_EA?7S|
zX@-ooXkM8?I-A7|{i#&&$C8xt24h?d26wKBmISbggp4i4STIj2>eVuYf9^<JLl(r|
z_J}&eT&hqo1G@-_C$|Fxw6+%j;Gk82xM!x^nj+N22}KymwT;rU6GZ3`px!du{q1L0
z&%)R|L7?>k5FYjO1kX{y7ynS%(LswloY!qjZy+IrkAn-z7683E=0`!#(@j1gePwaP
zwNeVy$#xnIvnvqbDrZlYX~Z#Rx=_IZ7k&%ZQq>!hQT&^bhS{t9JXC%j=dxnknloo+
z*IvfGViG*kIf|$@`|__)dbBDjm}=g!5-T04EZR#>!OkcmM4vXt8Pnqa!0bRUeRiAH
z3a1^NFSgeUBT1$5<h5EO`{Q~hj>J3^8Yk@r@y>2ga)KG{F2|Iq@5@%T_Svg#OLD0E
zcB;Ru#hw`ZSX%EAl#t265tjX6F=6=|xXGSRf(Xs{)v6tL*Fe8i?>xu-5uM>p?f5sz
zNI#hv?j~#BUW@GBpGkw)4b^Iy5h1KE?wVz@sRw1#HNO5^mGKFdh?`aab=d3Y%J;_1
z0`r$^McciW5_xu^xq6g;_+zeM@^dR$eU@nOTi5@5w}N!3KT#QA^6=nB(WnMbuyar2
znNmT&Em*{F^rU|i(ic0|ItJm7m5tpq0&Dm3M51XpfLMiq^l&sUVS>3xSJ@wPNM|b^
zikq#rZJSN7biF-U+Xh&Vfv_ImOv$A(|8tk70dwpQbB7DuGbpv>;ilDfh{0rijz-#w
zF-rXN*wx!}OM&cKZ{>Ay=lW~04eCin!(CY`1Iihh)1TphDEugQ5?rI_bQ>*?3~n5q
zAH1yOx4I&cvJim!zdP4v7BXsG#nX4RTBJ*oHmU#m?DD1<6NMXrt=R{A9-lh`a|WMX
z#t$ZKiZ=i3#&rE$Tg%|Rgl0L!UX|4c8$KTaWXJ0`VLuZP6Gs6TKeeTPPq^3edzCrx
zoQ3GkTiZMG7oKWtVDsX5F+EiwULl*lZx8Bz{0-O>Vtmy}qEl3%;v?OVgFrZR1BF;`
z{r%hBip?@T{zHoS|6v=+@BG~V;GHo4FHK=JAx5Zy8AA_jT9!17d5C+xttF{HR3LVp
zo5*&+B<puR{F<`c=5ZVydw%P^FHV%Wx$2!xzwWC)**T)|l&nhp;i=-r>&K3K)<2!g
z$>%;eecw1Dh&K5->1Qpqucp5&{PF4L0G-J12j==UEVqzxWeNQn%>&_{$cb_DvD`6H
zF8$I^DNXSMW0q^#?Bse`7)?C(@fwKQp#_`)d@iIV%D9btC;LwtJ!XuKj(VQLG-%RD
zC{H&VywBckvE$uO&TE*(F3y(RB3qc5iAPVpwoutE>5Xq%&FY$$rDwbRC#IKik&z3V
zFeFE9JPrl~CyBBWsym_y$c|OJGA66Gonou&gnCHFl~OWVre$&OuE+Usd7ge%AAh<i
zEg6o0mtVhm^Co6&Y^(ry?SLNGB{V~#%-DrF!bZ1$iX;jAx2mXEh2jggNJl=nYIMkd
zHn{ic1llC-&{VTnvJx{oHkP7Q0&4r{SU<l}3d%v-<S}4=U2gug(An8J$hX`sy_3p+
zr8TH;5;yT<r`tfx%#FFZxdA(V5;PHE8dCE^fGpm7R6hAAN@=I@>^sPvz{&E@$4At}
z1(1YFyfJcGue29L>AtPZ!MK^4F>+RRwvFz{wdcu71#ZCw!BygyMMXtvr#vlw>gA0K
zqOI~cF|c_$r-Q;!LK+xYrakbqag=pIySt%2&tYuR4HH00Aw6}}9T^V|_Y=<a_ox5T
zD<E1d@2WKPOyvX>rBw-{8=hOIsP~62&JHP$afg3R)vVGI&G>ruUiwT!Hsozb&VM$`
za5GXeDnMT>P}XtCor-i!=+k$vjDS5(Q|o$rDU6VeQ^Ck0^T+tP(PAL1O;!u%ts=X;
z_ueH@;`h>Krdd@9duT}tPx-^B;D)>T(qNcK+q-pL2t1Z7ba&}Jw|86p?Aa6ztH!4%
zDJm)|8pgO)@k||06=m7W72JkIu_OTlY&$^X#@`k;BMW7C+54AUFEbx0%3m&1G%9ny
zSm01x5mYT&e(9Nit^p$yORNdzdmmlu*tk?(n#QjT4?(T)zaNk#`EAJam*jb)tvO9$
zd=3r5i)@(G>Bu@}mm7go;bJol=@u=cNLZftUSl35!fP&@Y5uft4v-ai7udyfQY@pE
zrd4kkMk%SAFLpsXFo#%5u01DJWdpdmFQKSYU0odaytUXs;3`##cloXFs5uV+0!GPS
zeiqXfcPThH*d4}c)84&%7lPKM2&Y*%G|ZkI(m;RjCB1|Cqv{RY0}!pmshG{2j^rKZ
zGkSV@nkxf)_@%(InuWZ9)$(WFp?hf$A*DUS@BG5U!>!I@B;K&2KrC@*5Et?-V#qU*
zbxkL1M7P|y=Oyv0Jy4BkG{+09=|9Y|)z#QOvt)MT#tko5mepsnLH6Bh*<O#IeYUB9
zOj)@Ar=6NHzx4h^MT2MA7WvCat)6sFxS^q8i09C(13WPl)EeLjK3aGjNsyq0f5_x^
zd6u~tg|K5ErG@RKRpwfT3vZQ*=Vh~jcH|@)p@5?)G2P1S`75VhE2A=C^IXmX*eEiU
zTTozCnMzUFx;1|?!nDjL%vyIuNLcQltwutk6S^bTOOh3(Jd+ApOc^|NvsG(Ho2!RM
z?0y<uF+|1p+_|CWD9d_v3a^gWh}0G-Jy(@IG<$pLZ{?=;wcq2>KMh#_ZXlJNEN~+b
zDy57iqGLRtAiP<tbf{11W-ajDM7{Jxl|;SX&d#d%o(F0CJQUlsHW=JA`1<`{6cHZn
zwP>pwV0|VrebL1b(Ym<AY|3{`Sn9YRd|7b;e6d8vf4E`Y4`Mv^3lvr1u%SbqX?k%%
zd~|2f<7XKe8OQp8gHgX|(ORR{|1qchTSaRs`nMeZEr&m2t^d#L0c+h)b<4@gi9wM7
zMyS%pU(_8*edaw|c)s~|+|=yQ^WtO9F&1mtTD!wF@0l6EnU^UH9YVbs4U4K|xFuAl
z@l&Pf1SGE@weO|{Q@<lf<ym%Ei~fM84SqLDhgJpvWFS{4cDnp*5LEhZnEotI{w;Dt
zew?hXBLoGmXWQo-YS5!akOadZ(^^PVG~9V4d5eXmq2XLFq(j)*U5Df89V%alUXVd#
zyUI%+N7*`ft$ayg2=m47mzS5p?u0K<rg?4dhaUgYs``Hm7!MJDZ}e6TD~B1X8OyKp
z3kdkSC{1UJhGlF(O<ZmNj7nxk#wi=T;8>SuOXJEEeP!yUt*vdxnC0s7935vcGWV36
z2Dj!IrPL}Oa@8HVGCeVIaaGCgu5>n!gXF#C?eZrkLi7_JK76<=EI7<qM<B5z46Czd
zdAC<rl!L*_)b`Et(!d_Euz5VP&X2mmG)9g3`qi$%=?GI7rkfKcF0d%hM%Owy#}Lea
zhWvRo*>{>U#a;X`+?2{;+?c@U2C*vjLlVR+kP<YX#t3IlR#87f#|gUps6WO$S7q;=
z+`6q0Q`W%Pd^c1q<Y{0PZoU;Ty$ZNxP4S&Ud%q2d?#KZtLo;IdmeJ@d)sUT7)zI&D
z0Pxi5MqmDDtS5cJ1fUq2Aqy-|){Iwpx;O!&$Uf|tBF$w8#Ig4w$NH)1fqU8p?fJ{I
z@mw3T&GMI3EV&<sn==pnyx_JVXd;-`znNP#rrl@@@2jqENaK^;xYrXhZc>7EM@E26
z_~?$b+z-B*y4VH{oUy@|Giu)}wQh5yFzk!cGPO+dIePbJl2aGS5Q|`*%6akEm~+Fe
zoo;1nE2AtY>3!WNZF;gfEfmHWvfAi;5u=up7G@8J9HknTknO=5awhE28FJn*@^<E_
znXijFPa1_4?YNPM17%bYP&kVG_vn{GVV0MHw;d$!!{{0&VVvV|o?d+7z7Ms`hTyMg
zzx*tgPCHSL=IZg&xw+9+|F%)Q2(0vEDw4qK6<K!#6{>%@&9)nfLcp1LwazjH<l0H1
zY6h#A(Q6BzdM)|8Mn)+6Fc%oeO_(cw-`Cey3GwV;P^}E~QhT{QGT63v2$V;yQaGFG
zbB9B!sHj*Stz-qWU<fER;VelY3|so4Pid!V#Q2Bsl)RiAG0n3~7+b~^zi?xEwmy*#
z8Zl;gXdUmelI0@sGi$<ww6L*j$~yVef@my|g)WT@735@P;paX{#=&+<AsDxq^feEC
zbOCrMbk@Ve+ym2ah$gnxb%f5^@Izr1SSNcD9v{Eby*r@He+_wazP=xv*aZalYbs<$
z@M+1=XEh3Anp|9En>m!ff7G^C<lcq;M+V$35G@|%`4_}3ys}yuwu+DJT%$TwcpMtR
zt7hV1PGh`D^CwZw$8|@BP6}mm_sE`@m`KRyFp*=fnHMg2=#F?nBHaBMS%ugg)ixN^
zC*1=qmD$3n*QLk$6&?picDjY^2u>(W;}>gA%X`apxq0PsGKQWv85AuK?G~@=zCd9Z
zZ81toKV`Y^ogd<mD@ia4v0wlw2aff%)n;dBFO{}fR6aVVfp~P;S{?tu1%uvZ89P8m
z*R6~kS6$zISjlR6RuL4s$g4^!xar<5)r&_Es#BnZ#a4^100-#bEA6*CRnuDCp5Kou
z4<BnQIc3v8{YshkPq8IPF2LNXo<5zAD((<bAQ0vX)Uj3w)r)`vP%U=ITcSWEV7=>S
z=|qas3=L-|P72rWUB6nC#z)@|mX(My(fz}(MB5X``VrLXtp6%=8C0-Sw0j7gWO(X0
zpK|K;5QIbS*|Yr2rn#-Ht+6I8jJ}cmo&qwBVP>xiHH;B0uU%T;%WYSd{>`51sqhc4
zVp&9|)~$*GqhjlCbx!}Iz&quDwaAR9>t{k)B#Qd+atj)7K`iN^L|yKB!$i3%INYjT
zmw0Laf0-s5(xMwYd1e@rmpwlR#m?rWqx|6dddAX%8+R4Z5(?5VZ-9%{_5GSQFAAod
zv6_-&yl|sa4bK?1FiLe2!%4&SCcI@c?t&HF;;q}K-5)i0hIj!XbKSTS3WG(6wT357
zy)IvMXgFP(RYoY^>Oru&fPmeDYerbMx=LCTkSIMXD@)i$r(0EkE&47F_V)emvYZTs
z)OTlt9VQ0)`jSXE2GrV`OpowZrt}>zC-4|sU0*Mfz=$vpHkdE8pyKgGrW8g2hzrZq
zudQ}buQSfe=W|c__Z*wwaOcs$S}ojykDmjZmmA8)BTN(0eM0BQ+Z`W0Eueh)I3M&+
z18*gWPQCt?Bv1+<4S3m@RBxrY7<%bk)F0pMnRu`q)Y`a_`LmyD@PuMs7N~PG9;i!0
z9GC!8+a!xa1q`{@r6&h@4MKJk+SRo>=p6W5y7YktYFV)l+{+UdfYWy!%M+@(NK(Ys
z5a}*?#WNq;5e645{03_buo*!|9TBwgQTyT!JVsizTLN0`TKYo8)(wn~!XwF+?L6VY
z)5niz+kq)$0)P1)sUQB0RJ2dGem=YY(Ji>6iJ$7mjiJX~0+BEWWqAhzt@JU1hoClJ
z<UMaKkz%A~x2}!VM>VsO6c;w!EKYrsW5%lw|C-jc!*ZF9aaI}pq&!<>dJ^-^IwW<6
zap)k`mJnzU1BPp~ePp48!nUd}a4!`~_?b<y3p*1cNt5e6uAgkBGNRYN>i8LCAuDl@
z#;+5|ebHf{oDY|FX!8VgcoFjrb8w@p9Nj|LchJenNw>)S$-Amhut^>Xlnk70k`g@t
zf>TZa3B@1(<(FSX@rt8rz(R`&v5s&t99zp+8hZKiWis4JncQ^OPSscr8J%$}z(9`R
ze|qjAV`=WWI3Y)&Z0ym-q|rxyjSjtTZ@1SYF!wqpJn|q4qxnp4aax#mVxGx>APmU}
zs_L)wUDUz=*Y6Z9sCk;N6liCm_}8Hjy`oqmlPh6^OzMAzrLec>P=LJW>piM3J><1J
zc8|6KI;N882G-5&932yuzZJVHtk-vIjm>*XxtFNot=K7L*j(!EegO&1A-{)R?rZCR
zkcVRhRblgEW@=&xAP%g6d04p>mW62701Tq!=jT@|iBBMQOvJkDjs&3`pq>UmlBJLy
zsw4{XlQIok4n>1$Vm_bqi;iAm42|;GF}#L^n0z*vs~<@=*T)kPI|e`b6YlYwk>VZ$
z;jJ04K2ie0F6TTd81P{WrCyTB0${`~Gj-L~i_{u=KW-py;5fhCvkV8(#xSI+8CDgq
zRtkCJYb@3&hoJojecQ?vb6Th`ShIL1jD-1I>}4?!&-mf+@EPDC0c#$zciZO1Lw)tZ
zMR5T6Rt}BM>7nJuzr8v%V}W9y#-O{wn-T8Pu{;~;oD_;vwDI2EEc;9+xi5RFw^!$;
z`}OROS+Cu4jGogFBV^wJtF{h1fz=kakk0FW<S`+AP>}v^sRe=eIaXeW&W-4y>xx;^
z`Jx*m6`6DD{#WY2R+uDEGBg0=k9W56-2=!!DU?wwEs?XUymdEetfFs^D%O$8=^n>-
zOhmBv_0?l2Nk3A)4ApY~n*w*5N6nU+#Hm$yV7r`+_Eb5ND49@Enixr0F&UcSsar%x
zthz1#_=K?}<ps~UE8O+qTF4=NYR(>-%ul2}Nu3*2<3%+`zy)fIrMCeAisJIE8_2Fw
zH?3TA(`*h|nud5L!k0BQH9;tL76h<-6IS<1^3aTRxM`n+b6d^KkGZR_AMo5*RP^)Y
zmUxsZ^*gt!?NMKiW%+7yKfY_MdMG46mQQ19^`vo-TeogKv+TEc17wE--UsKfKQVr2
z^ilM_m1~NfjOz~;-iAW*^77&W0S0O^wX#Zx1<P$n)y?vo!lU=CP{X{u_Zb-p34nms
z7M*SyXc~DlCzWSDXBG@(p${yk>Ci3WO{DLY)v~5nzSSKu0*o}^C;bVW(VUR@_J{U4
z!3IUKKGj^_9s`5;bLkoycSE`R<Op4@<z4d|LUkD!`FspmX;ewh;WeCa1h%7Ka6oP?
zz`9D)sBu<aUO(DK)cFZAAsD3Qha!>a{8-~&FfJzw!;3!A)ai*E6#*!j0>{QLhTidc
ziJ1WjBW_h999SLm?}`Q0W2wpoQ3_nY^5zE(?}H$hZt-&O+Edq;k1*(penYKLlw*}?
zp>%!4AiT8C34mb*rsae*ioTPYRFl6&7p;oCWA>MiZ77T?eFKA)h6-P#pFG*m!blhM
zP38rJ>k&Y+CSyoUZF~YC^mDiPXgGw4frui>eB|vCs6($GWRd}KVLy0)69~P|sF~eG
zV5@PH>cZR=NaO3G<>O5Qt6wQZdy^hDZ}ynp<33iidG-fEzEMX<ht<Bm<ua8|aLSl(
zyr1VucpW7p`N$%VGh1r1MFILB{R95^rzaW?A6!5n)+43=7vMi{Oufp=;E28mrfcEN
zX|b?|WBJG~?sD&0I<C!W^;GaC7x6c;+Uiv+cqwmJ2pz&d|E?3i3a%Rzu84vaFtcaZ
zA+{BkoHPRcD+goXO@yf(SWVf(mL-NEtHUN`>A735UYVS-wqD*1m7nKUd_BEr^n7d|
z&g+*;U}OC&5Tp^08#CJQ9t86oju2iZ<Tf5X0()d~d@=L+m%pe4PcW1M^lf=fwqeY(
zDXT+2?^uVpXD>TPq8CskL7Lq4h(vAa+LiayU}KC}^-m`vJkvO1ZPCl!f4i$YDu>XQ
z#z}=($#h8p<hA3hH5f^W*(9Jcz28Uz;Q~wUC7qUn3m=qGlUE})AgJA+f`1%9UCObf
zV|1hr`do2xBO-C%E>J)u-Vz$2ZbMlb?Ay01!8UR?>C#!EKu`eJhj-e8H>>hx@1H*^
zE0Riyy7-s1e?f4-W^JSK+uiGAt00H(ZvcV-pIiT-f$Ze}tzXLTSJzn^?HfJzVg9)E
PnW*m^j*`Fi{^5TCFPF|A

literal 18027
zcmeHv3sh6rwsxvjw4fYo6%~cHZlH*O@)RLNYZW&Lv{(gsS3!gb5fB0d2x!$~6*z7X
z<gK-U4+1DcKoST(z=(i~B=Sl?!!wXzAcPQ-{A-7vo}T-U-p4rizjyrio{Z6g*=w)I
zTyxI#eRIym#cy34R(!JI6AT8k0^a|P8wRr^27_6AblFnyO;6^HN$_LoQ74CQz)$dz
z?x`b#AIneeKN5+-Y`TE{_fgv5hd>PGa}4~=?(bvr#`_Zf(&wJQ{m@Pz>QOW@$DX9F
zN=wb(b+E(gslMK~moMMOJ|nq5{ruQhCWcSogJ#K(UTVN6dQYDSb}L?zvsB{?%|D(0
z=KH4Zt1EQ8GU4SPwZQ6wUwqd4C+%&TlrNBp&JL{nnD8Aw%H5MPLnXHNvGGHxZ5+~!
zMArTI?c-eCVSUw1|Ka~teA9Ye%_A&5J$*RSjdP8+7uI20wD$FdN$6^tI`jSY39{6J
zCQDN%(IcoGzOTom<`j2j@y_?v5y?mY6!{T40esK5OE?!VUMxJBGnAHoSx0%K!-AOQ
zW|Ezg6F2gGS=Z?4HW$%E`pMj#r}`O()X#NW4dq0#eTbp15z#`=2J(leTDnJ`N1<IM
z&0W@rOB#%Ly2gEe7wF}<)z9@&@$vI7Zr{FrC5%eijkIUDnK+6R2kC;hZ#{x@h8{b2
zoHQ-63vbCZ=9d%~??dd0gLCetZ-iglOuaByb~?6MmIUWmTRTs2ii0_I;qRMPIMG+4
zV|`nABlZlnV|3mVvCeX{x4>fM_Z98Y<8SZ99zQ<R1G6^SMvv2S3H8#uEiqgz{aPDX
z!bo1Mol5?RI#D|#<Tl`#d*S<_M{`r!<fF4ric2bS+%engI%x%S{sI+cP&X<E+VX81
zvUonQ+0LDDtr46UFjJ~Hb91g$=v-r`nHD;gZS&{9ob-+AJe1}X0ah^Ef7UOL;A%Kn
z%qdK7i4~HBbgEJ~aI@{zCZ0Fio-wKY6N5DJ@<y`(++iUxp57aD?AS5C(TUim2o9<5
zVN|=oxS@t55iGtNXo9}rKbx`w$aNc-f{USEQHH7M10a>!*`4)KeAc~lXZW|@-aFRZ
zl%1W8+mE<CzuTRMS1RNLSJ9dGT+^aFuEn0n3^TN&Q=fpDX1ZBTXKETh2m*3?_V(?#
zpP(s4+aW#gaDN}@W#q=53t<O<s@N8nl;FBV-mZ=!YAX3)?-KBSQWgHI)D!z7Y9zz!
z)k8<3>$quMnz;GNdmXi1VzwX7%-npc(e3#f=Z^T<pD&rrY2E!{VF97*e=>-v>l0A-
z?c0a&=FiPsPE9T7^3t)-f4{!0K{0owPFOFK!(}E#<2t&!4MTJMa$z-=jg|CP9;=s-
zXD^UvJKBT`QwaWQk?{Nq?fOLCvnx=n=<DIPsVOP>evUAL^BYT;dnaf~?~R}AZ`5*i
zbv2!RGZE1iCGK=@Cl$(NlB|PLJSqsx`Q~pozN)~p%Vg;<Am&hOZsAUDft#ZNoKsvI
zY$(}bBH+aJGm<9^NT0^&4J|@nVU^ujMuQy@ssT*xN$B{X>*C^4kj=A%BY@K3>^d(=
z-U-yqw21`n{lJzdU#7*Li28N$+L?^pn+|F9#;`ThNMBz+F{(LrlXiN5Y-YlBOVPOV
zeS2mrX=c23c$S-=mlq#c(qYp(dfGmEJ}4nv5(v+PcHJL5XWe`1@s+FB)X}N*f0MIY
zvmy_4Rc+-Q*}O=X91YB@%a$mxi<wsg?x1{2(R?&wTU=aRz*Qb8>5%5a^)F@jaswL8
zi4ObrCC(S;hRw}Kv{tQ?Nrc=&-p^21P5Aq4E#H^{q--%d#G9ZK1^v#*=;&xdQHFtj
ztOL;0K&z(2EzBsd>XWr6Zm+6p-;YG8(;xN;XpQkpewESn8*yid#-fi+(LPFj_J|jL
zV|ac!sS@J%j{WE^Vs2$HR4e(`Fj-|o*WDJWUfEUju`l{O$#FRi{fW8yX^hG)x9I(&
z9avh8fSz1<ZMMRBpy;V`7esw2R(-L!%I$f(cXU@^q0aQ@7)(XZG8%?)hcyvR)@O2c
zr#EW!b~$&@OqeZd*kxc%`e33Jcdoj%<$3AaL_mFE#J{|+c$8DjoD=iY?dEV`S&Xe<
zQMZ~LaC5JE$x8WIowI8m#7+%)Cx|=G>Hz5;Huk%Zo?V##!qxx7;`?U@n15Uc|0%?O
z3K7`-f1V<RoP3G8>#e_JyzWhuO(?Gi=NPPvTe}qC1Cj}S`n9XI^FiR`Fr`+gf2|1V
z3vb?}JQ@(d3q4zKwy~#<D7=fzw_P8`By~2eS^->FNHp`hnt$hH;S$WX3kJaVCT)oq
zL>9IHSg;hk)M#XTTI_*>nFl2+;xsTGov5?E{)yl0BP|hQv8G0x#^IN>!v#i@ix}{Z
zpH+VQ|90#v`qtC_Rm5uY;eKm%XiwWFZEe{_XmwG>UF4)=1pm#OH%~4mh95opt_b*J
zCB$wB2niW;&Dx=jy3&e$D-QMNU%NKTbbGFAus)wuo+0Wj4lbw*?kM&fIlVLMfU`69
z^d0uZ>7DNS*;|V2#^lApE_0Id;G8QE-{nC`#_PPX187f}J*#8NYu^2&dm?A3?L|``
zH8m}*A#k`YU(+{s@G8#E%q-%V;o#4Z&{g#AAU&GegEccVyI&cc(^r(?n&Fm63(oN=
ziAVv5inFV01OIy=5G$D9Uv4a?@!92MA0(<MLlb=zERbVKGXUW<r#c-!e!M=PcPF%`
z8ap_YOHhp6_9nF#V0E2D_i78N1a9JPFiw#az$cdp)q#m|IqFd#K!HW)A;Tit;H)N5
z&?}`wLPzA)t5^8L(H7{SNzod6`{`qM?&@UmE<>@CjcN@$0L*#rOvf3xpIT_b(s;UR
z#f6od;^v<{+YitWfm@&8Fx+EqWi@jZ%&aIlC#ZBf9*gT3J*_x`fC;rix~4@mLR{Qj
z8#MT&A*VNr?PCgLMT@oHNjg&UF*@*_?VlwRPoJL7b8~bw^z%L8+F_v<*YbhlMNKvS
z>`c#V8A7#jg(E^cEZEUI^AproZj2cKWaYTaw9L6<ehM-TiQ13$bA8L6@Nf1Q2JRt9
zUw%yjLP~u>u@39q(LMJqwkSiWS6c@3{c>7bnzOTW3W_N^En2JK<H*Ur2f-a`?jbs8
zC;l-SPgm36l@f*Bu^TG@Ig*hnD%8p{7|SjwsF%)4>_u17;;or#JM-HVkl$BDw2|jV
z<#uGS1Cp($^WB~cEuE(frsAi&Ay@DEi>P8Jc>>W+00g{Y{dyRg(&{z`==?Y#Hl`>o
z$m1E%RV~~6wpO@0B{kLFmU$*wlHDFZ<UB=hAS)I41u(?OvG~lv;^f<k0X}^&of_7+
zlkh-CTRZns1G(Ppps`Db1vwMN7cJL=RHig$287Bk^N|G2*!7&`(4@DgCJc*CBfkXa
z#1{UzDZ}kKcfz>A=H+g5(Me-qe$H+P4GRN=mYb?P6%iqN0hPlD3VjBir&d=OmD5D?
zP(CNEa3gOk`p&L|^Y1h`kWJk5NKaE!6CjjpR<B;|#{tl=T-aMXH89ab#og|i5ICZU
zbt>UX7mvp?axxUc5zS!5Yg#Nj)PXOE1n(x^3(f)DBR&?G=nbfLUeuynnJAxY$t{-B
zIjl4nS65~E;6oMUtFY2=?xzoCdevjMZx2V=r-gH*{KbnG;(Qkau>7!=pyuXgzlV(S
zH2ouNlALVPFfVFG#g<%K2sp(FB*U!?Sonouj%ks%Eg*t&gCE%xFKai)*gb}WE8y3o
zr_It_^<zK0Q#z^yd3;67sul3xV_Q9&Ox{7btOsECy(%2NA<K^~*^n&+ZbC;aiIfQX
zxEnWZl3#%cDxy3eMJQHY<<Xrx><D0uHj%C~DuA2!6X2!9w(03<HbSeGcxC{u-C)X;
zt(}`^c@Yw}!Ic@YDs9tal1J#M)G?W@-(nk8Xt*L1U{Cb(pND$?i8%ffPuu@K^77AZ
z{C~KOx2+Zl=hl`jM=g&L-o1>*g+-Z=>xqGG^c&3dgH<uj>GN)g3>kNhfSn{NC*sl3
zy{w$z6j4L1w_xVO^=Kr(s_GODeIv;-xbz&p-jG?YUu(zAO`rb;sZ5_A=J1K>^ZSr?
z>M6qcHyk?2Ah$Ty$1w2y^`IY4IT})NN@+Vevcn?qWo5<v89|>+1mU!YY2|QH##C<c
z`ssD0i-5NY5_0H;8>csx)&LBK@jn$-H&NYSZe}LJ!)=?Uw`#Z;4qn>4c{71F{o1L)
zeB_9~Wyt19{YuwK7l|*3<(R|1g#ZiSI!Ouk%yOtJ-JxLi+&NlFhG@@OoimLgIYUQ?
zz2VKqu>JOtBOW&B?=gmh*lam*8U8Odb=9G7D{d&8D-k>VuA7)`7;swip&m7?+F5Y&
zRyrCMwvR79%LqUG?qn~?Tm}h;Aa_Xmo+Be0G0PyX6PRRmDP!v!e%@|P#@Of6<ye3`
zc5Z%WQhJ2XCY{ptH^%2Qk+mTBwMB~3WSRO2{&J3VqjLmaD4vCs7EDup8}rJKW?Sj)
z3TMPs-=_LZ<#!hstF7cUX%jXA0aXOqy<Z<Y*HLIcvfX1zy?lZ&oaq&Yimn2jT|rYr
zt$>Ul>aUjxT7dl30H8eyLNQ%6uMB;5i5$q&4j_hxw!UtF9ffOw1VNM1IHUD3vIH#C
z3jr~e9&?!m!Hg6p9E4XwQX-sa&h*nfy6bR>n<H3>1sL{+=2R({bZxNkql)UNIv=Hm
z$63elY5Q&b$$`fTY(>*Xn}~pyn|G}_{O-@^-&?{aE}}+=`)qT#bvCyCMs8P0oH7(q
zL*}O)H9@!m#0bt_)u53#GjZ+X9d$&#*ds!@GM%SoCM#N>1;w63{3ddRSh$+fpNF=_
z96Dh;nHn&Yn@S9v3GU?{5Y=#ks0OjVh_0ik4dPDFm8Xt3oTezf%#?>(A{SyZB^2`*
z@-`+87PZjDdD)QCo>^y3vU_}SntZ9vJS%D^+HuksFZcD+4Vk=fVS+w0*IYYPi}ZE0
zvVM2o8V&S$#ht2EF@A5U3JRn&WM*Z?P>A`89q1Fx<J&0=65)9JOZh28&2`EXN1S`F
zdZ1APEH4c*H1<|u+9dnXyig;7YEq?1R%rYwC4qk-D_n&WFxXJLM%j;$rr5Wk?$2^%
z5kYy-^O_pkk^fg<Y^sF9rT<yp&_}Hg$zDKA>pVk0<`~IX%7j%iTnCj?{i1SVTnZSM
z%l9G^R>?G{DdhOyx38RI_3a%uRi9Ty?=S{n+495OxW`}4l!G^jzi*10@WMr{l9`Tp
zDVb*Ns(EjF0Q^VY9E;%lMBz8)#eMMmA-_kmIdgh&qCH&6;P4Ue9xq+^e&xAX)uhh5
zX(vl=K&C>OG#|Fl=jj^Me#JCLXj>~CY<gc(g@yW3{@Njx?^)+~%-BcL1eH693gd8o
zdA!92Z<VI$^ShmXU0)FpUjEb!gv)U#WRhq;a6u-|+A~iT*#p87e=N*1HHZyIK0vPf
z@9R|#f8Z4IICc?-7X)2$Yl2}Lq|bDZTljb8?py~IS-`yf@?VDQy6w&qEto85HC%AT
zq27NYAo0P|?1lC0^GcT7hGzn;##EAdc?Jl52qI_lwkHVt`be8X1`<^b@#w-WV<7=2
zoo53J2m|&DUwdl%-Jf|~FLwHJK_wpioWhX3sTT)%)hx6KHkjnFsrIZ#VwF_(+a^mM
zC-BXG_gIltFJ5Ept&4V;^k8+2&{>og$Ja*kIv!zFgRp4#)|S=kF-y|tBM>Siy$cZz
zM34}no}qzMPI$0wHQI{tGI;Z3q9yZTePMGtZR)ech9jJC)$8XEYX2&kibdTqXCyYp
z-l{i}-h(qk7&4HfYPGQ$rPko{8>~ag_Egt3-Z;dM!*M%UDf$Km?Z!p4?aWJv9pE%|
z0N0W<b?jyv6J#@+eBK4-0%n+R3pm+oSl`jmU9T&H#{yZEiyyVA!l|V{gxWXuK(R2Q
z%;w#I$X9trboU0fek@)X<=GeuIJ1G1p`Q~Vi^B#sH&x6yk<d}>OzsBh9gpWxfU8`+
zdGjXstD@fT9cxXPngD5AwEjF4ivKl)jW?Q1HK;Xbwn1720mAig;yuaSha%48Pex_4
z$;H8f17<lq%KP{4gIoX+J6+eOj>H#dM0|%xOb5FG<JynpJ9l{J^5!|KX^RV0Sl6Sc
z`x6>)O1+%9RH)tQwU57WB7skFq7Q&Ts$W)skKy2+g9i^1x@#kc8=*jyiUFeMhw%0z
zcR_SuCgUJ|rT|N_x3aX15)1n3aN8^`#SIP)4w|LsYf)h*RZ)Q3`3{+&i@~m4mki%q
zGDE&iphsPloHQfN?{+jK*eD+7+Y%E^nRQ42V5>QLoMYD(0@>$x0i8UQz-I>y7!Zdy
zj(gz}46SJ@Cf#*YN7R2-Gcmj#P|(~svWv-}&SQ&(OwdMP{Ov+7av8yA?OXB#LPDEc
zSj=yk9&iK+N8(9<9@W3@nYvum``c)bJ=EED<%EQh&&b5bZectAMW!b*vKgg^)!&}-
zacQw(E95fVwO`EJ9fGz#zJ)Gz*Ev-A>Js>`>cw9>SeAkrL>XFdEqZZo;KJH&b93{^
zfG|2pb-g+_V_XtgfVX2F8;-3K?^QV-#uoOzjq?HAxa|%md2FiU$*00tkl;DvS{Yr}
zC&W?aCm(B?PM;&RgxU<xHNf@u?fhrU6IFh9-O*5xMKNMtisQE+@%@mDTuqSQDC(`~
znMlM31VC_`---5-k&$fr;K0=#1wYgdKb(Dci5yoi0a*t$0as#ffX@Hh65y}s+nEOl
z@>D37#FwU?h>1qn5|$=Z3*_Eg@xx;_sl|W*a$}qmSSu?lts}~KLy5;vojd;~cTN4;
zVye9nGb^2D222&(?jmZlo7;aLA2H_lJQM1w!n5V0dk0%{t#MK4*0~^nXLh&=wjWqX
zgg(AOF&%kmn6{CADjMOo-drp+K<4AQ?QFeTv4z0>kC=3DZ;;1db;c{%8V=hp;^ob1
zaC~5s{US~Wz3bIY2X1E9vCn<CZ{PmmF33v0y$1DA31m0JKmsC$!#I^e?bv3e0Oiua
z4hLLHrtWp*AF~-&dCeVHtU#EDONYyxMPpE!tLR7QCJHvo5j&c<^}|1+%>qf9LOwn_
zJL{x1D}MFD{s&rEy{75EMVxOLgu6znEIw&TmGomm!(&)sR_S0KZ^!ftSpQy-_jl=_
ze~mH^$z>pSi0&<8L+8wDno$^t7vW*w9+L5MV65CMUV?f;BA-IFzxK<KQI)-7Tmp&;
z7P3SB*;OAt7Ou#~c0}LyrF)-6`@v{bg7d>QFxk||3tBjZ{8s^UQ@)4Tn_$r_6eOWb
zq%RUzsltJnOK<k|Ck%Zlk$;N-5QIR5Alii7G=2e&K+i0feSfV50NLA&rBXdS*D)dr
z=|Q~HdBspGW>|He+<N}`Vgix@l}}KpdZQYr)}pgWx}&jkQlFVM$HH0O`U3q16H>Wh
z5kXm7m!)|Utx>=vJuf}au;oyrm#U6W#+Q`yj2^N;P^j_O=a;KawLbmk{S_!kud9I9
zmmq8)DJIZYF&oQM7`EvFeV--@SHsc9Q>wTOW)I$S<Exj>qVja_ZZFj#Ua<^-A(9XB
zDt#OTE|MW~HcuT+eXvbsmW*%ln<o{%>TFdku7Ahffj_I#O_<DGf0TU-jc0btydDZ#
zJZn_73A?Jgm)$dUzF&5Z|H(o~A_2&{@Y%`!zQ6F^)l1SKsW*b~9zlGWeVr}yfco!%
zRZ(_Io$jc7eiSNi1F#m(fpD2TEL@P!tAq8w6sgpDtEYA{zzyUU6x15Tx*#{3Ck$cx
zLLLI?x<(v1zGrf8cEalgRG7Xlcp%5LZL@><{Aq=<rbp<3@Z<Omn})}K>q??4q2Q>0
zmXlvm6Q#dLVqP+qL0lZM?OwfjDwFsbD&nMp&VBt8xIM&<FK2MogoK#iBf1_L%XaNO
zrLOU09oiO?hn@q}IhN2-Byp*bjKUnYWQtI|O%r{{_!dwrPIds1fX(5xBk|jqYhcB3
zq(o180Zzm*gVIa+6Y>)XU8QxXrZsui9A}Qd(y-R=Vv&S_boSR_E#u#<DEb=$bOlt+
zFVpG@?=@7ov$0;Dosc6DK^Hu8RL4AkZon?`kN`g<NDpBDW&<0lq>GBgW0}cog{nDi
zd(*8!ldsg5x~bCT7>3yi{;#|5S?%UJ8FnBd8i}G83Xj~7sl=y2v06+>=RJkXTq6eT
z3R`1C5o5Zz9J1+tcK}@^X3tFu!?u9Og4%taaLkeUqXhW?B=gEq(PjTz52BU|FNpT{
z(l;}wkkeXkNhEF@-f2qpszyFA*}ZVJgry6L5jcg-Yu_r1>cy<zU@U;=?qZ5x;km-z
z{KyJ|e0orK0nqONfWDARRd{nBZmnogI(4=kQTztEk|nfy@r{G?eJfga{7vB#sRV2{
z)pz99d2s^2+(h{|?jd8=i;t#y?<gn8FAi>6kYo%PfGDPzdOcATPI!?|{9XyP(_c)p
zg!}K;QVWECJ|AQdn?R87LVCP@M>cd1L^u-?4kG|X2O?1kB%s{+vC~&H*t|Cl87i>n
z-?{-m`p`OXX5<{vz{Zv-)R9O(1Wov6@mQJ1e?{@YhHZz>L#{5O<kjO{S9xb3VQBbD
zRCGx<-mq5FA_dOCYq=Z5#v=yMdS?)N2gq(9^Vgk5nMNUOR`3ffx=EL;G?2D%RLCkR
z5Tz`yUM%%hZGz{Uru8(*G$nU!nMy-;buXvsj(QP72If@Hf5~UadJ_0Xzrkj(W_Jlo
z2#a`Y1*(Zw3{`#5!+Rm|7R&}`krP~>!(*q<vmx&Oh-s)nnX>k{N|V;@1lw7z58?+T
z<$Z~#hTjX7*8#3-<15ujj89IyU(OkRaLQTr&(sb;DDR%bt6n~}@mC=I{n+Wk{)BC5
z@i)5X(!G@<!M`rbzC=5QJ$Qv-TX<p4p5QXIu(;{)Z&d#9->w_@HB-i5X1KN0PgM65
z*o@eCgr{wq#+0rE2sK=>zQPI(*3&>uO}izUHERh`Mcpd4q+Gr{U2cFJ&<NY$UzC!9
zMJPGNv3ol#%GdSp(AaON2kJO_vUy{>+wXfC2ldO-;D^n0ssJi%A70U7wF<1Yvzrz>
zb=?88bfmPhZNv1k(oc&rvNGH@xPE9O*en+EU8bgNRP_&_Bud6d;~5H>P`BSg<Ixf=
z@5x&RwMXi7V)~Qghi$p4SjJSFttp>HaYBXyNNh8e;3~!v-fMH8<r6tJXbfUFpIE?f
zLU<gKu(iZqe0z6V<0o}<V|V2ZHfsk?FBaOgn5palQ!)<zCCLm_om?1~Y-QoUQ=b1D
zVxbB@l0+9TiDj$J%X=fs5+?i|KKq)#HFWRmbE}FQL(Ee?|LM>^w;qZ2YGSZ@$fX}M
zSAPG+PY*F`R=40f*Lj`u4E9>%xEH?t+$0hqm975i@YtiI;;38umaM?%+_PHlyx8k(
z{@c`q_DWp5_hn)^xlgyb-KIQix>H&ZSs;wG_cNHA{UA#eN|r6uz--W4IKEG0B&klu
z>)$O>SwYgi<^OB|%<R8c4t?KgRj`JJhI~EobX(>o*!Qqw?CEXm;`>2yo~}*&?Br-l
z0Vz6qCZ#AtW@0AB&&d@fhYzaI>bBUXu}?T%U0uT4RrXhT`7o%!t76S1fJAX9w_bjb
z?n~hBYxi-?b~H=4L1B58Jy@XS5h<-}XjE>Z%n3wZ54?s&mjKrj78Dke6G0Z2yAx1j
zG9@Yq1g>d`pGC4g$w$M(KgivqqgDP_*>JD+3=u>DgwL|?{($QiO1PFd9C<5@Jw5Vr
zHR!1vj@ZIXrjsP~5uZi6Iy<i@8-JmCGt=7SBLXwq%itH*oM4KZ2L?1@%F=)77^RJF
z$x$t-|EH9_3=$}z%hK!0f!cA(`C~09V0Am<^Xfc@R0o4c%DoyjAv4{eyf7!n(QuIY
z_=0AkZ`c89kNn;1*KUr6<vOo=34C!Ju7KB*ZkNuJgiv(7y}joHkT@Gl)yV`M`*ac{
zkoh4L&~^RmYi`e94G3NUf>CF5u8MCdnY{G2EiMqDX1H-)^TjK5v2mx{2OeK=^}dw~
z(y`_^Z%64&Am^dNtxe_i*5Qt=nq+xeYZCoDf?r%*ZQv4B7vw?&HJ|xV)@PGt(E?9?
zWKl+)qoHOf$iHSwrP4-C`Abb7+jSbp@stK6Kmv{e9KvIq8v~(~Y+kC<?e(a{u4#ad
z^<%?lYodxiI@{RKAC8-!{ZN#WLjoIeW2zZ9$yf}+GXuWmM2I<zrx3?MsGw=jg_YRw
z^--#ot`6H;9}o)EDXNC_-tibnR96oOE@j1$&Wj7`xI%!iZwzdt`kpvR5d&^(UC^Mn
z>JxNLj2|~#GK#S#aOS6PA1pz5YmE99Q+p;j9(ZS@-PbtM05%*<!&69VXB$)1Q)B|6
zLV)uSh&MZ=&c7H;RB?fAI!*6B;ha^|Ow?muglcEH@dU6-l&UOHBCA%OS$m;5arl%Z
zy9KKvNY_rHOT7(qav`u#pWy}@Yip`8vnM?uIC#_=G?z^D5RW08$x&UO<36Zx7(Z%X
z^0I)Kqa7vOj)`cYi)aX#LNoPbm&j>qYN{E;3Y4No|5MwT#gL}4Wbg6F$Vi+gUVarS
z9FfSV<(<7~@nl)ggyN*(ARA9flgQ*+2(P*)tnN6_Zt7J~9=vg`=hCH1ejYtI^gPD+
z_olS0p~cP)TqvYeDRzMjvcM+%-4k`XZ~4llXHE407K;Q1K=2-zbU{W3s*gv4Vp3PV
z*dU6S48_*7`JKMy?m(#Vr5O?1nALf;5fbrTTav2lPg9&3xeyrb0-j<G#rgrohMZs>
zID(xDYLeU9+jV=M)oQI>TQGWhXSrhp(Qr_bKHoZ7V+u)k84k*1a)qRTeqZI_`y(#5
z8gFRx^!AS5!gO6hb-G_sG1&;|dZ*Ru0z6=<FV%efE7-i;xn_Z<rGK+fS^4s-k5u$?
z*UgU?9;AG_P(f+o_&>A3!<#0U{-chD!vf`-W}M=~;>}wxq)>z3pg4!o2^Q@w_7x#n
zd9rC)`-*tfsPwbFEHLhL=Ui_FN@glHx=${0mQI)8@=FR6<K6^TB$w3=<;?eMD*A7=
zfCfo-ktD4X&m-HhSwV#f(P#sgydB%eJ_XPE?+mV0Z3X6kx}5))sM9}}3A7sXzs7s~
z7i>3hQ2vzG)>irhRvZzrMQEiNq7naXvkXv;ZXYo_InZ%4hlJ&;=&*i=hfkxet*s1O
z#VSE)=FAkhn&sI5iW+Z#ECTrsn;*LAX)piCLioG3Nxz>ipXu>)bCd=kE{I=Vc6Jv_
zC@EQbw$T|@{K9%@&P>q8O5#{5(s!Wx-trjh(C+C}sH2LNHY1Z0Tfwp9<fgf4=kya$
zd4)&v93%Wep+R}?vs!lL0@%Et;>7TZ7@ASLT~#lew;moIE_4zpAaIHix*5N<dh~~|
zujkfZ8zl*Dse7m!xE9b7LqfKI>jj#+_VIDG+W`%~<mBW8>KWs14*9k`R7|@nr?oRP
zGmG2F@~2vU<KAu32W1%u2;rwa)Ufg}9OTP7Iyw}(*S4YeapvlEl61Dk7MCcarRJ8F
zgAK}$@2|XevQNNOq*rx}&I)k@R-xS)jsDxl%--%Ao=FO$-c>trJ(B%V>B+5&I4&iT
zJ`~F;R<rwtgN`;kd-J>KBqFhh5oJ6K=8~$5Q}AIyO-4?07f3&eSk(HcH-X!}1{JhA
zQOhsC1VxY~Y6oiAbV1ZjljKzo2hM9xCoQLfYQOKY7|Z%jcR1IM7U%BsS-N@pyfy8I
zH+tK?ruIfSh1CvbewX!;d(pj`y?ZF)@w8LEzar24KJ*@mo}c((4v<^f^6Bwgs_2eC
zqzHMl#Jd&6WRU4fU22(yKY)dD?=)Rp{e#_+<$*p*bLQq2nnE+(J2Lrw(^H?K4B{J4
z_xZ|z(kLgx@^u<ZOF&VkPSp0xPpl23O>WN*R<YKQPc+5csS4XtM6+XptZ>>xoqg}T
zkdv;W0A%&%vU4fc2l4bDW^)PcmmjX%chtc!5RaI$J^g4SCz>27QFz0_Imqu)QjlBU
zUjhyyrh2u%aGQ>gZv=&+0DC+5JnIYr5^nUL>;y?ms5ZQ4d%jCh&nd}MCpz|V2rA#`
ziEeRZ3mv`Gvl)&qi&=E+T~&F)#Mj8V@7O8;?4Axp^cdXKY4>fN`UzCknQD6y>-s9c
z;0IZU2*jle!U3bGfa+sEcHQ3t)xU29Su#oK0o&Hm_&}3~hN|^qNyt`XWBJh1HhRHd
z5x4guPM&?j92Q;*bL@aE5(KoVHe>ic*widQrmTa)p2=T=!svn>%D1((wYn4HIMbOp
zdKyLCd)Gz<hWoz;mYMMHhMC2Y<@Z&_mbB*I1d-amGymte)cvDUL176560}`FZK{Q{
zslq?4T_6DK?iwwG2lOxge;GH3a`3_6htFyG)(n~#N?)koT)3fBbGpPOKW+|GRmySh
z5v57Q%NcGu>!Va4j4=obdlg!6qS!Gt>eBh9t8R)wgls*wP|1=q{0XON0u;2q0yGEQ
zKct1cslxD&J5a=|0fm;MA=&J82*&DLidY-zwU3|6D$2lZ48&rzDXh7<0l_`D=gs>(
zH5g0pR&g=|ya0Dl@c-O$#2qB7$>BxKnU<xClcFX^Zl=wIP}Con2+_o2!-nA9810cO
z4m!29;Lf6Sa`au9PhH$*<7NJ5H`zq|vXa<<UPjAxK>M|g;5|DAgWF=!f?k_Lj<>aJ
z0=}%>yAYQ9Loj~}_*6{)Bj79xKrL)T9!kC>`+PJR`*O0grQLvAk!|1*3fXEb4rAqM
zICxSsJ{qk0GNb&eTYcc0z35^82cJ{xfqr4uSp(W-N~z-H^HL#GqawSV>on|7fvaz6
z8|~v~r`NPe7Q-`;lmpo!@l_cSJD7BJb#-#Z>wA@zm4vNDy~HbNG0qKK<(om)l8U@4
zK{9iU)D`#UWAt}xK;XK1C!*7Q-%XOaLJ3*Xt5v!rd4(HV1@}6PDsx3%usk&(kSrsD
zu*tY*JR@X|50x${+2?~{?8wf~mlt^Q55k;mUK8Ynr*PdJr7uw?-pLnN`?kf?eTBJr
ziXSY}hB<jWFL32Ab}vOtsbVQPeDT_7A*atrFzfdGdt_8SwCXT!d@<{Zr-V^?xTA%2
zS3hZra;%}&Uuhd8o0h71sxQ*M!RAl!Pwylj!6#I(;@lB}qnVH6ks0poEWD*W%8hjf
zFH&2vD?&CwZTKZa#Dwyph8^H$qyU5qh5`GPP=KmP*q%iFT2m(9G^#C(>Xnt5nP|jJ
z0GsxTU*4d&D*=JqQQvZEQO5mp)|@sR*HaxTI|nsXytb=i6?y<dsNCYZ%6~8a)#7$y
zE&b%?kVX4h<y<?EtT%{E=F)(F6r$N=KWP0%Y~1uQqm;_EAG13Ax9zY5A3w(<x2hDo
zDPj=GP-7nzbWXEjPxXgqwER;}c0U;1WD~jOi31>_bwk6$X?nNO+W|+xZBa|uMbD;!
zDCD^Nj^#<XWTl_!)~&s@72s1-Gy8@PCcOookjTKdTVEJ-;Wo}q8TmZTg}`#p9?ILm
z*h_KHwBruW+)uf+{PXob4JdSQ0T)*37XSdb_j(6@w?B5%&eJ2u<cZ1^Kznor5}ERD
z$Ef!hCD0@Uh!$a;S>&5k^<Bh}yS*DeK_VppI*rTX$<Z2b&V$3c4?%Ie(yBMV8(b!^
zrgYLUS-}vtqA@4wg9VZZc$?&ao-}))LrFV4wa9z#;SN885aX}<mw14X`^K0f2)Ir$
zKh=(_?!D&8Kl`}3wrZ}|lq_u-p824~3h_;9*?vdot*0|1X2YQd>39%M<{CNVbh*IO
z!(+DBvsL|6m4`Dq|3TGt(&VDj<a7g9U!0}oRk!EqMhWUce+z2)-fhnZw+^aUIZ&GM
z)~((nY<~`mFHW>$f;v5Yrm)i!R|CY!grX>8AL(~2onbf$0&kABFnH?^HQcUJOizB`
z4!z^s;CElQxjR~(UcL785>v(oVD)T0m?HjgwFzW6mR6W+J@gY4Jp`!&szyVAaTQCq
z?&aHo*7wSABpWI}Aj))mZV*T32nC=J@_1DhCO!0KT>s$o{qLp&{moK-=8kt&29YS@
zLIWnKWp^;O;&}e)#4BKRzfhJep<(dXSr21t>fVKZ3K($s*tjCo*lXuqDa}?$tBe%!
zKrnE8zo`D{)2BGk8)d^t?;%Gi2?SL75*u)xqo?cdWq<-(17w^7_Q~P!L3Ku)gkdxq
zsNI*9%(VO|(Hx6SXg(z&3L2D(hWcHhtTVfSG5Q&V8oxcgbH%AOzYC;cC9@yiJGsy8
zmh?upzqy%aKX(P9X1OKuJIfQrt{@T=*lc2AVt-nJ*R-2tu#xy0HXoD=NIm(RK-#9R
z*3|n4@b?6!(7y}t0i5No-oeg{Cthrwm~-@#kLS#`eXyc&-*_s;lOM87EZTq!dVvhS
zp)B!fRIgYH;I;dOI_olv;*Qai+JgF;*VrVJ<Ld;REZ&ML);WMv8%7IAg7w}^Z=llm
zAk7u{I<eyrX1`ULnn_TRuhsLMB<B5B^5Xz%TN}h4LRN!nR2lDT+cD#1-V+NeR;g#M
zH6bysM)uQHK_7^awJV;Ax{RckBOmqY!lIXeKKSBrxQ080W|?k^9Wa<xYUhi^2dYy>
zlC|!8n`%cnGk9-D62T1~j#Y1hFl6y^{O<mc>Kw(+VEpcnfbQ&wne2Cu7_vY9FM@lf
zM8(vjJ}!D8L{T-n{1`=Sd!eqqG}^>Jf<n}!T^o$+x`kh)-@I=omr9=q?SK;_cCghB
zo3krSzzBD$ejB0t)*r+JO@rS3P$L`2TTaB)1KaJYT1NaWK6u~n_0m%pSrj$j7fa&0
zo`o#ZZ#yr}Eh&f_tAefcq*Le4om<neUg74UzGww%dAF*rn;u#0vAM*xFwvFT1A_3m
zK|w*fD2z5m`3)D`y=eh)&c?hWmvIkzwLD(5w#rXX#O_G^&s-rlaYl>^)FmPYYywFD
zyxGBF-G)4HMa!EFE(2F*i1>gU0I#SHNA!eJeJa7tvK~i6-3=?{6F|{TOGVSk%c(lp
zZAW}ey_}sfS;y#c{ZeoRJ29+ACb-lYV7%->|55Cz1wR%D;!>B>L7cM>+!`yLoEZj}
zo5J!73vtm3Z8W}l7%%V4lR8aQ1^EfZamvq`!xvoD;KQ_G0k~;H#DYIEK#p$91Ltw#
z;Fo&J?+|iybfad)27KS*3qs1BET3R70X#OYzxI6cgnRlg8TS)@_+en=+}ORfA6q>S
zrS?U)0w}E%+L%U(sKgbsb4whnSk$d1CW`#Kfu==jWyi7BhWOpo6Bv&@e;3NkTOOeM
zYbdi!WSV*#>M#lgr<QR3Wf&MB8x$|yfxw@S0Ob!{4ZieiL^H#!zJV(wmoLFq9A@2A
zE33|;MEWhAeJB=K1MY@1>y?{osS{TN-~TvYChWL9=nHP~e7CYTM-gJIDuDh2RPrDC
znsL4-llOE@bvj(I->i^#A^l|Slh{OQ9@|#ynlhsL>|N}4Xn#rJmBujr$%GUtulq5U
zLu}*cw_H<x-ScQeo)HGK*}@X7Wc7a(>MS~i_}ny6mDeOSkOn8nc0xL0enWsypMPre
zzCPwkEQk6+W+Y`srpl^YcC5rCHPxH}T%E;!9ovpaw2=8Qgq&ecJ+U9rPN&7E*hf}8
z0yVA2_MxHKn^V>>htxLJfV=ZG26G}E1?~IzQ^Vur_s{NtJ8>|Y^;<{DZSPWc|L%)?
zJTm{rrd?O}&=(lY0~qZ|n-Gb3rQ78M$R?|l`8=o1cb8$3^ie#G!CbB_JRo{MY$ptm
zsv9RlaNl2k0=;UZeS45Qv|@zcxBXa0uGw7$3Rm`_(UFP|iY?45(A49JkrOQ+VXhxi
zHT46<-aQUJKoswpS*0Ig7;~N*Q2un;^gkMY{ts>FKiGKN7Yf}lx@%)erbRK@6};Ew
Ko3gL{&-@>*8~EG+

diff --git a/reference/plot_pit-3.png b/reference/plot_pit-3.png
index e38701506b828efa24f9ed36d47248102bcede43..d0a1a4e5b93560f3288b22a2f402153be0579a96 100644
GIT binary patch
literal 37900
zcmdqJcUV(-v@XnyV=v5%qo}k|gNPuAD81;YU_d}YK|@nIgx*36u`?<~H(i<vhzKD9
z2BgENFrh^$2`vPr38A+LA<4Ib&N=71_mq3S`~7jBo9A)DkiCEVSJryhyWX|Jz4JzT
zTQ}|5BqSuX6@B)!iIC9RKp~+uwj0*LZ+haNPQ%+eTLZn*@D8t~Q{@DB+vs)nim#B6
zc(~x-579$24njh|3!zV+ycn1=&LE_uI$24tQ2kq)?2+!@^!1gTOfdQmYXg4ST4=Dt
z?FS1NC7ItEqwjnk(>^#>*1vBPwJaJRdiul}bKK1O-}h}0Mi}}^F|o5Bl&`Oxw>W?K
zMx04Eq83wp;s)<;RE!#Zu0BP^MLouT*3PJ9kQ-Lnf}Ql`znP@sCo?ODlUbIyA6Mu4
zf8Kxc?6c@U2w{jcjhKq@BLQ4aKsonqy#}=)pl-KQD7B`hW-i7}K1)-Jl7WO0X~B4z
zp~{H>&UW;eZG8!b=${oUizOx;SbB<TM)5D}FP$eYSi$Gm2V(3pRP_Zcw<sqkCtV}r
zMWk8o@vdgJXZOUSQ`tm?5~b0VsB`@I@r6Eax6PNTW|WKcQYd2Vubw(l9x$=|S8YD2
zhb9uE=@A(CHNwKPa-k>u(ZUPyDc{A<kK;{qJIdrKcW&Lf<-`o*Oqhmagli}UHpVud
zQi}_!NA-Md=9R8_M_R8JI+(QuwOg}2m!|%q=%Jpoz5QTAZEdY*oJpUR{?a{E2KsJH
zsHe<O99moZ=+W8Ht)j{i@?L?v#l^W<C>$L_P@ojW<Qe+tBXd|{^`pTkv)qN@Chk73
zC+q{e<+4_qf<{tH#*P&7t|ANQDuzc1|5NSH%t{^MB|_akw!0xRhiK|hJXQ8Y@;h9~
zJoDU+<r}9`!m5ZveihWN2Uv{J9DBu!zQzk_rMJ-=8^&4Q_EE!CU!4Bh_gmnN?Yc5U
zOmfdeVJFtrb<BpXK49`1LsUsH=}Rr$w$y_q8e0}QHhYybJR%|@wsc^q=vhg98RIX0
z>V)E&8PAD;#h5|Etk@#h^Am7UOG``I75yMXTn`C`sc!j!Z#WgT_%$3A$tcJ1JD<+R
zn}j_r|2m9HeOuYGX41mW-rnBX%j;F#pcLA}c=jDC9rlIc_s0E4A=&7vzPC|y`kMa4
zFXe>2Yt8bFvVA+no3~rR3Ll}W5mZ*Jld+Le-z)xNvS(G$0blXYCB+z`jzX4YDSz?f
z$B&z_M4bdwr0&39sT#P+P68(t*B4}tFfFRXxZ!&u>qd}>$cTsvKUlaHXUbeZ$d0%`
zawS$H_O=BTqRMWU8KZtYD`d;IZB@7e?oOW?d@ci<tUt>`S%5W5N(Y@GE{wr*9#JS1
zPq{4ovOzf`%#5TY<Yn7R=g8x=grcO@T0coY4p;jjuX`q*-U-+xE{=B~E(ph1e)K3I
zbrE`ab2<}!m7HI)Vr`6kT#&6<u_H!CM%GLQIU*{`8iU0<AahD8`U7oQu_drt>(ub*
z>j%-i!t&6L8{)h*aCGaDl~G<j9m9-6k!eiLS8uDUt;z%NPdi+>ZU`>cq_5a}ppK2x
zV2n!WFh3INmKRvmPBNCx*i3F4^^>2J=S?W4QC*7}9(>E(j$k9Co`xZ)QCPwOHCLQc
zJ07Q|;;oQIBrlf?o1tQCXo&xE-70#FC+XEv?@tYKL8j0?cG+!9&G1w%_mlOPz4<Qa
z3D4d)-d?b?H%%Ih+c64Fy>zYWw83SQ2FY_qM!bQ2*{*O_>1St6(P{Damk<(4;Mdgb
z=g+I&d^qWp7BpP#rkt#kK3E?U;TPT2>U@mX(~_!DT^ftk>g;vq@KckMlGKKdOwD;y
zHlDEg0H&?+i4$xvkuK&yaPC?T=8S~)-&ilv<)&=IOqC9HM}&{>iDdSc`i_hs_HXZQ
z&8D`r@JwWe&Lpu)!d}MC+8mIU*71Kuoeb<1;hr%vd+sc>%s4r6?Il^cyhkMX#o!5~
zc64+!A>PEpU{(^*RQC0)IUX~JHV$t6@b}+;Zza=an*22-l1h!5&yZ<{iDoQM3nJ|?
zS5%9YSJYzY6M$^lx|LgNqTmb<k2aW1MC}$6Td6!Xp7!X`Bb;=S;nC23bJ3oO#Wh*6
zY!o`<W8lY=kAtk(Bcch9*+-fVEKwOlHcclbrq#5xxHr4Hx-`l-S_dePO&Zo_#bWhz
zb$2d1d~%-Tuc(nyQd29XDEvpIM*TTxWdMe1QLwsZ+j{Z*{VoJJlf!6jODs1D#Y`?;
znv{qc%yuR6Kj2?9+;|Mr&PWn9^z|O*<>jMiNk~O;?Cc$Ytdy`CMF)Vk!_AI!bLcR6
zUji>YMpK4zDb>iy6@H&%P%(Im5wle7Yfwq8-!Co1JqcJjb$Ws&?!q5D%Q@A^po^Y8
z`X+=uDFO#U^5bM>Bym>q0Z90&0E_Lg3q=~KE#lw@jW`aGnokb%;Vv&wt;XMKY-67x
zJc@bzqr!<<8fxlsQqpdhlAg%P6<bLoBxJI5gqRh(85wsywuk?m;UQeyGjYSD!4wHS
zPBD$C@Z{GyIysGKY_lnwDDJ5uw;zES^S<)&_2Rf*-xu8S4Ao&hPt9>kTTxLFpIH92
zOzy*y7H7Oj4_RTMQlCD3%8Gx?_BysFGTL9nSZyT5*GS{<d_w%7J6!Js&#L?I;lmgh
z`2n=dP)Q41peGg{$UjV>K@czlNQ66&C$XHL)0WFKGBS*>OfDM8*c{HPaAsEFS>=S`
zJ#B4m*-}B)2q&h(+s|))h9jv28)BDK&bN6%*7gR3e8KQgq2*msvc%$TkrEg%T=P>I
zxVNp$kdD;SA2LIur%s*99ietiyc3Qy(GFgXs~5u7t;e+DC$AX^nXSWX;VbR>=o;&|
zF;e)Qm1C?p6W%e(l=7ZPJwTAUzRD2#fu)SeV*VBVB|iz?Qk6Cu05YU7kTIIrC-8>=
z@CTz^S+O{X$l+SoZFS6H>DB7nvUz9A8N5-H@aWH4%O9h)l^8}4QNjZz!PCNe1_uXq
zJUU-}%1JdE$wwu)!;ck!#eJ=emCQ*+z2%N)(gD%@aw)O7t|mO0#S!j(g{9{wgX3tS
zb?GD<{%rx0UE@!17AHc`xS7-vMnKQ}FOJ$TK7IN`(wVMVgGRJM79&ts#uJ|MO8~)E
zR+2r^1e1Iav(l54lq6|XII*jqlMS3>#jiRH+`ol^=DGR#`8}a6OT)C-F_v#qCIDG^
zyc?0BbIHsVBLu6ftD9z_zjV1IW4UpzM>MidxC3K~#SgW&w~J&yAFjd&H6AV*N6)_J
zvJDZBkwtb)XvRlpO>#1p?^}uY^Lu>-%N@9J;|3vl&|Pn4x7e3gQ@+u@KYDE$WoezY
zA?4a|g6m%p3NxwC!ts?jI02hf!rTIh!0n3^RjCRF7mZ624O+>8S-1M}!~Cgp&w&**
zpq$eEX;-4T-Sn!(xV7C{j^G!+MD(vU(!8}kNSwrXw2E7!zJfk_<jC~+<Y$hwv@}lQ
z4Su90l{wzB6bIpxvH}2{!+5_xxaQq25(HzU9V4<w@gsV+2i*!l>D@mH;jde3$|7Oc
z!YhVP<6+^J9dzeoW`~QJ<KyF=hhJC4`WPa9>Zx;4__7M^&6?=og|UvQu{6$2z=%BU
z3*9<Hv8Ka>-77O(IvtZ;HlelT)eS&bam1I9-B&@74kQTNtf*&r_-!*zHQpyZBx(Eg
zfj<OTac!AM!pSX58*t@<JLoJw^=jY#vdI^-(E-u;Q#-$ME@8RpjC7~Nt@DcI3d;|!
z{-ogbqfFLHIYi&;%ae0JJ;H(>85$cQCJp2V;yrH4I<2FAVL7xi?=UOkLLX|>PknvM
z4gbsPsV(@dSX5B^uiHXStFTH$x;veZ^+71?yNu;7!2$3ZR^F=V>glC}zxg5M+h+Uo
zGXna8Mz*#@j{#nV?CkLb%2N4Z8N*tki_qm>a}RRg{vqpGIux%30P2fQ(^OI_()P7!
zvn*Z|Sn)gEjF~ocm>gvQ*&;<*BdmFZ-#Epds3v55d{fpox^e;G_qmdlfpl48xjg~p
zT1jiwspzm5oH;`nZ|ej5bRGJ3;jnuKMejJhMJYmJ7R$lD^9vIB!kl0|hR&5Sr;oW+
zeSkx@!b9a6XuvxWPH)vmhIYl@i*`Xa^CD`aGSfdwM;K)nov$B!V^JBr(04{z>kwLB
z#>Qs66P6KknLbgnyqPf&<@_t(q(P+Io&VwKB7?djb}8M$c>p4>G&p55TG`0Zkn4;@
zqtMxHK@KufS+OckHwZyGX3w*;C09bth#2yhnB?%gcO6B#{CW$wZr#cVdDuLGGL{e%
z3-!v+2P<Y@>1RE9`}VC<Z!n7}%*n#$OyH<9tRzu)za!0D=6G-ETr!ZaKsVw-B$)=(
zU&O!XvC*uJlamvH-=Z;9fvrA8!x5ByzWid^e9FxRk?I>s8~Ulgvz4BlK`MC?#$t*4
z^X+re&ybdb*Q{CdfLl2sh^Lp`I47q@AI(1!{sV$Z+K&paTE%Ax{p+<OuIez)ArMi|
z=$V6Q>&2P?+mKkfZzVINM5dvFK96RnR~2hawVCwxa+mNB0b2HNTbPalOW6k7;ckRr
zSkfXrB-(PFNdy1v>|>Oq+Em4ynHInk=_~>u#Fkea5O|G~pU6}-M4_ExK|U#Bvpi~3
zQ>gLIbZHMfE}g@+P<ePuvw6w6M-rlSC~@H>sts)oCahfmm>6=DeS56ClDV1ptPC)W
zf>O|diG(>iIu3JOi6g*85&H4uW;zh3r0D9o77fpd$uMIi*2G#>O|5jui<G^7SxQ={
z@+pa)MJjnU&`xKe`Swx^3sL=5Voe+I#8+ZXwVUUQxEibn+*=*h>YwYd+)&(%+JgyG
z{U9^kOwk+*7*chLy%Um$A&-f~2loLwQPIy4`>DQ4{>*vKR5zI=mBi|ubSGodF>ck4
zMWvPHUk6tAD`2XRn=<L0Uct&QpR{ysk#Y_g8P*?8ix0`ubry@^RCj7R0+Wil+bVvz
z%}p8CO!6%0(fNR1UOZ*JIkHa+Sp;sdmN5{P!|)(@(etfv(p?FY3t<yj8;wEwNzd|$
z;!(f>f^$g^q1~f|@H<7JCH46dmNQgV8d7B#?n+7!EWiFvmG%5WnbC_XYZ6CV_9>~P
zo#XRovC$XoouMZ+u>8k^>8tEeIN4yd*O#hwm|`s9G62z)E)C&*h><O)(9X*(_hb{g
zJt2lygp7NVF+@r2&LBbL{{+bZ*O!K28RHW8Q}928qRs-aWmT|^sL7u6^mI@6y^QtS
zDrC`%lMK#H42$HaQaO5cDas@R!g&~Rp%+z@70cAR>n<6?VFG$7ZC>F|TQHYtE8DYm
zAY+hJZGC9qUj5NIbQQZR62QPoYB#s21c*rLRC{=NO~0k|rcV{MsI$Bc+<i;u4UB}M
zHVdNv_CgZw*@E$M0buM@<GnCAa7h2E%=Ckt1g6P`bgkacTt(Tfb9xYXuCdcbn=G9V
z6D3UTOZrQVsNK7EeJux!*HrW6Gvol3Y{uz4|CpE<eg(e(-c$tip{H7q)pA##E~?^@
zLTytO5GhE1s+r|D&vM!_!i`RU{CF9tJ_Bun<+gwM@`W(U`+VUWC6A7ZswxCLobtFU
zT+%b4qtkx81H%u&&9D(oCCd2Y<5=yALE(}9$1FNY2X;)0)ZvPk@!h`(8C(l<(3^if
z`d%!9!(RjQs}x+bKS}iX2UoJS2^Mc&GGU|MkZ9mqO<Tq>-ZNpQc>OXyNdn$4LfT`B
z(B=IQI5WN`u&-A5Ve8W5!316S=p2x73pSUpQcYm9o{$U#SV0hj`#zQVvxd{hQhi(0
zoC1LaZ-nRi!@rdb&W0$<>cOA-zX}o)1EY#6YlUTL*;gc7)1-l<rK}ik7^3!fI;Er0
zsXY-8;y*mRFI8ta8bpAU((K=v5D^d*!zr-WQh$j=rkN|W(?0D26nzh5ty+arJt<($
zOkNnz1u#H*CuLa@LY9s>mYacL<pNmRXdOQux1{XoC?R%zMwm=X*P8w+5K8v!-n|5Q
zPYFbNMIZ{I>n)iTI&LLBB+6gLRIOLdRLILbGdbrUVOTwakai~BFI08g8_#CV6-i+}
zF1xMN2mFsfI5u)+uvZVyH;}1@WP$-9?z35L9MEwmesgu$>(YeoFkptKAz8nSIMFaV
z8U#Z|DGwPGK`Q)jm~tPOG<ec5MK$Sds8WZ3fW^HTuAm`!0&`PBDu67TgVPC1l;fuQ
z{L-gWQ98x3SZ$iXhu3G5*yopn!~G|>7nA6>c$E8NEO8dcXP=F%-1_~XXf*flwH7WA
z5IrR&rD%n$6*ENRD0_3QQ%r>*QyC@GETRrHxtB1MK}?~N$ku6tZ_$&+N3F1oM}t|Q
zEODl)IFp>6Y88-(^G2tqr$^B|Jc30+wR}_OdSX_BWQMN5W$PhX^|0yx+5)Ul)XC9N
z_b6MTRkE@7R*7R-h9*DVD0{9GAo68X^T%LSb#$L5|9O`iZ|0f=?%*8TN!gdK4AEW(
z$R!(WU0N)98^OnIlK=8jZr?}H9UKt0)RHbziPnxC)D!HCn0wuR=W1ZxPEPp`ImSul
zPBG}d4(DTOy&wtsJ)HxYf_F~F!4%Okp6zJD?_}!F!Vxw~F8!^iJqmGCbNb{CGB9Kx
zu;x&NiJo1W^kyF@m7yd9w6?($w(2eH6{`%RY^-@Y%EBj9_3cgR$5%Q`r02b^>hpf1
zAU#h0|5IYDDKHNfzn<aSW~jn7LVS_I*KI~2UF@(a_T;GhK7ovRwRFq8x{%(48afR@
z;h1=KV@S$c!3URh1-hHy)&KXw>D5Wi%dE8){NlQHR?TL?4<E9B3;aRw!*cA83WE0&
z=_dWFcYiV6R3Z3<5MKX=h~WK+F|Z@S5C6^}>w?9FmML3RdcC|6+<_}hmYkNKK~dMr
zJxT6INp2|Vmf*gW@!tZkxr8aAbLo47$7g`y>ckD|EU0gQ&uS<}H$>u}#1=lMJw}y+
zWkEy_PH|q^vSo{X^W)=lo^&xc8^Iv);Xf*fJ;mmi9uU@?K0&gRO9po`P^f%OR}_5Q
z86BRtx}JY$lK=HeU7yf@h$1&9tE6AS;@y!E=_@w+rKN476$4=EA<QUojYaV@$yB4+
zUX)IXJ&IzL6)TSJEB0s6jOX{QZjzUe#fANWD(>C$^B9&rLPx2oP#$AgO8PQ{f`Ex`
z<LJmp;uY6qJ3Bj;E3q7@P@yz}NJ8;K>f13kVCcyO1qDPAORw5z?`H=dtsc?U_rC=y
zRDJ#2W*{SJ*m`dEIV5$a_4W0lkd}F)wau}N#6c|scih#hPvT8@dNM<W7}j0T37*c(
z%;+Mf`Z5aVj>g!Jx0@PstK*ecXIXPU>De`B=Pw=Tby(t5dSYTAcV*eVr<f7^euufH
zZ*cJPBr3tiha%<V$*kfozIA7l_{$_vUGJ}<M@L8NJhX4@?Civ&i=~&Sg2{S{sWSeW
zX;`(JS0=jC@lA$=hZvUVw%#`mjq#q+OED-H<zxlzcxefM)w8$)!CvU49xN3c+4VTj
zGqA$ymDvR}4MpLynf$Cfi#cTLHg4(C<une~hSSYU|FlM0UtVze)gg;t*7iX<`x?oN
zEu5>7`E_{fmMx)K<y2vbMqlA=+tQd;E3Gu5#gx_spH)+kW!_J`Rnp~_b^&WhT!??a
zD3jGBViT_LrvK)R_nYz!ykXIKX=!O;o{;DfmIH^EIn{iQNP>w(b<OhyJw#ma*e?Xx
zXEhdBs}8Hss|9%A@lNa34a?&ouWLr>UFxs=d|&o)$l~}C3$-83^Rl)5Y+A2m4j6LJ
zMNDw}buRHwnLWt(1kbXla~qsotu1hya;NBFdvh!QOLnm3crxvQfehbnpf0l7Z{Ogm
zs~O45+WzDT!6f&iGm29BU;<d5@r3-O*5lFL`DJ4->zY-VD`_Z&JhF$lghbfPC3C9|
zwZ2kFZ3v_;x3H@>3_V|v(<*Np2<9MU_UDcZ{Z?exOCgSq2hb8lotni!j7Dgfn86~w
z;56?2a=tQUkuzC7wRZdV?GK{LwBk$C7GG!M{hltUF#OsSq%W*u!}XXC_Y<Fe8p|d1
z#o4Q)jWi}+-;p>Sk6~Q~CY`T__==tH<xW?!`$D>&>i(#pw|Z{<gXDdiD$uTo&RGO7
ze|e3HM_<KKJF5C23Xp(6Zm%nU&!}#)^ky%E@fwD!HNW+4UbU!{UNik0Dgtf805L)m
z+`C_o(n)VV-|s;CbU7rXUcy%HrJP45dWzl47i#QOJo{E+z`sK3>+13~yn4dkGZIi;
ziojNSODZSal^VZ!RsU79cVZQ2p9wO(uhOOMxl@9Phg_EBaV;%;M`tHwzw6Tpi%Z~Y
zB8Pnfl_*!)3qPwcL+1vR%zF(HCrD@*I?Bq$SNDT%E<?BUZ5D~Twz|ygXNZZvRG^P4
zDi&nW%Jxzu<4pn%BpJ?@psisJh2D+vhuPvoGJBRh_I|xaGRig$j$QR~)s2Gt<_^Fs
zrs5?>UCyG!1Mh_3<X^johOQW(cgpojOj)iT<NBg!y5EpE<=lUbY5xo8{ofd5D});E
z$X!v7z*8W0$67m$-Y5!Lbr7Lpndw3lM1Z$3$Vz%;%STx<&00?|;p^*@p52fFT_KyT
zzDDqiBlR1sqdIl?b2=jflKemb@myI_#ZjO#l~Jo6i~cQ8Z4;RWgxk7)b%dCBnLUCZ
z{x`$ae`BHlajg44c#u6`{IA=&&kZ*Nle9@5Ttd^ps(}Gdy@80@S7KKmh1ctJD;-1(
zTEuqk3dA6WWLhIs)n-3ItT0@!;nnusP~lKk#mLVSI=*0bZF-4Y1-q__CcgM_>bYTZ
z8ic4oWO$`)HKb85LxGSWCMp^-{PpYCH4uZ`+{Rl`17uowo<f$`?3`@YF=UH`TRH}N
z#r1$-pzAwK`zybGeqgMS6?)9~Jws)yuyFM?flt^3k7h-?AKOC!ehKjmZT+gf7#J5M
z6`dyfT^KC)G;u*DD>gTwRhPIh0$#$i&lwTap@l9t-Itl(wsmWzI^`;HfeXk*O1G(o
zXl5Y*2`3|CV`l2$tbu&#;P^yO39~A;xWv6DzzSgvGi?I89HK!&GiU($E<Qjono!N&
zyx8!(I1~vDg2n$B!L0k7S1+oszkhJrI&5i(6_bZXI8)VpgUu7B*vjV|-?^i@t46Ns
zVfOB~tq>AiK**0Z37G6Guw(wg0YVcBL|!>7R?>1Z)JjK5S$VMmWqj7h35o8&Fx3<l
zXFj49`omY@U$Yk;qnzBV<VwsT_Z4<(XT^q;KX@w$xiz+T`UkQ&4@-iaq)Q89eTE<)
zM}<sPvP(4r#uK$WX9R#vO)yETI8p8-&dL)^`vl?gFQ&)ZG3Ndvl9%1mA%9I;8tTY3
zSBVaJk&j`O#8f<sJ*uv$sd+?OyU$rUIT`>l$zGkZ7L*k{^J%wCmL=pcU2AO668;sW
zRZmq_B>o9-w-`Xkzv31SxZY2mgv@L4jr7>##Kg|9{xadP>03au69AG8sUF7v^a7|z
zE!h$mmJ+OkM@t3B?O!Sg)&_G0(Rep|NOXu0BRRpTS?2mOIGlPaU28;*x^$6LBqZc(
zcKSw1+h{M4dStNo{UR`n>;MtAJbQl2gWZyn{8reBMO4DE<mR~;IZQv1%)<~D*us`J
z`avPcT$J58m;wcKT4OpU-9~@ZKh`?<kBpCB7f?EVtlx!dQvQb+xl;BoF|sjr>Z-m>
zI#3f}r@GsUnWpK>?No)=yywXtWm8{KaPEudSglI>Zlj;FeoJ2?lyU;*$`-`Q<I2j)
zsrFw5<UJHo5PM@x^p^u&kuHa;x|Kr?Bj9FQ($>KSZ0qr-TP0{rzR92~#y3NjGdr75
zT>;$o`r5od!}Fwa(7fN#AkDIbd*1}j4V%WS{DCw<iLPT-*P=HIcGCkXn6&V{3`0q0
zPL}Nb;rjLu_jlDMqXglz5t<0FF%@p$<t2I6mCGe1Ze4|(!bW+G+_q86RQrRJ#XA!A
z#V~Gq9=S@35;-t1AR?K24w@%?sk=bh=rCz0#}Z2-1QZY@V&(Ty1~6L$iX_j<S>fQ5
zRiSw;tI;e)%<S9^b~lz(83GTZ%4hXn)-QDhlJgSyEx`Jcm9~zi=E}1(+dL2K9<P!Z
ze)d3K-J#><C3aKwl%IMt3ienN;wvr;Ph4oYckkYMy$wL^lwFCW9CFnGbX{Ft;dNuF
zpxRG1G6Ku4Nr^jp<`h(<ZEf|4{-+QfxvcSXN0SFVV=8Q+GBgl>*dl89hW#y0<(|z%
zg)Gg%xn2+@QlUy$fg&Bw3e}|G%95&52BEGdDGoODY3Wg=U|b=C0$bhbmgL05#B;MR
zCt0uD7-b8o=)Tg<MPb6%w`^${J!IsT<VwFevf{yBWh-7yc`ttXmN`jp|KG{B1fdpQ
zT%Zw)%PbPXjN~g<g24jMs~Xh{8`PVe@ZTxanZBB~8O$hvF|pO(`N(Doghpm>6`o_K
z07ySIuP!Xqq;y7rN$ZcRs&=wANZ)S@4-Zf8WBn}jN@u?y{Zn-V1)&9UeHv?5<5-gt
zwF$)BG!)>iMEwuf!_Ny?U;irxB$rFc)l13N^_R_z3CnGga??N7+^hR-HAeg|<%$1|
z*eO{4hXnbmPj^MNJ6S(B5^3rnvC@mu7V_LHTLkd<5R%&P(O6TXF8+iXgO1LMMNlGy
z*_d?Axs^F8lflvy>>pI*wlz*qdy)3195fPeQsW7KLB@Ig-3IIa4FYw2smV#ZO|bBr
zo8|uovj`R!Y*znX5dNfkikyt0bebkuGPn@S|0lt%;fkyK`JX1sL4OMvi`cuG8XUnt
zpUj(CpxY2Ug$L$O%M9h4G_d3=rvFL~PF;IFd^O>|eoK`*-^M0{7|CRrUkH8)`d%<n
zU_Bz#|Mc8P7)WUSM>pjV@lUrufa>1C+9ITxvT9{si69l1K{=pE02r`N>*dyjMRi?j
zZ{R?rC{(PqDC|Xb15&KX<hfbd=Ofp*C#+hG*mDe4J=I8%K!Z+at@kAd!=j=}^XO{9
zIke?>sOtE^AlPQPrf;XaS%dqbJTFv$YAH2POM?li-Ujzp20^hhs=&>RWQ{j5rvAV^
zy($*;|15}Di|TcSYDv%c-3ZHGRU5>jYNJC3_d6|uoxUXJ%Vf=~ud6u;OiS)#e}8}V
z@)tdw<!0-V%O0Re;WPz+uS<eXvJawUS~k>Rttwl7s5u3fi`_krrH%6!_e&47-3Dc-
zk438Hl~ThNi#)%8PYA`=_xE2V?HBUe01`&{wZOnP^k0mAIt5gH37F*%LPo!GjE#>d
zDaCk(4h$~;L5DLxdgO_pPhr_iooF~{R7O+&QPzUSgO)oY)`2!8UoU5#>{_o~i&LF_
z(B;%s;%Ot?CP#u<0Jxw)qII8F-6kZY<wMnC+lPh%UdMNAww}J361I%CU~1%85x5H@
zbd2=6b?YR$SaOvn*wiX`6k+VI%?hCeQjwPzOw?c=VxJt8xdr<PBBj8)Ks4C`6=nlh
zqO`kr309S23@Mma?Kg5-6Xr@3HafwL^bFk(4FeMnDGm@5eTI{q4g`G}skvuxL*#mv
zvj-H=FOg2kmB8kHOwHC_$WelQ6cUoFHpY*cCOv*U<{#kz%}b9f=Cc$g{I|rPfZJ|$
z6L_CcIvRk{MU@S$vdpCxoTKNPPj^eSSy!mRmjrQGsD=ldsro`sd)7WZ;1C+5i3dF;
zo_vJ_OCJg@SbW`+1nub?!pG=D3wAq(g=R-ap+rF^66!wzs};Kl$+1@uHBGCO&*Ow7
zIxZ2d%9SD1qD}7Jz1yU-Hf0`ic$^jDBsslRSV$-s4D0&a#W+%^RQlI<(!t_(SFC`&
z1<hwjGFZEWq;CU*J&sOeupr$IT{Us9%P^3HSaG=%H7#@@*G8~EIH*6^3+w^JA_?2~
zI|3nF40R+fkkM)B_Rvu`3=AepkO1R>MRnqCH^&QjV-wV<v!%K&Vk<Q%-6%;3Nl9k=
zTgb&eYzH{;)aK1Zv%%6aB_65ZT3&y-hy^F`3LvM<h9i%uI6x^y)7mlV7xb4NKtEAf
zu$!A(^P|@q%E|#QMO`T`$<|OaKZEE=K^vQx*^QOJp>L_bhwSs3keT-)>DKy7&ckVb
zynk2t(Q6IJZc9QJX2tqAAim}M14Gij<&&kvM43IX1J<ebuV~9Wh@o^UiCOI4Lo~Pa
zey^0Bojqmbvm^zFwYWIZ<7uPcBcNgO6#uy?h4S&jdg%X48~JA@4nKKPdgs?e?EO>|
zX;JL96oBZU5yzq>+wT(2)&E$+v8S%a#k?&bpa{;w3`Bx|%Z3HYP1^YPaju<ds)d>~
z^#%DkEx6H^1F$tAzs<(0cI}!6J1mXcEd{ie)PYG4=!d#;X;!RV>@ER1ev$r<<(c}*
znh-hPIFqo0l+J<{0rm^clXfTqU#MiHV|VsEZ5!3R+ytzZJ??FMY~=Je)ge#l-<Mvj
zqDFT|`RBvh=YZS?P5j8xNgUj$m%L+j8ajeSx^}!=Y-&1(XgU@IxvgKfu7yZGKyl(I
z(oNH5V<~rNOush6kf<|ef;*n@^FSENuUa0nLO+jLRd-kZ;Iw1vLEZ9crhGfh%e#R`
zs-O*L*Q0@XS`+%~1RCXIC=v*$GU=OEWOg&oq`w*Bejkz<Gw6`4?7qApoBzhvAF%{S
zaq7T$o;@0C&jAbY(3<Xq0r-Q!l>2~?v5Ydon&Gv+Xy0(ZAp*F*IBA%ytW_TCV<$88
z5)y@6U|jI%VcgORDE$vW^-r>Y#R*P)AYb++_EN4HLy2G2HhL^3lNLN97<Fj@iqg^t
z4oDpRm3`0`;06H4^?d{k4u&XBrfo&@Ud0wFQQVMf!`b^F_z0}3bn4(^$=<1s%g%Ti
zhg#~K>UaAyk#+oOsse7Lph#m?+Zq8Fm`Q`F58V0F;X0uNq_B6@dW5z)o{3xyO+u<k
z|6c^}HI)G3+-I?~T!D;H5|V0H8fW5BJ^k0_>OKLkK+FXP7kf9>kt=B_pJmB>dr-?1
z%Lo~WRIv{IbVp=zHTnoiG(gPs`WcG%l~G>%cIo?EA)SJor@cV6sW};IB4NoJGp~LO
zVOQe$6>BUl51reg57Hwy;|PH2#<LJXx3$&l@Mgxlin0Y!{q9y#iPd1>WqWkbs-pkI
zcz5k)p_-f8OBCaL?Rjh2Ht0QYvibU#2(ZF!Lf5TdlDApm82-_d{!v54(0#HT;iNgh
z%8Kn~$%UVS2V4*TTC-mJ$ZHQq84I9*kV(H1?7%4VFA}U=Qd0o(h60eE*OwVuffg+d
z!Dc}EtN^(9wS7GRyHmyV9R<x#lJ9KqLmDpeIYOx=&=H9#e$&$dR5>Q-H)!zm!GPEZ
z#0%Jfu}z5w!!#syz-yOa=BL^*lH(o5#>OK&&eRkjGy6|H&3(;|N@#(K>aqx=wZ_Nw
zSe1r^A83`#uXQTH5F%0ox?QeR22V1-f;4bGdO&QafMD-j7*ZgprSs;qLm(<>@t3I7
z?=0@g$Rr5&Zau|BK}1$cn~4Zzvhg-RdhY#SswcIdAPLB;kk=m}4UsB+UvZV&3R0l+
zVP1WWnEbbFM8fuJf4LNtKXagW2Iq49)^{>1TtK)XqFS)jBk1?9c{W+b%6DT{RSI@)
zE{$EG`T3XVFaBFp<-V1L<ba{tJ32mxk1r9)^k4d4B|Tep^Z%?&^bh?P%u)=Stm3I%
zL|g7gQH=|sW6BqLW$N*42D?fejLc%PSWrcIX3`L{?c=->)H`(W$GQYdyB<Tmzo)kR
zKaa0IC06<_&~M(EOT$DR`&OEn#VGFopDHOTkOB|!pnx=A0Q``IC5pyWNK-Zke+>98
zGY$To52ygF<&#;{hCXO?G<WhiHqL<w*;GSugM+jS3MZhNtTdsN(DKy+ac%NUPo!>!
zuf6%IK=tYs0Q2FbWc9sdnu^#PfdtGnnw4JdB@6wLyqZ0Ey%Q(}vyBQi>Gp<5FM-ds
z!8oU5SjQRZ$;Pq8t2-N22Xzb_FeajI#v_x4mep-?P6u*A0tw)G?CdQqSB?_^wRO_q
zMl{g<0t`z7Fj)I>ro7a4!5U5!{;{b7lHLdm)|FTyS8OaZ)HnHgdaa1%kyn=s76Zu)
z{LeiVMKP9F^h<q!rqn~e#Oa0SoRgHR7nB-8$TTPUSEDoGzN?yVO+(SX=gu$4`D60h
zuR67XyDH5!=9-f{<js4?G*4)^k#X<~3=F&qoEXv<Nf`Dh#Shs`%eUfGz5R3c4N6ob
zkf_#+U0t79{0_yNDwMuS>X`?rf)WXqA&AL7WGxWWxiws!ueHKqvJ{)B$Vf))-~%#k
zL?9ZDc=Qxc(4@ZCOMLzb+U0?hwj9uaOv@jH2;+k94lou$qf9{Dri#6JwTp}*C+Y)p
zNLt2jkW3mzwy{G6`%}|UxKDRFA47hr1=#^A`ik4OUf5n=ED=51wLNPBGU8C+CD7Hi
zTdgx}B%Q#U{1(kqv(i>K7y|XX4Sl(!s$lcs;i6`j?9}4^Q}wPyJG6;@>F{Zn@{n;h
zPGrh*tYGzV6nCIaG3_YALlmrcVZBFhMbI3$<z7K^8h`Ax3#ZjUK}&^lnLUvmpZJZ}
zQC15{_Q2{rq1z<T{aZmK9%MZJBFG#80!`Bi$XjmA7OAeP54W0>1qBf@twg%i8zkM&
zb{&hp>6~6A$VI})v9psH)?tu&gi(|;Am}tecifq**y=Bn%TFQRDoFMi3ld(T%cEuu
z8~Q9mCdfEY%~Ihz1-XJ&wq2s4xiIl4l(FILzpa({H+4fNn;R+!>TQxSy-+DJgKEqe
z$XS!JS(d4T?M;!r_}Y(FT7t2?jtcxTMxfo)gT$;HYZ3-zHi+ntq8Ot41ce(B&yb!T
z$M~WNqD4vHQpJeYR0RajkpvTE+(?g>6orV^#^?u8ffnebmAd>c)PMUD7GDR|=;?UK
zCLMEQG@sIz$tDeq%56mgFpM`@P4QK?$iA#eMuA`)gSPI#i^s$8RZ0|%XuTd#bk<iR
zgSc`9p)aCf&|zMsv9oK?VNY-_!6Wu^erXZ(EkG4q-wSecF^lXJAAQI-VJ#bEJ$`~5
zV4UGyFTV6yq`Fg}E2vR}j2<^TB7-2~P=TNc3fn30C5xcm0SP)JFG4lf1QN@(Q4Qvj
z(d3(fD1q8Q4#VAFfi7T2kaM)o6;-sT4n(ZhQ?|wY7qM%aUxb_T_POZK<1@3%LgeqE
zH<TKh3ya+PY`ao+_Q&#dwLd*Pk(hM$$c97D^52LsE}gXUc4(e7u3%m;KWVkr;Zf~|
zB=pZak>4M@LH_Lj@&2yuN3FEbw}19YYH9LIWlCDb_^C6Lw6shaqiOWHnJ@JzIqLpn
zZI|NaN2w|5YlTik?t>8j?RES<Y{$1(`@vr;zW;bW<LtKr7C_Sf{TFZAU)%HZAIaa>
zeqHk#Dv&QX8jQa8bN6a|fDeB9>+G~LdO0yu0u9~t;;mudFTVQ6=KV9vXVBb}K?RxX
zQSJ%z)@ajHos!=_K6e(zBE=m$V^W@Jv$oMuf4MA^f$&UQWlwefv+d^&UNcPnppawH
zYJ*m{Zk3jr5T;e0YTaJTvYk1v{C%K<22#Uyj02<U=r+{hNAp)-t;8W_G8!-0OsV&>
zKmT)>?UDN0D{kjoD;u=^4(}ZuKo$>j&KI*#LCEBzd3E%oO?9i89*lWB2ie$XnU=|t
zN|hU>qzsM7YW#&uDD+*L{ntl6+q2PnzsrqG3#lx>uWRTMv4gUlhY*sIzu<9MYgS<g
zf<5Lu<)<k{gm<#*V|n!1GR@1pluQD0%&Kct)Oz3OcC{G>%Ee{vz(1Ei{yE?#{n0$V
zLHnG0LZ;_g*NrrW!sr6xXi#>tb#v{kwu59@v0Qd4VXK<a_qD%J`}nXsb?@i|!jn2E
zj5b)#%k-4h(0c1N*&>s!o&Rw)Hi4mvZ`fZOl8_ly;Sg<5X4I8AapK;@AT5{nw*UJ;
z{V#Ut-emukGD|qSd_Ob0(T!zcC;Y7eM@VSQbl1i{_lB0IA>o;LJ2%!;y~HZ5fP34s
zkNjGp5MzJJd6lQYZyxTCD0N3qu4dox<h8dIZ<-^IeH-utm;brs=M8p0dH&JL>tEph
zIz~{+=55^jZk74L?fx<mKTQ>;%$69u;4@NYXT78TIhQdrn8gF1hH9CoyaBbLtO2oq
zIa}h9;eNcKQ4nt90(&slSe(_Op{_oB8LJ~VYKuNxmuOfuMQ{QvDuHh*TziFa58wTL
zD%bDHMEp|G=Gzdqal!qOyB?89oFLP>Gc&QM=D1@NOh~-q@Osm1Bd$Y82q)9^wYcNA
zWK&4!Kt3#7!>1uqsy%ur*5)8C+aO*}jkas<dgh;*)MLkw8;*{R`Ghic(5(Zs?j^({
z$FJwwKQ?yNcJuyPPGia}MXFs#CVS<953c*ehv^^Cir5KvD=RB{KtRA`oIO47N>OP}
z&ZrxD4HB<lRW^THzxdzGpcrP5>(fAwJI}XM8!B(;&At$@yygm)M74nsQ)<y#f|B|5
zo~2oB@!N>+TUGOui0;i20~7Uw)#z4EtfAjdpSVofo3qz5TYLiEgf7kQL_@m^#OG?+
zi}jcg^XZ}tlYiYUq1%u8!6FZ<RWpAdHAngmxm0Dgp{(-9FK$yAZV$=M6lv`GbYsnW
zOZ3sYc{jmUpHFo@wp{<D#U;*Fqv_3eTYJ*^clXS67}`drU{Vi-uj&=|_w%#7O02YK
zy_=t(|NhRHW$4U%rFHAq^X1TxQQg~N@FvQsXJ9wIE>g5wVRYrkT*pI!=b=H7pLu7q
z$YR`VRf9kDj-?K%wos*9yu8kHx!h}$lP{E9S~G7A`KcdfvU7JCI=r`4d~e(3w9jW!
zjlCT`SvtjH#gL2df4sPJB+XaBu*sL%-j1rLMBO@4ckW6>;7s44aKH99BWWQA>TL@u
zkwH#X>*)O1<o38;>_E+pH4XMhFGaoZiBNR)@Q8m`I@0SqDZQ!Q8(IZX_k49ew%}2}
zUa+&v$h;RcDY|FRZ;`rE=&Gr7JbcSQ>+;Y;-QTtrOpx?3`$vnAc#)hU{Z=xPCeav`
zM~*z{O(;-@7P6~f29K$z=%M0b`^61l`3Z`SWn^V*{ibV2>&8V7>1BMpC9-n**!)SA
zU%UlN>iCe5eD6<RYH9kDWzO>Qx|ibbKB8hMTk32J!q8iz2E-P_AKr8P@?h$;V7A5=
zQ$kv}r;mN4az98GM+aPE)6yUL-OD_H-iY>g(A&z^?|y#DcOk0d!z<*OdCC-LJriqV
z!*u42AB``rKP=B=b0QQ&Hsw9e^LpNW@C&re2l>Ibxl{~-GyW3)A?#uyO*cX@lGYue
zIDCRUkaP2Z1F7V7<~^m`$7T1q{FN4hbuUt^yNqqCOS~5_nStzy<*w;{x2G<#Lk@QD
z2P*d`K28C>9)))qZOz&aUwZo_2M$mJ$+4M!8K(6%rx;80k5Dv3@-oj)rD*KAf)$NW
zyx_n$mUF2&C+>|peg#WZbo?td#6d?`dazV-8K-lBT1hVcxNG}`MPKjFTdp<x#B#Td
zV^?O)kK29ewZIDRvOCaSnoyX&fJ?(Kc2(yD{GRoq-7DgCrp`K&_Xvj<b+vP)*VcdE
zqrYRHblQx~Z?SU^(Hu~R(uacV>riNYiq=;re-Yi~RJTiZv_XFYlwTpe1n3+dX+yNF
zCR7FBc&Xq~`ZZTKxA~i&XCCZ4d?@jCly1f;G)+&oBOuZHAGo$c1)38v9l2NEth{z<
zGm-~i?2$*;C6*uhAFhf^6&c6{%jA)lWC8M?pA9CV9F8VQn!lPjGVmROkJR2t>W@6D
zZr++8rS<!$3z~57)1`ROMHCKc%q;YDcXw+S*uKB-h{hDVb$Qde{TqxnRR=)hV)9(c
z#FsB`3M(rst1BzHX)_)nay{P%t?~Lxe>zRhB{`FW!k=jELagOS9i%E6eAo=!)N5Lr
zEX+0zY?u%dQoexaUqNq1af<=Un$IXlpS{19<n103@h{8008Ahoy-dywY$=2<TgCdF
z+DIz&$KgC*(Teq#A1glBGtq73>z5sph^9`BkM~+c{u@Ml=kSN~oE?%c+>k7lDwCsf
zn|q=aa6&T9?<H!1%{Ioo@w#Rcck27R&+ok!(chfv(Qmk~^q=tYR1h3}jFw9Fz7`yr
z;B{VgV_#Z|&w<;E-;YFSlXPu}o>7qghKV&ohR;Mj|LFzzcKvTgH`&Q*LsI|m5!{i!
zU9q0A$7FBg`ya}qbso;&g3Mo0>N|5fawGBQ?uhj|Oc@|lE@;VkziMDW;~RRi|M+v`
zw{eBWj3DTRp$in<Bz2+qzxq0l`z!IAUIG7m{)dl0cWWx$R{m$A-!J-~-gX@}eaby%
z@K{D>=So%SADn-}YHaP<;FwH)jQyN-X`jNVQmt%6c-23H={`lbQvL01OU~b9cc%<(
zKD0}aDF|Lqu50)obSXE2qd$5mrC)2m(^B}qe)>B#R9wGP`*+I5cy_MWO?G0+5M#&v
zU%uTcbiPK#@8C5J9jBmzpFEE_EyZto_Yc)L_*n*SR34JkdGgmr>mfO&NU-1cyRJW5
zhP$vn@d-%3_!h0z0yBK{VtdQqmvkrYr1+q+$3=3O4^8{jd7r-Da{><pmg8j<WVUg^
zxJgFmSk=>iKH<VA<%hCEJvJ`zm9s;8RO`Q+h|sP(6!l}mQ*+4kwT<l2ztsXohUG%6
zP3(sL`p(pZek(i|{%eVbl$A`5j#JkkXqCYg52;j}xU>K4n_gh_Hsk(n>|k7+?Z4dQ
zoswyBQL_O4?169Y&Q%Qp=F)yda4ctiq@+-11ROc9(0AdViF}{x|Md3rUEn*F7rKH1
z8W&UscC1zf1Va@8A>bmmcAZrw!eQSD)%C|aw?1vI*L9{p8q|NBR5Xru*#1`N{Dpkh
zqJhbQn4tGiK;d7^v)=bem5vE1$y`VqIZVM0gdlx8SAwfYf<(3z84Q$7?nKY@v$2}g
z@~7h}UcbE@8u(N;X^I%Ry1G6lL(c^xqgR=Y9~n!S*Ffj4U%$S(68g0!27W#S=kp`o
zf9la|)twJgUqQ3+#mO=njb?{u@}1Bk2I@v8?=YN}i-#66T;E}YE$R5Pt)uV<8R{<;
z3LigLEzjJaHj=7l#E}bjAF8t>;%yGal@**91)8>z$>yV65=Mf2>-Oi*C<9GU`{+Dx
zY1edz)~yuBm3Z|QQ(n8vH)AA7_>zY1lbvdNUk8<C(znbl8=#9&&5!)_UNsM>_55%`
z>itdnE1d9Utf3L(?Jm1#YAkuxnRnZxWoSkQ28}XEl;lfK;?@EFr<X5h1{8neycnvS
z>ZRMG=cKq6$M#IPySrN*(|-D+(Dg$HJ0(+e2w_cZZZ-;ky0ua2_}Yb+jFKD|T6ZDG
z(N6JP_2et_pP7L&f$l@~b`eXn_oUN0FTUvWVqS?_m(Oa6(9Nhs!t9Co<jjOa9M7RZ
zyQUr~ZZxYbcSp<B(@*j*&AoLD&oq!?NRJ*x<L-^_7HTh!MB155f_K5Dx<uRvjp)05
zHgK0+->0B73jFB;vb27gAri{0&c2)tC4lzl7Z3U0&FrFeZ-rW=sV3D|ld6<+aV*}o
z{_gCKFLd!m{VWs0rB4q-(J{*$4%7S-H)^w81ew;i*ClB{l{gbvQ9Gxp*J^Lql!dHA
zk<0Trq>jknUTAgju+QS=c~2@XnuP4I`{BA`0WIv338w3j=$+G%ic@j(7UE^Mo_ZLz
z4}9WOrI#Fx(a$OlPQ%^F<RW{3<=I%>`2ae?uAnqr)JV_){?1^#bDCe5Vn9HcytG&@
z?`nA?%*y$`l|>Nm@YkiBBqb-KoYR&AGD+vO6ZEsp0@F&$va_X20a|}R)8;C;6#K$9
zHrL@~2jQ2Dz#A9T_V8*oQ%%FdGdnKbUn5jEMx}Lo?y<{{svnTfG|3^B%34@h*o<((
zZjl##>k{K_Bb(B7bmH~1cJ8+8vnN`~$?P4aB9kh0p~vBg%ICKHLicB)MpKcNhwTSP
zqmeL2B9{sl+`fwyUH3*Tdav(rQ_v;UuS@IEA_q0h>5@|Z>QTO@jE$*t?~Ac6E=eP*
zElhoNo7Rnp2{IWV1~QPAR<uZuKs@nPN@~)RIvjJX4a1Q|ryFLO=*WJm6gm20Tyuf6
zfG4=}8a}46!GRo^Y@3IEze7OWp#HbspY;PGyX?#?u+_=>Sv+k5KB@kA6KkE&btG<1
zEnZ`U2hCGW{d2s1XoW|w``BFL#l=N!6Z@XG!)Qa)uOe5l1Xp6@PnSHrtk?!8gKFj~
zC~C>cPk2l;=Ag{J(S*9hWP~z!$U)C(o2VynK*hR5;k7fxd+g-T9f)B^4+WmhU@2UB
zM(S>D^(h-r6_!bK{e4g50V}#zU}%J*h;$6)c!o>SuXS_Xhb~Hgi4JSsH8*ovZt)j1
zJ@V+g3pTe-P32Tq6YA3WD<fJ#tm1>5h%CwAG<JTkKR7yR5l_g-OxC1^c*@jD{d$jM
zk`s$k0|_*(wn+AR>!pyNNU_|%GzM2*p7}v&pa#QXJo!rdk;S`!Y!g#GGA>zYlG83C
z#lRBr(S^Qm3{3*kx^WmR0WeMPf!iK$EA|f#**{+1=MwyCf*&(~&8{g19$Fvul=mXC
ze&(V5=b5h}`yc)7DaEba*4%O2xK*FYHk}C!J$Yk7jP74OtlGjyFYBR68;V16G6~ao
zenD4}lSt^-)9R^JK)y9a-Eb9|K!(l$dmEVfO5R^NwQnDCS9cPQX~>xx0%fFh#bFNe
zJTMJsRFJyIPWyn&&U>eM`w5re5a$Jl_-iaj?Z8^!$?UdK*`rBKN=P5k?Wo`V%x(4F
zVR2_s`7(v>ycOW;Y3ggo#PXmkpyF<zW>pKOC(G?G&$7%Mq@{!wP?pcB&Sx3z+1ppL
z$FBHnu+4Oso!Y&j`wIz&I?bE4T7D4^vkZTZ)9hZB8VGXVaZFY9>lO5xy2R+{;mWv>
zt*~SJWOhDyzb(YkWZ$F2r{j`qY>~da3K<n5c|=CTX_}E$C-}<`xA;+^Os36PfKXJ_
zPo9&Sx8W<NPo6J;xk<_Fj>uE}N*HfSQp&!JO)wY><}Hobh~;m!Yf6t$^c0A$Gnxv+
zL1{5Y+1cpej7&VDoq85!)WVdPh^{`_nuMgird9bk8|9*74gkHkkr{muHPGq9GG_(8
z$>&7ZCFV+LxkLoHLto$A2gQw5myhkTlSqU{7lYYi&Yy)g7ng}-Pvh*iHbB15Gl91T
zRHY4;BL-C8dz5qXGeyDm#He#A(01Tx*A$1?M=Z5_2}jA+q<H<c(VOn?d2#UuRya6~
z?t41xXiLa}m?g}siTX_wf$(P=4kIndyc>A9r!PG&#8TFq0gd8_k52t;@9!@Dayxys
zsJ>$LNttec7}2^FP`}<Y)w{nx>0Cfano}%3KLW)yZ4o1GJ+%PeELni<*oUr+8d&k4
zz9M<NF6GduF|zY)ng4AD{!}X)D)OIC`K|zZPF;@DQ{!go&tD1&36b0w#nMG@YWY}O
z>KQ|^XFK=gy;8KXc>j~>_0V$=0c@^}1=zh=H}wcr?;SrVKaVGT5nflFaE#*5p!F`$
zx(x(r^R#(ZT_>Gha^xUSp78p}^-tnkO^*$>lye@ey4G6^R>w19={~EBFdEdCfwX`4
z;PnW#+pbF8WDUiBwsEQPChCjTOyJ5Nc1_e<QOn^zEY&FAGO`)1yV@Yj#CUCq%4m{P
zn%eK9IR!L@w!|pCX{o_RB+XaM=_Ggo^FvvGVsGqO)IM4_|4-cv&qsb4sAh-(81+=1
z->YaPO{zv<?$sl4wUVf<D!;O;HL0BNr#6tYAn|@htTF>srNh;${pZlwP43PSrCt7v
zr;i$=GXJirxsIApth2;7Db|O+ovYYi<s9qV2YZ%&4)tp?kqkhh6=KA5cPqF&e*ei=
zzKoN6jZbv$#R7rrJ*%5liyzrYos&27vIzdrg4=%YbUTIvf<m>Ni?@v8Zo5z{@nZi@
z_SK)p%Wnl|v*+i<R!(^yO$yB;ll^Z@aWqhevF`5pD_|Q+X99d}yhK;7asSk3CmiZ9
zk3(UJo)H&pY;0<E8p0V06c&Al&z+GV9r;(Gz4^pH_UzeH)yHJ=JrUl_?d?Wi=QqqP
zMvd-)?)R^Tr%v6<BPYmesN~-akVoKDc`a=2Lc@V>&zQ`F%5jHC4tt6(i>A7RmZK*#
zC6gF7u6MF$YEO>#^|yBvZ_;Hx?T9;O(8a&%(b%=vBaq*AbijF*qt=S|?3H|U>_cno
z>|DWXt)SV#r`6YO9#*$tk`a{MIkfQf9(S%!<WS%{+k*EEVIHB%>_+|ka|h<OgXrV8
z<uHrI(oi^<zx$fRo0|Yu*_#fl)b$FKk7~Eu2kv?lJQ$Qqwk5g~!1o6-BApWFJi>K7
zPG@|Sou#R5s*25O8|@L^oZmEX=+b1_4?<583qqrYO3l{(TG-s&%oHxRYKMGPlNOlq
z@k{B+nPU5+LGP?IA%0RJ_8owyD2PALi90ksUF^D$uxU57%2no2yTZM#bBE$hH!gH{
z7djYpV>l<#d!RqA_v}Kw&(?MOUHHF`X2Ew!Q{#)qe6SOw;NW11*w_n;r8|&JJZG`;
z(7~PhVIsfI;Q84NMou$i`=d)<_j-QV0K%pU@z>!<nf|@#+&Z<LLNDGx5$3riodM?O
z+E)=YMSu<{C&!~n^s~#s1?#U3LGTaC8c>}==GUR6(XJ0Oxg55z|33acf@r<B*-{nX
z5N(6ADtWuhH-@}O(4rc=Jh9&Q^+MzEx&iOt={co*mbFc0%Lm1vq1v@kzL+EThoy(!
zi9L8<@kb+(Tz=EGKyB7OB1rFO(#pxnXYhAWj3!tttV-QV&91q6HQ(Ws3Rq6_Qct{Z
zAD(Hm<KAfxvqiH?1sPQwM7(THm5x=b``YKV2U&0Q<H|jRgm&q6rP~FCnL2+`qH6lq
z_%rIIDudyhdS|+N==GmFkYl};nUBs6YBSk7&csxkJR`uwQ;#m~m=bUS%>7p!skO`g
zS(~`sK2~fU)0}pvJhCTjL1}(T39j)-M}6?tzC)wi*F%^#Y+a6zkH3OAsz=SaIK-4=
zV^g2KeKaU)*AjzbvdawuWusVSCHfd!V&(ogef+E2?m~lVdxrZLq`1k2CDPOHe?D;u
z?D20e7M>}&{PxoG*u@CN^n~BKpl59#ldTW~4&rgh!Gi9u^GUU3JCc8)SV{ScUepGy
zeSPrdxe@XDpd7{Fooj^+9?s{n$~cyE?tF@xuVZvp6{{@zO@2>0zky@G3zUqG@6XQo
zyk9y>^#HB=O09N7@T>=fn}Rvb&te``QzL}<E@)1gG6LRqF(9h^EU-W$UcgWq8`@8%
zV&=8bZQJj?+pXqpDZa8|!c#sp_eMXX8n$}FN!aRAaQM5)18p&xm*tdu#j3I%?*-zZ
zBa(ZSh}S;YB4?IeT3TBD_sjc>0`xe1rjy(CBex8>__=%2I{gZBM67<d%bqs{*0<Q}
z{N-aa&!UdDk5uS5<Ki-(!|o2;8uHxsIoSU88{jPft~r?<!2ORt68W*=*E*>C9%~AB
z{(ZkoLgwLymItP-e<}-Ys4YUGN@y+T7ik$c#p>@FAc~HiOT@eIHRSwzr}rWA7qAmD
zNWh@Z;LC@n_jwnR1G4^~_P#T$sjOWaXGWc|F;+k+Dgi~LN$)x;5D-vMniMrEz4sEZ
zj4&ca1A=swst}M8ItdI41`re^fl!quHS`t&-?IbyzTY|TIoEft>zwb`$q#?Tu=m<)
zul1~_+|PaId7d?$V62{nlu=nEGoYn$-IES>pJ;7qiHMw?->a)Mr3}$Wz;(P?#@Jdt
zU+B4`e2ska3U$kuo_YqUwb!0)Z`#I%5!DuqNJwrQaBY4eCaV4j>`Q*H+{r`;uk5UT
znK5cA>yxv99cf0PWeS-2jOEtqTq#U0PllDHe^<LsIUA=Ikg$nOys>6hUgjno3j#Rv
z<9Ve58TmP#^R7x_=$H$))pEBH>3o=wCw+2F`!gjQs{cMgv3TufLD=_nq$x}-HCv=<
z#=+BU;So*&oIoL5us~F;+s<7(b|~>adWRZzLGw*vNgp_qF=6t9uWp`t1sS!i{7?Q0
zk;nmy-vzT^iyEI)%CY*xI0#qfm`lk9Z$3KXZIu*T*gi<Y$9HhBJvZ?`GVF!Uc{*Ev
zeI!8`>=%b-6wh@4Yr_&8k1RN9G`st$%(Ckf2{mxJ1tZ9cGgL=_476Ff^{<uQ54y(`
z^uo5dcP*exkL-EqXWG1*UF3KwamYJ|%enS^q_0wlL|hhO8cWjFBU>1SNRW2%7Uy12
zA0BGXZ&K;V=w*+`DAU1ut__t=7@&p2l;_)Cft}^pT4ZXT^HcI+n8+nHUt}VAYerp9
zJ}a90Ye{41LmU<1aMBlm>cVUkpLG;rzNm&1t)Sa1a{h+=vEwQVw0Yrni`ND)mDd+6
z)syw8Q7MoHR;JKI(LDtebK{I=qns)vymL2OK%;Z)ASk*O;+>z8qzaKY6ITFsN(2%V
zieav~De3n=IMh2HtF7wKPVogRateK)pLmt+JmQ?Noo?mC_@mGiTKI<~RZ_3T%zgQ^
z?xW}@8aM4<*~5{Ln4h$E;ucs}^F4M~CmRE;q<pkrF`D)(wHx4cb|w1@)yeb1x!?VK
zwIduFS0XgK3{M_l(4Tubi^qP+_&rpu=l6XrFis^o8w_`2*%gmcerZ`*3(;YV)U#Ai
zqD%Ow#^1%Tf8WcRCt;w-q_>&47s1>`Z+#b})YM_&z&ML+Y6naJ0|0SD{W<JuTNh7H
zj>di}v>>nQr7~XEJmWyU%FT8D+`@G^-$L(|triYqEfaJTC+2|Avx7%^yiD)cJnKLz
zIew>`-FQ}Of3awWW$yuIjVGH2jHT-3c}6P}f^%(@=UoZ@s_obQ&=QRB6lOTxkAWS2
z7xup4iQ=;Ba3bw|c<w7f-c{XwZgzHGLd_9Kr!1EYdDqM+>3_2(JTS5iN>o}|g1!9u
z-dZ|XsvTf3`G>3D6+)D_wkN!{CsqSx;{I*d&RX}bKAzeViA5%<9<ORJz#T4fwXpo~
za565Ee3MrTwI{j}>}TD_W!UezLcHNgUm@HwqnbS{OOUvIoV=u=#7lY$9@^IA^RTym
zgJG2(_U0>vL+XMKYL8<c#h?GLVf{L;Dy66y71+PeC9OseQD`L;8UdpFUi-uWUh3hr
zU~^eEw*Eu)Lq>^IhyapbtX2XB<)A6GqSJ(dMNeY`OnK)EAn6n-F&Kn{3QRyNjkaua
zOLa7Kv@}N}y@hnygLcOk=DVeua79Caq4jhubBq7p@*LsqJ8R^E83OxD3V$N@$r)}Q
z(B*h7u0!n&&}zRcQRQnCU9_5h<`7k*{mH;)8mH!jxdXoI_wZV=lXFKtvjuV>eI*yQ
zPQlVb@YfcaKKMR4-QsLvG>u?u=ui<PV0OSmJaI_5zjUHWAi*Awbvqu_>)|WJIQ+nI
zU;E1bnrFHuw$cS=iI(6)anxKSod9n)ezY_nWro2sozI@FuJU3q82dXN1aBpM^}Bc~
zro0<oi9kgMr)yA3b+@IHa9`*_O9<S}s7!~(mu(+ce_9ZTAXh8&V3g>%yrRZ;eN%Cv
z5{~Q#upY(B1H6W<@7;f9%W%Xjax1>)oH=4U&cR(<(%{~;LhL!_<Lj&LbNVv*MsC{`
zD6NS_&l(cxss_E1vk&3vWS@5Ur!H&lpUA9RVDia@%|KkAs`qNtl=?wqI&?k!A1IJN
z2+n{y*h6D)*II*V8ZHG6-mP%ZhNkF0YULW0w!&kT78bg@yI0Olo|PZC{NgU;#1gZ{
zl!&Kl;uADe=l9DyPGi?T!!M3KdiQg$6LfNjf3fc>l^*c!hI3L&z#YAupI5{+r%~F<
z>6$p?Z5c+jzs&yzGKiUWLyHd&hoz*@6~kt1nr4V*cGZx8;Kf{sOIf?@aoI(!JxH<B
z5|3rf4z~;;KC?y1T;_YoK^&=d)`^<+Mw70{P?T{pn#);qhNV|xv}$nWf)f3N?f6f(
zdN8_gy7F;E3|<*q8^)9DV>5faUxi?TTfK>|XkTEShXLg9JDwG0ln<30SzNkt;;D~S
zTkI`f>cLVl9dr23)v7b0p`kr+o?XE?T$#U%VdDG+huss0m<+EiWdPJG81mkZC)5jc
z30o(n6c9Y_rxd~1qQNkADh8TeeoeZZON43%MA_CM#>cG*dF&C7j(3nw4HOy&w*8vJ
ze|eg=&8TOgN7D2GxCd39FJ7SN6?LOcK8Z!z&Hm^-O4KQyxa2S4nL<RNJAT(!?w6eZ
z60(En*AMyI3#Q<93J#O92z(I5ZFc<hGuyS7u4W7DICIM9&u$HaAL2X>6?s3iXN$14
z?om5B|KY^6VU`2}qdDyymJ$%1VkKqL9Px&g8YhTBb^mOgpw}S*k@v<b&&Nc#4VsnE
zJLN}Z(3(LASYz)QsV5qM`I;ZVl>*#*@QUJLB>#Q-7Sx{2vg}+I8haNk(94H-G$%CM
zUmekT;uewepsw*riy#gVI*|}|&B2o<MTB`3vuY^U?)}<##CA7&jeq#3Tk3;}Lc?or
z$4fGwucV~p3o!p^E~yi|SN)8(iS3Thkd}!f-t2d!URM77PM+CoyPv2~())8cr{UGw
z{O{ijYHT6BqEMuWI(1YQr@v--<a<@0jkC%DzK`8g#Pzy|!{8z^Zuneu{2QQ;htrkd
zLSkg0g>cuC>;J)au?V0u-AVf9m;PYN9%zT|AJ*sM^+#@2bq~&<bGXtbG9!5iikH_q
z-`xL>kl1JGws@bC1=tfnyKvylj|}g`IW%Uv7HKBs|3FyawxDg@ynga5rUq!)WND^s
zT>1__=dE<x#nj+q9(+&S5;;q*7Fw))<@gW|Cx3q}udiJmA*yeqo^avtr{6mYaWWdq
zwvaZ2;=$gmjw9rVf@5!)y1xSL8wt89<*wj({TIKWam%l{_pIOJ&bHqqtKw{P;Nbo~
znrqy0t-Cfxz;-cFGql$}N=jev<RaGf!Tk3soV#$tFF8m7`YCR3zmaS|6ZiGZQXg*1
z*^nQ#Fg2s!x&_e-nhKeJmPVkMt`~`Cn`<85*eI9WDcwWiE-xCs=;B$CLe-jn^tnPc
zz5IBJ*T$WnPrwpN5VJPExLT4jb1y08=v-2xkl3~(yc<8r(1JQ-&r^M($vgFlUF~uD
z{;i8a-)DUe!KKK{4;*u;E7V|i2EG1paLwual~o=*=~b3U_q!I^ArppyR6FZ)kqWPu
z_*g%zKk|`w>&Jgtm@0`B;?VhjEljx!!x(f0QtwYwQQF(<9;8_r9UmTfUar;5{5Jp^
z6?F9Gfni+Y6Sve9Bb|f?IERm4^iP?leNv5FZ+HX%OO}<!cz5d9=j}c4GMX1=C__yI
z-=`_QXM?|&21oQb`MrWVkAsV<%AMAD0)p0Moxhwn9`Cf-Hrrq5kL_{Uk7s3ZrT`n*
zjqVUo7Jenx&)k@`(_KjUtr#!`YR&ccZ!vL&KM-}5a!_?;xZK(cyO#C2^ZNsdt@Oe8
zyx(46zkl^Nda^r4Kk?unjE!&D9z^sq9dwpn{!aT*F)SB4xGH!YGV+VR|ESjPHRohE
zex$V=lcI|t)9^TzW5dTF-M&S=O+`8?!PCvH0csRR&F^p~;3E6=tTjRljbqoF>(v?z
z-s}&q0%pgDFWl<s+l6V^f|kvOe&ZK5Z#Sj|z&m|Mtk&ynW6vNRVQw4ZeNnIfFSXG2
zn|S52%uL%gE1I37W0H{d(66nf^WgcF8@-eXDGs#jES|cAPS@1?0bOa?*cW$bd%<V8
zl6#Yk!-Pt(ACEi7yr8e&5}BPXTD!oEKq))m6U2t2J{2AcRl`dR!njV^+S@AeI(`F`
z5lUO8#`J{sX+1`ykp3zG_oJ{5uTPh+Z$3OS{}SMrNLr<hCcalaJr@Q_Mh&uzBt#N)
z=#l&CZ=O1q4J53ioS23#`U4zAb9!N{z2%u|u<Ted((u6gTrh>gIre5qjteK6s(~ZS
zFRwgJNJuCXz-r+D5pnC*MDlYlC4Y;HZfQCR3Vl`&H~#K4GxL$q!6-3{?xWT?6cPq|
zJ{U2wrEl~12pJ**n?aQC4)!?pqS)_)&IrJ0PfqC1zp!DRr>MJ{zmRw8y@S9B5^u08
zJW@*z*fA0>lgJ2rcVqkguXO0$tQh*In;o+KDt(|lxIHOfC1lB80Eq;mkPM_{aH~Yk
z4grg}10p8-M53H{LiY<BNI^!O4-L;^cD-iqJ%6|#cu(ecuiyvXZoHM}^A8mA3ibX8
zlXLjs2V&RY;NV{=5&~X^Sw_2u1^xTJV*y~KmAV4&1hDwBGR7D2J{RuvcP5ISCnYwv
zLxz~U#MYc$+PfuQ8Ex)aU>D2>n9eCodwbne;RH7YY$n0up~PU=(#%kR_*iikf$LTT
zDX1bu!s9c*4OpMkW*LummlNr9CvJ6IaF`QB&-(XntQn<Q8kM-D=0geWD$2|TZ`K7T
zGjdhtn?92_*Q%!fsF!u<T+B$(*ZN);W*bp63+c807f6))E?pg$1rZztIDQszz`6P|
z*+F>!55U|jQ9zv;;76WDMy0_05==9~2!&?25DCwx`QhdL6F{IaW-QaJ(X07m!+d@H
z=$9On@bJGvs)QF#9WG&<g*axRwr{F%PrCQ7(xMwryZ34Z$_x6w7tf=KDMbVV0Sh>l
z{U2}z(D%@nMHod%YnYTl_b24EJODw~4z6KOcelUpYH57`!~jm_ZQs;($MUI~yFZ4)
zRxN?)BwFa5JRWeq{oL*BOxL;x6b={dlBxm%LF80TmRR0$k+!z>c11<Sg~1%)F7%2f
z%$MOPp6>1m5oOr`gAqe1?_U|wI-xL>-6xmc4U`AEVTPuu@3HSq{>84=pRBUDwOq+b
zWfxMc>gejKcmcV#R?eB-m<AVHd(&n$bkD=uK2^>S?>mli@01(?fAQXLGuwVl-A@Ob
zc9-|v%97ANzTUYEQzMw28`hg3EqZBVn%LCrvwrLD>f7_(ZeRNx+<5H!xhhxsWWBEp
z0I3Jh-}`<h+wA*|#ij<=$hd@B=TQSxhuZ)C_<v`^=5hgTQ%F%*SeQJ8#;#Rrr1wbT
zb_AW#(+*S%cwjr_JehBt?>5|wKrVKm?J2YYT$*NSQD$bQeeq5}ly%#Ml-&ArzG-4)
zWcgN1?glF6+4GSTRUH({yov4jecaKr^;rZ<E=*%A0Ps}_MMX(nt*wNBg7LzlA`8^3
zdK2F^7fi!nwN0FT8o@gaYfWs`4zc=FV8wpczUN8C3Dh3M@LrTDVZ@}RrHvvx7Fa#b
zi@9?TrLz#XPu$2>9Z)^4bdJ}-G}zdVpQX_BLnSiRS^^34^s=%tk~9tb-BXcpkLb0x
zvRb^8@{BNFI59C{8x)meoPT>Jc)xI?PI`-?fQUt@+wgCY`07%KEd*$;Yj}!ppg&T8
zg1lY$36)xBwW9C%pOEL~ym_}#X8e$v8l5pj;Ji_@BIw|AN29OENNw!fU)2HU7yYkJ
z#>?VAexq%6;S(Gyo2bs*To@nVZWTAm(2+XG#0W+_kr*t{tmW|1g|oc}t#H2C1Fc#_
z@y~(7XA|0EQF~)By&CRMdf}kR&m!beRP?jLByQ_ov@<ENx>R6vRPJQe+3vvj!^7J!
z4g1H(#}9$G;`#-eHk+&MNnqX;vk3N^4WvVCpKlD>Mhymy{|89*9|vME@_kvTgdrD(
z0*o;ZfYh5snjdv$AWdrG8<BDzfYcRNnx}Wi2BU4bU%(mLX9I1Ue31p`+LtDdaogIH
zBnHzErd4+W<`opj+QA=drO;+Fd9(<em|YsB&Qo9Cb+xh(;hZ`!4uQF?s>k+c_A*wh
z7HCvz^l!>9PlJ0yTe$@tDnr|G{%zge#7oWk6xt)P(r+(IpkKA!G>gY}TnF7WC?;D|
zU}VenZaW*NC!iL4G6k@&XwH`-B-C(?rOf#bOQb`K$={m8F-pPvhDr=>Zw*V1%_87t
zOG6LIHX6gZ3cyxml7nBt`3AK<?%F3QCGSSg9epBnCYRLQBDt@nwRNrxz!n5tl-kiJ
z4{*VPUK}TlCMSNJ$7i?gW)r_FAGSBgV~6@f>y~s~N6HJ;<5XqLjv*D>WhhM^6kwU;
z%1fDW@IsoP8!2e=SlIb@SxG6#az320O6@xoCNW5*&{B!?ne*G&oId+352q90?b!rf
zx&)nG#M=$eE{%)7o>(BQRroVUEesVyf=hevZI{9>&)Nhbj}V!%aje@rI_hfK#LC#y
z$hgFY91Pw*8ii>vdm82_2X)*X!5x_gAwI1BMyCV!oNp$N+W94-#rW@G+xAX%TAtb;
z_W?O~I3?4;fb9L(C%&Of!B^<Kep!N}-M8S&zWqHKw$TNL<Y@{5OjS$%#IEH7u$&G_
zNJl4)0^<?1ShqV9={dO1hpp{jY>h!6<JR~Jbn`Mm#s&KJw*cFPobbWW)qIJ;+qUD9
zfTCvz3<H-+QYX|ki2&n2)0zaB(zWa5-$Tx8H!|XWp~IW6$OqUzK*xMA4Rvk`6##fu
zUYM`zk&%|JD0x4)8=c$h{^v}SYJdv{)0h3C2ip@44zLFrwq4cMY$My8Zxa(La(@8A
z4a}u&7J;gv2dLL=UPjFVsU-c4ee?cxw9r8mx~r?}P!<7?Bi;@feiHd)mW0a=uI-xv
z<5hd=uIb64fqf<pA@XBh5OVnVqV{;lq_7{t@J7zw_`QV>g{|eBK^|8!NNs*2=&iK+
z{NdN`BXC(Xn$B)603F723n*+(+cNNJc_yLsb(D~;z$bt`^8JjjEPj=_LGlAAg(r{y
z;T>G@)R#K!2No`Y*4bJ9ae<kahR<*h>)WBRob>JyM|vjDhEpLZv!|SsMFi{u>GEWP
zN1DXmQzRgAN8%J^$jN8&Y#?a`MboS*Ghj!wpgv@IaI4=nR9)=7FVebeWBhv$@q1k=
zgy;j%Tn4ZzeTKCxUXvq^Y2g5C|Cal2=v0Hx2n}%GqPY}eHRs+;kCRt!;0c}yz(d^g
z8Zg?2s~+Wzx8<H?S6y);RA*#xN#5qZu+gIRISyef&P#z*XT)nBEomq$8G$qq_7R7e
ze;O?}<j;+EG&VL3Tx?uix@PQ0a@UcV3xek3oSId?e)o{nf&VNBTJ#LZ#ZQbtv-n4l
zzR2XM$@npMPJdQaof(IgciQ#KmjeO<2xmE^*E_~8{$z_chFkuD&dKmN*?I%tB@ZN!
zNg_sXu05yaf=_mG<9CR>g6Yv$`RgVKAx)*CDoE=;;mu<~h6UbGh=nu>?wuhL+z|eT
z9bI<`?_5Im`yBAGT!tBZsFtiSUeNNIN39jYO^HGP<;cp1xw?Q!J<zk@xkwBcJ=8rJ
z$TFi&a_cu2*91nSq}j3jCgdGhTUkB(_!~k~YHA%)LN>WNe(`k;XR-#$<#h@P?{Sjt
zPW49HHCy!?$|}jG)=5gd#)+G)_c-tpv0j+wY#Szg7`TTrJvjI8ee}+aVr+RuaR*za
z_(t>G#g~^)q(bxY736+)b|?rl)#ZWiX#Mj)>8S3<dq7TZy90?i9|%ywLSx&Z%ZgDp
z&fG)n>`IfBrB8(om)DONHny8!dhhIMcqY-OkIR;*xvUuZI+rqnsCX!%n5@?a<Gphd
z#(ifGU|rM*7`cN+w({%IBlmeEcup>G?o$JjSLD^m4?e@V)d{6vhM7P<7umoE=S)pc
z^wAu4D}|+V;pmgY>j8J=rQjb+^SK&xZM@4jB*hXdwn*2%($yk4-*BzL-fy$3{OPF7
zo!fqIyNe$4O3UO4Y<qn2q@LSp^2_7gNB(}9UpF<eP2`4d!6xgbyukZ!?;dVbnNBUM
zRq@KTjQ>3wqiBMf+=ThH>dvMh&TwUFw&mJ0UdaokLzJ#Lk1JRey;nW2eMVn7eqelV
zX<8W>|4z5OB~k#PhhpoMLlApOGAb&os?b^6;uhtv=(jjos;Ai^A@?)VYF}wx))%I@
z`aE)4HByfvoshbTgK84j$ab0!XxP@!__^@7#!tVz>#9mdIF{n0pk+FY38Zl=qzJpq
z-+sCFtktAQDq)k(Iwy_^l4_g+&W}*-A+|gIr67SY^YJXZG*7Z4A{wzWL0@(Fac3^y
z#V<jUii#a>A;b}nGWoTYnYge`r<0Ud7yA$nq!x+g+g+uXOdnXkVKEX}W>I3RgV<$x
zl5<XKZDPwhk55~74V^qTI{&w9uuNCp*<hLGytqy$!1nV7PwBPO>*PjkJC1LVr)}Dx
zsuq9#A73N^Zd7S9|3J*|%L<5R%$tn1#5OkHv6SX{4zys$3PF`)&a>0GwrWe?>=pc6
zdo9Tyn1sg<mi|PCUZWNuLDrtGK{Xvu?d%$zQy~KDaT8K&d~Ymswa~F=brxuW>QK-`
zoq(1RD8Yg&npb7YmX!y$dVvZ^X_NM8R9@&jR;VRJcMu2qgi2hX&BH)YS}<%i;kz)M
zh>8|yzX*)dLIV*9EsJJRN0*b#xJOwR4-ll+0@Jz;p045IkIU7ov)=#SP?&=)GbeKT
zUYg{`5FN{JDqF!&1%CXv(lqcoGuy`n_`^`+D8Be-&lW3ZXJ;|Mmf&RoI()AvKHfMW
z#cPQvgs0dA9Pz_?br<k+bQ`>wNNQ)Bn<(3|Eg-ez{T7|z)eQp2he6^@{E!q6M!T)T
z*E5WpD?zo`dp<uYp<c8sYnT&Re4MdA(V=m^EFz@UL{(8%c3~A$gh*K2MMz@Y0eG@#
zj7tF8C(<nCsv|Q<(%Khj?~`<=moU_vlEGNZ1)qu<*G(TSkmh}t$=>%Tt7DHT52Ll!
zjU<5>lCS^ci?iCb-{QbEAPs|4RB3%MO5mVom?**@y;Y|i;i7(m;?u@?lrayia|B%%
z-?zCN3g0UH<Uo`+z!oO10>kL2$ad9qHDy+4F3ufz<6+?at?cHYYM`gN^aWhN17?SF
zT88~R{r!PNGn)NIwwV9aqeHMR+Kze>3{flxO57>%>kdS(<1_FNc9TBCFV#6@jukp|
z(W3vJ*Bmr~GbkO1ANW+JaQ)|n<5)YGbKluZKr)$qwaw4(6&8RvlW(bsITj?Mm}NzF
zt0}U{E1ful1rFl4878AUwZ=b=&Lw)`0b7=2F?UWKZ<KAGvtR*YSH}I>90Kw~S$kQ>
z`p>q<x0P3z)6?+=!f4T}Hc1|370%aZKI)Br$sRDx@p>n`>X+NGFwy1fh>I3Ib`0*q
z<<$(*ee!x1d(3PrvLc(^b{uq}nt4`S^Y>pZ(H^q;covNAD9J$cmG;(62V`Lup{&dN
zHu<F*?Lvl=S@F?+P2g3z1!^uOo}UrYp7@7-kK%x5L9isXM2eYStXPUDl=#lf%*aY7
zBwIryyXuKvjZSlAdvqvR+6&1OC86XWqk!4NzxxKT7?lC<!End9oA^O^D7n2r+7f^0
z7cdjc9om*g+ckw&d9Lj|6xGu#O%94E&!LW1sFFLvNUqQl0V0ED=u@hyYx2lj2Zl=~
zV<v&_mp;sio$DM@kp<lfM};jr8QlxSXmvhp0elbIMALtB5Z?@Q!m^F}$uW#uemWZH
z2`bRt3i9&aYoO?-Dy6Jku@(#T@x|29u4gxu$atWz_33?E3yjVc6_n*%R2hgu<d_H}
zccsJvh+-hqf@uTW1>tJD+$#VgBSq7__t|B@VglnOFxsEqp?!Cuu)~4T4mw_idUX?e
zb#t&UiisWLY1C1kj)6nKFyLtZ^~*R4vpqWCOPz3zk|o4!Bsq@?>L>xBOuACkq3kvW
zRtp552K~n`blyVd3uqUbKLS6*O$P^uCb%{bGk|L27;t@~^JK}jK;Ah`KIBmRW%5+4
zMZu!)%6Mwzd8(B80Bg-<rcr1#I1A3T$S=Qw+hD7X-oKFn1l*@!4`0KnUM-G~0?i#=
z)Y4nM1w^_tT)>!8wrt0uOwZZ`G@BES3g=XXCQfMtW>;>{w1yGu+5r>mRuJuJ+|nX-
zI{@b>*iLnO?EjiTEFMNqW9BK~jdlB_S4wE|Z!S4dw22LSb0ACC#k=JQ@-WJHN{+z`
zC6|5;|D}2hnP+KhKur-3R?uySoXECvr!8S?dqd$}&Mu18AM$~KvHtB_4UrP){W*<8
zPRW>r1bYQF3+5_)%^x|e(lC|QmwF%MO959$Enh%6EP<Jq92Libp^_AX0>(OGL=JY`
zcIUB?9MIDd#bC6s)T1zv(~`jKDCyDv{>hUQ4#)wObp@|aXH9seNy@q|!hd$y{I{G~
z|DP_>YR&5LdNU7{X`Vv~p8nax+}QbPI4<LZSv-($G7LtktCI^WD5~_n&36QhSY2^O
zm(%;o#qSUrHIPzQe6uREMfDaO9bl?6!Q;FI6w}(ZY;x;r61|bI({Mq2{>AVHq*YaW
z+<v*EpLdRR^#6As^#3i>2Hrwe0f@W+KVMH=e<(Wfl4eO6brkCdpoJu8J!pF3U!*KA
zzci><m#*_^b`;21NO|#gFcWN*-?&~&dVyq|1yGELlYMcfQ8&*5e_(2)t^@GW3wJqq
zo+WjNrWW8r62|mwMJ(f3;U1^iS=TLeVCSFf*Q+x_)6gn%K=gbUh}CL=XK*UZAk$qR
zKR1m^(wt5cqybwfxwErVy=kHq$t$p}Xd!=%oWicq5)O)-jl=*z*jnI9A<o<iR>(ZM
z-)a36E_evOjsbnloQpNuuYswRS6$qO|NLyC>hv`M2ztU%OIC=;YEjmUbylH9+Qofd
z0D@QwK^?;N=hpH9&JK&P1FgiH2-~w>;>p^JSyjH1y;a*y@CKSfs-4Iby_*L{_nw5L
zBp?wQ*hyeEIutQ#(dLi)!6VYQWxj<6Fhh^LazXT=q8po<98LSPAGUb2diE+Oxq>CV
zK*duZv3_e%bJOEBYm0A#2-TXS(D3#61F;*v#$g9;ifK<H7S*B17aFs!>Kdou3H~XO
z)1iCC(1`)1Ijm$or|7>qfSnfG=E2r?Fl|6eNdwV#6$M}E8*n5Qf_{rPD8vfLKfXB%
zTr(=JrQXOqv4wH54rr6blIyQ{d@8#4=D4q|GG~}Jz77r(l)Ve2rNOv0d2)JL*F-m}
z2MnGwpj4vA5kDw3F`2SFU^03`>zti%s+~&fBi0oIn_yafE4p>zb>K`3KZAA=qivZG
z6lJ1S)MF3tu$wyW;i#Qg(d9r=AmcW716AMG`;WSaAm!s5Q9kRI#7fk1@}A8-R}J&x
zU+~78tY_Qnp^$*tO!K8`b|92O$Ivj?C`s(P`$7wRS&5L*t)kSa`5ypOl2p{DLZCPR
z;i7}Fu)UblyukK2_Jm;!va0TUk^v6@g`B)8aPY3ZRRp0P89`9~ZD?u=aKINJ?T7U^
zH;!!or5)=Ge*6x{DZt>zuO7B?3&^@0aYZ2g(+y71GP|~}E}b+OYb!c2=OZh~rRy?!
z!{R0+e-Kx^7B`j%TB|mxG+;T*Orn8=prPiHE`h>;qn1w>2i*VD@j%}xtR<9W+BQdO
z`0xR1d28)FGLLtz%SvtT?r<m@w*;oy4o#slEg>qOoKx?zLs6W0j;CF;<An<s%Do>3
z6-9)Im>Oh_Lu_M!pHn7K7Eow7R-L2!zr~a}DliSVeT>(ut4ACdVw&qLY`4GDB;gK~
zdVy)c0_5uI>I=$kz<v+HMYk;R?plbb5D|bwy4_CIJA)YQK<U_UmbBnYT8Lj-T9S>@
zOFKa6P(DWf89i|#)z1dxr5s(|xPHAOZ4K%l563}X3RnKR1W7_yP6Z-FLvRar!KM5M
z_119kE}g^aW~p@`ni8H7D~t!*D=Xy}I$h(!kf%O*{Tmp%tVF)G#oDF$B2pIwV9WTP
zWjt)$Gz@aZmk<ld*^?IEiK%5*GGeLaX^8jC_9n%kg=I3><P`r~Oy&Px<bNn){dW`V
z9=q#JGZbM!Vgc)jlNIy^f!GNM4W_^c#omUqS3TCoS9*OTCR=AT_9n!~`#~&Qg`%Tf
zjB%i>OpVf0K{_~d+{(IkY8O<*B0enab7|N?nV=R!PK_+X6-4-fI?au~u`#<$>Zr0J
zdCd@^wM}2{nzvbZI@LPB>4Ye!C*dYI>cE21z5uO7bS*tk=&Y)I`}XZ9j$0?s6#&-J
z^ZDbg4J;!ZQg~fxGK$-5^)P|L9`7{-aVbAqB^Cu%PXufCecWI`wv_|o&QA@u6Fm*-
ztuB^<x_ZE0XD)O7QnEHent)*QL!Ucy85nl06)H95ye4}$Ll^|y>=Q^(euDuF@^m;-
z(j%v*{RtJg65{Gy4k;^g`t$h-nioy%_5(y~uH@UxN3lN3Y6T{w<qJ1~bWh%E^4fjt
zWm|AwbnTKLsmbp-rX{3nhp@}ayM4O||5u;8)|!iekJabf5NYm|a3+xBJ?2+Gz`UXo
zDFgL?+m2+9F_@jWF!A4OYHQU?9pj-7XL!zMXWDQMP^KL#RA;uS4hZR@@q{WVCgFN@
z4sov^aOw4jF@YF$Z-=gToxZ=lzxWs;g$Tc%!UnA52GCj(qrels7Q0FVwu-$S5Hi5%
z!s(3rH(<(O7GDv7Lzr%1lcZOdyghC&sKbn(yy9d=E;Kv^TK1*>_=Ex9Pv<#8JL6Wo
z9KEU!#fE)+_DGB<$6?{;+#+!>AH23SeyKs6mM9rGf1BMxYC@)SNwdvbuEVLxWu`0i
zl}5T0&kKnrzd^|vAVkj}=FB<+$%?vGLkloJHNY9XUt8S@fhzJacDqA9JajfUU_jWS
z-R;jBC7>bo7*qe8!N)iwu}=#mCa5BID#Xt!NMnkhoiA{jq%(8tU1;VXOtpnll^)+D
z4s(OD*kpAQbIg(vw<|YFY5G0?R@}-P*7xG1i(9YGAi+4QcGcXYT2FD-r{b9TC<K8`
zDW^-~;i0LfxB$5+kP?O5$s>`jHzxq-INu9m)wvasdHm|6a}FtxE~A=QJnpYTu}deU
zZ;BRd2lWaAp@B)E1>U2nk#oTL++PJMlyQwMceEnxbf!O`l4m?e)a^k}niM%*jbg2j
zCWyD{{^Dhznm{8DqEbukpy48u(R`3=1o~med7p|W45y+<7jd$M1u(AiFVJ)cfjTp;
z2Qn(M<N(i3@(z9wVI<h%#m44*7@9&owhFqWr+SGdwiU&ZE*5xqa)eRfWmwDiP~j94
zQjgoSElPNb&n2M&5G42|?MN$n`ft5DG4-i7Q1dXbt#JB+Rkh*s@xYafllhHPcqPYx
zT~#dAw0LrD?ULd=u#w}-kjc2#az>|i!wKP_s8!g#!h-vg`>uai!o>8BdV601S2IvB
z;k!CE4{;(<2HgYEVIHiLj)>0grlDWhuGxD*ooj6Yy~=u9+|ux7v#u6RsBe#p1|0<V
zPcPlmTjULHRuoy-=iC-$8Jic!C%b`oUKQ<f7>y7QmcIifFOLY|GS(s?Zy)&)8~AJ5
zdJ%qdi1WhGEoX)MI8Ooc%F4>na@!9()cIzgmn%eRqzlbj(-cIv)DzU2vUPC~z7Kvi
z8BIXfM06r9aKO~Kr@p?vBi*AcsI+Th$gp&LK#7F}?Y#+yyKjem5i^TYDuM`ZAil$t
zL(x@m&De3K&>!{B!TElB-x!wBWQ&plqlt=%i|YVYm*Zr0@Om$tO6t!<cjtmjP!E*3
z3(}u{jMNaKta>2bLp(Pr^e!HSf-kK({&TB={H((md+M^~dNCzr&%aA*Drj%LHUOzy
zQZFKORt32fHm5-(by3;wm-KfuaW-`hau|87?|%XP7VvdezP*Fgm006o?zZykfeC|p
zbUPc{eJh}LoJK9g5BCB8XQG9sd|&M*wl^N!N?wx|$Ckn^XN+2*{yjR^9n;?<o51qw
zzo}YI?kYAUPCvCE0YQ_@ho&Y*e);Z;n(ZZ%t`<4evWj7)kdSoX%@#%j(L8F25jZ2R
zuo|aVH*=tn1hS-;lC3$pK-w7P-Te3bg&eA4b9(yfP`+O$KY?<{Q0g~ZJx~RO;0O&%
zG6CJ3LnQy2NeF|qB>BO3fM#BL`?-){y}I*g^Ae{yqRFUiTtoG@<EiCJWJ!n$mSxfY
z-Qh#_MIAk=9ko&W8#ZO-!7`;h_~2*(rfBW;C-IOFRzAy2b18r$do>7gpv^+J+_*Ug
zn5+Bf?jonTP=xCIgZAKcH{`nsl$b7VZj|b{656vEiruMx4@HLhD;+(ZhvGVPNO3_x
z%$tnasea=ZKd7MjOhdp`4MS&63!6^`8_Fg;{Nw~RZc+&xiVrf^M)w<<PH}v{9|pbZ
zj>rgmE)7Jl*;hZ&A@5lSiT)>fZ6r2#TDHLlJVk*Ts-?E&bvH0`d>}DPGAUm63=O;v
zCXCoNI!6()3n+q@5!rOo590!-4y4$(gw4d5jGCeIu2%>0YX@DI0`e&Nt?WDsvxX2D
zEcsVEa~^^8VFx;lsoGz!efxR=Tnkc<VOa7+_`#j56~Nkz|L!00zcgDmT{8j%mog;8
zx>1f7rb7_!Omli4(p1#|$<)S?tKl59&ZzzokV(oDAnR%g`Qkc3QDMeYOWDK4W%>ie
z!wFx7%w-i78Bg=cHMHDjtfRnkqfI%Vk*~qDoCS8SbrfW5lG95VYbx*~$8b{BGucyh
zz@(DY4HV}wH}fc^lH@d05rj%QqD4~!v(|mQ<hq5QUBRl^gs2YQkKg5b=HNBa;^&0s
zoj;KwKb8rrRM1-&bjj>82*kq3X)>^hVpBWD9pqYMAac{U!K<RDsp#(5eq*#C&J<oP
z-4<^#=20feI!@;o^)!XaZDdh`0pIeD;?{4z_2Y_}PaYC1pN7q#8C}lNISGrWG2K?*
zJjxWol!k*8&S#UJC;clZ45Z!yI1(TJ-=hpgMj2dkSX-Q9<>@WbwO*>3d+QZlP{Vne
zlF)x2EDzr^kR7yxShC56U?SUUBBne23gxH};h~tI1EKX=6I!KpbZ&wOI0Jv5>YcLO
z!yaX$T0)AjXcAJ&ocn4>8q*wF1c<jIpsz7pv`-*U4R6q+IA{lRZF$4qD74QHtRGJU
zdbQaK@z_8Zm40kkdHGv%l7)>brrT+B&L;@6;BG><Kuy+@K#7jhb1-!%s~>KuQO30n
zZymfBuc=zYH*@9yMpQ$;VROJcuJSj3!y4>KNKV?~52cWKCZrXB1{n&_%HojKC`AQD
zj3N3nsg}(X+Ol1<lMpx&K+;1J34mC^RK#cSDr-S#BQ3%-|IH}LY@{I+Wlb=t$oI3(
za?WgB2$PQ^=F?(~4#mnfg7bq%V%_WD18<F@H9eE<M0)&};1Z)C?5&~<%csJl3|ksd
z=dJf2kNf;x4g&-7C;ds7tjdR;APc67>rX~EI~4Wg;dINvs*9bpD+J7j+p1>ahtpPB
z^qoe62If8CGa%g3qd7ga-Eb}Esny=F{CE}nPJZhQzGvIR$_WaqS*@m7+^%N=Kb&sd
zyx;5k=5df_=R(V%$e3_tj~*lo?r6LyPrx-X!`BwWYe`9b7UV)>M(mjH{m6$;sc)T$
zpdt|<sZLV*QU!sbW7fGirv{hh0_X6Q68~$Wax}`wV!nZEu1mvaE}8oZ90gm%vY!7_
zaYL^sxNX>tsHmz6zG1!LbO})Hnw!~b;}|eII`fvl(P4!L#P@GT^J-4Z5-7S5Uw1uC
z$-XJLoUV(dhb-$v(6=t9KkNhVNOrsGT2Z`D=cz$wwWyZ#P+!!Cxu0FDK2d7@Emhv<
zNO0V2+c9GL5thI`*Z}{?-h^ovAJZvRp~V!>(`>0?ez|I}Lf!JP90z9ojl=v$r&H10
z2Ab_%;~sQGc<HTX|BJQdPYaC&bg4Q2=cHw9d=BIWK#i_P7Y6}+Q=&}(hS$<Y9!o1$
zZ@!=^&XHolZ1j7)H)i`OJ7~<!sN61`K`C{(?n+jDqzKTiFPILbARx9N4^=7JeAWG}
zj+$VX)FGWtEI(JMjw?@&boui3?Kw*ikHS{aGZs!Q7a`w8T}3#?V+ICaaAmeHG#1jO
zru?6;!DwI#yNlm=Gg!+P?ZP!9C%nG0RuKb)Yo&V*-aq-y^zn7c4A(PBVO4bs1(U8D
zZfdtDtG3e^fvCRV%RqTg5}gfcg(0ZP4y@gV9rFo0G&ty>JXILyNL)&a@GI>YpU|Gh
zgWR0wSetV@0LgicbVLv4+5p8fHa&;F^<WDXc{@o5{QbY(WN^(G1+okcnvZd}<Z!KV
zEAn#C&ycxs5&gEq+~^EaI07$CgrHm2g*=GWD`{h$+Q0u?)kkM%5*iUNP6XtLuyi|G
zETm?{$oTC`=Gej*D;-rv_mmFBGq&zrz}$=+BLC`*PK@ZUg#A3B(BTo{9lx<Q1DKVj
zkNc$MSD&$m$so)B`TFvQ{(Y8*c#;sM3GU&%8fn2UX{h)&CZAx5J0w^gV-pk2DaTYn
z$_@vKCwpTZMX6*9tfPe1T!NtFCsn2rvf~rB$MKm@+<gD6R&RYBGFhf5V|c38&&lnd
zmch50Lyjxxu<W4-Xc|{#r7pXGrn}QcrpXzecOI!H#6Dg8oDQn$4?YA%bsS()!-++a
zsj%#0z9sqd40#e~65#lDdyx>HWUp9z>(pRw#=0FJL0<NOmf1y&id^Z&?z`Ok;OUkB
zz;|sdn0oLVGu_qgkAPaB;ubW#@dxR95>0gZ9e0*@CT$_gC79TvA-J3^R=ZTSmCx@i
zZVI*L4qKJ%n5xWEqoi}4wd(VNk@=0VvRjeh<MtlBw<GRu{Pty>w9$3-Fz%&#C`iM}
z3@L!}00^%5U{1ROK1fqdJ4fdn8a`QQ7Ks)<W<Y7X!xqn{mFEjq`gI>&(czD^t>}4#
z@qUC=IC^M*auC$%{B0ncXCXA8+G;{iN=U#ep|i`U>s-Oe=bTLSbGc4kLyATTx61tv
z797MnmS8RAV>r1pr6+hmS)LgVuS)I;FNQbW<+Xni%Q(t2z;L~H$)eCPGiP*QpPX^a
zBg+2eT9t<j-K4IqqvIfwI%fWLf|a|2Otw$YL(dD^+RRJ%In}{$Y%_s<^ihwpCf#ZL
zgUkw)R(3uG)aiZdMT-1pianA*gOV(CBS4<)$j2C{y=F2@EZTz|vXhiBGQe~y-oB1o
zZGjbF7aJ4fJ$utQ#5=j~A0@YmG!Oqd4jbKlL>s(gTwI***OL!hhCS=K%0hfmmQ?nJ
z&BocSi&#6A_rqXxezR2MxS-Ruu*F`~>fsin%}A2a2G1d|q#-mAB(Jb(%?9U*%;xld
zFa9E2h=@Dn0Z8*-J|LMwSj6(^?yH(h9Vn9N``H8(AH<JEAa$EFRc}9uck8+Okk}Hm
z@Uv~D4et=xJXO=Sl>i;-7rG(d{NEOP?&9A$h^59)#e1$VhY@5soX4w?B>DQ?o^5YF
z?0V+kd2avpFRoeHUvRN{SPvmGDn<w7WhhCv71wH=i;8rcS7S^aOeeOl=h$TqMSN~4
zSBmQQ)J;wZ=l1kv?4YT(-@3ioT$qh*r;Rj{)>rEcL=_2?yc|Bv8pz6WOwroVZwpVk
zMMj86$P3{e^s;oTA^u05q)1ie3Fozu9lTrFD(^UN5rYz3_R`*LDk}U~i6r@vtJ}js
zRLM#5JK@>*3IQlnaGpZaWgp`$1eYeAsYxA|e`jOc5`omOq(-mS+9y$s0CcehxBi89
zt^JxS24n#_HP-*i6}QsvOr<-XfDGw<G@?(o>}ZBFJXWGePJ7U?dW6h=Q0o`A%F`oK
z$WsJ%CJ8R}^<@|3FFrgF@GNCs04NvFBAO{|20d-)3B~y2cU@w0LQvN^%Mv?VrVpX#
z1!K&Mm1uU5)Y$SA8Aew2+T(sR?I1vz$SvWdq55eyqYW^QYhM3#cnfPb|3Ce6XZO_F
Y&)I+a>^2~aux^PyuX8T@k8Ah-4=X+9Z2$lO

literal 38034
zcmd?R2UL^UyD!Y>C{`H90#Zi}Afh55y;*?-0R;sKLR6X(dhfA}GAcz9dQ|~YA%qr?
z7C=WZv`9;&gn%?5kkA8!kh@<T&v*WJopZ}~&$rfH>&u!kknDHAdq4XrzvuTnFHbI;
z8t&d9wnIQbU^m?8g1LaemOuf4&9}F01)uaKyqp1VTW=d1UI6dlwQ{}+1KzfK8d>`Y
z2uMai|Njy*I_oGP@P`2W!r7~VX;XAeTDr5X+#1ENt;GS`djNq@aW+RH9Jl!I*<EbB
ze@fUjr#1WXas2~N^SCeGFdsZSs>9S<n%%Qyzb*Zf^u53S&@u8?n7*`WwCJwD%FD7B
zg_{&|N@aXx-R^HPNrlV$6pY7cT6Flzl1CcT(#Tw1a5}AjeBFzl$?b=Sw5ro8Sif$p
z_5Z$qu9Fdj%>t!Jd;lz{=hoCJXWE}D7!#0TKi#52Su41I|NgjK&s9W_6Sfhy;(}j%
zZQc~pJH6D1>-rS!K}4dB<!N0=g3;V&cnW;IC~od0lBwM{y<{b#$?}7T<Eg8;78Vxz
zKB8B|@s{Y~R5qiuZ`v%NFUSy^*p0;8c5#{1<q~VZgr&keOy??KnNz-6HUX^um6eCK
zsB*j_Ea=0`%uI4QBb-|<bcA_DdtiL(lj@0F%RE1O?6a+cf|QE+Htk#?qSQTHA}ND3
zfS6)>IYf_<zB_;TCRQUPa2avXNICG072fZK^P!k8pFe+AefaPp<9JSWUfi5L+yJ*r
zZiQvuq<{-`b#NH5VA2TEwu3XuT3V|+;Ve9H-5qO;O-mSwgNbw@F(=5jcuS^9GWX)(
z88%$`I%4JRJVhVVeanHj4r5|N^XTsUkLM4Od@xc={K$)}z7*KFTu?Zc;c{AxpV#fe
zBUt7whDB)bLT#ES0y9fQXr*am?3JMc!9iFKA(xOC-$NVzEYTX)PWkiV(V0zhk>*Xc
z=<+_tB3f5hS1rr0A?FSe>2WNlTERzhuDlzWS?$!CGUQ)H7d?3J;5%YQ>WF@vc}CZS
z@4D~(VMfRFlpI;ySl+q3Z~8tMl$M^Z3wE8tDnq#tk(l_+>!|~h)3d^Gv8n*}dhGep
z6FF-)0%lF%S`i`Xg8k(UKw4i02X88njj{d7BV@Rqu9jBitY_b^0+v!?HHH!df?%Ce
zdjS93j{F<xrsIX$DRPxyhv(tp@^FNF;w4j4_dX04Up}CPs}P#CLE@>I@#Y0R81DD)
z-!BrWnQ*K@mREbUq;?s*mdC1MljO+k0-JKy`6z?o5_p4wfq|Wts_G(($5F^xbHXnw
zqVctH4!4V{!7-ueV3)SoT|z=6=OW$#@4B!&Iyx0GM}no=;HlCwK^O6hmF?~Av1Brt
zpqLZ3YFtGpy{8=%&`_FShDDI11RAlM6u(f!fL?0Bfl7ZlryUs`O)93Y3c?HA%ll~T
zS~vbmrayat_r|<QQMTeRdDXnhM@~-8W0Hk5Ff=5*A(E8ITav|dX5X4O^~uV~5t@$}
zo{58D+HFZobwd36VAhB5bqD@18qdCjMKTG}UxyfswsdWMh6^!Vi`-`{??ErT^^sed
z&hm}PWPjI3(<}X)S^X9B1CM$qe6<q&MEFah6lc{w_j1hZZWmoSGMKA2X=G7QdvL0R
z;@;ZUMmkB3hf}FY{z-D`>({UKeCrAf{rJKf<9jQ|s}GX{WGX5;C)QTMUHR*tiVMHd
zO0$>LUB8~vP*wH2B|qBef){aQF*8uc-WKws9I!pUSFD99kYS9>$`csHsQ4H;5f}P~
zy6Oxp8Fl^pi;^}iCQ2=}HPd7+3vN)^=;{u%+pRHLl5xF>DCO#2%Hy}SwzQ;qCvjHZ
zkNo;V@jG*ZnLZO%le1(NU94Z~8dk&e(OVvT7~`&%N?^C`4MSpuRmGxEj3WC+XPg6X
zb<w${t!@3B{OHAGW_fsV+#CuBS3jtb!V2sc;axPfe0@h?m40^o);pqFRi8-kn~^iv
z`iY5&!~}De(VR3^N6*KnP8ByIZx+-9nS2?N891gT71MSjcdP@+90NL94vxKbJt`w?
zsb^e9Qx5+4@#ApJTanUm-l`ld=mw-$>#Rfm6mWGFwng@!-b`K;aE<lmO;xz;_B++z
z|7fi);LfWjrf^QR$gWW6qnVVmQZt&my1WNHJv~*GT-}r86!WG>IdQ_#k&(w2nm@Gn
z^H+TARJ652Y_R+zKIwH9I9K2pFzUeBq<o6jzG2w04$M3i7y6#MY5^x{`^?k9BxYSm
z{!8xV_|4Fn7%Ho$j1{-=tZi{4D48HP`O$e`3;M(N?3=(!Q&U^V#<6Rc#@5$_<$@4c
zw03%17x=t0F_0`+B{z>5PYBM8F*FreeF*u=C%rW7j&*YEQba-(LH)UdCx=B>1S>+Z
zmW&NmI4ic0iFpCHIC132k#S$G^eTjWEtb7tqxhB>ej8{i#dhkWwh-qcCKi{nN%72F
z0*sxOoP6+Zd0!N3?Y6WjmJ5vfL{8jJ+>|TOk=Jy0;WFTg%$qu~p)%x7Ty+4y!O7Wq
zTw4fLJshn~s^~hEZtA?g%nzg42G9fE@T#&j#tc1lzL7giN=o?ns^!Y#UsiO-(|ioD
zYfM=B%a<?3!2x)l7K@7U7ctZ1^!u3Ue=fkJjNrj|XJX%U!<R2#Ce52-<lyq7>TSB5
zenZKKK>i8xZV&|Uz$M-}ErwR6wO_(71^|oBEbE_6G$)xOg4#)8T&f?xMfG)Vu9?|f
zHB1<`(h9tZ91GI|1RCSOTE~#6t0S?ou`j_i*GrsQJ@JcV_(1=i(^5-+GJ%H8=fv^j
zM!>($Lc4e4GxV*>_v&9otb9&(&RDTETsbIl2hOR8(ER24!G*<z#poauTXexJVZnOg
z9l;8*f`ip|f~h*)%f&-DEZq}iY5#H_#k?tZ?lXn62=BOadLaE1)3kAc^j*=rSmoko
z&ksjIOam_B+b{dL6+qPdQ7?@}6mb2Sm*xtE+U=#eUrRcs9pcQxZvtD@9xFF7HC-QG
zU1VksO!%_k-lO103S@$A^j5P8DW7cAiC}Nea;?n=rA%_aRTYU-w_7`aRZZ-6p(i@{
zg5x*qt?(Hi=P*bf3{s-62*ayUGT@T*t+9t<Qc~-^4~6xTU_B+y6eH}X@Pn%ufC?yW
zwG(4OR(ai($H}$n$;m7AFq>$vYK%*HAH|HrOw{(D`kFpsW@u<A=z_=ePxyu_k}o#*
ztdES0uqD5KRwvCydB|0S1C6Z6iHq^qepjneTho4L?F?R(NVPIGF$ufn;=<}nET{YT
zWx!BhF$~y*zdJ^eOcRf4=&Y(((x>%-V@@Sf^~FR*m&@S;!TU1n?+bDDI2v*Sjn7E~
z=NA?$SBZpx$W=Mx)#JfmX(7&4^!3pdbriRk#$o_potOLiwOw*WY-6Ttd7q*Qhi%2t
z@o*&~375RvdYm(P%Y^X7Je1uFy1hSYL8N}0-QeFUfzN5Ps~%LM=8KP+u+5t{L&8$m
z)T|h;n+986HV9}V0PAWVrzM)3PM?%&-z*zFv&_Y2Q!Bsx^~4m*KwzCpR-SG)vqJMm
zSwW2Azu!bmm-V@a+$nf(kX7VT!~;fzv46Rk6UW45cTS{XraDI#tq0*9c~*pQXHLIJ
z0jf59@kz(Jd~TXDzj-i*RZF63b-S_OMs&6f4h~*S=4vm#=4sb+Q`6IVnRGRENq5r6
zqBw4UbDDsJ$ZGIvaLgBvup0Yzed|f*_B@hZZs*GMW}2U2JSID2O1!v^K|nJ3Q|me*
zv9Du#!rD>49tpg^*GPVp0V&yvcV>XU|B~@D*I*ZHoVninuSKm+^910W6uXe>ErdMc
zKqd^}hdRwdA>-W6o@)p=w(<B-p`l+%IQnH200VUJtvVX7Uv=x_V^j)8qY(fI3p75)
z#2Yx>e!s3Zeg0bg$`wiAJt()iyM=_xM35+SI)C+B%A+#r9L)7Qqi*H&vfOT$Fg3D;
zvy&6mk%dlarmk1|F*AV4hAWd}U_s}wFJVD5^@8J=6<!99<8fMy=BJgee`$^ZJTCyg
zeLyKeAeJjRGp<P%_QjBjx?O5+#ZJwS2hY_jdfiP3%nV7`d(Yk&<q=I9II?&P#eTz;
zGv*x+?xu;0i6N(6hH_-dQ&Us7Gy7;&eGwu(;m6s})(^|+Zm}Euvwgz%z+{p{064RO
zs?V<*VZddfpGn%F(=op@dwG@^Qb_+ElW!=*%DM7sX^k!tH#swFvSg=QKd@N`U{k8(
z$8X=hl{THD!8;~=?W_@C0u$>&)Idx;h>a0?nb-0xPppoN0k7F}QBAVk_5Ats9BBt1
zZL)=+prCY09)wWC4lD<c{Dj;`!Y7a}T)NMi%b0<<<Ru@zI;5@*z)?t}(7aY86PTTG
zm79s&3IX`r+5iqcfKkFY#H`d8*IfAP?bx9*Me^@#ABy)TPuB@%Ydg*6a_!<`bS0*^
zi}K1UtOjHkpVkNbu^^xxFXO%7EEuhR7JOOz8E$r8<oblrivG-IX@kN{4;Uo0S6K>L
z>1|mE`MamZgu^}+#LZ10(O8|8NR>7Xyf(YjyuX(RVnap<D@$l`CVIEH+AiP`+z<eF
z$sOn;P*+odLyjF$g|Dx!%~Mc|gAavChv806P6H)>M;Zf(Wpvwy4n}LUu6vG<y}Unj
z^;J~nZLLEy-uuzRq`rF4&j`((9?x>a`1&q9AyUz}OL9SQU<b<pS+Se@_r61O!WEaF
zO#<I4i>qGi_o49KCMPGe-xBSfkL-ur(OlI~AyjK;-B}3f)>LuB;2_Wp0pB+Eu_i<6
zdt#cv!ef$MB5B}5v;~SF$HT>U(X=0My-&$45I=aI5(Hp%iJh;wBy7o_{7am$=tKWB
ze!R>~e)JpboAVeASX#I|IT~h7bm^v?SgZzUB@=f^hCEOc!XJLQ!~nk9SeaHq?{gvQ
zReTL^b@D+6%o<Xlm^eL9!_}k@>v=)H)pVs9f0=4)jc?IC!MPswMMIW@DGF-In2XEZ
zm7|S(-Q&tiLowXTYYE(i>4=&GVl^SK=>V<tomGvzbHG{FLGmyuR-`j|lUB9Cp9v2K
zKw#*^;c)yf?&bVKx;qAi&CJv`nYUVM_Cs?*#f7`=6c>$61)_I`)qH3ArC$!KvE!Io
zo<n)O<&6I3mi~pdqxtN8k?+o`WPKSjq<u<>9YN#i0l`>l{f>r+$B*3+So5alDl==l
zH$1BV28&<HKq7*IIA0kcU(ydOoKFj3Rea3K%HodagS=zvB%c9(;X4)CgG2%oj$fRn
zm+J#m-g~Z6SR&9FJ4LVJY#~xnlkUO8^>dOr26^49HArR<+{swJ)@N6ro-C&pfa>4_
z%lskr+LhK7Ia5qE-CaL0`%q$Ucvkyz0qiaDWT2wp)~$9vzaDgFoz(LRrvpzLfveDG
zIA?|}?M>(%^PT`_{=>b>IiT1Vn3)UqDE9S%#JxP^ElrUBh2i7oqG2EpV7n8MG0Xz@
zd{piF+s@9;%%<N-2+Rv@cD2vDxH%EH@~KlZ0RRYMT*lU`tU;bt`<1{Ghp>DQ-yX_X
zOy?r#<ro!!G($G-t_Bi-%v`TF&GAT(`(tVY*aQCfMH#NNKFGH~4qpX=&ng)J9)1V!
zQx*&oIz?1cGRzSuJcGVV8stY9|2OD&#B~nq8(TKWAKRmfriKQRrQ7ONX?U9er@Zc9
zJ0P{&V+Q+R<edP&$9%ur`ApM*!{>tag;1d}%IYpJfC~k;YIE<a^^8+W>;h-Ou6$Mt
z63x|8Up@+YPXb`70Y<R>4uj4t2t@xJ@B?!`{mRKG@D&Jwvs4zB?-9;t(axe0VGvZ*
z0M~sWqG@gV-P({#m;s;-2ky%R|8Ftm?2)JlCvUHteF-JqX@NlB7R)SfqS#6FM7{Tx
z8myN3ooS~K@@yzg4lWD>dE8`6>qRY0i!>lDlzhz2l1jQ=WZ;#QnfFAHouKgqyEpic
zN*Wr=PisOr;Q?6826QIzcx#<4EiEx8u#5U2xePC(u0{Z)LIGJ@u#3yFdAF7poznut
z(k^)Yu)$e~DN#~Zp8I6p<N??Qx0^QyX+rNwK^R~+DBPpWjF6kDnRkgulo{vcVImbm
zId0GNCYC)VJQ*oLBl=S7Y|~ty){RW>PFu<6ikQ{)MDy?>>Z&$~p`Y-U3Xp?dPsJ?W
z@lho5{Hu7gKz{8oN&zw4X+5RVhbKd($OJWS%3T%7i9->HPWSb5;og0~`F=F$2E+}>
zT9MAOK_(y*2Sz8Q1#;t>3B4h0c({W+Ei@mpVIfl#duU-%h}kdaiXsT|aQnMy+EmWy
z1(M};Ap<~Zq|~3UmU03Jv<igb3_zho!2%pvA`d9Apl)zTDrkJUq)C!C>%Dy?-5qrq
zU2}{afzx^JP)V)SB^v_}$r%V?*Hj(QtTqMEf($~{LHK$YmWc~J;gdeEja#z@4iRjP
z2c#Kr(lgW<P)zFJRtCx(#SZD#{)gpGm03FcOw-)?WiaerOKW?uh8BF_G#_}j_4Zp*
z0Y~RK&T2k1H9Is<AK*p~NLl10V}vl?t}On$Yk30zF0VUcIkGFk*c!NA+(-ma4}zpy
z!;w3+V5qUO)n{IA3%!I4V}qo}%nevd@s}V#Rbk%66)TZ5WmXVlde1y$SC~j<vo!C!
zq5dcc2zoPT_Y(nVzn#?{cpJ+ATo}uXBUd3qP6a_v1>B)5YxTEiyV{Xo2fr*VlFp;p
zznRXE28x%=ERiU-Mt|boCFC#vh4kX!SA`m(y7r3E1e<Veas}Lbbt1QRs;ZV-#aU3>
z1Kp_b#@1lTds$0qFf(H;I1<jNZrdj4i{n`afKr5Bo(}s0Hx{)Tw@DHD;7q3Z;1=kI
z?-o0%p?85W#C;Ly{e_wQ%*NFV@>}en5ANyZ)a`_R_>%jdPqG;>d_0G~GPgt-z$p6F
zv~`>>S50k|<_<}(4^bAiwt?&G$foy262``HrNNS~8Z$JswYhuY>lTEE(Bx@T`vPxm
zN64dH@Nlecm48m>#DYfq+(A*%&nc?tF-C#rR}wVtxbPM`jdL^<CQK@Hrrxgcx*D)1
zs~cVy3HH#}h0?;-8(aQ=wXd!-gv}D@R5drPJnCvMjAB+?5a*4~?F`};*j5cxg>jd{
z4BN+K;DFu}#6`j?a^s41$u=JEhaSgp4sS@$)7$3CkwA6#{^b@V(*a9^X)2OakjxT<
zJSM#NZf=;1tE+3Pd7p&1xQz{(o;;!pw@VnAg+IpW6i`>^kW5BoR21EF=#j`t<Z*qe
z3TR6LmY42xiknYrXT{CM!dhV-5>irnexae&0H<a=A1NVHcf$v2y8J<ibCDm##F-c5
z6APpP4Jjrq?E%xwiJSWXQ-L>_>MnNi(V9LRYLK5KF9wJfyL5*_>MH5UlP5F^fl=((
zJZ*&vTP(@V&1IW-caJ8b!rMT2VC(ez^Ct=GlT3Jlo+od4dHEq+EhZ{z8d0oD#*k({
z?8HphO&o}}Iti`zUcB;yatHU-hh0d9=U_u?syar2EG-8w=?R@J(wPOG1r;D7o}2F9
zYi}BN1G=KFsr>McYAaj6?n)RMJ91x8YWdqaE+Dhn>;1mRDZFk%xF4S~Q@>YcL&Sx1
z_`f!>Gjv0c0Jimb4er}Mzjf<YQa^p18#7t~0JP%wli{<E#ogcYFOWBMJU>+7w$Ol6
zNstqip7tH+%Z*b;g-Svz7d)Ww{^X}_D^yKzaPX=w+#Jm#4<|F({B<_!1|sO&rc|mV
z+#g!y3qH852j1^mz-;}erR0o^jDw;{RsG$0R_Zv3Yxg;mc?5S>Psum|8wp!^2t;Zo
zMY#dIz07Mb=yHLj^24w17!{XueOhm8gCsNJX%8TKGjvmMwjtBRD3BO%s7OUXuf_nt
zy!PSA^9`HRsL20oPc=xoG7Pd}VdE#t%u0H6h*j@7cV0WcU+P7Rmk$Ipt8mqY)ZmZ6
z13x3wEcLd{wy6T!ssgDUZ>MYUdHiCU%#96`E<BeHL<d}FuZoHaeD3|Hu_8%PQ4In$
z1(vDVF+XM1&nt(Qv3iNjEqnLweHLA*n^2K48|fVC@N%)d-@92+_R5CWxQF}lH0jk)
zC9|L}*+C0#s?S?^By}MH$-EA-$^w0?kHqDE-VBe!z>K~$*raH<p%{ZBL~zV-7c9dJ
z3w-|SW_9=8(uppZ{svIMRCF~|eWKTXzTw4r4al6R2+h`yUacGFm)38IxUbG#nhx>7
znqm{(`rpr55lE|(-GD2Jx5}_Z(-RzylBp+b%KF0xi-7sA&K4CF)!5tHt4qCitNq3@
z#<~K72x8eVR;kIOpEoA;JOwEQY-LbTKakxJC9Wu+(|-d|;SbcH7hLVDC9KPz^7;7j
zqat!-#tB=`V%~DES)|H26}(S{Q8H3ef^Py_t~U?^ghoN_`YJ{G<Ayd1JVPoS3B)<v
zzI{6c?-z#EKbkXTq?|h9fvdjl;Nb9atTi=2a#UVy#q`kf9)fAEMR44YlirR`+EYC*
z3tqVDcU-OG>iwP{wXxOqFWrWRhf$Y`Zw@U5j&B^JM$&%O2OG99=<w=)?wkKh9Q@Bo
zTwc(A`K}KfeaLyN0xr(ZdE&lLu;hk@kIByz!=P^vSbhmjGdPFZ%Y^*Cz|3zJrwbuo
zA=1wI$4Jro<~`hT%=&ofl)W_5FVfMkRh8y&=gu3N5;WkCCqQXlQd3NGJDH-;2c8|u
z7dPDce|6mYxvl@B;P-#eBp?iX=DPv2%11KMI30a*rFjz`2!6WCe>#}GG~LIMyj3zf
z*Q}_I2AK9BT#bq(AiM$V9}^Sfq4V{=xVn4j75Pzm#2tPQ(7H^m53Z2->)<XI_y3G)
zjo8!UG*I!nh6tkG%}rmDZft1C(ACphA2jci6c?{LOb)~^-eMZdN35+O<fC8@1^aTT
z$b^*b&`dW!?F&?_nkrU7$V-!fsYd|=^Y~gFG`|_d@K3`4<)ONf>ad^ytjKZVv}vvi
z;9k`d&E4M+!>g&QrRGh3#&@%*t17@fVXC`)Hgw{+6POgh;Qz&Fb_3C8Q|4NsM2?zg
z6RPK$<bX4l8AYEKGuDPyC@}b&Wz%^o0>o}OesQ;;dp}rRId;tpdpfmMMA73kH0Iv0
z{l$HOgv!ZxesC&`yjfUh=F_7kKrk?P6?YG2ZNyj!b%oncAM1uJm5N0fAmp{n06FXN
zG7O4UI@;Q)=1m;M08&Awn7DX|BUVnPLYr&|s#8fxNy|pq;F<Fqb4c9<w*j?;WFoZ-
z>0#rVY2L)6e;V4USnYYU=-kX~N7(1SY9H(k3q*wz@J3wVa{2w!edT7-Q_D8)6{F6^
z@@BR#=p`|Jme8_a=;!|>FXn>Jt9&MWvW&8@u+XwLH`$$++NgWg3cqLrx4VKkC@Cc+
zB?_Pe*sfusc>!EGedI%@srGbDgiCImw>5q-S2$EAsAZM}bpZ$r{#|<5U`Dp5k5}PT
zBncq$d0Hjl$$N$~-k^aCWZq}$&D<9bRUxAyqe^(`-9LK)qUhL>okZ$5%Inkqakrl0
z(WTKQ;FpiL8tSjM$AZF5IXD*fgkORk@XLLGMdPH6>x|=S0J3ed-m_%@<X2`{?MB#n
z5Y2mBGkR_NtNJ@tjyr{ht~~>e-v)PMz*SArsQAwPa_XuAk;>5a43i&qgSY8NEBzvH
z56VsTt<hoO@;S6C!4q&0sI72ocOWe4V|n3zK=-?A(||LA1@+&aXA14wm1c!D!BUO~
zkRHnCgjQ06gK|3$YJLqj!>o4dmw#CWvUa4+!huAl2-IK$f=b#pguI@hfX00&;QG;Q
z%YDG{Xpv_oax>TehGpt|X8sc_QLIow3p8(l^^LB(8+dn#GI5V;r2x906~X>^Dj*6+
zM-?$a5V+lWeUGrUu{mxn91BE4lYZ~j`Pw8`cBKNA47lRv(WtWb3I*@KRm1G*<zt0l
zDpMo6$N5bHaxeBPJy3^3#y<vlKALP$4+wW2HZ{SE9&Q8GL7aFiyCDDE&We-d%UJwQ
z-OsLIUVS@sm<=|yv9VE6+cG(W+drP638;<coVb8)q=&e)H0BiiWX(c{1wfG}LA5$4
zfF^8m=~C##_4JLvq@kIgw23XKzg!;zz;GZSK*!9Rh=4GmMMp&eY~RA3s~8#@5;@@=
zs6xKUS=^Rr7d#=KXNj-$SP#qXyekWAuNVsA^Gr0JcInb3ozv$6Jv|Grpu<4Ys|r_E
zQ%l$qs!ZM~#7Asc%2<A!yciY+{6ZfZ&%~{rm3K76_P{WyBl|(kOkB-lAoR2_OuV1n
zef<tlgj<>8yFq{r0BA~|40<P|jpwG_Un|N4g~*RafL&ue!yPsfr8@Kc|2QlcO-rxO
zL`WN%N5d$BK*ts3R?=A;k$f=88sq>Qt1^}SKZy-u-~AvHTQ$TlmZrK_PJFW)zwS<?
zYU${NHVgB<2-6wN;{t{H8);g?H&mRt`=sArqm~GWX{;^sV^SMY;VYvImY<YBlJir@
zT!mbK3~75z_c`wp7S=DJ9TK>S0bBqGuwMa%kPedlESge6x3zP6JRozA0Dvv88@*XT
zI154<b+`J6Xsh)xkM-BkAeGFOmI$yt&sR$S-^6Apr(DQsE_3T!x^EW&?L67MNn4S$
z{Ottf1_T7217<E_BF@6Jhd*6Gr@?dLu&~6iZ+kN}m=AWI{7Pr)K;i=>YN6(t8BfBI
zw4<gq{B^F`v>yaZ?)?o)LfiVoxzj5Px%zabX!GQUor*2JM0AKG6jugs?F+mw(^ukb
z2GPVI)3gK#l<5cc58X37^*`To&}T6)RO0Wtx`1%M0^)kRi8|dEBRsrtL4GvDyoq_T
z^7|{B;Pfr`A}k<Keea>haHPr-Cd6Gn$LdP(JHTqXV2kX=1qO#U((b`OQd*tB9)1IW
z(>^%;r3i5E3H9`<0E`vEXQ2s{N&Qj2?H4<1+I~3>@@#Nxq0T7vfh0+7{{^jv;)7CB
zZN{k84!g$h$!-9oEiaYV*2s{pp-jCB5-}SREQh1Dt?<-_LTOwgQh}d=GHl>0YofoP
z6<f)+{U%s8V$K4-FL3DBAGAPW;megr9^vz;Af540EIFPB5*5Jr$%4L<^ATSqzaQF|
z?cVw2zvc9x77%Dc5cDg?QqaMYn>BVt*$INR0&#x^rwRSAAa8M6;CMUWXlqwL8|uwP
zL0S*;l>pswP0C6z0rZxA5hqgFU=w|b!Rhvz-AKyuoCU3|b!P!{A@LZ{pHuuLf>}fE
zc6G6+8X)veAH|9htQ({>b#r&rQn|H~rrzN(uNMCNcJhlbkggEOogn?#_816v9tSNX
z4JLGS<^z0mEY(9QBRN?!3jkV2PtTb?5nJE(0=x0SFB{jb^Sv>E|8`=K!gMc{R8#Y<
zaUgjC$f3J`T0lVVEnvIlAhH5D^E{F2!7ZA@k+?EafO*x#>(rBPh^D)C?b4~FSp%%B
z6*k|h&j0(Oi0uN&&Ut7MOrP*QOPY5F6uv_w?Rx-gsdp7p6pe$R%AVMTH0%B5PV>ZO
zUV2p*@?b-S1n%*=kb>ihwd;!nXTS*7M3&?O>;bw&@?pY!2@NkGpTd2uc$@M9$WbhP
zZPY892f82jnx+X9aFjzf3s(kgT}^b6E3}hD(n3y6bp7u@9-fy4bu^cABO)R&l~&fl
zqkW9^r73U#1KzWrL~7sBtii$E!~ky~(Um0~R&1sn54$V7vI1_kp4hBdyVh4DwE2rM
z8^h<OqG<(T!T7~&u7M$T5tz7u=N~{tgx<B^(J8HVrex^M)NKZ`Tmz7!6o>}~KPV=q
z%-`qPVFN&H6#S&-ptuy!lD`0v6*1k6TKCplpN-)F6|bCYQkaMK1d<5TZ2-Qau{S!H
z5&{CD5fJI+c^$+mDY3FDaF5VY1=NINnqvT~V%#Y(@I91vNI8817}u@W*Al--glA+r
z*vgMmL87_@*r=KSVEf$Y!XEFH(Zozu5D6n;;%K~;jOGb46fV@4P63Ns_#5D_%$Tq#
zdmzDlYJqJYaGPnKVi5y57*~+A0u)Dv5#Zi-SU^KK$D6OOm3EqPyS{$?nsVfBfUE1&
zQ>PNmYv>x#&+rr$1GTkj=5Jb=g>Vrd$ol_157eSNu?Au1+~}AXb%46|VnfTQm50bx
zG1(Am#yR-NkAj{zKf{?NosEDDh$$tD)4O)>uF)dj#4m<{KuO3%)jBpO^g*e#vzeJ0
zBX#5*bXIBL)CMZz%F11PyzQ~#;|x^LXuR2XUoBnA&y_T78^Bgf2%DqBK7yRVV_*#r
z!=%9g;qb|OBkn2GKRr5Q=huCWiOq*&1*HN9pY5Es06XLQbdobRqU8015YD1x=;C4W
zIa6PO{~|qF0(9T9cR(gvzvl-}rSS1-(!+9q@2t@v?3aC70$`9~EAf-L$#_!%c|p`$
zqTX=bB9S_ekZ(M?kwJbj1GAS?E8kTu58mcpGBcy>+SOvqg0L<;I^;g*`q;6YwKLcv
z`}94KaKED<tPTsQ(x)9SI}ZrIH%Xv;0HN4BZBJ5utoIkozqkEIuv4N_MS^()?A@6&
z)id9_g^KxujEh!Nfad*z`R~Ex*M9?(3-$%My3==YJMp!E+g*})mJeNVhHt&c;`9vw
z0JT9~le~c!q0Ay$24JXFobJY97AgZ+>48ld;Xqvm)$4Up5ND|hY6eh!7WG46@_$vx
zsY?Z%WsN3Wj@+yUILSPS-kXF~07C(#26toSI0t6J2^)9)nXe|PYP@Bji30$1iJN8r
zpfUtY4v!6*V1>hH9&+4)0)yIKYfc;*@Ph=9=^5vCCNoC=g(BIrfe9_~NI%9J7i8~H
zubm@+<00cT8j_Hlloa^Ys6VS^LyZdG>{kJ}6!0BoG$c)K<)L{~G^_-^u0~b>{f26;
z0GiUSoYsHO$Se8+`u!ecj(fHJ(7$ZUmTdz;K!b5`yY4W_1VHr-Nx}z;zAg!4mOl!^
z<eSqsj%@IzW!JVJ95xOx3LE16-{HMgzc9fODk9_=RUl+~ENV4+E9CSCwO<qo!$TfZ
zAP@#Tm&RX!1Jo}C+|k>JE5Wa+l^hT=NK>z#o$MU|_)I`RCrw^VVDOK}5fLGhLl;iP
zjWpST-e`fr*vP>x_Z}Y2(J5VvCxps?DEl*nSmwopvgNAbE#Q^+J_jq8cnx_K(8FT}
zWd(~&n*K&882sDeG;p90ov#cCp?g~(@N+%vdyVC55cXS0GmjMe`ydZka}_OFIolvE
z4Ss8OTb%4(w59}8X$W~D&01)~I||RQb80bnrhAF%CT3>*%Y>4JCIE$;%K<dtil9Pg
z(EzFlys$#Zi-3;20Xj4f2zouhZsi#Kb&fvskz(l3cK#m+K=c-vIpv%YG**I3EB!<-
zaKc)pw`3*#au3%DsK;Q*p+7I9!^XhmnFCCTh>y?-Nxi17UM7SG2Wx>$`i$c6@hU^z
z$t!o&Kr?bb*mMakNI)RLB5O8%_60x!E335Hfsdz~*$bU!l*w)%egk8|_i?bqcp((I
z3*RV)O&G6q<?3}qm~MrF2?wsY4<eFu13-1esX5^U<exJZ^PNXSNaG>FlJ@XR?T{G%
zhb4~x6D0p%TWtB?SP7^MLHoM5x3|-sv@%x91abc=TDP2baM1HdooD9M=FJTGgWX6o
zlex_`#+xM~fwqEIU=nX<b={Lz#s+h6zz=jJI54UB4asq;$49Tk1YafC*WLQT2;aPt
zZYl?k2Kw3s3=)WF-M1T>=6|WK1XekCy|46cr!|oZ^3!mvUrwA2=t}AfnC@L10DK19
zhE)a?RbGLvPvQ235)g9r{Z98l^WVc!o{H61c?~x>Q;~AGYFTm+9FQe|o?4lQXbTJA
ze$WY3)8>dH<iUD7BGo&);>UHgrw8jpdK?qeuOL>8uufQYAz|U#Tk~VB!hnj}9|O9M
zFxSxZ=Oc8u-Ir%&py^3PNolsx(0?N-6No9jZ7>5G3Tw1{vTiZd0splMY8`BZITSXz
zfQv7|wZJMSA+9Y~#0?<PwpiPP(2eUp05%&`NGAXM4=N<vyVN$u?6CnHM_xgti!D|a
zK3BI{IMnN9m<yg3fN|Hl(!0UDXox`s_Wy`q;=GyukLjh#xJ%Go1jsw&XF&fjDD}Q2
z4wQY>%ZcMwJ^LtP7dTXR`%~|5w?F5s>wFvB+XR3K5ErHaq0jdz_h9IS6-q#b2R0R%
zSyT5~PO9yPOcW@fEb#K*N8J`hf?Z8z#SJ$avNnEXe@<LG$olWk&dwTudXO)KQ99u#
zb*4)@2SP-zWIT8HxQU<|5prTf22I63nWZ~LC7lyGzIdo$Vp~uq2SSF_dp?wPgX--w
zureEL=u7JA+6Mmtl>tb9ZUnG8xckM(!D(MDj;{y>uxFqI<PzY;9x|nPR`*FsOY?7I
z@pQG58ed5vOH)C!03Dd=&H%-k8p;rcmT=t~iur(qA%eO!cEsyh>wWE0r#$WOs&GA@
zSEcE&oH#QYQjzVebwsd_4dt9Q8(dIMNte$?E(7q3XnhFXk2c-XA6ll#Tj&%C{eB(q
zcN}C#w9KP!0WllQkFo9n=yn<vP>Jb34eJ?4-8R@VnS)FcZ{+tM2uZGDztY9?*ePHm
zt)QWiU@})6C9XbD7Q8sg=mpFrO9614Q-CeA1wC~(BEiplUYn#sTxuCi9gw~#|NP!?
zmQQBLSY2J6Vxsu~fS+!FL#u~XEOZ!~0q9*>8aFpix79xllt0Iw`vh93fC3$Hus(GI
z7l5J_9N+?o+YtubPB*xf6*gCHr5`Fsf#!f@heEEFlbf)RkQ%lcxcL~kl&H9P9<WM3
zK(c{GW}1wqs?U)Cl&7!NhS1X@q3{j^VBsqXST*RSL><eyaSbgA_cq3s$3#Z1t^BCt
zxN@N)A2)~+GOz-ICq*&Gb+WA-R3G|qq1kbBEC?3ZC#P=RBw%_S;B64Xr(0{<idrn?
zg3R%Yc{y<*A(Bo|_NV}5k03!1wY++N1Cab{1!g#~lm{rx!De`PE_c&CQW21CM1rwA
zBXNWqocKDxwBg)1Ij|K(4}xV2@OpJceogYZ3tEr{L==95_?5;1&_*2(0_4sO|2dyF
zp55ar3kRIeYjARe#wQ7smGR6>Ih~>HHPHY;OKb(eV(>YMH|;TAP{o?`i*&rk40!z+
zbOjqOvFLpu_%Xp_K~oSLF?cHZ)duMP4=VSI{E`6nE$^#->y7%kl_4jez_(SH&~}OZ
zawU@aYCB5|CVTI~hn5SkhF%?>d>YcexM}jK0@@$*jTc?HV{*sQXF>*_FFh!H{_CDw
zGJEQ`>{F6;s0_w#J2a-8o;;ydt6wQQ;alk(99&u1smrF#FQ_=bGfiLhza|s5s-0^Z
zyHP6yLbxY(@khA|{Oq!C`vxKdKfbvF`@xa{V%p&UPlNXxJSac(r`pD$0Xb-XFq&Dn
z?jT~3=kJJv-=WuU+m6)t!6xjn!7_MU>8WhqkB<fF{ybptZSfg)>w~%5a1Olp<b*Ev
z_2WYQf4)w74(75uarbGq5WIFrLTlWDAG}J2mzbUSxDfSo%;gU$TDQ)a$7I*W`c`L~
zA$q0rL#nbHAJQIud%02W0Rt~%5iL<wCLOBS*Kp+FiS?T+#xi(wr>ks?(Nn8GHoO%W
z^+r5Cbh+nwc2#`PcVu36jB#&P%lV#y*b#W8aR(Z9@tEjFx&lUaWg}z*OM<Wk@xJe}
z<%$zBb6d7CY@1p}o6m1VaWLe?Jr`JDRQ3wiJvOduNcct-3QK4?->a8%=@rK|Ug$S4
z1bVrgmDyX}XtXNlXu#GxKGAV=r35=5cehfxQ@#33hg|Wus<Ry(I~q2sO<=g{p{ZX9
zM<zVtgOYkr!WqxIe$TJ6>Ojl1Citdjk3SscHR<@eSzBuJ4|u0-%n$bN^Xd>$`+P?q
zjEBA~m4IEahBGXkf*;5BCtuUc%5VSY+|EdXGYW~wZcB5Ffma$NXHVaZopzw*Q9lm;
zGtuB1aMGN${5cF_^=Wo4!j)+=Ec}D*5)hcQ5ZylD*3|McBsDv1*p(^WD7^vY!L?nv
zr+%-_H7Y(-x{=0!Pd@+N&|d}%_~*_qHhmDSSAe%#kFG0IH&P-n<X7-n-?6)xnbq(;
zi?zwu_<#QP?Vs`YD)$*Y2(Y$X>ATp)cee}*O1A&EZ=e-E$e;6g`i6hqGU)EV|E58r
zn(+A;`{b+jjQqKYVR{gFKtuvP9U)_T^zA3n5;BL!hT1a|pa$P-dDEwcH$t9(z$v9i
zN)G~pEqfGSkxmp&ClSlbYkQHY&wb;waWIo&YKCKEh1X=)TVX$6-%IZd`ffG+vxl4C
zb{wg%I+E9a{Q~d0<;seAK&1m2YagY=fj>1`y<}n08OLI|H*$D58C@rQy4=JVqFmBx
zBhmu_$d&=A2|LD{op`lQwjM<#RCWTwN7(48HQHmE#UfSa<)I0%D)5|MtBHxpPMO^Z
zB*}N8WACvZ|0J(}<dO6h*iS*zF_gsxUW{Wr`>0o@LGpyK1HMKrz|qQz<_>3IvyEWh
z@GX5pDjQu5;KqAzfRl8GzkRVL3uj-GZBM4`$DZ8T^blK#=oy+DZMK1#V6#CU&O3>c
zL{CfBQ`!*3|7Y31fupI-R(Ixwr)Q(!!OyaTuy*nj=8mtu-|OWKXv26pagtIRo^|ms
z`%CP{yTu>Rbw4r4!t9!?E;e_IIQY}R_K8{B2(c3v;tw;!FLaC>)31QjV&aJbh~6)B
zdW{Au9V!|^PvK<B+y$&l(Yft3?=suA_Yf}-1|J`_R<)>qI0>qgvL~(=zNk&&Tzdz|
z%kkrlUg0O<pg|oxr?Jf^z%U(-la)N4Yc9V+LVdb(OyR&Y`2)>5dt-el6`4lGlrRjY
zHlKJc@^!HWCoSae?F{bg*Rq<XP7;4AIW>Lz^o(PvnVr=Df}yJBfoIe4Fa{?ckz>%8
zUEqKc7Zj4Xtd(BieIedH@`{g739Jn?k^AUh(TL6hlfcusPyAU&V*dgKQyG&MAI6t2
zFYcT^2A>PV3{`;6QcV1Lv1_OQE~`CuH!7QgK-#@%I|jEh4tCMeE~ct&{lt4TnQi{I
zehAmu-!Bn+_3DEF9k)c4llCpXXt(O5q>m1fJ?b~24YC9km!I5Oe&~MqTfYxW?%j~0
z`rbXzGS2Q^;BbU3wg1Y3Yo$v){d9XXR!pK%b4vKo-qslr#*@Dw3A_Qj=03UeDw8{;
z?<zh0frZyUF%gALfe{f;6;xc_n{}v3rsM@wKaPmsSN$!Np}%a$<#HS6oD*kDZOh(=
zGT7=3Q{)nt;t$C#M+hYpJ#H*W_QAt5yLSZqx_pI<6jy3EjYM}gByH}F&fSp`S219Y
zm2LZt8u?fD_7L!xY-@w4J>KM?y=-qq48rB|n6T$ww5YuSkR|U|9nRi0Mh#5z`9e;P
zaiZya{=8u^Fo4Y<tFw`xK*0C1`U;3ta07TT_FRn*>y887?Nq~s_!a%4H@~R({T<uo
zfG0L2Z4Y3@ujr&b9}iz<@Gj_bsJ#aEmo6kzK0Q(SQ}yGOG3`Ra+$h1NVWy&-Ds`&i
zQmK!kqaM=(_1<&H<DI0D(lJ$uBQg^)^?6gF491>asrqj(ryIePWsVB%)7JeM*mu(T
zhI5)%gUm{T2cVqxz;i`HH#<5-7GX~ky4HcYdHQN{r%HfaJG((?N<iS?e({xquNxW+
z6pu>P-f4eT4QJjtt(E2wmi_PTqwC@WsHhvZ<8iZDs@il{s9!!&66+9Ik@eq=vPuq8
zA{~R1*r@cX;*t`dr`a4MbFK9CEB|WpN!thh%JzYUS$BkCGq4L0hwWqlSYCalycbnV
ztf}GcRj|yzlaLMgRY_;OZEj2U44Rg2U7{J#*Vfk7tgWTBRZB~Ybu6H7xaVgyHTd1~
z0p}{b4@N&Su^D>+5geOs@iZ`gq)nUMe;2Q7h>cWIuz1_G9vv;j<gTk=$Bb9qfKU9i
zMJ>kY=@vq9nQq&AAP?yE;sx-9oiHBKTnF7F3#S=Z7W`#J1}$!C*U^2juvXDd1^E*b
z7#Z6=F^%JXety&Ve{sXghe}tJ;Iyryt=P7>E|Cx6f7~5zh>9`Nvc>+JgJ?SRkI#Jb
z{FK3rlbQOAY;6BOj{M%`KVt`VRrA*Re(U`i?~bPZ>fvFjDS3{&vBE>n$lE`A0l){3
zaDz1`@^obWT>58LfgN)7+}8X#?7z&wR)c$NYoZya?Ej$WX;hvqE<@LF>plJY?~OS@
z4ESyDxzyRgac!8bg1pScM=UG8)CMc*$aUBTs{R|VGxY68>b3unSBswh_}5RAFlj0Z
z(N;fR$NmsIm+JiZi|o%2{<C*`$77|LPH+{OPY)RCR(ERu8F%k@QMd<o0fTjIQ|RN$
zgvkazn*E~w&zE(gr{Sv=(U+#ijA!i9iJ{?J?4gPx^cwN15|VMF!I4Mh@oU=$tQ=49
zFaNu5lXA~*@8iNW>*szMHJg#b5vz00Dg6_21?H9?3(2^SJDyOC8;o)~!7#tS{9{t^
zHyd=`f(u{gj|#@=*XQMra?i#51R9qw*Mh4|sB^b}uRj0M@aIwaeK=Utl%$(^E!rWT
zw`ozlxXGsZU%`p6Q@r#3EBwFn=QfY-{)s>)tw0>SW<BZyums0?)TUYDCw#fNOP2XX
zr<wv!IdGhjdt`jj_BnA%6CM@6bMHT^d`?l8MD(-bd@Hi==R59%*(NSl#rxV_JN$EU
zi6Od{&VB6;rtP(-8<ah`@N%6O*4=XGZ<ZM>{VMVMKU2MUoV;T?_bgW7?KSfY9sY32
zQ5CBV?)Rra{-gI>+w6q~nDNQZ<ySFQ1!sOvmGLNcQ1bac@y4b2;F+J;ysPxiHa9&j
zL{&7*s-q>}(d;NrbnCsR-nIc}upJW%&j3NPo_lT)X=6hvq7KO4tr-t8_U>KI$S&Vn
zJHQMYSUED)@xGwyqPfm8?-*siKw)LK#3PrL!1o2*>}*_WN{Z7)TZU+PYdE84D~xB_
zF$ta&^I(51vJW|U4hZDhU?s~Id!X@l(cUt5dP<8OjY{>;XLI?7C(gke;ZLul8L^jS
zG1sH6gZ7(BVIwX3J9oanM=ZTG_)Z%Q{yJZ+(p*Y;K(qAMq4siQab)H^v7Jh#PIA{X
zVbgOu9;TY#j+H|Xi~^uBE^6OaPR)4R!nr3&?cLpw#MGsPyy#WhMEtz{WvdFT=Jua7
zlyQp8-RSJT2X0c$y+Q)V<PTIEREN|d(Z!Qe4WfHnk(|}NW;(M;=cY?bOVea0mWNiU
zy{Fct3*L$8OMNVgv2NqGg_|UtHtVQ0%xWi<m6cr_-M(Ky;Og0d^+CX4+$OR0<tL6{
z?O<=^-&jqPOiW6s3H|EgF?&}1B5EEiH_5Px@!~N`W6g3IfwwJ4!AHWJHO1Z+Xsd`t
zEx}h`5|{8;yEiSh@xxI{9=L^_G(RhawEDl!*yfEzv<ksR;FX3QGFaAcohkxv+{9(>
zg4VbX=k`Y`v6~8P#$`ulPc>B2=XG$4PmyS|INv<j3JGKN(o_@VKPF^{xocP_8?-tg
z#Z6~*DBwRBj+~k&DUL|YMFo6SNWQdg^631Wj0wO`fgX=IHU}C+(fAsZ4&_F#-fy#k
z566R|v&juf&tSU|lb*I5#yfHO%OUT0v<nIoyt6@Xv!jnSY!Vp0jpx0~j#ECOKKDe-
ze&EPNxt6csd3ws5#*dsZedjA$$(P(0b=b;qu8b;F<E!5B5;Qq^&FQdRDf5?DCl(r#
z0%zvsz*EEhc`X69UGH^xm0$K%mU*v*%*<P;IJc#lKY8CIA8VwOdP&wQFr&P(yj;EL
zZs+T>aO!+D@5zN%%^aTN*>24Ktibz9nquqqI`P-StFpVVJ>4wOz@$)nJ;dw_A{&S0
z!0z!q3Vpr3cEjB8hr}hHhNOgZQ7x(JiV2rwMGo2zIN)uMzrEIxDMLp`uDBeI4fOp}
z^l$-_1n<y>)ga8Xvh@+G$;72vEO`VsoUebtQB&&USWEJJu@P*}3~NVdxnYZm>l)Q)
z^Q)<;q3rK1$jNam7}mfjPUyhpj1Y**POmf%&eD>e<((?<D0`bpZl`p7>7|ZK=xG8h
zObwhz{$HF7%5&xUuG(G>?`?%bWAFf2Z%P;gW7ly%P)F~`_dlc$NyXF-h?evsxpW8R
zm6!^qu};A)JK)6k;o~SpQ<@ymXYRZ;lOtjjd_Q}U#Ksg82<)LveNt*>E@(7!3$-^B
zi9T{Ro_T}DGruggpyT099g}^-7sehidTJE>S=8tJ;n?drE#F)A3{SJ!`VFG<FaH=&
zvu{~>a^c&+tc37gS6&Wc;hGH++2$!R{>v7N-50Xj|72DQ#sEG4aN%%~({VS;bD9Kv
zsNaM2?{ZN}(lMWIOUL@Qik_F4SJha{0D;<R4HQ2Ha3$`~lquI&?~ezuH(bS%H-n>&
zsH~r_=Iq;h1FTk;>|l;fgA9JKXaAYmce)R(nnH&p{q4G63EWdIq=u^>CCN}KF$vk5
z1%)D)Bl3YrF2VzB&<uQwggY)SId=*81_tn7?VC~!ZqRDXlHUp>7^O(La;>m>u8~2n
z@c8MkUR}~9-V(xaS*aL|((g~!EIafkzud6(k-;y7=^q3*WW1-%%a((V9$w4L#w09!
zyICcISe0XNq|<dn{0Ab;0DM<}-8td>U^+VSG+DF+2+>(bsy4WkV|(U8$INFsW4u6u
z8N{BRo>p%NMRn|>%(tB^wXkn#gnc?3`x!G<aBTMd<8V3%9DkbYH)h{#<6S-GXRVbU
zj_jz4gFiNn?V8YNBLKfmI9BM!T6_J`CqpY}QX*d+n9bur9a0;z4}a-{M`iX~Y+DJ{
zzoLL!Tfj6GlNl%%wJ$RY_LvE`w|+AEbTRSRSF2XNwmp*JHzINabb421Myt!_!~i_Z
z&pr#5(>OPPsM(4q>c{ST_UU$nuF6{?(eJS`X4hZv%IlgI?npG{np4}L_OmE2@P{A-
z`I0N7+S;$v0uqwj2d09yB|j87W5Q%IL(I(&i&}fl@V_RjIGYwxPbPNppod)@v(+M%
zFv<-#bWWe{`W9r@GOr^Sk1)Y4Y=v`SVU<MUVpsq4x7lUzcN^p<wxg$wBgRWpPTw`h
z5tnkXr0*KhCy3$1=m=&d-!-zrykl=el9l@D)2D@{af8n;K2#bbS(iA~{NY>`b9DZq
zV;z!PcJ-KQE4YVLjJig&&%11jMzqEEvMM&BQZL;U@`!iu+_|IHaN{JrA;~OiVeq|9
z%XYc6-vVqo!H!bz{cFdAY7DO)8>-IjoFHuJ3qQ_?c5Y4SK0I+8PFYx5T2~9PecMR<
zod(KDS@MfM#~b~ymW{WSX!AOaUTN?q*oXt3cMfRrMAoNRBUrEwMWso{$^bi%sA`D*
z=vf20H^M%OmKd350w*CJN;Qm9$~9iaK|Iqz#{kCj1zvr9{ql@NZN@lVH)Hq;0`vCw
zux=8G6!CDl@(T8WkFO`I@514B6{?zoYd_8MQ2o9?q;Vi|wK`Bm*C?p3f|0$_WO{an
zF{IZY*0W_yylu-*gwzn14#S}nzgDEQ#NjAzds^Epd1BX12g8s1cK6J)){9-P2%HJJ
zj-DQmQbGZ>CxEbe0Sm4;`5b27Vr7W_Bi1NXGr;=!uQf-cVlq7<SAxD=;a~QbZ1id?
zq&6YB!XWgJ2bW}ZK~X^vrmH!yDkDal-A4K|)ViZ2dbkBL18F-E37j}k=p)o#exyN`
zM}lt+EZw&Q-f8}}EB9eob<=$H!}1+lEe@`o?Jzpx#qXQ1#)z$hq~(M>T)m>8>Ow~<
zaCDToQd!31hNRRRG)bw7DWj(#*daQk>|5d+C45@cbtu~3o*BrG*8msz1mK8PYVRZ9
zOEPdF;x@+*R2@IMuJ72o<@%&o{4syX_`U?{HG1o?f(9P#a5on2U2#XF(OpRH2?nS3
z_+2k0rGxgNXnfrG0nW`|r>Y(VI|C(<T08FnLPaSa??z&CwP7c~CE7XV<(!F4RTXmY
zcdr+QU&!Ga$4}-VutchQ!<*V#eYW)IM^@s<0&}jSWxiT^)Y`5Lt&j;QA%gI(U29d@
z7`>rC_v-03Ro4txf`<YHF`Zw&{DGd%X=;hedy&2qI#$d~7^X(~$&!r>)Ma?LzZS!1
z7zO)-ULftKL1y&}_WHYe!^iI|{vvTQa!<AC9>SY!>!vk9A2hX$A#_Me;LRIwJ85I{
zFGZ3|d*BaGoH(H`ZPtgAGm~21xy<yTS)*|v1(l!t#m*j&dN(T(bNzVP^<+vE#R+3g
zPB6{I90n%SHsr|T@y54)Qq5}FoM;77(W|7<h9qqmAmA(v?%rNLP-R3dmZ$bc0~X|@
z9yQQFgY6$UVsKil=D2fu)}XQVvG~>5TlMjQH*%V6p%A8=K`D~3QZPGLR#CCStuK4l
z3&LNcI4F4P${ig@F@i<Ef0he`tBIbbL3hbHa++u!BsE}(UpLnqoyt3qb&iMqx{Yv7
zGyXCG59)2%`9$fsYyeUzQI;4T+RangC1I>xU<X{zq#5b=mbhdEVl~GYp)lXS1Kt5z
z2J8;(o0qAbf3i&CTR6ThSw@1xE&M#aL1Lzw@0o8HnZ1{N%`tk5z%h3~Przw@>HuFP
z0sO+{&n4EjT*~bvJYQHmC;XwQyQ5>yywPhwe!>rHn$YzJoXcTLvo4Te0pg8bmMbK_
z2^<w*Tiz+^1JDRYmtuU>Owb2BA*<Rp>}4=BuTsFYvw6`rqZBkzza@IU>E^yxGsF`>
zP6e$b*q{Tpd^+bdLKdB0x^URbe(4p?RO_w_27`%E8Z)qOksfbJ_L&5Y)#~H!HcdgC
zFw8aB?-lT5n?+LgU_X&aOdsN|`@+k|lDB4M4<Eb6s{BRZ=C3lC=+R{p(jznQ)YLCD
zKWW^;9PmH~9NkgUF^R?<Mn16yhhsxR8T=PSf4i4j6TYj$TP&|t*Is=>a#8vKG<t%z
zUts~=>__J#WO12AR1z^KXEX(PtP@ABO0!i1pJkg#Xm}3BW2xn(r7hUg01%le;QDQ|
zTJ}js!x*AI1pvV^EdI9f0-^e1$M5KAH{eSiUUc33M`QabL4x80BfdiSzT#tncZUx=
zyJ&!9bJu623pzfPI!CTlXMYCILLLAR_z6ri+ULAdUu6Mzd!v_BqHnx%dJ$R6`@;7<
zvr7Pfjcc^+j8alp2)zCv+xZ5KK3>Vn1p-qE(OI;mg?;s--|x3-L(MgZ=^!6uM(>D8
z)A*Kj9qqh@dvt<awR`vOM{V<OeBQ5+GP%}HpvA0O8$h=+$(pRgu|^}>6@Br+10c~N
znRRerCLV5XVb?4rthVac##KcjbP77Y5<s?g?(*FD^)g_Bv17{%F!=+|qE1J66ge4T
zmBuKKj$KD*?tH4Uv-_^QyVTLbcdr26TD<mbk3UAuthV52{QI&8_kU-{!Wz8{Qn$KS
z(+25h)8}R6e}A%zqsiXgYb*2iEfD+i&6Q7QZwSZyo;X~|YQ>(!mI5elcX;mH893nM
zLmn9{|F%0@W*S?1F*(TVsaX3~fkS}?9kbNlgH8Rxaf!nkUU21xNR8U=LSQ|ru|^lL
zdM<cflaA>Ah#RAwy}iNji2gas0q1Xk<ZN?TSXfou3Z}E|2b$jL$Lj9uOMSjj24z=t
z@Q0KX0SLG}#vhFV8YDa(Eg@;odxNMtpEWZ4vc>sPq>_lt?@tPprsk~wTYKLf)?~Kz
z%M^xDN0}%JN^=4zC{-P#V~IjQsnVO0L8OTgI#R5Jt}zHm00jafO+b2+5kXpzB1lJ2
zLJJW?59O{Gan3pSJLfCUz4y8IKF^nb@QKNr{qA?~z1I4z-}<fbx7^XmXW%F?xyHAr
z1uiG6bkn<>*6%{jfi0=!K#w9yD$Kvfr5$S59cb<9DiUisKoKy);;N`@Y@+!texT~X
z$9Mfk3T<2F)h!cKhYME4Td)9=CFnlA$zav-;6BPx^Y2WAeSZEN@rU}2B+gQn^KrWA
z4vlaYmu0igVB=@JR*qrs>}>pc{0f3AM086}iCnB4yc*UjQ^&{r1AQOx>bI*fEwwRS
zFA55r7W~$C6Fs~VjHI7rp04jdZU#i-2ldCFy{SIN76D0)B2q6m+hbFIv2ibK&coTa
zKhII`gQ;eUG)6o}s&}&|J+P)I@nu2??`p{#9wA&4$FATSddVcvoID(lDJ%!{SD&u-
z-&I~YUOMe7V!<7ZQIQ-c1nuF{A7qvrg_Vwcuc0hZ8*{$X6t<9L`k0h>ZHyHMXP<Fu
z@+^KF6s7DgeCD9eBrFUJ46NfVJTp>u*9rJeb&QUVZo6eBnPZTk`zuAD%N$pMOk|L&
zUj#d|P(Vg-ysre$>p|~vFBLw6zF*lU)cm}2t&>LxOVcPVIU#P#@mj66$6th{`EDX(
zkI>ZGl<|d$9wJG$w!GK0S7JP_v!{`1u#fS3F`h=P$evXBZJYwuCLbQT=JL`=q^in(
zY%gq=;nL~A5jk!RpF{7bI{Q5G_|8phl)}c6d~od8L>arz+g60wej!|DG>8ap6;hb>
zW$ky>7~DiNkt-O}xu>_VV1CkswZyP4bm611R~LFS90_NWjdqu<J61n%5*hE1p5RU<
zji5qbPw$whq8CF5R`TBOwRP_LmJsfrlJ`bzZM`z<ep03Y26`)5w&B`9){%DpbTCAE
z*!2VarIq+7A9J-`weCN#oydmifdd@-X-R7ZQHt5eCHPs%IUt?R>2T!=Pi6S3(}tT-
zrV=AC^h0%y!`B&#n$FmVONdUXt*w>Ow$*9&$5OH-uo}WU9YmC!!c0;w+TS++RQyx8
zOp%73@Xn>clyQZX$SsvAiU0@$>U&I2u|e1~;BnJJhVRIXSDE_s_akR-a=*O&P+fTX
z!sv&ejy|aW=m(Q4WpK^&74q1?zp{$|-SdK{v$J!x9iOYngZf0@1Y2-Kml6!)A`3hg
z>t7|p5v6QVz22Ucm30(i!mt!h?w&$$#jQqW7R}%ay>4SWw%57$kMP%(s4T7qXk@8c
zQurdlxGG7FClg6pX@X8pP8C-m6xP4nmFP7bd>+|9UGAzArPDX3=wULiu+$*97_UVa
zzrVDpFgT5G;ruv%ubi`=ERR1>R9IM9Z^{bapQ<h9@FUZU%6blFC;M@hv$2DAUT5}s
z;ho;jO!2wmpJWVX!BFk*Lfo|SKi&HUcPwy>Qsa98b%K&kh9;50vhjZ34q1Z-wJ}ft
zb8O$L0o*`i)Kcbg2;}9WG?Y`CUVP^XhfUg9`@@ME^U;U8jkd2G@Un`g@rJo@EodHc
zbz;9>=aT(%Ln77a$+@nfb@kXAGS$Au(fMm>CwYyqvG+fD_wZ;<TiR#!{~B4>A~pY8
z2iw^_wLq`X(QL~#o@#=@2MJg7xRC2Cx}t=Ew|Ka0;1*GGiy}`8QviSz8gY-QgW%o>
zk>Y;K1|{_myzH_8xiX^<I!(7bM~{G%%jzAMj-0J6OcgLp;#eaShk%Z^Z?8!Y+_>=_
zdr=%VHcd{Dq%NH8Zn5BnWgMyPJ23uyVQ*i9S4Be;5B|Yq>0q-d7w>@8Py)CH#x!3t
z%|dUqv&2iu|AEY?Q*}~H#D%Jkk!Sl#uClSsIq@apIQHUHU{Bo~k&7<Qj_X}uVXUkM
zuW%9$nSzsS4Vcpz;#%#(sv7h9T@@7&FiF+x&AAYe{->V-7~A9#ISB60ABiomb?<+H
z81YMhALwhu;qC*>^kWP2QPfdSN0A%}>;#=&(c<1esIE|<6F*~ltFxyrFMu&Q*?9g(
z)q4h7qr6w(Ii@!>jU2``{rSg{lAM>`4;ZYpg}FIpT~PtHAl#2qfm@srr>ljj1yP%D
zx>;LdR7vwwqc(}!7`$?eV!nBiZup-*37!#0;z#Ok$?YUoGUTX@$}KF9C);tk@5{nv
z=)^V6SlxO%8-eL-kp=Tk?P=PH8}bR1&KxvAph1v1GialUtQy7|FR1s3j^|f0svsEY
z)BUlM=51u`Ks(G7@V!M(xIYUi)4d>e#nY~zT(_T<{*E|%(1Cuwk|71)mJ+`=uW3C^
zxz4@1r3#QIj<cC2!!B7*CVyhn8PuQ)@&UJgk$-T(>cIFO8v!}l{o}So%MtH^0MgO-
zh}){vVs<ghLUQ~n1jdxi0$$;Y*Ua*&0T5^!aG6TdTcw1|8Hl7qShEG!^gT8MI0WXH
z*tICJg65qyXTABK5W5UtE?2Hk-#-+#u<w`3ZQBoeGYqu?05WqT&TA423S-9K^MzkG
zKmXzUcOSQo2sSluwf*BWCD{o4z+qR5g`L=Sr>s@uBe6}sp#qRdj-M<IxaXz;NgbIu
zaCt1~UOKA9g=`lzd4k`E8Vuzak6;j*@1K0w_200qS6)0lcnQgXvDsSz^3FlRewNjE
zlvr?GC)P|ZGo^&MFR#;yNZR@vB$y7|yaFLIE3-g~*iqrsE1T2v-9|p-kWko<3~L)=
zSrJFox<=dJxQ?q)!@2D)k5YJrC3G%FkUj2y1lsQvID6{gPBYzVUY*|n$9G|PI88#;
zm+z!>9X@(w;T(27?R32b`=^7#5~Nv8EKT&m&pUATkWrY3m$IuU@bLc}_%$1#;c@@S
z7W2d7v*?McV;O6`r^3Ev38X_5-9#^$H4*yM4ObKI{KXx)$W`VpddVTA?{cw!fbq$1
zd7O$Z0NGb(e(8V<?axrs88!Zf+Fu3KzT$XBc@kmSS@ih^nGpW#@t=Ff-;%bSCR)mz
z>W>yXYVTRRCVz?JThc?cY3wscmI<~<9r!LHL`!jqC$2W6<I_Sw9(eQ0zY>4KI=Fd#
z@0rrIu5O>Hk+S)Ipb(Eus)LvMon2*|nhHi?eG=VL0ZWaeYJz@lUy>tL6uM0tT{hD^
z2ku=VIGJI2g;%puxYb=nnzwY)?|sDf>>6G>cpH3Gs~tObBuHX8L6=n+yK1mwGevdc
z&h6Nt=0c-3{pxS?rSp?k!N%i9MnOu|6Y`1^X(n-P^B**SvbvcVI?=!NXtjfIg>IV$
z7SGJ&o%hGiJYI3Zp8l=&TiVH8DDy|>2PtJD@dlWFEOAXlbnt*3PW`WZnZ*`|9Rp_W
zq9tXPz?)fOpBRWUu;kKH+PrriPoPXtAG{D3T{mwlscvz*v8-(q#eL`34Kg2msDlEz
zNJi5ktMosGJcH=XhpIAHK5TyZgdZ?Id_4@8W-{|6bW~YS?DHG`G1GMLaujEnOK<Iw
z+Rf+A<}LwCmTB@y-LT~Ch3(r-K4pFSM^X((e#7f?CD*JCmu`Hy`z=8>cNo6)Ec1&z
zP+(Y@U$T(c^6P8D5H?Suh6ekm9LP_RnCDxcpJ<CZ=^dTodz$cVbxi2Nhy1|2G_;E-
zON_f>b2AknOO^I2?T=y%E<p5KZs?qG>%E?e)3NE?l@fNTDGXX>OT5FvR;*_b*SE(8
z^8v(Rn9LcXzx1hi1BhB@<9{9591U=FI)5Ix)GKUpV4NXbLNpaQBkUAC8<w(mZ{gdF
zx+C|316ILatkl+r+tqt^mT!KW%?9Ws3uhYsBG$3}wiR|svx&{F&Acmo5XrSFXDI#t
zkGa<V%rES}z4-PMI$rw6RxETXPqxkcHuY_h9VnL;$lDV=dlZ{UoRz_viTG)?X$y;p
zY!A=#b5&AgOiht-S*y=ewuuz*;AFvbi-#hbfjp3kY@Juui=(M4ShdAAl-Kds#%w2M
zgXGHG5AO(@)AF)rPwnZqKMB9C&GoC7*`N&Z;Hy&Jpu+d%*PE6-FDvt1H}07l8;cRM
zqlvyNg<6~Za-AgEu;ROj9th9`G~0l}n!>(q&unztW`Z-YXBT)32DzIoBC!XU)qln!
z$}Qv#Kbs{p3h?_82PZQn0q3#lX%FG^i1oA6(W@hg>91uGzj1_4@CbRe@X+Qtqw{pt
z)anA%OKVMi9iYO9(miRIX)GfVr9mfgL2=AM2$Vf0<#0^{j3r{H#upfJdyHbp@u^~-
zZ+rNh`K7QSKQD5Q9uL+CODvf0>5H<kuqYGp(<B09Eg~{4lmi}xwvJn>R=mQ<mEg_U
zI<cA&Dz@*Tl+^&*jwp-}<UN-rj2#$1`+GdZ4?<;RqsrZ&=47+xZ-ju3s+j{`3t~*2
zf*apYCY$8E1%zOOQsn%EBmPWstLjhgmL^m`NGY<PoY_#m#J{N7+#!$G!jI4F+h%!Z
zs>M{W_Iee<fI8A>df0*9erJLQclSZg`OrNQA$v56BVLE>KKuwc3Y;R~YEq>~d=@_h
zRXK&pA8&b9sAs*onRHmE_)W|Ju&RArT%{9#;n2IBq@HhFfZut-FArSB<MEib;+^F{
zq97nxMjCOLSlQX$&b%1oA|KIiouzE2B(u3JpGvvQaOCVbkCI8(VIN%JJdh8Rn)Z?I
z?uMwtaB=t00*_#s(L!(LsGsyyab|&?4Y4YyB231k$$4|voUnKriYl*UFs|{b5w}KM
z3&)`KZw|loO0_jEai!!#8Z1u40`gY<9UX4<aMji4Bac)A*1lYCL~0TUDea;D-?qra
zS&OhCCr8KGo5M3ODGhNifKv1d+o`6}uWjhM&`j){2FhGGJY#mXe#1kOn<LnzZz1eu
zlx6zvpqL02%I_6_2dbD;knmpIw5W}sd_8;z*1O}2`!A#ZwqXpow^(8g+l)j2qRYS=
zd3bne-Jcb}YNWNR?sFJ9>AJuHfea(OZiIeDi}|bl-*|F9*w6Nm5e_9@$2vPXJFnLu
zPff(GJI|)h#GwmofA57y^P}J6@I~~gsq*sVWZdKA<dN(?ca!>lO%W$T-3ZMpV0G@+
zUqfL#Yh7Ei8kl0NvlenBxXIDzauhSGh4xJEmYuV+)gNIdRu|`-A+Np14VWg)7Omag
zO8_x=iexdSr&SbvEnpXE3M=njnb15WM}u5m8X{n<O*))y{fFuwAij68?I!!SWlg)y
z^>cz=&l<|v>35Ml85BhxqK}VvzlCu7a@Z0<=5k4atfXCX3ZrIZhPUp>mqIo*p)j9$
zKUL>H%fJ7&gJS|4<Zl+4*YdXC{_|^n7!<D#?_>NtU}6*e&0t7h9y$McT2hm1^AQEg
z9F|K&%DIr&&=`b-U;p{vjOPF5cmL-I?5*}3I#=W4<MDu&KXiMNvIp060CFC{&>t{E
zWI!)i=U*Ogqu`TT$1;Qq%p9UM9~~N>hS&UkSx0B*s`os2rYK!%YI@rLY0dl1)pt(E
zJ!%LxaX>)<_;&X{*mU*)>SKF8USZCUpPzqQSy|aaWI=yKM8q~>cQXf0v#Viy=N9+k
z_^`S6!D2KU<2Dz3U@7*qq)FaKV_cIToExg`aM(Ek<+XB~Wbx1)NXmAYDr&bi69?ov
z3nAC##+TN(ksl8%n0Lc9J<EM@DQmS3RMDKn-3K?vrla03>;-zuD*MGQXH^uWrO_rn
zd~)c_>@K%{9dalAf_+-bC(UR{<x3E~o7J_=gV5hJ(XHz<1M*Lb-3zNSuK`1TFS~iQ
zK;4eU)16@{8N&6ZzE0B(+1c6s#7Yq?z`?|ZyBj4@@W~HPLw{uaQ_}6$zdrLM9gOD=
zKDi-<2%&!<)k0d~NNssu+YI|ae{1UuCsFIc3yZdvtb)PG@$nLEBEPr(=3wOX;J_BT
z|3=&)E3-hG`x!2*dly&S9tl$}3DVCyG(X=fx^z&z6o3K~INT5sAaWJBK^^M7&LGYt
z>)_;@V6fe0`{vnlq8SFAQ&1=)IxTepQnlO9CdU@_ZqQWk!}}gZGNW5@O+5f;(ZO)6
z@eX{xYgfrAGIMag_u>i=sUc)h(a@K{h9`G?$M!fAfSHApSWQwa!~f6R&K*3st8ze?
zG%`|rsFBzC2-w=@z$VP;$-NLNGa8sO$t`8sr%y+Mr@;Hbk~C^N@fol8WNRfu@Olm;
zp2pt_Z3UsA%w`J@7QIMSYMcX>6|ac0T~@_ssCbgILrlfC0$f(V*m!IoB&PN_NF+mD
zhm9vx43?X>ML*+L`+YA?csx^ZeX%ZI3REv{v`%z|f;>sZpFGSv_I0{|reb}XL6nBp
zl&Bi;yk`cP<U25yu7y?ph!OQR?41b=`o7pnm&a}GN4eVi`&Y|gK^G7o0?y4&+<L%^
z){?lM{g*jDy%ohKeM3HCSB}RoYQc5@J~($iQ5Ee{#{EK5!7YPy;69O!dXtyB8In88
z!^6V|hK3#s^IKi<hdu;8deS>b?rYQ?--&C=$tuwF8>EGDu?;<)>(JK*SK$1+9mz>R
z$y92NkxA~kbfF(a4SZLjA~oBCXZIZZwS?cx@!)}5IH5a%t}e5G_pA~|OG-(FZ^s`R
z5AIXw?AZaZf1mF7;#nZ}Og=8Icy<GL^EUx{qW6~-!Bt5N*1H`ao=T%y{1hy5yAl6B
z&;|dD|3s`Y)(W9b=TI#`BuV~Eo3F4~yw>{DyeGRos@b{8!s$YbbJh018QOR~p6_`5
zv*ku6+8S`W=P5;nMMbLpeSIq>uz!$DFF%d7b_lzj*g5KZ&jdK*pE`SZg%Qhash1st
z$hn&UU~3g7bKJxs5ze9kDJ<w+419!0l?iYDWDtP3WkIg=O>_9NM(6mJ0Ynek*4LLM
zUmG)58xsXv-3EFI%+sT)ob2sOi(ihSu(?C-x1nQ@vnvialKo-OZ!l7wSm1$$?NZH=
zZR{WgJ5FBil-!<B`Ocn<%mO2<o|;<F7cZSQ-q3)wuQ$hbPghF+7^r3aOI~ZZ?e`_d
zosIQHeLVR0Gv*L+%maHrB!!a~Hyu42y+2h|eK|sAw4lGg-*!jj)TYB5nENyiRsISt
zyOTH|SRRrG1z!y8vgMOG&q2Qh6>KWW>uS~NJP?&mL2cK@D=;rT1NAKmlfHj%=-Z|C
z8F=a++kYKIFXDUp`pRD}tP)eRp1Mbj{Gze^8VK?F@4f}1yLh))hShIU<5ycv729LB
z#12Lk7!n2YI}@&k<-Uv9bbxk&;}?bnyq1y@`z~Tc^z#=MqpIqRUS1EZBIHHt1^Mu!
zLOX+Nkj<CUZs!a7(o;{PZukj%>E=OH0!*(9ieVu>AGTRNt$%P(lL&l(8oob&+4%NE
z4~X$!jdF9j#?8DvA{Wt_W3~Pk(VaFe_nOfu7}>jhizUesgoAxRqCbQG>>1bzBNsRQ
zp~nLtjqRUWFh6Wx=fbsUhGmK<U!b2jd-|Kd6etWiw3@qjkK8hZr28E7&M}lvYIr2;
z^ypDsoCp>Q$_}JH-Ty5>8u}S>z8zo~p26?LyLX!S)N5Mm(WPr||0snFmaE>>p|rG=
z45x?$B*AW5fWvM@mxC(X-@eAckAFs_mhF%}MauZmCjDppng!zK#h*)%^9PQ_mNjev
zQVp|r*Su5<az*Jqka5i-9&mI$el0U)!9WqTg0B7iRnO_M-1%2`AddtL!N-qsFOWv$
zkZ}vw^R%0Px(54w&(}a)`fSzWYpGc%&_gB}*e`EEO4X7s{p>QZsB=NGrwX_Q`WY~i
z#mcHz3#(?93#DE3Y!7j^z*n1fa2J1siqsNVwYLfid!9f-<}8>wH*A0va!?21c6_a&
zJ9{x}HN(E{+A-<cYkX`&qM&Gm+yYz3>{xQ-1T?|Of-`AX=xaAnyEjn$h3)0f>R%VE
zPRbug(xD*I02!a7qcBXjnmh<4eSQ6(tk>I}<pC6sZT>4Gg2$W4^7;LyifM=`0&-3<
zqmtE`WnR*j12ciV_TPes%yp?YY;PjbElMU8nTP_z96!3lpyYV1ecji+cO(YNA47)Z
zs2R`rU$wpjq;2ENL!yXp&yx(*u!!gtoT*KaQ%r`E@T{Io8=nYNS_|-95ENQ48QikM
zY--B(q<fwB|NLqR+Z4Ie$7sJyD{9}hYrKi{cz2xoV`aINi4$2m28S^6ZfUX;Tlj~w
zOK-Y)zPYAue&xp^<%8WO2Sr3?QYjyzA6y9PxP9-(eZ!xHbRNBr=$fBiC{b0C<@M~c
zWJE4adydW#&$8lU8A?onfJE>48QV%$*!NN|O^{$2=jxg_#lw#Wjfsu&{Fhs8f;U{7
zz>rN1fv2LH8%*_=zsS>Gfa{o{uaP6JsF?<;4OLI4ej)QCdP^4vT&d%rwq8=6ljCgA
zZK9AR8p3p?+SIW<{s9Cx{12lmFZQKvMTDgSy~{wzdtE!r(6?@O)~#EY?PB*wa*l-J
z4^7dCW`?vBQuRFq-4PTUsNb>0BTK9H#t$3~5v?-NAtM56u9K#C$m|<GYYASJ4p7rV
z34~;|$l7*oRSVZ7YAFcqCRbHeP4Td~y$3jSpX$m7&PMn?Eis_-d*|1PY6F`s9jbUj
ztClmW4k?~h>%aDDy9?Me5=e0kvD@2xJ^3zzBp2*eHnv0G^g5INcfRpV7uKe6edU*^
z9I{o$yMZ{oKepRM<C=H(cMT*xg%XR5cULQcWD`N+NlSL6=EN*5!~8{MOl~a)jX9}i
z`2aceRRSTJLJ|-S8P4b66exEG4Mh7J&8e{0Dqgf+G3Q6UlBdO^WVleCMW$IFposf<
zlYK$YLG*CZ?lPYidFZI*sz{lw-kA({*_58~@CuKBfWJP~I{<TC6fKLXnIJO@#3(JC
z+qe5wbUS8Ss)Nig-nYVX{StL4g3M&0hgS$=nqtivCo0WL*o;Bn@Nfh5g#5nnY!)9M
z2;H2rV5&w#?t>hyS1N^`K+Vj6CSYWEM1(*O@NzyCn9>W$OaqjQCZ>GVu~#x=*f`6O
z_q<s6C-S;1ZDpC%Hl2t6?yU)kueeOT&&|WRHNn{?kOP8BjS;zMs0Tcrn(9O2ylOcU
zVpId=Y7Iy^bJOWAE-yf3RC!Q-@-!*XEDD&dp(8c-!?nE5*BfDR9Yy8h0k?|ohjtR{
z*JPa-Af?=hn)$S=%X8}eOh}TbMhp81TjiyX4_rQ~w}`*mC*{)`+7R*FS_ye;cB~Z{
z3e-?rc>u0`F`?8Z_%2snv?_tjtkbH%L#L2#!lD}nQ(+kpy#a)rI6fBgJp&Z_t@fZY
zgwQ+q6=vT!vOba-SJwH)=e~Z#b1Xz;Uj(1L>!mEiP*R&d+;qO=y)Nhmm6(taL28RK
zXqrRBshyw~Vce(eW#a7(tA8l%Fm))aYf>tL@J3!$wo65f@&mp31}N;fr#!D9l>phS
z>h0j*dcM9CYDOwr_IG4O>+H7B9cq@(v3FKc?NWgr9S5uCufw?|WLuHl@pr>P57%^=
z<|H@-C5dWPcD@SY)Ryv18hI#gq{}sk74!3ZShi+^$uzlk3c2XxywaqXW?Zkep<Mu`
z=5EP+de@{wZ=4#<l+#m$<}<=OxEPe*k93U+LPJ$?J>(j$?Xh$Nk(Bw4Xbg>fJcZN{
zaT1H@-yH@SGAjd*h@Gp8Ap6X^3|$5Je~hs(;zD4nxmoCZgGNjCzp&HC6`rz7Zhe0b
zA3^YX+J%S|ki%nNH<Ote=;4SjZEgv}R~bDZ<lqdVOoN6V5qaS8E<_3qYs2Z%lzq5!
zs2@P%NVN44G}72CC&4L^j6aG-Q#x{k-)JCu(AQ*PG0PqQnQBU1>S~}q-<d*MQGB_z
zQ=*E3$6P;xR?*{nttK?Ae3Zercdtnte(eON1_GGqk8sEs$H0u$c~-IdVYzf<Sk67;
z;o}om{7Hm0uu#bj1&E@GXX`*-&5Q0}W#;kAMS)VvOI}b!3+TCWOob1{0BKZzM^J&=
z1${4qU}3sob-Bufl2p;tAXC$_D^0}%nqpMQ(^@b!1BMv|Hk6GYwc8<}#51^9dmPS8
zUC@ua|Gm`A=U1F{YpX2e6a?9Zt>ro>3;m&r$?UQOQ0f~X>-EIxm3q6nei^--zxRwT
z`TeI}_!xsc@?nlJ=3uoJJDF4E-wbj({{F*0x?FPvSm+Ny-U7CjJZ&K?GO}vuT;G||
z4GS5g54%qYU0YXo5HyISw8T~~Y1r9Pz5Os#p@bIbNVUcUWi^FD!l<pp8ns-Ra%}QZ
zZ6i69m_k~62aY7OJrD0@n%ufLfaop=lHvjE45Yr12gBB{sN)f#btoihdfT4#knBUB
z&tl-W8IgoRXRA>gP&C7%W=cw?tDyZH=b4(Mfp^>LUpGLDwnq<o=X_SzAM#L+fo{ZW
z(5v)BB(FwnY+=1T78)SqRm?pIT6%pHjmyd-TrvGGPiXsHl#e#WI7%rZiL1sm#Gd^8
zQpvr4Z~0rXNacCsbLY=T@h9U9b1eBmOw<m8;X30oSFtd_Iw+zx@s!N8v9X0Oj*(31
z+_WE|A<=7mi?WgRBdS@^Rf~?vxjfv<a!~yH-Vb&K0WnNqhVvn2f51KfJ00a>Qxc4P
z{dVHoY+9>*0rDsrUUiC4CE6mP_)}A;$i6`D*pfAA_z>ISTSvWa<QE34uPstNEA>6X
zB77#z1}9oTx{xh!x7rPEIR^-i&6U;`M{MfGo;u`I7;Y#w8oP*p)gjW+-PHKmz1e%=
z|Hn7pi7FzK@%+mtvXvj4DgNnGeEc3Z-QTfMF&noVV~{m}Q*}jI6@B%P*j6_Da|{Bb
zOJ|&Y<$a0maQuBlhe$3tV5Ke~4D|Go9Sh>=I5-u+kCBr=91gajFzB>7Y^%yL>6THj
z2PKNBpV@5x_%|-o#%J07&WHNH^TyM&-QPXzfffjwc&cCoffqe&BSzr}Nm`t$zQy4j
z58BTYZoQ!D$2cd}?|L_Nu+6@pvNFaI?vw2VWBl1sFVJ5cB;f}HQGy|a?WkM@+I1(5
zaVAt}TRF?dWhnN0Xr%8!`K6V8B)6aXst;3<;Xobght>|>ViXoSNdP4%2pW6up>5j}
zNE=x}XP~Q%@>aW1MVOl2#w&`nl`;@cOtgHTv60bOq?L~TGO)I`##uX)f2ry{bgY81
zRUc{e7XVQbc%Y<76T=ruhzHAEmKj#)P+*$-+-SpJl`gv~_BmCq((TkovZ}{KJRF!U
zXKT`g&=YgZWrjMv$UU)b>)g!_s<WPz;%GxP44;DA*!}MLdilZDHTgWfuPR5zVcEvF
z7T(+!;_kFBP#07FT}DRc_g)pyYo39Y3F=}NCVIULxcRDGC?iY_*pVDOH4*fQtEA9h
zVqKM>xoI?b#yRnyvTrPvZ=?kuv)o?qEr;XhH3}t9s~hb(NO5~yA^#lE8E@{w=x6kK
z?%aysBDnvM3MDM0kmf-k$rq7@tVKIfm-NLb&WS$3gs!HAf{k0~rySl|E>AO{H`8c6
zws2s-52gb($tSIcDzD0mVrYt49FvxQ3*D~@d@B5}YK$wRWcH%s06w`I?<{~^B5+r!
zb?{w+T+dpGPd0x@lH8Ihr=W^be|jOPTrrf~*L$6-Ruj>~+rZJY!YGmPm6w2k&N~lc
zgEi(Y_wj3{RPRC&v}_uwGk*Nc*FmMhpw8&7q!?zL!8pb^B*ITyul6M4vnajeh0sG!
z2;>T>g!hIyT{*!7`EBK(`@XX1Sj2!t;uC_`>h;8py38>Fi(JerFfs{Ooh=o*_7%gd
zV{XhpiE8t}U^K?734|ok&P4i#m_*<J@+={?T^EhWvE>E+knbGgPsO27ViZL~jb3_P
zWp>5VNWR}jZ1=VoJQH4&JX&QhN*C<4b_E`qBRFBM??+oh2L{c<6gn(VX9D3eYod#K
zWCL7!J5p-BeSkoK-g>G|$M(Sn^5g^+V_D!z1w(_ZSHKb~P7w|1N3{x~JgM_%E`w9f
z0H^!_dcM#!OG5MAJRqZ1&}2}BsJA=_j~-f3F;OUxXI*^+OMFsaBhi_$91ZJ#(4r?`
zqsa7eulxBM3!uQ`11h0jcnv#ejqy<sv4#Gwm%+SPM>|6;K{-Ams7oj*$Im3MkhQ#Q
zFHhtAL{{ZOwK~db_lpcK#+ErJt{-L>)z%o_4x*B)>fQ1RVzleJ-H066MM<#x;RFy|
zPXtfDQjRhkBu0ssr_G^qrO+y6&;_@J0sT0P5B3W1^0H=}S8AQhU15gqlG@aLEBb#&
z&0sKn;L9V%)_HjG$ggaNkvw=koy?3w51;=;LE@#$6VJYP#?(OPmNmlS>b#95K*=iH
zs6?}fLRee#bwa~cg;H!$ctD9r{Kyw#PIvA0`p#Gd3Dibf)dVFVBwK|RF!mdU5<c)U
z69TW8qQvzTnzGT5A2V*y9C5taV)V17;19gSA*MzCo;dSWXO8W{x7I`d{l<n{_W#U$
zTu7c`({w_*xxva+tJ#j4$s@J>a7Op~{806+A}tVKk9s%n`qh{8(Km{(R#Z}2NhLF3
zE$Lt!350j@GzZu_f>7<`lizJ_akjTHc7vxdW&vyaf3`iw(Du;&jBYs^y=FySdPXRw
zv^z=JZY07V7rCDgexS?=u-^Ar%*hKkimIyCZ=qVF@oU!5!47QJST*4d5;h`qNePsT
zINIYXsCwmT6;_Z-ki`b4kOcd{zX<${IC?FqUqh+L$$uUEUHCTq$GRSYlxrW~+}{Rv
zErpcu2<-DCr$p#-w+KK<3MqT)La$~lICjgOym}f^37@hHnEo=Fj*t*>0afJaxmn87
zcm6Z6ZvZ<T0ShLQI-UurV|<?qMEO3`@6$+HSFOB0+(Dw(o$YQ9aqFSD004gGrgE*y
zJe;%zSs>XWotk=i)nFhBBrI7s>6VJ~kdm=cdwLrVtb}P^@KgP{&gg;{3+2UfCH<3i
zYmk#6knzEG3J!A<R@V=DSy@@N@Ux&g<UudD`L*h8qa6KLVT1>Zd}sVrneVGUibw_f
zR9GUH*tbuiL>Y_~zRv&hU`|N3=;zMKIc^$Rh_>?HoRvb_@!~j#tW!@%OvX?t$w|s4
z+ce9AsiD}oFdQm1U2=TsoU?ClZ$pI6BTdN?VM&nO&cl%|gPx*|nrJg6+HxP_mUqi&
zv^UnRK|ia|M#;(kPs0JLT>%wSKU2Jy8m$Bn)?8ym^;Rf1<O07OjvYJDr!n@p-Ii%q
zM9pRB0{U_lCMF}N5<ESMJfob+#}arqSyy=oF`{JVv-wdbo_<tb-g~NM9wfxeRXt43
z!2`_@7o!|RTbEApv9a-g?l{__gfE`@^>95%tSIh9X-7d!^Z19<Uias_b0ByRC+rH4
zD*HQYd|j5k;q?oM-D`!01+^+<D6`pQrlaO@@lZtbu?j|2NFf#0%}Cv!6ceLVlbO3w
zxvFS?bB~0b=6g_+u2jx6QbfE9Lz1YsFO1NIsaYd4A>h@&*@(8bz^wRTW~qbef<+3n
zl}CUxpjz>DDwNqTWTwvSSZ*YtD~05L5AUBkB_00|Us-OL5&uAMnUy5!GySTc)OMe+
z$b{|y=`>5$c%NJQ`F5}|PG@VXVqK1$Zh897evZs^2u+fU(Ob6c_EwOf%np;Gj)kJ;
zO2sGCv{Ci}LWCYZL0w|m1c_B^x>C8*NCLs_<d+?keW;lpQX9U)Yf4{3s$Slz!+&t0
zCnzDD;N^r-;?3{Cm0D1jC}Nb*p7nJ`0ZtHgA5Zm#P7CE#Q~Q~%;0`2-`nOr~y1z1K
zf!){@TF)QP`0`7rldl*antd50#?t2jOt;`g@gIi35d?pirU}Jm4_dZBOR#Gonz#y6
zW6=$D_Hx;WA6_wrcB<|4W=-(LjybfJwq7BN=Bad~7e*i5@D5G@6ajGPYBC`vj9^@J
zwLinQ_aLn=jG!*o-}(=q5<;uh927!>+dY=<2`%9>!wH-i48Rcq>9_tmit5}oZvd2)
zx^@g5LhxDuF+;&YuaWRuC=}vm2-B`RV`kNA;^8n_vK{n44$UHw24gR}9sRx2do7#3
z3T$bmIczP5xVG!4JgpgVgyNtDXd-y4C88k*g{*Oy>%MT1#p9LA1yFwUMq**bn~n2{
z4L|W{(_mw$OT&c43mW6;gqR2d7;zS)V*=hGN8yp4Z^s!!!-nIc3)1032UD(s9v*8=
z&d33U{Z}Gt8>U7fC4>4kBQ+-SZvxBkydR#sEwflTi=)S@yi<@JLaqQG>)@U*ep?aj
z_aeYK&-r2@_zeTBFfHJdk^d*!Ht)>uojtuD8{+k3*-q@9TUdY~f-`|Wvb?-(`9M!$
zvhF*!=dOUAeH=Vfvu^qZXYj+}Y|)SfwVD74#SJv~tv8@##jul?%@DHd+u7L{s*RK1
zf%ijC<OoeFE}K%C4U2}Q|2rB@Tfx9msP`V@Uq@4BC+6q7@Ko*SM|TtM>-m^r%AZy!
z8*;tA5R|Z-xM#807EdjUKDOns9zfb=)TJ|so1aS<yD*p%^oh)JDCoK7%Mui_PU^$V
zhLGA`qg>os6GHN|DygHK6+#p@8%((p@{3)lnRYEDXMk4MP6@_6l;(Ke1T!4n8yAU4
zVMl9hKi#s}SyVaaAV#UOw^|iMeCQMDH^%lNQt@7x>nVP7cM&O0%gX{jb%3o35D15b
z&;b5*t$mSN|58m9I*Gv`x}DVouhST$7(hP9w|{a+6ko8wwe{77#4ZFQM1Y;H0l<zd
z^*s*wyh1t+QOQj1UZeGPNSLob?Km+ThEz5kK(ARKPPqiqM{khCHy-vMLkUQLE*&6G
zQw=4+InhFjzcT>7_yfRzTY8Mv3p=D|Nu)NT&?Mt`1ODakG)QK>gE4$yfa)4{{BX4I
z5x!CiZ4F}5`ZhsNj)S<-=$qDLwRLd->t<JpBX3WKBzYY}Yptk$bg;Snq$q_{3CLWq
zRJUJg#|BD*80>%YG9WeVVC%R4cl;6YF|){Cr>hZ!EY!>>!B`NbiWNJLsmTT**sA!+
z+`j_oG%rnX8<tPy<+_Ty51NHk=JPQ)3==3y8<@|#E5~*;;TP5+$X<KZ#2e<VEQCme
zQXt5gAn;J-8DWGca2&m}-!4@+^Lo~9L;$5Z$l1gm8~KSRpot@v6Ms_UqU3u+vx%Q>
z5J1)T_$M-~?jW@FX-p<n-YQOSc?F8sx}ba0M7~pgx1;4f{F*Li#T!$uh?a~b+#nPm
zqpdjOsa8(1aX8R#w{MzXB<V?@m$f|W<ymjjh94#~BnlbgY6lDxY!s+zbkC!bgGzHZ
zYRd=SI?5JxP0k69k{NI0A=0Hr5-Le;nN{0bYOR-FxI_{t435#8O?9r}s-w5a!6o}`
zq`6n_oT<7vbf*y1i7hq8D+y24YU-e6q>3!W^H*TWMt9j4^tw`w&eTXi{bs&hQR|hD
z5Mfsl2xjmchoCjeN<9!-Qy(>tUethu@=}XXQVCS7sfG|#A7#XonG6Kh_d!YB-)&#A
zJg{ZkxME{`aMUIF2qxsT-G#N7(PuYc0?V8LRdfg_NnH2NUJdi^htw3Vz|`p&<=8{~
zS}&Oiaab95@ATIj`7{nULdG7f6=a^gjFVJ1X<`qk-dp~>1`aG$o;ua-EC0S{m~RJN
zW3RHhLD@bWBxi9D+_6AZ+Fcy&3Vo6Np()cbw8sn2RZe_-2@0i~@h*hD@|a=qrwQMK
z;wcPq&jW^ck3`@hYv2i+%dV@YrsktLYcI337x0hDKKp{Jphm|!1*^=m!ZMMO=$g>^
zug49YvmY9UjHC^zQI~LO1M<)r*%5PH0i9%-Q{^C*=(V~$XJzNy3D5x5s-ZCbcH6S$
z>m#|d3jSngknT3ctXyJEtec0(Ub_2rHHysrfUiy{(yL31dK5~5$tg<zXs!c4^7fyn
zHB1_4Yt@|o(#+MB6}`gd2BXV^8F~jETJEh|dJ-hCY~-@ixN9Ezdiwc__1xaIr#&s)
z7enfT74O@t!5?5~FZnQ{WPg9^GyF-1&wuVzaQ*n^PJyX}o`!bwTo&PPfJ_OgrN67o
z^E_1^J$xUwWc#YgpDfq~3{1^#)XWHBagxlW0T|5)C6tq5$H+V8D)J{3Z7K!F$S~E^
zCC(nzbv)U-@5rc&;olF!jDr=wv|s8Xa-S3CZ($MK1}rI52<=LZY%YM{y5_d;klSQ(
zJT$i|cX_`0dLLjT^mcl$<y+Qvmld<<f5dZ+J?HWsE7zK>$fqmXR0xh`Y6;Sy@x_i$
z<Ute)N?sRUAEi4tjW|lAYV$4-MAHOgwzZ}E+p4@tZ(<*pHHhDK2qvKcN!U=AY+WE>
zQ>Wz!BFCNFEb{RIFWf3P{6%MKmI;d;5O#NszL|H#luJ?ET3^0Ak^Ij79SprhjIw~K
z`2*t!j(4ehuD58N>wLwX`S>FPoNktBk$A8O+v&|^&Smt%#P|czGLh^i=MrV7QLRh<
zFbOqZ78IE$N3A!fBMtZbMMPmh_GV^?u>&BaikO3COQUfPU^8B3p>jniv)s_?>yscf
z8aG+s2aJr_(TfmS0zwE<(Cy6+HFJ%+bQi&0-XqzJ(E2lC53?8&Nes8y)f09KK4d&~
zJ$IG4E3LOBrVcH>F)?EoRi(NAxNAfAgp=kjvOb9TleRE5heC!A(wn#A>H0ZcU4_RZ
zsRaibufI4r3qa#}u*V>(T`>ub5+wJu8NyB!Mo&A^$}T^N71ch-8aS6Wsc2)9l&|1T
zhQi$W34)gm=A3|%*XPPRg}9~>x39~>EgPQ65&hVa$fb+UN4ZmaFStuSeYs8Z3h4Z4
z7af<1caVrWo9r?izV`N1qHXQ%wIcK|^%e(brJtLdn{MJt3P~W0AR1!5YTfgMFMh%|
zKQ@CwiVdcYe{{+2nq)XT`@7~o_zfq3-<<YXnD#g64FYiZ`$li>1a?c~Q;^JB)8(^j
zIfxMNCJ+GgxDZ-T!&CgDyjJYnLBFAwH~2MY(zU5ez<w5#DwI&j+2!XfiJyx<a<o-G
zUV)0V#eu}_LJ$4A#}j(fn?IUxhN0u_waVy+Rmd&I<<$u?s|RepK5Ar#8%7Yc&q9J@
zrd59<%f*WlBoJZ}64u(fWd{=82K*MGJ;pGp@?ZU|cWw+Rlm89<h@O$}FT-&dI#pMc
zke@I}#Ndd@H!cHJ0qebJ$Y50-m6esH6}3@=WFn6mZ!EOju+^95_u56BR%8UHyhm@k
z#c5F4{p$CL=l(|!oXbjdTYSF6M-{{`CNpi(*3KF4dX6wF_wHIA3?l@CsHJm1bcxE<
z?~Q5x&!ciRjcJ(1)`dO2o_ihyPZ(#rIH#Is%`<xA1QqoKa00fjss2SVDo%Yu)*q0O
zzc$Vutov#J4ctmAjb4{rG;OF29Dyv@aSCak>atJ-7BG7+Kfkiq1V5mKRq^;WE0^nm
z-n~sMuyYRR;Sclg(CATf+bqM3inR#E=|$FFdL`KH<fAofa3c2vTlrsT*ITHdj45tw
z#uSLx6?>lYC9XXn5J+ur!RXv*w~-m0i})*Br!tc2vob8zp|z*dmCD#FD5&cFu;%)5
zzjdu7_!rwI!P<#;x~{0m3jU$e#4h&g6zwL;cG}k<ZkuV+cMq%KDEV7JAgA)(`dLi5
zGuRCyiE7>wK>qO-V5_Txj)SFBg*BWLMu5O`?EpH#!FKs7=DI9pw$m_U?s8S%Gv{>H
zfZlBUj&ecc?;bMcp!{>y(3iPRwmT(wscG`$(EyIl%Vo+Gw@&=PS8LNafcQS|TxXF}
z=au=row3_CVvB#`!Tnd>vT$(1@Wqd~3ZoMUgjT|0Gl7R}tGMn|OP_0h5g;4gAL{pH
z<9hv<oiHE0Eq9!$sY*~&Onl_xyDHwJ+Hd2y#quy4o5-^$q|t>>-vpufFk1E@|3-CN
zAa{Z{Yf!sH>h(<0bT4|Cb#&f$<rn&O`-d#r7vG}-3!1hwz!f`GySE(jROPAKB?&^q
zON8R%Xn&Cisn%anxxb+$O_FHhqwjxWd)$cdR8Dv&gc7u=#cR*<ZbH`nJQ7anWCz(g
zCYqUUg-c<P%vQtP#NOWZ3xD@U{sFXqK<!*z!&bP&?+7A#f|eCRfE9Lwz+*o+&xxA(
z!wwRRPg5X81S!zQRZGkYHEeP&W)%}U7((c~k6~kT+m58IjQ#!6N*Km`Qqu#2PXJ7J
zLm}^ul_#F9sU+|?S?wV4a<BzHL+C<*pXOB2!#~&r!4(ykh(I(jwroeZG>?7%%b@Ik
h<)QvBe|DCz&e?r3el4xFwhOr!_KX%L`_#po{{tMU<f{Mx

diff --git a/reference/plot_pit.html b/reference/plot_pit.html
index 61ab5ba93..2cbe837ed 100644
--- a/reference/plot_pit.html
+++ b/reference/plot_pit.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/plot_predictions.html b/reference/plot_predictions.html
index 83caaf359..f1b9f9daa 100644
--- a/reference/plot_predictions.html
+++ b/reference/plot_predictions.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/plot_quantile_coverage.html b/reference/plot_quantile_coverage.html
index 98c22b9a1..783de4e0b 100644
--- a/reference/plot_quantile_coverage.html
+++ b/reference/plot_quantile_coverage.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/plot_ranges.html b/reference/plot_ranges.html
index 455605d0d..4a264e243 100644
--- a/reference/plot_ranges.html
+++ b/reference/plot_ranges.html
@@ -14,7 +14,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/plot_score_table.html b/reference/plot_score_table.html
index 69fba4a47..a82cbbe77 100644
--- a/reference/plot_score_table.html
+++ b/reference/plot_score_table.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/plot_wis.html b/reference/plot_wis.html
index d705a0af1..66bf47981 100644
--- a/reference/plot_wis.html
+++ b/reference/plot_wis.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/prediction_is_quantile.html b/reference/prediction_is_quantile.html
index 4ea9d7c88..04d0e41b5 100644
--- a/reference/prediction_is_quantile.html
+++ b/reference/prediction_is_quantile.html
@@ -14,7 +14,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/print.scoringutils_check.html b/reference/print.scoringutils_check.html
index 73c4a2e4e..2e666f8c2 100644
--- a/reference/print.scoringutils_check.html
+++ b/reference/print.scoringutils_check.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/quantile_score.html b/reference/quantile_score.html
index 8f3262363..4618cbe62 100644
--- a/reference/quantile_score.html
+++ b/reference/quantile_score.html
@@ -18,7 +18,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/quantile_to_range_long.html b/reference/quantile_to_range_long.html
index fe75850a9..9c5069266 100644
--- a/reference/quantile_to_range_long.html
+++ b/reference/quantile_to_range_long.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/range_long_to_quantile.html b/reference/range_long_to_quantile.html
index 6472ded33..f00ce6721 100644
--- a/reference/range_long_to_quantile.html
+++ b/reference/range_long_to_quantile.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/sample_to_quantile.html b/reference/sample_to_quantile.html
index f9228ea7d..0155c771e 100644
--- a/reference/sample_to_quantile.html
+++ b/reference/sample_to_quantile.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/sample_to_range_long.html b/reference/sample_to_range_long.html
index 289607aec..f31caf100 100644
--- a/reference/sample_to_range_long.html
+++ b/reference/sample_to_range_long.html
@@ -12,7 +12,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/score.html b/reference/score.html
index 05a3c5655..bd4675483 100644
--- a/reference/score.html
+++ b/reference/score.html
@@ -62,7 +62,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/score_binary.html b/reference/score_binary.html
index 0c3069292..3aaf2af1d 100644
--- a/reference/score_binary.html
+++ b/reference/score_binary.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/score_quantile.html b/reference/score_quantile.html
index cb9f1fdcf..46bc440d4 100644
--- a/reference/score_quantile.html
+++ b/reference/score_quantile.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/score_sample.html b/reference/score_sample.html
index e9115733a..00b44b69c 100644
--- a/reference/score_sample.html
+++ b/reference/score_sample.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/scoringutils-package.html b/reference/scoringutils-package.html
index e628c7966..146abe46c 100644
--- a/reference/scoringutils-package.html
+++ b/reference/scoringutils-package.html
@@ -24,7 +24,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/se_mean_sample.html b/reference/se_mean_sample.html
index 463bc6efd..f34cdd642 100644
--- a/reference/se_mean_sample.html
+++ b/reference/se_mean_sample.html
@@ -16,7 +16,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
@@ -101,12 +101,12 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
     <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="va">true_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">30</span>, mean <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">predicted_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">30</span>, mean <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="fu">se_mean_sample</span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predicted_values</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 4.860613e-01 1.744323e+00 1.360432e-01 3.302945e-01 3.751945e-01</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [6] 3.798420e-01 3.073065e+00 2.825546e+00 1.037927e+00 1.759771e+00</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [11] 9.024846e-01 2.191300e+00 1.669601e+01 2.655281e-02 2.879499e-02</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [16] 1.966896e+00 8.706269e-04 6.036063e-01 1.630220e+00 2.489198e-01</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [21] 6.122734e-01 1.001614e+01 8.163432e-01 2.706157e-01 2.772148e+00</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [26] 3.985060e-01 2.621120e+00 3.539384e+00 2.818409e-01 1.450891e+00</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 1.5369388190 0.7926418467 0.0064358942 8.0875579457 0.6223434196</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [6] 5.1973894672 0.3780707728 1.8730178893 4.6789601717 0.1668502145</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [11] 0.0270798542 0.5391113037 0.8762152838 0.3347385895 5.3358123896</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [16] 2.6739807804 2.2938975472 2.6748938964 7.3483374250 0.2775697450</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [21] 3.3677055015 2.1256807972 0.5024278826 0.0082441616 0.9248873844</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [26] 0.5373266162 0.0382242675 0.0239365969 0.0003035308 0.0570072019</span>
 </code></pre></div>
     </div>
   </main><aside class="col-md-3"><nav id="toc"><h2>On this page</h2>
diff --git a/reference/set_forecast_unit.html b/reference/set_forecast_unit.html
index 0a8bd923a..fd91124e0 100644
--- a/reference/set_forecast_unit.html
+++ b/reference/set_forecast_unit.html
@@ -32,7 +32,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/squared_error.html b/reference/squared_error.html
index 446532526..072b5e225 100644
--- a/reference/squared_error.html
+++ b/reference/squared_error.html
@@ -16,7 +16,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
@@ -95,11 +95,12 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
     <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="va">true_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">30</span>, mean <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">predicted_values</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">30</span>, mean <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">30</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="fu">squared_error</span><span class="op">(</span><span class="va">true_values</span>, <span class="va">predicted_values</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [1]  0.51162387  1.78113912  0.79705667  1.83329995  0.30668430  0.36035257</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span>  [7] 10.35323665  0.78037062  1.98496901  0.03794845  0.13034237  3.26910428</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [13]  2.84648268  1.00105946  0.75844437  0.46828924  7.78161096  0.63237370</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [19]  0.06595388  1.17345957  1.54562777  4.11031694  0.06182022  1.01685836</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> [25]  0.61568034  7.35156133  6.54269939  0.12839408  0.25734304  2.38597297</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [1] 1.815208e-02 8.494946e-01 8.870800e-02 2.570784e+00 1.112817e+01</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span>  [6] 2.011951e+00 5.608241e-01 4.543362e+00 2.312538e+00 4.374467e-02</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [11] 8.841450e-02 1.002675e-02 6.565734e+00 3.997209e+00 1.425201e+00</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [16] 2.282389e+00 1.362692e+01 5.913843e-04 2.354307e-01 1.745345e+00</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [21] 2.374719e+00 3.503143e+00 1.722154e+00 6.022786e+00 5.777315e-01</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> [26] 3.155219e+00 4.461945e+00 1.634580e+00 1.122590e+01 8.894234e-02</span>
 </code></pre></div>
     </div>
   </main><aside class="col-md-3"><nav id="toc"><h2>On this page</h2>
diff --git a/reference/summarise_scores.html b/reference/summarise_scores.html
index 2d839eda8..4e636226d 100644
--- a/reference/summarise_scores.html
+++ b/reference/summarise_scores.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/theme_scoringutils.html b/reference/theme_scoringutils.html
index 215839ef6..4a75c613d 100644
--- a/reference/theme_scoringutils.html
+++ b/reference/theme_scoringutils.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/reference/transform_forecasts.html b/reference/transform_forecasts.html
index 3006fbe8e..edda314df 100644
--- a/reference/transform_forecasts.html
+++ b/reference/transform_forecasts.html
@@ -10,7 +10,7 @@
     
     <a class="navbar-brand me-2" href="../index.html">scoringutils</a>
 
-    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.1</small>
+    <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">1.2.2</small>
 
     
     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
diff --git a/search.json b/search.json
index 7982166f7..b1ab13b42 100644
--- a/search.json
+++ b/search.json
@@ -1 +1 @@
-[{"path":[]},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"our-pledge","dir":"","previous_headings":"","what":"Our Pledge","title":"Contributor Covenant Code of Conduct","text":"members, contributors, leaders pledge make participation community harassment-free experience everyone, regardless age, body size, visible invisible disability, ethnicity, sex characteristics, gender identity expression, level experience, education, socio-economic status, nationality, personal appearance, race, religion, sexual identity orientation. pledge act interact ways contribute open, welcoming, diverse, inclusive, healthy community.","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"our-standards","dir":"","previous_headings":"","what":"Our Standards","title":"Contributor Covenant Code of Conduct","text":"Examples behavior contributes positive environment community include: Demonstrating empathy kindness toward people respectful differing opinions, viewpoints, experiences Giving gracefully accepting constructive feedback Accepting responsibility apologizing affected mistakes, learning experience Focusing best just us individuals, overall community Examples unacceptable behavior include: use sexualized language imagery, sexual attention advances kind Trolling, insulting derogatory comments, personal political attacks Public private harassment Publishing others’ private information, physical email address, without explicit permission conduct reasonably considered inappropriate professional setting","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"enforcement-responsibilities","dir":"","previous_headings":"","what":"Enforcement Responsibilities","title":"Contributor Covenant Code of Conduct","text":"Community leaders responsible clarifying enforcing standards acceptable behavior take appropriate fair corrective action response behavior deem inappropriate, threatening, offensive, harmful. Community leaders right responsibility remove, edit, reject comments, commits, code, wiki edits, issues, contributions aligned Code Conduct, communicate reasons moderation decisions appropriate.","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"scope","dir":"","previous_headings":"","what":"Scope","title":"Contributor Covenant Code of Conduct","text":"Code Conduct applies within community spaces, also applies individual officially representing community public spaces. Examples representing community include using official e-mail address, posting via official social media account, acting appointed representative online offline event.","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"enforcement","dir":"","previous_headings":"","what":"Enforcement","title":"Contributor Covenant Code of Conduct","text":"Instances abusive, harassing, otherwise unacceptable behavior may reported community leaders responsible enforcement Sebastian.Funk@lshtm.ac.uk. complaints reviewed investigated promptly fairly. community leaders obligated respect privacy security reporter incident.","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"enforcement-guidelines","dir":"","previous_headings":"","what":"Enforcement Guidelines","title":"Contributor Covenant Code of Conduct","text":"Community leaders follow Community Impact Guidelines determining consequences action deem violation Code Conduct:","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"id_1-correction","dir":"","previous_headings":"Enforcement Guidelines","what":"1. Correction","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Use inappropriate language behavior deemed unprofessional unwelcome community. Consequence: private, written warning community leaders, providing clarity around nature violation explanation behavior inappropriate. public apology may requested.","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"id_2-warning","dir":"","previous_headings":"Enforcement Guidelines","what":"2. Warning","title":"Contributor Covenant Code of Conduct","text":"Community Impact: violation single incident series actions. Consequence: warning consequences continued behavior. interaction people involved, including unsolicited interaction enforcing Code Conduct, specified period time. includes avoiding interactions community spaces well external channels like social media. Violating terms may lead temporary permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"id_3-temporary-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"3. Temporary Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: serious violation community standards, including sustained inappropriate behavior. Consequence: temporary ban sort interaction public communication community specified period time. public private interaction people involved, including unsolicited interaction enforcing Code Conduct, allowed period. Violating terms may lead permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"id_4-permanent-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"4. Permanent Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Demonstrating pattern violation community standards, including sustained inappropriate behavior, harassment individual, aggression toward disparagement classes individuals. Consequence: permanent ban sort public interaction within community.","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"attribution","dir":"","previous_headings":"","what":"Attribution","title":"Contributor Covenant Code of Conduct","text":"Code Conduct adapted Contributor Covenant, version 2.0, available https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. Community Impact Guidelines inspired Mozilla’s code conduct enforcement ladder. answers common questions code conduct, see FAQ https://www.contributor-covenant.org/faq. Translations available https://www.contributor-covenant.org/translations.","code":""},{"path":"https://epiforecasts.io/scoringutils/CONTRIBUTING.html","id":null,"dir":"","previous_headings":"","what":"Contributing to scoringutils","title":"Contributing to scoringutils","text":"outlines propose change scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/CONTRIBUTING.html","id":"fixing-typos","dir":"","previous_headings":"","what":"Fixing typos","title":"Contributing to scoringutils","text":"can fix typos, spelling mistakes, grammatical errors documentation directly using GitHub web interface, long changes made source file. generally means ’ll need edit roxygen2 comments .R, .Rd file. can find .R file generates .Rd reading comment first line.","code":""},{"path":"https://epiforecasts.io/scoringutils/CONTRIBUTING.html","id":"bigger-changes","dir":"","previous_headings":"","what":"Bigger changes","title":"Contributing to scoringutils","text":"want make bigger change, ’s good idea first file issue make sure someone team agrees ’s needed. ’ve found bug, please file issue illustrates bug minimal reprex (also help write unit test, needed).","code":""},{"path":"https://epiforecasts.io/scoringutils/CONTRIBUTING.html","id":"pull-request-process","dir":"","previous_headings":"Bigger changes","what":"Pull request process","title":"Contributing to scoringutils","text":"Fork package clone onto computer. haven’t done , recommend using usethis::create_from_github(\"epiforecasts/scoringutils\", fork = TRUE). Install development dependences devtools::install_dev_deps(), make sure package passes R CMD check running devtools::check(). R CMD check doesn’t pass cleanly, ’s good idea ask help continuing. Create Git branch pull request (PR). recommend using usethis::pr_init(\"brief-description--change\"). Make changes, commit git, create PR running usethis::pr_push(), following prompts browser. title PR briefly describe change. body PR contain Fixes #issue-number. user-facing changes, add bullet top NEWS.md (.e. just first header). Follow style described https://style.tidyverse.org/news.html.","code":""},{"path":"https://epiforecasts.io/scoringutils/CONTRIBUTING.html","id":"code-style","dir":"","previous_headings":"Bigger changes","what":"Code style","title":"Contributing to scoringutils","text":"New code follow tidyverse style guide. can use styler package apply styles, please don’t restyle code nothing PR. use roxygen2, Markdown syntax, documentation. use testthat unit tests. Contributions test cases included easier accept.","code":""},{"path":"https://epiforecasts.io/scoringutils/CONTRIBUTING.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Contributing to scoringutils","text":"Please note scoringutils project released Contributor Code Conduct. contributing project agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/LICENSE.html","id":null,"dir":"","previous_headings":"","what":"MIT License","title":"MIT License","text":"Copyright (c) 2020 EpiForecasts Permission hereby granted, free charge, person obtaining copy software associated documentation files (“Software”), deal Software without restriction, including without limitation rights use, copy, modify, merge, publish, distribute, sublicense, /sell copies Software, permit persons Software furnished , subject following conditions: copyright notice permission notice shall included copies substantial portions Software. SOFTWARE PROVIDED “”, WITHOUT WARRANTY KIND, EXPRESS IMPLIED, INCLUDING LIMITED WARRANTIES MERCHANTABILITY, FITNESS PARTICULAR PURPOSE NONINFRINGEMENT. EVENT SHALL AUTHORS COPYRIGHT HOLDERS LIABLE CLAIM, DAMAGES LIABILITY, WHETHER ACTION CONTRACT, TORT OTHERWISE, ARISING , CONNECTION SOFTWARE USE DEALINGS SOFTWARE.","code":""},{"path":"https://epiforecasts.io/scoringutils/articles/metric-details.html","id":"applicability-of-different-metrics-for-different-types-of-forecasts-and-formats","dir":"Articles","previous_headings":"","what":"Applicability of different metrics for different types of forecasts and formats","title":"Details on the metrics implemented in `scoringutils`","text":"table gives overview metric can applied gives brief description. Note table shows metrics implemented scoringutils. example, scoring sample-based discrete continuous distributions implemented scoringutils, closed-form solutions often exist (e.g. scoringRules package).","code":""},{"path":"https://epiforecasts.io/scoringutils/articles/metric-details.html","id":"detailed-explanation-of-the-metrics-implemented-in-scoringutils","dir":"Articles","previous_headings":"","what":"Detailed explanation of the metrics implemented in scoringutils","title":"Details on the metrics implemented in `scoringutils`","text":"crps proper scoring rule generalises absolute error probabilistic forecasts. measures ‘distance’ predictive distribution observed data-generating distribution. CRPS given \\[\\text{CRPS}(F, y) = \\int_{-\\infty}^\\infty \\left( F(x) - 1(x \\geq y) \\right)^2 dx,\\] y true observed value F CDF predictive distribution. Often alternative representation used: \\[ \\text{CRPS}(F, y) = \\frac{1}{2} \\mathbb{E}_{F} |X - X'| - \\mathbb{E}_P |X - y|,\\] \\(X\\) \\(X'\\) independent realisations predictive distributions \\(F\\) finite first moment \\(y\\) true value. represenation can simply replace \\(X\\) \\(X'\\) samples sum possible combinations obtain CRPS. integer-valued forecasts, RPS given \\[ \\text{RPS}(F, y) = \\sum_{x = 0}^\\infty (F(x) - 1(x \\geq y))^2. \\] Log score proper scoring rule simply compuated log predictive density evaluated true observed value. given \\[ \\text{log score} = \\log f(y), \\] \\(f\\) predictive density function y true value. integer-valued forecasts, log score can computed \\[ \\text{log score} = \\log p_y, \\] \\(p_y\\) probability assigned outcome p forecast F. (weighted) interval score proper scoring rule quantile forecasts converges crps increasing number intervals. score can decomposed sharpness (uncertainty) component penalties - underprediction. single interval, score computed \\[IS_\\alpha(F,y) = (u-l) + \\frac{2}{\\alpha} \\cdot (l-y) \\cdot \\mathbf{1}(y \\leq l) + \\frac{2}{\\alpha} \\cdot (y-u) \\cdot \\mathbf{1}(y \\geq u), \\] \\(\\mathbf{1}()\\) indicator function, \\(y\\) true value, \\(l\\) \\(u\\) \\(\\frac{\\alpha}{2}\\) \\(1 - \\frac{\\alpha}{2}\\) quantiles predictive distribution \\(F\\), .e. lower upper bound single prediction interval. set \\(K\\) prediction intervals median \\(m\\), score computed weighted sum, \\[WIS = \\frac{1}{K + 0.5} \\cdot \\left(w_0 \\cdot |y - m| + \\sum_{k = 1}^{K} w_k \\cdot IS_{\\alpha}(F, y)\\right),\\] \\(w_k\\) weight every interval. Usually, \\(w_k = \\frac{\\alpha_k}{2}\\) \\(w_0 = 0.5\\). Dawid-Sebastiani-Score proper scoring rule proposed relies first moments predictive distribution therefore easy compute. given \\[\\text{dss}(F, y) = \\left( \\frac{y - \\mu}{\\sigma} \\right)^2 + 2 \\cdot \\log \\sigma,\\] \\(F\\) predictive distribution mean \\(\\mu\\) standard deviation \\(\\sigma\\) \\(y\\) true observed value. Proper scoring rule binary forecasts. Brier score computed \\[\\text{Brier Score} = \\frac{1}{N} \\sum_{n = 1}^{N} (f_n - y_n),\\] \\(f_n\\), \\(n = 1, \\dots, N\\) predicted probablities corresponding events, \\(y_n \\(0, 1)\\) equal one. Interval coverage measures proportion observed values fall given prediction interval range. Interval coverage single prediction interval range can calculated \\[IC_{\\alpha} = \\text{nominal coverage} - \\text{empirical coverage},\\] nominal coverage \\(1 - \\alpha\\) empirical coverage proportion true values actually covered \\(1 - \\alpha\\) prediction intervals. summarise interval coverage different multiple interval ranges, can compute coverage deviation defined mean interval coverage \\(K\\) interval ranges \\(\\alpha_k\\) \\(k = 1, \\dots, K\\): \\[\\text{Coverage deviation} = \\frac{1}{K} \\sum_{k = 1}^{K} \\text{IC}_{\\alpha_k}\\] Quantile coverage given quantile level proportion true values smaller predictions corresponding quantile level. probability integral transform (PIT, Dawid 1984) represents succinct way visualise deviations predictive distribution \\(F\\) true data-generating distribution \\(G\\). idea transform observed values agreement forecasts data can examined observing whether transformed values follow uniform distribution. PIT given \\[u = F (y),\\] \\(u\\) transformed variable \\(F(y)\\) predictive distribution \\(F\\) evaluated true observed value \\(y\\). \\(F = G\\), \\(u\\) follows uniform distribution. integer outcomes, PIT longer uniform even forecasts ideal. Instead, randomised PIT can used: \\[u = P(y) + v \\cdot (P(y) - P(y - 1) ),\\] \\(y\\) observed value \\(P()\\) cumulative probability assigned values smaller equal \\(y\\) (\\(P(-1) = 0\\) definition, \\(v\\) standard uniform variable independent \\(y\\). \\(P\\) equal true data-generating distribution function, \\(u\\) standard uniform. also propose non-randomised version PIT count data used alternatively. Sharpness ability produce narrow forecasts feature forecasts depend observations. Sharpness therefore interest conditional calibration: precise forecast useful clearly wrong. suggested Funk et al. (2019), measure sharpness continuous integer forecasts represented predictive samples normalised median absolute deviation median (MADN) ), .e. \\[ S(F) = \\frac{1}{0.675} \\cdot \\text{median}(|x - \\text{median(x)}|), \\] \\(x\\) vector predictive samples \\(\\frac{1}{0.675}\\) normalising constant. predictive distribution \\(F\\) CDF normal distribution, sharpness equal standard deviation \\(F\\). Bias measure tendency forecaster - underpredict. continuous forecasts, bias given \\[B(F, y) = 1 - 2 \\cdot (F (y)), \\] \\(F\\) CDF predictive distribution \\(y\\) observed value. integer-valued forecasts, maximum minimum bias respect data point \\(y\\) \\(\\pm (1 - p(y))\\), \\(p(y)\\) predicted probability mass data point. Bias can calculated \\[B(P, y) = 1 - (P(y) + P(y + 1)), \\] \\(P(y)\\) cumulative probability assigned outcomes smaller equal \\(y\\), .e. cumulative probability mass function corresponding \\(p(y)\\). quantile forecasts, Bias can calculated maximum percentile rank prediction smaller \\(y\\), true value smaller median predictive distribution. true value median predictive distribution, bias minimum percentile rank corresponding quantile still larger true value. true value exactly median, bias zero. large enough number quantiles, percentile rank equal proportion predictive samples observed true value, metric coincides one continuous forecasts. mean score ratio used compare two models overlapping set forecast targets models made prediction. mean score ratio calculated mean score achieved first model mean score achieved second model. precisely, two models \\(, j\\), determine set overlapping forecasts, denoted \\(\\mathcal{}_{ij}\\) compute mean score ratio \\(\\theta_{ij}\\) \\[\\theta_{ij} =\\frac{\\text{mean score model } \\text{ } \\mathcal{}_{ij}}{\\text{mean score model } j \\text{ } \\mathcal{}_{ij}}.\\] mean score ratio can principle computed arbitrary score. Relative skill scores can used obtain ranking models based pairwise comparisons models. compute relative skill \\(\\theta_i\\) model \\(\\), take geometric mean mean score ratios involve model \\(\\), .e. \\[ \\theta_{} = \\left(\\prod_{m = 1}^M \\theta_{im}\\right)^{1/M}, \\] M number models.","code":""},{"path":"https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html","id":"bias","dir":"Articles","previous_headings":"","what":"Bias","title":"Scoring forecasts directly","text":"function bias determines bias predictive Monte-Carlo samples, automatically recognising whether forecasts continuous integer valued. continuous forecasts, Bias measured \\[B_t (P_t, x_t) = 1 - 2 \\cdot (P_t (x_t))\\] \\(P_t\\) empirical cumulative distribution function prediction true value \\(x_t\\). Computationally, \\(P_t (x_t)\\) just calculated fraction predictive samples \\(x_t\\) smaller \\(x_t\\). integer valued forecasts, Bias measured \\[B_t (P_t, x_t) = 1 - (P_t (x_t) + P_t (x_t + 1))\\] adjust integer nature forecasts. cases, Bias can assume values -1 1 0 ideally.","code":"## integer valued forecasts true_values <- rpois(30, lambda = 1:30) predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) bias_sample(true_values, predictions) #>  [1] -0.115 -0.845  0.735  1.000 -0.800 -0.810 -0.105  0.855  0.585 -0.005 #> [11] -0.795 -0.355  0.200  0.950 -0.500 -0.745  0.190  0.190 -0.435  0.045 #> [21]  0.955 -0.455 -0.400 -0.320 -0.470 -0.240 -0.660  0.040 -0.470 -0.090  ## continuous forecasts true_values <- rnorm(30, mean = 1:30) predictions <- replicate(200, rnorm(30, mean = 1:30)) bias_sample(true_values, predictions) #>  [1]  0.64  0.18 -0.08 -0.06 -1.00  0.72 -0.14  0.86 -0.25  0.14  1.00 -0.24 #> [13]  0.85  0.70  0.08 -0.37 -0.01 -0.58  0.59  0.06  0.65 -0.20  0.44 -0.01 #> [25]  0.14 -0.26  0.92 -0.37  0.36  0.70"},{"path":"https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html","id":"sharpness","dir":"Articles","previous_headings":"","what":"Sharpness","title":"Scoring forecasts directly","text":"Sharpness ability model generate predictions within narrow range. data-independent measure, purely feature forecasts . Sharpness / dispersion predictive samples corresponding one single true value measured normalised median absolute deviation median predictive samples. details, see ?stats::mad","code":"predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) mad_sample(predictions) #>  [1] 1.4826 1.4826 1.4826 2.2239 1.4826 2.9652 2.9652 2.9652 2.9652 2.9652 #> [11] 2.9652 2.9652 4.4478 2.9652 4.4478 4.4478 4.4478 4.4478 4.4478 4.4478 #> [21] 4.4478 4.4478 4.4478 5.1891 5.9304 5.9304 5.1891 5.9304 5.9304 5.9304"},{"path":"https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html","id":"calibration","dir":"Articles","previous_headings":"","what":"Calibration","title":"Scoring forecasts directly","text":"Calibration reliability forecasts ability model correctly identify uncertainty making predictions. model perfect calibration, observed data time point look came predictive probability distribution time. Equivalently, one can inspect probability integral transform predictive distribution time t, \\[u_t = F_t (x_t)\\] \\(x_t\\) observed data point time \\(t \\text{ } t_1, …, t_n\\), n number forecasts, \\(F_t\\) (continuous) predictive cumulative probability distribution time t. true probability distribution outcomes time t \\(G_t\\) forecasts \\(F_t\\) said ideal \\(F_t = G_t\\) times \\(t\\). case, probabilities ut distributed uniformly. case discrete outcomes incidence counts, PIT longer uniform even forecasts ideal. case randomised PIT can used instead: \\[u_t = P_t(k_t) + v \\cdot (P_t(k_t) - P_t(k_t - 1) )\\] \\(k_t\\) observed count, \\(P_t(x)\\) predictive cumulative probability observing incidence \\(k\\) time \\(t\\), \\(P_t (-1) = 0\\) definition \\(v\\) standard uniform independent \\(k\\). \\(P_t\\) true cumulative probability distribution, \\(u_t\\) standard uniform. function checks whether integer continuous forecasts provided. applies (randomised) probability integral tests values \\(u_t\\) uniformity using Anderson-Darling test. rule thumb, evidence suggest forecasting model miscalibrated p-value found greater threshold \\(p >= 0.1\\), evidence miscalibrated \\(0.01 < p < 0.1\\), good evidence miscalibrated \\(p <= 0.01\\). context noted, though, uniformity PIT necessary sufficient condition calibration. also noted test works given sufficient samples, otherwise Null hypothesis often rejected outright.","code":""},{"path":"https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html","id":"continuous-ranked-probability-score-crps","dir":"Articles","previous_headings":"","what":"Continuous Ranked Probability Score (CRPS)","title":"Scoring forecasts directly","text":"Wrapper around crps_sample() function scoringRules package. information look manuals scoringRules package. function can used continuous well integer valued forecasts. Smaller values better.","code":"true_values <- rpois(30, lambda = 1:30) predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) crps_sample(true_values, predictions) #>  [1] 0.701250 0.516400 1.278100 0.522450 0.621800 1.001350 2.554350 5.215725 #>  [9] 2.457975 1.875300 1.109525 2.505550 1.681925 0.973200 1.333575 3.204275 #> [17] 1.325825 2.290900 3.644650 3.761125 3.134750 3.563725 4.126400 1.799000 #> [25] 1.304850 1.328775 1.551425 1.240325 2.940300 6.601350"},{"path":"https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html","id":"dawid-sebastiani-score-dss","dir":"Articles","previous_headings":"","what":"Dawid-Sebastiani Score (DSS)","title":"Scoring forecasts directly","text":"Wrapper around dss_sample() function scoringRules package. information look manuals scoringRules package. function can used continuous well integer valued forecasts. Smaller values better.","code":"true_values <- rpois(30, lambda = 1:30) predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) dss_sample(true_values, predictions) #>  [1] 0.08618786 2.98559786 1.46976495 2.34099849 3.62407591 9.73895158 #>  [7] 1.98313057 3.90895141 2.96015505 2.11907238 4.68271144 4.47945581 #> [13] 2.59996394 2.77948300 3.66267936 2.74522043 3.80306668 4.97143605 #> [19] 4.31862922 3.89337389 3.26161140 6.10044027 3.19136140 3.46318559 #> [25] 3.38871634 3.84638962 3.99732263 3.34833904 3.85810706 4.17401075"},{"path":"https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html","id":"log-score","dir":"Articles","previous_headings":"","what":"Log Score","title":"Scoring forecasts directly","text":"Wrapper around logs_sample() function scoringRules package. information look manuals scoringRules package. function used integer valued forecasts. Log Scores principle possible integer valued forecasts require kernel density estimate well defined discrete values. Smaller values better.","code":"true_values <- rnorm(30, mean = 1:30) predictions <- replicate(200, rnorm(n = 30, mean = 1:30)) logs_sample(true_values, predictions) #>  [1] 1.2493238 1.4910234 1.0449596 1.5655875 1.0975545 1.7011817 1.5305268 #>  [8] 1.0834896 0.9961612 0.9489941 3.6769227 0.9405285 1.0693227 1.0154917 #> [15] 1.0614725 1.3418976 1.4577452 1.7541831 1.6804492 2.8461110 1.0636520 #> [22] 1.4800348 1.0343293 1.0278244 1.7132284 1.5020562 1.0706665 1.0368028 #> [29] 1.0395886 1.2623404"},{"path":"https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html","id":"brier-score","dir":"Articles","previous_headings":"","what":"Brier Score","title":"Scoring forecasts directly","text":"Brier score proper score rule assesses accuracy probabilistic binary predictions. outcomes can either 0 1, predictions must probability true outcome 1. Brier Score computed mean squared error probabilistic prediction true outcome. \\[\\text{Brier_Score} = \\frac{1}{N} \\sum_{t = 1}^{n} (\\text{prediction}_t - \\text{outcome}_t)^2\\]","code":"true_values <- sample(c(0, 1), size = 30, replace = TRUE) predictions <- runif(n = 30, min = 0, max = 1)  brier_score(true_values, predictions) #>  [1] 9.975442e-01 7.032452e-01 8.554911e-01 3.057581e-02 1.032670e-01 #>  [6] 3.699259e-02 4.383884e-01 1.552730e-02 4.501522e-02 2.735159e-05 #> [11] 1.333007e-01 5.782685e-02 2.755121e-01 7.752817e-02 5.448333e-01 #> [16] 6.876504e-01 6.058707e-01 1.770059e-01 7.403249e-01 1.341089e-01 #> [21] 3.608961e-02 1.156040e-02 6.170601e-02 7.775925e-01 6.875119e-02 #> [26] 8.468132e-02 8.020570e-01 5.994507e-01 4.433694e-01 7.353284e-01"},{"path":"https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html","id":"interval-score","dir":"Articles","previous_headings":"Brier Score","what":"Interval Score","title":"Scoring forecasts directly","text":"Interval Score Proper Scoring Rule score quantile predictions, following Gneiting Raftery (2007). Smaller values better. score computed \\[ \\text{score} = (\\text{upper} - \\text{lower}) + \\\\ \\frac{2}{\\alpha} \\cdot (\\text{lower} - \\text{true_value}) \\cdot 1(\\text{true_values} < \\text{lower}) + \\\\ \\frac{2}{\\alpha} \\cdot (\\text{true_value} - \\text{upper}) \\cdot 1(\\text{true_value} > \\text{upper})\\] \\(1()\\) indicator function \\(\\alpha\\) decimal value indicates much outside prediction interval. improve usability, user asked provide interval range percentage terms, .e. interval_range = 90 (percent) 90 percent prediction interval. Correspondingly, user provide 5% 95% quantiles (corresponding alpha 0.1). specific distribution assumed, range symmetric (.e can’t use 0.1 quantile lower bound 0.7 quantile upper). Setting weigh = TRUE weigh score \\(\\frac{\\alpha}{2}\\) Interval Score converges CRPS increasing number quantiles.","code":"true_values <- rnorm(30, mean = 1:30) interval_range <- 90 alpha <- (100 - interval_range) / 100 lower <- qnorm(alpha / 2, rnorm(30, mean = 1:30)) upper <- qnorm((1 - alpha / 2), rnorm(30, mean = 1:30))  interval_score(   true_values = true_values,   lower = lower,   upper = upper,   interval_range = interval_range ) #>  [1] 0.14146594 0.19799671 0.16409937 0.16358175 0.16151246 0.37690919 #>  [7] 0.55464477 0.12684004 0.15252842 0.24688999 0.31001926 0.25944660 #> [13] 0.14553859 0.01066308 0.21402588 0.12998395 0.17358867 0.22675278 #> [19] 1.48765122 0.08471481 0.15074407 0.63991832 0.25178951 0.26549263 #> [25] 0.19100429 0.15244351 0.22823608 0.13784232 0.27805907 0.11023177"},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"input-formats","dir":"Articles","previous_headings":"","what":"Input formats","title":"Getting started","text":"time, score() function able entire evaluation . need pass data.frame appropriate columns. columns required depends format forecasts come . forecast format can either based quantiles (see example_quantile expected format), based predictive samples (see example_continuous example_integer expected format case) binary format. following table gives overview (pairwise comparisons explained detail ): Additional columns may present indicate grouping forecasts. example, forecasts made different models various locations different time points, several weeks future. scoringutils automatically tries determine unit single forecast, .e. combination existing columns able uniquely identify single forecast. uses existing columns , can sometimes lead issues. therefore recommend using function set_forecast_unit() determine forecast unit manually. function simply drops unneeded columns, making sure necessary, ‘protected columns’ like “prediction” “true_value” retained.","code":"colnames(example_quantile) #>  [1] \"location\"        \"target_end_date\" \"target_type\"     \"true_value\"      #>  [5] \"location_name\"   \"forecast_date\"   \"quantile\"        \"prediction\"      #>  [9] \"model\"           \"horizon\"  set_forecast_unit(   example_quantile,   c(\"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\") ) %>%   colnames() #> [1] \"true_value\"      \"quantile\"        \"prediction\"      \"location\"        #> [5] \"target_end_date\" \"target_type\"     \"horizon\"         \"model\""},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"checking-the-input-data","dir":"Articles","previous_headings":"","what":"Checking the input data","title":"Getting started","text":"function check_forecasts() can used check input data. gives summary scoringutils thinks trying achieve. infers type prediction target, prediction format, unit single forecasts, gives overview number unique values per column (helpful spotting missing data) returns warnings errors. unsure input data look like, look example_quantile, example_integer, example_continuous example_binary data sets provided package. output check_forecasts() can later directly used input score() (otherwise, score() just run check_forecasts() anyway internally).","code":"head(example_quantile) #>    location target_end_date target_type true_value location_name forecast_date #> 1:       DE      2021-01-02       Cases     127300       Germany          <NA> #> 2:       DE      2021-01-02      Deaths       4534       Germany          <NA> #> 3:       DE      2021-01-09       Cases     154922       Germany          <NA> #> 4:       DE      2021-01-09      Deaths       6117       Germany          <NA> #> 5:       DE      2021-01-16       Cases     110183       Germany          <NA> #> 6:       DE      2021-01-16      Deaths       5867       Germany          <NA> #>    quantile prediction model horizon #> 1:       NA         NA  <NA>      NA #> 2:       NA         NA  <NA>      NA #> 3:       NA         NA  <NA>      NA #> 4:       NA         NA  <NA>      NA #> 5:       NA         NA  <NA>      NA #> 6:       NA         NA  <NA>      NA check_forecasts(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #> Your forecasts seem to be for a target of the following type: #> $target_type #> [1] \"integer\" #>  #> and in the following format: #> $prediction_type #> [1] \"quantile\" #>  #> The unit of a single forecast is defined by: #> $forecast_unit #> [1] \"location\"        \"target_end_date\" \"target_type\"     \"location_name\"   #> [5] \"forecast_date\"   \"model\"           \"horizon\"         #>  #> Cleaned data, rows with NA values in prediction or true_value removed: #> $cleaned_data #>        location target_end_date target_type true_value location_name #>     1:       DE      2021-05-08       Cases     106987       Germany #>     2:       DE      2021-05-08       Cases     106987       Germany #>     3:       DE      2021-05-08       Cases     106987       Germany #>     4:       DE      2021-05-08       Cases     106987       Germany #>     5:       DE      2021-05-08       Cases     106987       Germany #>    ---                                                               #> 20397:       IT      2021-07-24      Deaths         78         Italy #> 20398:       IT      2021-07-24      Deaths         78         Italy #> 20399:       IT      2021-07-24      Deaths         78         Italy #> 20400:       IT      2021-07-24      Deaths         78         Italy #> 20401:       IT      2021-07-24      Deaths         78         Italy #>        forecast_date quantile prediction                 model horizon #>     1:    2021-05-03    0.010      82466 EuroCOVIDhub-ensemble       1 #>     2:    2021-05-03    0.025      86669 EuroCOVIDhub-ensemble       1 #>     3:    2021-05-03    0.050      90285 EuroCOVIDhub-ensemble       1 #>     4:    2021-05-03    0.100      95341 EuroCOVIDhub-ensemble       1 #>     5:    2021-05-03    0.150      99171 EuroCOVIDhub-ensemble       1 #>    ---                                                                 #> 20397:    2021-07-12    0.850        352  epiforecasts-EpiNow2       2 #> 20398:    2021-07-12    0.900        397  epiforecasts-EpiNow2       2 #> 20399:    2021-07-12    0.950        499  epiforecasts-EpiNow2       2 #> 20400:    2021-07-12    0.975        611  epiforecasts-EpiNow2       2 #> 20401:    2021-07-12    0.990        719  epiforecasts-EpiNow2       2 #>  #> Number of unique values per column per model: #> $unique_values #>                    model location target_end_date target_type true_value #> 1: EuroCOVIDhub-ensemble        4              12           2         96 #> 2: EuroCOVIDhub-baseline        4              12           2         96 #> 3:  epiforecasts-EpiNow2        4              12           2         95 #> 4:       UMass-MechBayes        4              12           1         48 #>    location_name forecast_date quantile prediction horizon #> 1:             4            11       23       3969       3 #> 2:             4            11       23       3733       3 #> 3:             4            11       23       3903       3 #> 4:             4            11       23       1058       3 #>  #> $messages #> [1] \"144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.\""},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"showing-available-forecasts","dir":"Articles","previous_headings":"","what":"Showing available forecasts","title":"Getting started","text":"function avail_forecasts() may also helpful determine forecasts available. Using argument can specify level summary. example, see many forecasts per model target_type, can run see ‘epiforecasts-EpiNow2’ missing forecasts deaths forecast target UMass-MechBayes case forecasts. information can also visualised using plot_avail_forecasts() function:  can also visualise forecasts directly using plot_predictions() function:","code":"avail_forecasts(example_quantile, by = c(\"model\", \"target_type\")) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model target_type Number forecasts #> 1: EuroCOVIDhub-ensemble       Cases              128 #> 2: EuroCOVIDhub-baseline       Cases              128 #> 3:  epiforecasts-EpiNow2       Cases              128 #> 4: EuroCOVIDhub-ensemble      Deaths              128 #> 5: EuroCOVIDhub-baseline      Deaths              128 #> 6:       UMass-MechBayes      Deaths              128 #> 7:  epiforecasts-EpiNow2      Deaths              119 example_quantile %>%   avail_forecasts(by = c(\"model\", \"forecast_date\", \"target_type\")) %>%   plot_avail_forecasts() +   facet_wrap(~ target_type) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. example_quantile %>%   make_NA(     what = \"truth\",     target_end_date >= \"2021-07-15\",     target_end_date < \"2021-05-22\"   ) %>%   make_NA(     what = \"forecast\",     model != \"EuroCOVIDhub-ensemble\",     forecast_date != \"2021-06-28\"   ) %>%   plot_predictions(     x = \"target_end_date\",     by = c(\"target_type\", \"location\")   ) +   facet_wrap(target_type ~ location, ncol = 4, scales = \"free\")"},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"scoring-and-summarising-forecasts","dir":"Articles","previous_headings":"","what":"Scoring and summarising forecasts","title":"Getting started","text":"Forecasts can easily scored using score() function. clarity, suggest setting forecast unit explicitly may also want call check_forecasts() explicitly. Note example columns contain duplicated information regards forecast unit, e.g. “location” “location_name”, dropped. drop essential information, example, “target_type” column, ’ll get error informing us forecasts aren’t uniquely identified . function find_duplicates() may help investigate issue. filtering median forecasts EuroCOVIDhub-ensemble, can see indeed two forecasts every date, location horizon. function score() returns unsumarised scores, cases user wants. returns single score per forecast (determined forecast unit). forecasts quantile format, returns one score per quantile. second function, summarise_scores() takes care summarising scores level specified user. argument can used define level summary. default, = NULL summary unit assumed unit single forecast. continuous forecasts, means nothing happens isn’t specified. quantile based forecasts, = NULL, scores summarised across quantiles instead one score per forecast_unit quantile get one score per forecast unit. argument can specify unit summary want. can also call sumarise_scores() multiple tines, e.g round outputs specifying e.g. signif() summary function.","code":"scores <- example_quantile %>%   set_forecast_unit(     c(\"location\", \"target_end_date\", \"target_type\", \"location_name\",       \"forecast_date\", \"model\", \"horizon\")   ) %>%   check_forecasts() %>%   score() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. head(scores) #>    location target_end_date target_type location_name forecast_date #> 1:       DE      2021-05-08       Cases       Germany    2021-05-03 #> 2:       DE      2021-05-08       Cases       Germany    2021-05-03 #> 3:       DE      2021-05-08       Cases       Germany    2021-05-03 #> 4:       DE      2021-05-08       Cases       Germany    2021-05-03 #> 5:       DE      2021-05-08       Cases       Germany    2021-05-03 #> 6:       DE      2021-05-08       Cases       Germany    2021-05-03 #>                    model horizon range interval_score dispersion #> 1: EuroCOVIDhub-baseline       1     0        25620.0        0.0 #> 2: EuroCOVIDhub-baseline       1    10        25599.5      184.5 #> 3: EuroCOVIDhub-baseline       1    10        25599.5      184.5 #> 4: EuroCOVIDhub-baseline       1    20        25481.0      556.0 #> 5: EuroCOVIDhub-baseline       1    20        25481.0      556.0 #> 6: EuroCOVIDhub-baseline       1    30        25270.2      816.2 #>    underprediction overprediction coverage coverage_deviation bias quantile #> 1:               0          25620        0                0.0 0.95     0.50 #> 2:               0          25415        0               -0.1 0.95     0.45 #> 3:               0          25415        0               -0.1 0.95     0.55 #> 4:               0          24925        0               -0.2 0.95     0.40 #> 5:               0          24925        0               -0.2 0.95     0.60 #> 6:               0          24454        0               -0.3 0.95     0.35 #>    ae_median quantile_coverage #> 1:     25620              TRUE #> 2:     25620              TRUE #> 3:     25620              TRUE #> 4:     25620              TRUE #> 5:     25620              TRUE #> 6:     25620              TRUE example_quantile %>%   set_forecast_unit(     c(\"location\", \"target_end_date\", \"target_type\",       \"forecast_date\", \"model\", \"horizon\")   ) %>%   check_forecasts() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #> Your forecasts seem to be for a target of the following type: #> $target_type #> [1] \"integer\" #>  #> and in the following format: #> $prediction_type #> [1] \"quantile\" #>  #> The unit of a single forecast is defined by: #> $forecast_unit #> [1] \"location\"        \"target_end_date\" \"target_type\"     \"forecast_date\"   #> [5] \"model\"           \"horizon\"         #>  #> Cleaned data, rows with NA values in prediction or true_value removed: #> $cleaned_data #>        true_value quantile prediction location target_end_date target_type #>     1:     106987    0.010      82466       DE      2021-05-08       Cases #>     2:     106987    0.025      86669       DE      2021-05-08       Cases #>     3:     106987    0.050      90285       DE      2021-05-08       Cases #>     4:     106987    0.100      95341       DE      2021-05-08       Cases #>     5:     106987    0.150      99171       DE      2021-05-08       Cases #>    ---                                                                     #> 20397:         78    0.850        352       IT      2021-07-24      Deaths #> 20398:         78    0.900        397       IT      2021-07-24      Deaths #> 20399:         78    0.950        499       IT      2021-07-24      Deaths #> 20400:         78    0.975        611       IT      2021-07-24      Deaths #> 20401:         78    0.990        719       IT      2021-07-24      Deaths #>        forecast_date                 model horizon #>     1:    2021-05-03 EuroCOVIDhub-ensemble       1 #>     2:    2021-05-03 EuroCOVIDhub-ensemble       1 #>     3:    2021-05-03 EuroCOVIDhub-ensemble       1 #>     4:    2021-05-03 EuroCOVIDhub-ensemble       1 #>     5:    2021-05-03 EuroCOVIDhub-ensemble       1 #>    ---                                             #> 20397:    2021-07-12  epiforecasts-EpiNow2       2 #> 20398:    2021-07-12  epiforecasts-EpiNow2       2 #> 20399:    2021-07-12  epiforecasts-EpiNow2       2 #> 20400:    2021-07-12  epiforecasts-EpiNow2       2 #> 20401:    2021-07-12  epiforecasts-EpiNow2       2 #>  #> Number of unique values per column per model: #> $unique_values #>                    model true_value quantile prediction location #> 1: EuroCOVIDhub-ensemble         96       23       3969        4 #> 2: EuroCOVIDhub-baseline         96       23       3733        4 #> 3:  epiforecasts-EpiNow2         95       23       3903        4 #> 4:       UMass-MechBayes         48       23       1058        4 #>    target_end_date target_type forecast_date horizon #> 1:              12           2            11       3 #> 2:              12           2            11       3 #> 3:              12           2            11       3 #> 4:              12           1            11       3 #>  #> $messages #> [1] \"144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.\" example_quantile %>%   set_forecast_unit(     c(\"location\", \"target_end_date\",       \"forecast_date\", \"model\", \"horizon\")   ) %>%   check_forecasts() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #> Warning in check_forecasts(.): The following warnings were produced when checking inputs: #> 1.  Some forecasts have different numbers of rows  #> 2.  (e.g. quantiles or samples).  #> 3.  scoringutils found:  #> 4.  46, 23 #> 5.  . This may be a problem (it can potentially distort scores,  #> 6.  making it more difficult to compare them),  #> 7.  so make sure this is intended. #> Error in check_forecasts(.): The following errors were produced when checking inputs: #> 1.  There are instances with more than one forecast for the same target. This can't be right and needs to be resolved. Maybe you need to check the unit of a single forecast and add missing columns? Use the  function find_duplicates() to identify duplicate rows. duplicates <- example_quantile %>%   set_forecast_unit(     c(\"location\", \"target_end_date\",       \"forecast_date\", \"model\", \"horizon\")   ) %>%   find_duplicates()  duplicates[quantile == 0.5 & model == \"EuroCOVIDhub-ensemble\", ] %>%   head() #>    true_value quantile prediction location target_end_date forecast_date #> 1:     106987      0.5     119258       DE      2021-05-08    2021-05-03 #> 2:       1582      0.5       1568       DE      2021-05-08    2021-05-03 #> 3:      64985      0.5     110716       DE      2021-05-15    2021-05-03 #> 4:      64985      0.5      92649       DE      2021-05-15    2021-05-10 #> 5:       1311      0.5       1521       DE      2021-05-15    2021-05-03 #> 6:       1311      0.5       1422       DE      2021-05-15    2021-05-10 #>                    model horizon #> 1: EuroCOVIDhub-ensemble       1 #> 2: EuroCOVIDhub-ensemble       1 #> 3: EuroCOVIDhub-ensemble       2 #> 4: EuroCOVIDhub-ensemble       1 #> 5: EuroCOVIDhub-ensemble       2 #> 6: EuroCOVIDhub-ensemble       1 scores <- score(example_continuous) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. all(scores == summarise_scores(scores), na.rm = TRUE) #> [1] TRUE scores <- example_quantile %>%   set_forecast_unit(     c(\"location\", \"target_end_date\", \"target_type\",       \"forecast_date\", \"model\", \"horizon\")   ) %>%   check_forecasts() %>%   score() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.  head(scores) #>    location target_end_date target_type forecast_date                 model #> 1:       DE      2021-05-08       Cases    2021-05-03 EuroCOVIDhub-baseline #> 2:       DE      2021-05-08       Cases    2021-05-03 EuroCOVIDhub-baseline #> 3:       DE      2021-05-08       Cases    2021-05-03 EuroCOVIDhub-baseline #> 4:       DE      2021-05-08       Cases    2021-05-03 EuroCOVIDhub-baseline #> 5:       DE      2021-05-08       Cases    2021-05-03 EuroCOVIDhub-baseline #> 6:       DE      2021-05-08       Cases    2021-05-03 EuroCOVIDhub-baseline #>    horizon range interval_score dispersion underprediction overprediction #> 1:       1     0        25620.0        0.0               0          25620 #> 2:       1    10        25599.5      184.5               0          25415 #> 3:       1    10        25599.5      184.5               0          25415 #> 4:       1    20        25481.0      556.0               0          24925 #> 5:       1    20        25481.0      556.0               0          24925 #> 6:       1    30        25270.2      816.2               0          24454 #>    coverage coverage_deviation bias quantile ae_median quantile_coverage #> 1:        0                0.0 0.95     0.50     25620              TRUE #> 2:        0               -0.1 0.95     0.45     25620              TRUE #> 3:        0               -0.1 0.95     0.55     25620              TRUE #> 4:        0               -0.2 0.95     0.40     25620              TRUE #> 5:        0               -0.2 0.95     0.60     25620              TRUE #> 6:        0               -0.3 0.95     0.35     25620              TRUE  scores %>%   summarise_scores() %>%   head() #>    location target_end_date target_type forecast_date                 model #> 1:       DE      2021-05-08       Cases    2021-05-03 EuroCOVIDhub-baseline #> 2:       DE      2021-05-08       Cases    2021-05-03 EuroCOVIDhub-ensemble #> 3:       DE      2021-05-08       Cases    2021-05-03  epiforecasts-EpiNow2 #> 4:       DE      2021-05-08      Deaths    2021-05-03 EuroCOVIDhub-baseline #> 5:       DE      2021-05-08      Deaths    2021-05-03 EuroCOVIDhub-ensemble #> 6:       DE      2021-05-08      Deaths    2021-05-03       UMass-MechBayes #>    horizon interval_score dispersion underprediction overprediction #> 1:       1    16925.04696 1649.22087       0.0000000   15275.826087 #> 2:       1     7990.85478 5440.98522       0.0000000    2549.869565 #> 3:       1    25395.96087 8173.70000       0.0000000   17222.260870 #> 4:       1       46.79304   44.66261       0.0000000       2.130435 #> 5:       1       53.88000   53.27130       0.6086957       0.000000 #> 6:       1      116.12174   77.07826      39.0434783       0.000000 #>    coverage_deviation  bias ae_median #> 1:        -0.38521739  0.95     25620 #> 2:         0.04956522  0.50     12271 #> 3:        -0.29826087  0.90     44192 #> 4:         0.22347826  0.30        15 #> 5:         0.39739130 -0.10        14 #> 6:         0.04956522 -0.50       208 scores %>%   summarise_scores(by = c(\"model\", \"target_type\")) %>%   summarise_scores(fun = signif, digits = 2) %>%   kable()"},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"scoring-point-forecasts","dir":"Articles","previous_headings":"Scoring and summarising forecasts","what":"Scoring point forecasts","title":"Getting started","text":"Point forecasts can scored making use quantile-based format, value NA \"point\" quantile column. Point forecasts can scored alongside quantile-based forecasts. point forecasts values NA metrics designed probabilistic forecasts, important use na.rm = TRUE summarising.","code":"suppressMessages(score(example_point)) %>%   summarise_scores(by = \"model\", na.rm = TRUE) #>                    model interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline     14092.7647 2192.26967      4996.92898 #> 2: EuroCOVIDhub-ensemble      8852.4196 1930.80064      2005.56357 #> 3:  epiforecasts-EpiNow2     10659.5125 3084.85850      1604.96135 #> 4:       UMass-MechBayes        51.4781   28.09387        15.32315 #>    overprediction coverage_deviation        bias    ae_point     se_point #> 1:     6903.56605        0.002102273  0.21851562 19353.42969 2.883446e+09 #> 2:     4916.05540        0.050752841  0.00812500 12077.10156 1.945118e+09 #> 3:     5969.69268       -0.057861612 -0.04376518 14521.10526 2.680928e+09 #> 4:        8.06108       -0.024886364 -0.02312500    78.47656 1.170976e+04 #>    ae_median #> 1:       NaN #> 2:       NaN #> 3:       NaN #> 4:       NaN"},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"adding-empirical-coverage","dir":"Articles","previous_headings":"Scoring and summarising forecasts","what":"Adding empirical coverage","title":"Getting started","text":"quantile-based forecasts often interested specific coverage-levels, example, percentage true values fell 50% 90% prediction intervals. can add information using function add_coverage(). function also requires argument defines level grouping percentage true values covered certain prediction intervals computed.","code":"score(example_quantile) %>%   add_coverage(ranges = c(50, 90), by = c(\"model\", \"target_type\")) %>%   summarise_scores(by = c(\"model\", \"target_type\")) %>%   summarise_scores(fun = signif, digits = 2) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model target_type interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline       Cases          28000       4100         10000.0 #> 2: EuroCOVIDhub-baseline      Deaths            160         91             2.1 #> 3: EuroCOVIDhub-ensemble       Cases          18000       3700          4200.0 #> 4: EuroCOVIDhub-ensemble      Deaths             41         30             4.1 #> 5:       UMass-MechBayes      Deaths             53         27            17.0 #> 6:  epiforecasts-EpiNow2       Cases          21000       5700          3300.0 #> 7:  epiforecasts-EpiNow2      Deaths             67         32            16.0 #>    overprediction coverage_deviation    bias ae_median coverage_50 coverage_90 #> 1:        14000.0             -0.110  0.0980     38000        0.33        0.82 #> 2:           66.0              0.120  0.3400       230        0.66        1.00 #> 3:        10000.0             -0.098 -0.0560     24000        0.39        0.80 #> 4:            7.1              0.200  0.0730        53        0.88        1.00 #> 5:            9.0             -0.023 -0.0220        78        0.46        0.88 #> 6:        12000.0             -0.067 -0.0790     28000        0.47        0.79 #> 7:           19.0             -0.043 -0.0051       100        0.42        0.91"},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"adding-relative-scores","dir":"Articles","previous_headings":"Scoring and summarising forecasts","what":"Adding relative scores","title":"Getting started","text":"order better compare models can use relative scores computed based pairwise comparisons (see details ). Relative scores can added evaluation using function summarise_scores(). requires column called ‘model’ present. Pairwise comparisons computed according grouping specified : essentially, data.frame scores gets split different data.frames according values specified relative scores computed every individual group separately. baseline argument allows us specify baseline can used scale relative scores (scores divided baseline relative score). example, obtain relative scores separately different forecast targets, can run","code":"score(example_quantile) %>%   summarise_scores(by = c(\"model\", \"target_type\"),                    relative_skill = TRUE,                    baseline = \"EuroCOVIDhub-ensemble\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model target_type interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline       Cases    28483.57465 4102.50094    10284.972826 #> 2: EuroCOVIDhub-baseline      Deaths      159.40387   91.40625        2.098505 #> 3: EuroCOVIDhub-ensemble       Cases    17943.82383 3663.52458     4237.177310 #> 4: EuroCOVIDhub-ensemble      Deaths       41.42249   30.18099        4.103261 #> 5:       UMass-MechBayes      Deaths       52.65195   26.87239       16.800951 #> 6:  epiforecasts-EpiNow2       Cases    20831.55662 5664.37795     3260.355639 #> 7:  epiforecasts-EpiNow2      Deaths       66.64282   31.85692       15.893314 #>    overprediction coverage_deviation        bias   ae_median relative_skill #> 1:   14096.100883        -0.11211957  0.09796875 38473.60156      1.2947445 #> 2:      65.899117         0.11614130  0.33906250   233.25781      2.2958723 #> 3:   10043.121943        -0.09785326 -0.05640625 24101.07031      0.8156514 #> 4:       7.138247         0.19528533  0.07265625    53.13281      0.5966310 #> 5:       8.978601        -0.02312500 -0.02234375    78.47656      0.7475873 #> 6:   11906.823030        -0.06660326 -0.07890625 27923.81250      0.9469157 #> 7:      18.892583        -0.04287176 -0.00512605   104.74790      0.9765276 #>    scaled_rel_skill #> 1:         1.587375 #> 2:         3.848060 #> 3:         1.000000 #> 4:         1.000000 #> 5:         1.253014 #> 6:         1.160932 #> 7:         1.636736"},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"coloured-table","dir":"Articles","previous_headings":"Visualising scores","what":"Coloured table","title":"Getting started","text":"simple coloured table can produced based scores:","code":"score(example_integer) %>%   summarise_scores(by = c(\"model\", \"target_type\"), na.rm = TRUE) %>%   summarise_scores(fun = signif, digits = 2) %>%   plot_score_table(by = \"target_type\") +   facet_wrap(~ target_type, nrow = 1) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected."},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"score-heatmap","dir":"Articles","previous_headings":"Visualising scores","what":"Score heatmap","title":"Getting started","text":"can also summarise one particular metric across different categories using simple heatmap:","code":"score(example_continuous) %>%   summarise_scores(by = c(\"model\", \"location\", \"target_type\")) %>%   plot_heatmap(x = \"location\", metric = \"bias\") +   facet_wrap(~ target_type) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected."},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"weighted-interval-score-components","dir":"Articles","previous_headings":"Visualising scores","what":"Weighted interval score components","title":"Getting started","text":"weighted interval score can split three components: -prediction, -prediction dispersion. can visualised separately following way:","code":"score(example_quantile) %>%   summarise_scores(by = c(\"target_type\", \"model\")) %>%   plot_wis() +   facet_wrap(~ target_type, scales = \"free\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected."},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"calibration","dir":"Articles","previous_headings":"","what":"Calibration","title":"Getting started","text":"Calibration measure statistical consistency forecasts observed values. common way assessing calibration (precisely: probabilistic calibration) PIT histograms. probability integral transform (PIT) equal cumulative distribution function forecast evaluated true observed value. Ideally, pit values uniformly distributed transformation. can compute pit values : visualise results :  Similarly quantile-based forecasts:  Another way look calibration interval coverage quantile coverage. Interval coverage percentage true values fall inside given central prediction interval. Quantile coverage percentage observed values fall given quantile level. order plot interval coverage, need include “range” argument summarise_scores(). green area plot marks conservative behaviour, .e. empirical coverage greater nominally need (e.g. 55% true values covered 50% central prediction intervals.)  visualise quantile coverage, need include “quantile” . , green area corresponds conservative forecasts, central prediction intervals cover needed.","code":"example_continuous %>%   pit(by = \"model\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                      model pit_value #>   1: EuroCOVIDhub-baseline     0.025 #>   2: EuroCOVIDhub-baseline     0.525 #>   3: EuroCOVIDhub-baseline     0.000 #>   4: EuroCOVIDhub-baseline     0.000 #>   5: EuroCOVIDhub-baseline     0.200 #>  ---                                 #> 883:       UMass-MechBayes     0.950 #> 884:       UMass-MechBayes     0.500 #> 885:       UMass-MechBayes     0.100 #> 886:       UMass-MechBayes     0.450 #> 887:       UMass-MechBayes     0.100 example_continuous %>%   pit(by = c(\"model\", \"target_type\")) %>%   plot_pit() +   facet_grid(model ~ target_type) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. example_quantile[quantile %in% seq(0.1, 0.9, 0.1), ] %>%   pit(by = c(\"model\", \"target_type\")) %>%   plot_pit() +   facet_grid(model ~ target_type) example_quantile %>%   score() %>%   summarise_scores(by = c(\"model\", \"range\")) %>%   plot_interval_coverage() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. example_quantile %>%   score() %>%   summarise_scores(by = c(\"model\", \"quantile\")) %>%   plot_quantile_coverage() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected."},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"pairwise-comparisons","dir":"Articles","previous_headings":"","what":"Pairwise comparisons","title":"Getting started","text":"Relative scores different models can computed using pairwise comparisons, sort pairwise tournament combinations two models compared based overlapping set available forecasts common models. Internally, ratio mean scores models computed. relative score model geometric mean mean score ratios involve model. baseline provided, baseline excluded relative scores individual models (therefore differ slightly relative scores without baseline) relative scores scaled (.e. divided ) relative score baseline model. scoringutils, pairwise comparisons can made two ways: standalone function pairwise_comparison() within summarise_scores() simply adds relative scores existing set scores. using pairwise_comparison() function, can also visualise pairwise comparisons showing mean score ratios models. default, smaller values better model care showing y axis left, model compared shown x-axis bottom. example , EuroCOVIDhub-ensemble performs best (values smaller 1), EuroCOVIDhub-baseline performs worst (values larger 1). cases, UMass-MechBayes model course excluded case forecasts available therefore set overlapping forecasts empty.","code":"example_quantile %>%   score() %>%   pairwise_comparison(by = \"model\", baseline = \"EuroCOVIDhub-baseline\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                     model       compare_against mean_scores_ratio         pval #>  1: EuroCOVIDhub-baseline  epiforecasts-EpiNow2         1.3703452 9.164893e-18 #>  2: EuroCOVIDhub-baseline EuroCOVIDhub-ensemble         1.5925819 2.608666e-32 #>  3: EuroCOVIDhub-baseline       UMass-MechBayes         3.0275019 2.627464e-20 #>  4: EuroCOVIDhub-baseline EuroCOVIDhub-baseline         1.0000000 1.000000e+00 #>  5: EuroCOVIDhub-ensemble       UMass-MechBayes         0.7867229 1.244731e-04 #>  6: EuroCOVIDhub-ensemble  epiforecasts-EpiNow2         0.8606607 1.881520e-02 #>  7: EuroCOVIDhub-ensemble EuroCOVIDhub-baseline         0.6279112 2.608666e-32 #>  8: EuroCOVIDhub-ensemble EuroCOVIDhub-ensemble         1.0000000 1.000000e+00 #>  9:       UMass-MechBayes EuroCOVIDhub-ensemble         1.2710955 1.244731e-04 #> 10:       UMass-MechBayes  epiforecasts-EpiNow2         0.7439673 7.253878e-03 #> 11:       UMass-MechBayes EuroCOVIDhub-baseline         0.3303053 2.627464e-20 #> 12:       UMass-MechBayes       UMass-MechBayes         1.0000000 1.000000e+00 #> 13:  epiforecasts-EpiNow2       UMass-MechBayes         1.3441452 7.253878e-03 #> 14:  epiforecasts-EpiNow2 EuroCOVIDhub-ensemble         1.1618981 1.881520e-02 #> 15:  epiforecasts-EpiNow2 EuroCOVIDhub-baseline         0.7297431 9.164893e-18 #> 16:  epiforecasts-EpiNow2  epiforecasts-EpiNow2         1.0000000 1.000000e+00 #>         adj_pval relative_skill scaled_rel_skill #>  1: 3.665957e-17      1.6032604        1.0000000 #>  2: 1.565200e-31      1.6032604        1.0000000 #>  3: 1.313732e-19      1.6032604        1.0000000 #>  4: 1.000000e+00      1.6032604        1.0000000 #>  5: 3.734192e-04      0.8074916        0.5036559 #>  6: 1.881520e-02      0.8074916        0.5036559 #>  7: 1.565200e-31      0.8074916        0.5036559 #>  8: 1.000000e+00      0.8074916        0.5036559 #>  9: 3.734192e-04      0.7475873        0.4662919 #> 10: 1.450776e-02      0.7475873        0.4662919 #> 11: 1.313732e-19      0.7475873        0.4662919 #> 12: 1.000000e+00      0.7475873        0.4662919 #> 13: 1.450776e-02      1.0332277        0.6444541 #> 14: 1.881520e-02      1.0332277        0.6444541 #> 15: 3.665957e-17      1.0332277        0.6444541 #> 16: 1.000000e+00      1.0332277        0.6444541 example_quantile %>%   score() %>%   summarise_scores(     by = \"model\", relative_skill = TRUE, baseline = \"EuroCOVIDhub-baseline\"   ) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline    14321.48926 2096.95360      5143.53567 #> 2: EuroCOVIDhub-ensemble     8992.62316 1846.85278      2120.64029 #> 3:       UMass-MechBayes       52.65195   26.87239        16.80095 #> 4:  epiforecasts-EpiNow2    10827.40786 2950.73422      1697.23411 #>    overprediction coverage_deviation        bias   ae_median relative_skill #> 1:    7081.000000         0.00201087  0.21851562 19353.42969      1.6032604 #> 2:    5025.130095         0.04871603  0.00812500 12077.10156      0.8074916 #> 3:       8.978601        -0.02312500 -0.02234375    78.47656      0.7475873 #> 4:    6179.439535        -0.05516986 -0.04336032 14521.10526      1.0332277 #>    scaled_rel_skill #> 1:        1.0000000 #> 2:        0.5036559 #> 3:        0.4662919 #> 4:        0.6444541 example_quantile %>%   score() %>%   pairwise_comparison(by = c(\"model\", \"target_type\")) %>%   plot_pairwise_comparison() +   facet_wrap(~ target_type) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected."},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"correlation-between-scores","dir":"Articles","previous_headings":"Additional analyses and visualisations","what":"Correlation between scores","title":"Getting started","text":"may sometimes interesting see different scores correlate . can examine using function correlation(). dealing quantile-based forecasts, important call summarise_scorees() correlation() summarise quantiles computing correlations. Visualising correlations:","code":"example_quantile %>%   score() %>%   summarise_scores() %>%   correlation() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>    interval_score dispersion underprediction overprediction coverage_deviation #> 1:           1.00       0.46            0.28           0.94              -0.34 #> 2:           0.46       1.00            0.15           0.32              -0.12 #> 3:           0.28       0.15            1.00          -0.03              -0.33 #> 4:           0.94       0.32           -0.03           1.00              -0.25 #> 5:          -0.34      -0.12           -0.33          -0.25               1.00 #> 6:           0.11       0.11           -0.35           0.22               0.06 #> 7:           0.99       0.54            0.34           0.90              -0.38 #>     bias ae_median             metric #> 1:  0.11      0.99     interval_score #> 2:  0.11      0.54         dispersion #> 3: -0.35      0.34    underprediction #> 4:  0.22      0.90     overprediction #> 5:  0.06     -0.38 coverage_deviation #> 6:  1.00      0.10               bias #> 7:  0.10      1.00          ae_median example_quantile %>%   score() %>%   summarise_scores() %>%   correlation() %>%   plot_correlation() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected."},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"scores-by-interval-ranges","dir":"Articles","previous_headings":"Additional analyses and visualisations","what":"Scores by interval ranges","title":"Getting started","text":"like see different forecast interval ranges contribute average scores, can visualise scores interval range:","code":"example_quantile %>%   score() %>%   summarise_scores(by = c(\"model\", \"range\", \"target_type\")) %>%   plot_ranges() +   facet_wrap(~ target_type, scales = \"free\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected."},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"converting-to-quantile-based-forecasts","dir":"Articles","previous_headings":"","what":"Converting to quantile-based forecasts","title":"Getting started","text":"Different metrics available different forecasting formats. cases, may example forecasts sample-based format, wish make use functionality available quantile-based forecasts. example, may want use decomposition weighted interval score, may like compute interval coverage values. can convert sample-based forecasts quantile-based format using function sample_to_quantile(). , however, one caveat: Quantiles calculated based predictive samples, may introduce bias number available samples small.","code":"example_integer %>%   sample_to_quantile(     quantiles = c(0.01, 0.025, seq(0.05, 0.95, 0.05), 0.975, 0.99)   ) %>%   score() %>%   add_coverage(by = c(\"model\", \"target_type\")) #> The following messages were produced when checking inputs: #> 1.  3312 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                        model target_type location location_name target_end_date #>     1: EuroCOVIDhub-baseline       Cases       DE       Germany      2021-05-08 #>     2: EuroCOVIDhub-baseline       Cases       DE       Germany      2021-05-08 #>     3: EuroCOVIDhub-baseline       Cases       DE       Germany      2021-05-08 #>     4: EuroCOVIDhub-baseline       Cases       DE       Germany      2021-05-08 #>     5: EuroCOVIDhub-baseline       Cases       DE       Germany      2021-05-08 #>    ---                                                                          #> 20397:  epiforecasts-EpiNow2      Deaths       IT         Italy      2021-07-24 #> 20398:  epiforecasts-EpiNow2      Deaths       IT         Italy      2021-07-24 #> 20399:  epiforecasts-EpiNow2      Deaths       IT         Italy      2021-07-24 #> 20400:  epiforecasts-EpiNow2      Deaths       IT         Italy      2021-07-24 #> 20401:  epiforecasts-EpiNow2      Deaths       IT         Italy      2021-07-24 #>        forecast_date horizon range interval_score dispersion underprediction #>     1:    2021-05-03       1     0    29379.00000    0.00000               0 #>     2:    2021-05-03       1    10    27857.47500 3012.07500               0 #>     3:    2021-05-03       1    10    27857.47500 3012.07500               0 #>     4:    2021-05-03       1    20    26443.96000 4109.76000               0 #>     5:    2021-05-03       1    20    26443.96000 4109.76000               0 #>    ---                                                                       #> 20397:    2021-07-12       2    90       21.21000   21.21000               0 #> 20398:    2021-07-12       2    95       15.00563   15.00563               0 #> 20399:    2021-07-12       2    95       15.00563   15.00563               0 #> 20400:    2021-07-12       2    98        6.28890    6.28890               0 #> 20401:    2021-07-12       2    98        6.28890    6.28890               0 #>        overprediction coverage coverage_deviation bias quantile ae_median #>     1:        29379.0        0               0.00 0.98    0.500   29379.0 #>     2:        24845.4        0              -0.10 0.98    0.450   29379.0 #>     3:        24845.4        0              -0.10 0.98    0.550   29379.0 #>     4:        22334.2        0              -0.20 0.98    0.400   29379.0 #>     5:        22334.2        0              -0.20 0.98    0.600   29379.0 #>    ---                                                                    #> 20397:            0.0        1               0.10 0.70    0.950      73.5 #> 20398:            0.0        1               0.05 0.70    0.025      73.5 #> 20399:            0.0        1               0.05 0.70    0.975      73.5 #> 20400:            0.0        1               0.02 0.70    0.010      73.5 #> 20401:            0.0        1               0.02 0.70    0.990      73.5 #>        quantile_coverage coverage_50 coverage_90 #>     1:              TRUE   0.3828125   0.7265625 #>     2:              TRUE   0.3828125   0.7265625 #>     3:              TRUE   0.3828125   0.7265625 #>     4:              TRUE   0.3828125   0.7265625 #>     5:              TRUE   0.3828125   0.7265625 #>    ---                                           #> 20397:              TRUE   0.5462185   0.9243697 #> 20398:             FALSE   0.5462185   0.9243697 #> 20399:              TRUE   0.5462185   0.9243697 #> 20400:             FALSE   0.5462185   0.9243697 #> 20401:              TRUE   0.5462185   0.9243697"},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"available-metrics","dir":"Articles","previous_headings":"","what":"Available metrics","title":"Getting started","text":"overview available metrics can found metrics data set included package.","code":"metrics #>                                           Metric               Name #>  1:                               Absolute error           ae_point #>  2:                               Absolute error          ae_median #>  3:                                Squared error           se_point #>  4:                                Squared error            se_mean #>  5: (Continuous) ranked probability score (CRPS)               crps #>  6:                                    Log score          log_score #>  7:              (Weighted) interval score (WIS)     interval_score #>  8:                 Dawid-Sebastiani score (DSS)                dss #>  9:                             Brier score (BS)        brier_score #> 10:                            Interval coverage           coverage #> 11:                           Coverage deviation coverage_deviation #> 12:                            Quantile coverage  quantile_coverage #> 13:                                   Dispersion         dispersion #> 14:       Median Absolute Deviation (Dispersion)                mad #> 15:                      Under-, Over-prediction    underprediction #> 16:                      Under-, Over-prediction     overprediction #> 17:         Probability integral transform (PIT)               crps #> 18:                                   Dispersion         dispersion #> 19:                                         Bias               bias #> 20:                             Mean score ratio  mean_scores_ratio #> 21:                               Relative skill     relative_skill #> 22:                        Scaled relative skill   scaled_rel_skill #>                                           Metric               Name #>                                   Functions Discrete Continuous Binary Quantile #>  1: score(), ae_point()), ae_median_sample(        +          +      -        + #>  2: score(), ae_point()), ae_median_sample(        +          +      -        + #>  3:   score(), se_point(), se_mean_sample()        +          +      -        + #>  4:   score(), se_point(), se_mean_sample()        +          +      -        + #>  5:                       score(), ae_point        +          +      -        - #>  6:   score(), logs_sample(), logs_binary()        -          +      +        - #>  7:               score(), interval_score()        +          +      -        + #>  8:                   score(), dss_sample()        +          +      -        - #>  9:                  score(), brier_score()        -          -      +        - #> 10:                                 score()        -          -      -        + #> 11:                                 score()        -          -      -        + #> 12:                                 score()        +          +      -        - #> 13:               score(), interval_score()        -          -      -        + #> 14:                   score(), mad_sample()        +          +      -        - #> 15:               score(), interval_score()        -          -      -        + #> 16:               score(), interval_score()        -          -      -        + #> 17:                          score(), pit()        +          +      -        + #> 18:               score(), interval_score()        -          -      -        + #> 19: score(), bias_sample(), bias_quantile()        +          +      -        + #> 20:                   pairwise_comparison()        ~          ~      ~        ~ #> 21:          score(), pairwise_comparison()        ~          ~      ~        ~ #> 22:          score(), pairwise_comparison()        ~          ~      ~        ~ #>                                   Functions Discrete Continuous Binary Quantile #>                                                                                                                                                                                                           Info #>  1:                                                                                                                                               Suitable for scoring the median of a predictive distribution #>  2:                                                                                                                                               Suitable for scoring the median of a predictive distribution #>  3:                                                                                                                                                Suitable for scoring the mean of a predictive distribution. #>  4:                                                                                                                                                Suitable for scoring the mean of a predictive distribution. #>  5:                           Proper scoring rule (smaller is better), takes entire predictive distribution into account (global), penalises over- and under-confidence similarly, stable handling of outliers #>  6:                                           Proper scoring rule, smaller is better, only evaluates predictive density at observed value (local), penalises over-confidence severely, susceptible to outliers #>  7:                                                              Proper scoring rule, smaller is better, similar properties to CRPS and converges to CRPS for an increasing number of equally spaced intervals #>  8:                           Proper scoring rule, smaller is better, evaluates forecast based on mean and sd of predictive distribution (global), susceptible to outliers, penalises over-confidence severely #>  9:                                                                                    Proper scoring rule, smaller is better, equals CRPS for binary outcomes, penalises over- and under-confidence similarly #> 10:                                                 Proportion of observations falling inside a given central prediction interval (= 'empirical interval coverage'). Used to assess probabilistic calibration. #> 11:                                                                                               Average difference between empirical and nominal interval coverage (coverage that should have been realised) #> 12:                                                                                         Proportion of observations below a given quantile of the predictive CDF. Used to assess probabilistic calibration. #> 13:                                                                                                                                       Dispersion component of WIS, measures width of predictive intervals. #> 14:                                                                                                                    Measure for dispersion of a forecast: median of the absolute deviations from the median #> 15:                                                                                                                                            Absolute amount of over-or under-prediction (components of WIS) #> 16:                                                                                                                                            Absolute amount of over-or under-prediction (components of WIS) #> 17:                                                                                   PIT transform is the CDF of the predictive distribution evaluated at the observed values. PIT values should be uniform.  #> 18:                                                                                                                                       Dispersion component of WIS, measures width of predictive intervals. #> 19:                                                                                       Measure of relative tendency to over- or under-predict (aspect of calibration), bounded between -1 and 1 (ideally 0) #> 20:                                                                                                             Compares performance of two models. Properties depend on the metric chosen for the comparison. #> 21:                                                         Ranks models based on pairwise comparisons, useful in the context of missing forecasts. Properties depend on the metric chosen for the comparison. #> 22: Ranks models based on pairwise comparisons, useful in the context of missing forecasts. Scaled (i.e. divided) by the score of a baseline model. Properties depend on the metric chosen for the comparison. #>                                                                                                                                                                                                           Info"},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"lower-level-functions","dir":"Articles","previous_headings":"","what":"Lower level functions","title":"Getting started","text":"metrics available lower level functions extensively documented. look help files understand better.","code":""},{"path":"https://epiforecasts.io/scoringutils/authors.html","id":null,"dir":"","previous_headings":"","what":"Authors","title":"Authors and Citation","text":"Nikos Bosse. Author, maintainer. Sam Abbott. Author. Hugo Gruson. Author. Johannes Bracher. Contributor. Sebastian Funk. Author.","code":""},{"path":"https://epiforecasts.io/scoringutils/authors.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Authors and Citation","text":"Nikos . Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van Leeuwen, Sam Abbott (2022). Evaluating Forecasts scoringutils R, arXiv. DOI: 10.48550/ARXIV.2205.07090 Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating Probabilistic Forecasts scoringRules. Journal Statistical Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12","code":"@Article{,   title = {Evaluating Forecasts with scoringutils in R},   author = {Nikos I. Bosse and Hugo Gruson and Anne Cori and Edwin {van Leeuwen} and Sebastian Funk and Sam Abbott},   journal = {arXiv},   url = {https://arxiv.org/abs/2205.07090},   year = {2022},   doi = {10.48550/ARXIV.2205.07090}, } @Article{,   title = {Evaluating Probabilistic Forecasts with {scoringRules}},   author = {Alexander Jordan and Fabian Kr\\\"uger and Sebastian Lerch},   journal = {Journal of Statistical Software},   year = {2019},   volume = {90},   number = {12},   pages = {1--37},   doi = {10.18637/jss.v090.i12}, }"},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"scoringutils-utilities-for-scoring-and-assessing-predictions","dir":"","previous_headings":"","what":"Utilities for Scoring and Assessing Predictions","title":"Utilities for Scoring and Assessing Predictions","text":"scoringutils package provides collection metrics proper scoring rules aims make simple score probabilistic forecasts true observed values. can find additional information examples papers Evaluating Forecasts scoringutils R Scoring epidemiological forecasts transformed scales well Vignettes (Getting started, Details metrics implemented Scoring forecasts directly). scoringutils package offers convenient automated forecast evaluation function score(). function operates data.frames (uses data.table internally speed efficiency) can easily integrated workflow based dplyr data.table. also provides experienced users set reliable lower-level scoring metrics operating vectors/matrices can build upon applications. addition implements wide range flexible plots designed cover many use cases. available scoringutils depends functionality scoringRules provides comprehensive collection proper scoring rules predictive probability distributions represented sample parametric distributions. forecast types, quantile forecasts, scoringutils also implements additional metrics evaluating forecasts. top providing interface proper scoring rules implemented scoringRules natively, scoringutils also offers utilities summarising visualising forecasts scores, obtain relative scores models may useful non-overlapping forecasts forecasts across scales. Predictions can handled various formats: scoringutils can handle probabilistic forecasts either sample based quantile based format. detail expected input formats please see . True values can integer, continuous binary, appropriate scores value types selected automatically.","code":""},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"installation","dir":"","previous_headings":"","what":"Installation","title":"Utilities for Scoring and Assessing Predictions","text":"Install CRAN version package using: Install stable development version package : Install unstable development GitHub using following,","code":"install.packages(\"scoringutils\") install.packages(\"scoringutils\", repos = \"https://epiforecasts.r-universe.dev\") remotes::install_github(\"epiforecasts/scoringutils\", dependencies = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"quick-start","dir":"","previous_headings":"","what":"Quick start","title":"Utilities for Scoring and Assessing Predictions","text":"quick start guide explore functionality scoringutils package using quantile forecasts ECDC forecasting hub example. detailed documentation please see package vignettes, individual function documentation.","code":""},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"plotting-forecasts","dir":"","previous_headings":"Quick start","what":"Plotting forecasts","title":"Utilities for Scoring and Assessing Predictions","text":"first step evaluating forecasts visualise . purposes example make use plot_predictions() filter available forecasts single model, forecast date.","code":"example_quantile %>%   make_NA(what = \"truth\",            target_end_date >= \"2021-07-15\",            target_end_date < \"2021-05-22\"   ) %>%   make_NA(what = \"forecast\",           model != \"EuroCOVIDhub-ensemble\",            forecast_date != \"2021-06-28\"   ) %>%   plot_predictions(     x = \"target_end_date\",     by = c(\"target_type\", \"location\")   ) +   facet_wrap(target_type ~ location, ncol = 4, scales = \"free\")"},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"scoring-forecasts","dir":"","previous_headings":"Quick start","what":"Scoring forecasts","title":"Utilities for Scoring and Assessing Predictions","text":"Forecasts can easily quickly scored using score() function. score() automatically tries determine forecast_unit, .e. set columns uniquely defines single forecast, taking column names data account. However, recommended set forecast unit manually using set_forecast_unit() may help avoid errors, especially scoringutils used automated pipelines. function set_forecast_unit() simply drop unneeded columns. verify everything order, function check_forecasts() used. result check can passed directly score(). score() returns unsummarised scores, cases user wants. make use additional functions scoringutils add empirical coverage-levels (add_coverage()), scores relative baseline model (chosen EuroCOVIDhub-ensemble model). See getting started vignette details. Finally summarise scores model target type. scoringutils contains additional functionality transform forecasts, summarise scores different levels, visualise , explore forecasts . See package vignettes function documentation information. may want score forecasts based transformations original data order obtain complete evaluation (see paper information). can done using function transform_forecasts(). following example, truncate values 0 use function log_shift() add 1 values applying natural logarithm.","code":"example_quantile %>%   set_forecast_unit(c(\"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\")) %>%   check_forecasts() %>%   score() %>%   add_coverage(ranges = c(50, 90), by = c(\"model\", \"target_type\")) %>%   summarise_scores(     by = c(\"model\", \"target_type\"),     relative_skill = TRUE,     baseline = \"EuroCOVIDhub-ensemble\"   ) %>%   summarise_scores(     fun = signif,      digits = 2   ) %>%   kable() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. example_quantile %>%  .[, true_value := ifelse(true_value < 0, 0, true_value)] %>%   transform_forecasts(append = TRUE, fun = log_shift, offset = 1) %>%   score %>%   summarise_scores(by = c(\"model\", \"target_type\", \"scale\")) %>%   head() #> The following messages were produced when checking inputs: #> 1.  288 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model target_type   scale interval_score   dispersion #> 1: EuroCOVIDhub-baseline       Cases     log   1.169972e+00    0.4373146 #> 2: EuroCOVIDhub-baseline       Cases natural   2.209046e+04 4102.5009443 #> 3: EuroCOVIDhub-ensemble       Cases     log   5.500974e-01    0.1011850 #> 4: EuroCOVIDhub-ensemble       Cases natural   1.155071e+04 3663.5245788 #> 5:  epiforecasts-EpiNow2       Cases     log   6.005778e-01    0.1066329 #> 6:  epiforecasts-EpiNow2       Cases natural   1.443844e+04 5664.3779484 #>    underprediction overprediction coverage_deviation        bias    ae_median #> 1:    3.521964e-01      0.3804607        -0.10940217  0.09726562 1.185905e+00 #> 2:    1.028497e+04   7702.9836957        -0.10940217  0.09726562 3.208048e+04 #> 3:    1.356563e-01      0.3132561        -0.09785326 -0.05640625 7.410484e-01 #> 4:    4.237177e+03   3650.0047554        -0.09785326 -0.05640625 1.770795e+04 #> 5:    1.858699e-01      0.3080750        -0.06660326 -0.07890625 7.656591e-01 #> 6:    3.260356e+03   5513.7058424        -0.06660326 -0.07890625 2.153070e+04"},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Utilities for Scoring and Assessing Predictions","text":"using scoringutils work please consider citing using output citation(\"scoringutils\"):","code":"#> To cite scoringutils in publications use the following. If you use the #> CRPS, DSS, or Log Score, please also cite scoringRules. #>  #>   Nikos I. Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van #>   Leeuwen, and Sam Abbott (2022). Evaluating Forecasts with #>   scoringutils in R, arXiv. DOI: 10.48550/ARXIV.2205.07090 #>  #> To cite scoringRules in publications use: #>  #>   Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating #>   Probabilistic Forecasts with scoringRules. Journal of Statistical #>   Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12 #>  #> To see these entries in BibTeX format, use 'print(<citation>, #> bibtex=TRUE)', 'toBibtex(.)', or set #> 'options(citation.bibtex.max=999)'."},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"how-to-make-a-bug-report-or-feature-request","dir":"","previous_headings":"","what":"How to make a bug report or feature request","title":"Utilities for Scoring and Assessing Predictions","text":"Please briefly describe problem output expect issue. question, please don’t open issue. Instead, ask Q page.","code":""},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"contributing","dir":"","previous_headings":"","what":"Contributing","title":"Utilities for Scoring and Assessing Predictions","text":"welcome contributions new contributors! particularly appreciate help priority problems issues. Please check add issues, /add pull request.","code":""},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Utilities for Scoring and Assessing Predictions","text":"Please note scoringutils project released Contributor Code Conduct. contributing project, agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/abs_error.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute Error — abs_error","title":"Absolute Error — abs_error","text":"Calculate absolute error $$   \\textrm{abs}(\\textrm{true\\_value} - \\textrm{median\\_prediction}) $$","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/abs_error.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute Error — abs_error","text":"","code":"abs_error(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/abs_error.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute Error — abs_error","text":"true_values vector true observed values size n predictions numeric vector predictions, corresponding quantiles second vector, quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/abs_error.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute Error — abs_error","text":"vector absolute error","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/reference/abs_error.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute Error — abs_error","text":"","code":"true_values <- rnorm(30, mean = 1:30) predicted_values <- rnorm(30, mean = 1:30) abs_error(true_values, predicted_values) #>  [1] 2.01879202 0.69207765 0.91175498 1.71395928 0.74646413 1.51484910 #>  [7] 0.15624959 0.35110899 1.47857662 3.29085515 1.07546222 0.22773463 #> [13] 1.33546528 0.28994391 0.25914987 0.26462259 0.41897401 2.14605889 #> [19] 0.82280576 1.75390591 1.22319941 0.99206998 1.69741206 0.50146093 #> [25] 2.80167576 1.37368546 0.54469432 0.59079704 0.06504385 0.07585973"},{"path":"https://epiforecasts.io/scoringutils/reference/add_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Add coverage of central prediction intervals — add_coverage","title":"Add coverage of central prediction intervals — add_coverage","text":"Adds column coverage central prediction intervals unsummarised scores produced score()","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/add_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Add coverage of central prediction intervals — add_coverage","text":"","code":"add_coverage(scores, by, ranges = c(50, 90))"},{"path":"https://epiforecasts.io/scoringutils/reference/add_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Add coverage of central prediction intervals — add_coverage","text":"scores data.table scores produced score(). character vector column names add coverage . ranges numeric vector ranges central prediction intervals coverage values shall added.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/add_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Add coverage of central prediction intervals — add_coverage","text":"data.table unsummarised scores columns added coverage central prediction intervals. overall data.table still unsummarised, note coverage columns level summary present according value specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/add_coverage.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Add coverage of central prediction intervals — add_coverage","text":"coverage values added computed according values specified . , example, = \"model\", one coverage value every model add_coverage() compute coverage every model across values present columns define unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/add_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Add coverage of central prediction intervals — add_coverage","text":"","code":"library(magrittr) # pipe operator score(example_quantile) %>%   add_coverage(by = c(\"model\", \"target_type\")) %>%   summarise_scores(by = c(\"model\", \"target_type\")) %>%   summarise_scores(fun = signif, digits = 2) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model target_type interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline       Cases          28000       4100         10000.0 #> 2: EuroCOVIDhub-baseline      Deaths            160         91             2.1 #> 3: EuroCOVIDhub-ensemble       Cases          18000       3700          4200.0 #> 4: EuroCOVIDhub-ensemble      Deaths             41         30             4.1 #> 5:       UMass-MechBayes      Deaths             53         27            17.0 #> 6:  epiforecasts-EpiNow2       Cases          21000       5700          3300.0 #> 7:  epiforecasts-EpiNow2      Deaths             67         32            16.0 #>    overprediction coverage_deviation    bias ae_median coverage_50 coverage_90 #> 1:        14000.0             -0.110  0.0980     38000        0.33        0.82 #> 2:           66.0              0.120  0.3400       230        0.66        1.00 #> 3:        10000.0             -0.098 -0.0560     24000        0.39        0.80 #> 4:            7.1              0.200  0.0730        53        0.88        1.00 #> 5:            9.0             -0.023 -0.0220        78        0.46        0.88 #> 6:        12000.0             -0.067 -0.0790     28000        0.47        0.79 #> 7:           19.0             -0.043 -0.0051       100        0.42        0.91"},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute Error of the Median (Quantile-based Version) — ae_median_quantile","title":"Absolute Error of the Median (Quantile-based Version) — ae_median_quantile","text":"Absolute error median calculated $$   \\textrm{abs}(\\textrm{true\\_value} - \\textrm{prediction}) $$ function created internal use within score(), can also used standalone function.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute Error of the Median (Quantile-based Version) — ae_median_quantile","text":"","code":"ae_median_quantile(true_values, predictions, quantiles = NULL)"},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute Error of the Median (Quantile-based Version) — ae_median_quantile","text":"true_values vector true observed values size n predictions numeric vector predictions, corresponding quantiles second vector, quantiles. quantiles numeric vector denotes quantile values predictions. predictions quantiles == 0.5 kept. quantiles NULL, predictions true_values used (abs_error())","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute Error of the Median (Quantile-based Version) — ae_median_quantile","text":"vector scoring values","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute Error of the Median (Quantile-based Version) — ae_median_quantile","text":"","code":"true_values <- rnorm(30, mean = 1:30) predicted_values <- rnorm(30, mean = 1:30) ae_median_quantile(true_values, predicted_values, quantiles = 0.5) #>  [1] 1.812709892 0.414207758 0.093280028 3.162829089 3.906720909 0.850222619 #>  [7] 0.304749051 0.001270822 1.193929248 0.919874521 1.182473820 1.831074557 #> [13] 0.264693752 1.968146319 0.809231613 0.713629639 1.924207600 0.594296163 #> [19] 0.373904488 2.058168624 0.688137071 1.124096332 3.128342150 0.056254366 #> [25] 1.070399093 0.278518244 1.339064990 1.052953923 1.033862842 2.321194807"},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute Error of the Median (Sample-based Version) — ae_median_sample","title":"Absolute Error of the Median (Sample-based Version) — ae_median_sample","text":"Absolute error median calculated $$%   \\textrm{abs}(\\textrm{true\\_value} - \\textrm{median\\_prediction}) $$","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute Error of the Median (Sample-based Version) — ae_median_sample","text":"","code":"ae_median_sample(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute Error of the Median (Sample-based Version) — ae_median_sample","text":"true_values vector true observed values size n predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute Error of the Median (Sample-based Version) — ae_median_sample","text":"vector scoring values","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute Error of the Median (Sample-based Version) — ae_median_sample","text":"","code":"true_values <- rnorm(30, mean = 1:30) predicted_values <- rnorm(30, mean = 1:30) ae_median_sample(true_values, predicted_values) #>  [1] 0.06347216 1.53636267 1.26550790 0.49051927 2.07635562 0.23897194 #>  [7] 1.28540707 0.11143626 0.01528276 0.57545113 2.99596646 1.30559601 #> [13] 0.46210542 0.65158364 0.21668173 1.12933887 1.42363120 2.18281301 #> [19] 2.08457395 0.74101704 1.31789897 1.57850901 2.44268252 0.24034947 #> [25] 0.51951770 0.31845583 0.08615476 0.68660076 0.30681817 1.44981216"},{"path":"https://epiforecasts.io/scoringutils/reference/avail_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Display Number of Forecasts Available — avail_forecasts","title":"Display Number of Forecasts Available — avail_forecasts","text":"Given data set forecasts, count number available forecasts arbitrary grouping (e.g. number forecasts per model, number forecasts per model location). useful determine whether missing forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/avail_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Display Number of Forecasts Available — avail_forecasts","text":"","code":"avail_forecasts(data, by = NULL, collapse = c(\"quantile\", \"sample\"))"},{"path":"https://epiforecasts.io/scoringutils/reference/avail_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Display Number of Forecasts Available — avail_forecasts","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary). character vector NULL (default) denotes categories number forecasts counted. default (= NULL) unit single forecast (.e. available columns (apart \"protected\" columns 'prediction' 'true value') plus \"quantile\" \"sample\" present). collapse character vector (default c(\"quantile\", \"sample\") names categories number rows collapsed one counting. example, single forecast usually represented set several quantiles samples collapsing one makes sure single forecast gets counted .","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/avail_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Display Number of Forecasts Available — avail_forecasts","text":"data.table columns specified additional column number forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/avail_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Display Number of Forecasts Available — avail_forecasts","text":"","code":"data.table::setDTthreads(1) # only needed to avoid issues on CRAN  avail_forecasts(example_quantile,   collapse = c(\"quantile\"),   by = c(\"model\", \"target_type\") ) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model target_type Number forecasts #> 1: EuroCOVIDhub-ensemble       Cases              128 #> 2: EuroCOVIDhub-baseline       Cases              128 #> 3:  epiforecasts-EpiNow2       Cases              128 #> 4: EuroCOVIDhub-ensemble      Deaths              128 #> 5: EuroCOVIDhub-baseline      Deaths              128 #> 6:       UMass-MechBayes      Deaths              128 #> 7:  epiforecasts-EpiNow2      Deaths              119"},{"path":"https://epiforecasts.io/scoringutils/reference/available_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Available metrics in scoringutils — available_metrics","title":"Available metrics in scoringutils — available_metrics","text":"Available metrics scoringutils","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/available_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Available metrics in scoringutils — available_metrics","text":"","code":"available_metrics()"},{"path":"https://epiforecasts.io/scoringutils/reference/available_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Available metrics in scoringutils — available_metrics","text":"vector name available metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Determines Bias of Quantile Forecasts — bias_quantile","title":"Determines Bias of Quantile Forecasts — bias_quantile","text":"Determines bias quantile forecasts. increasing number quantiles measure converges sample based bias version integer continuous forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determines Bias of Quantile Forecasts — bias_quantile","text":"","code":"bias_quantile(predictions, quantiles, true_value)"},{"path":"https://epiforecasts.io/scoringutils/reference/bias_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determines Bias of Quantile Forecasts — bias_quantile","text":"predictions vector length corresponding number quantiles holds predictions quantiles vector corresponding size quantiles predictions made. contain median (0.5) median imputed mean two innermost quantiles. true_value single true value","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determines Bias of Quantile Forecasts — bias_quantile","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determines Bias of Quantile Forecasts — bias_quantile","text":"quantile forecasts, bias measured $$ B_t = (1 - 2 \\cdot \\max \\{| q_{t,} \\Q_t \\land q_{t,} \\leq x_t\\})  \\mathbf{1}( x_t \\leq q_{t, 0.5}) \\\\ + (1 - 2 \\cdot \\min \\{| q_{t,} \\Q_t \\land q_{t,} \\geq x_t\\})  1( x_t \\geq q_{t, 0.5}),$$ \\(Q_t\\) set quantiles form predictive distribution time \\(t\\). represent belief true value $x_t$ . consistency, define \\(Q_t\\) always includes element \\(q_{t, 0} = - \\infty\\) \\(q_{t,1} = \\infty\\). \\(1()\\) indicator function \\(1\\) condition satisfied $0$ otherwise. clearer terms, \\(B_t\\) defined maximum percentile rank corresponding quantile still true value, true value smaller median predictive distribution. true value median predictive distribution, $B_t$ minimum percentile rank corresponding quantile still larger true value. true value exactly median, terms cancel \\(B_t\\) zero. large enough number quantiles, percentile rank equal proportion predictive samples observed true value, metric coincides one continuous forecasts. Bias can assume values -1 1 0 ideally (.e. unbiased).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_quantile.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Determines Bias of Quantile Forecasts — bias_quantile","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determines Bias of Quantile Forecasts — bias_quantile","text":"","code":"predictions <- c(   705.500, 1127.000, 4006.250, 4341.500, 4709.000, 4821.996,   5340.500, 5451.000, 5703.500, 6087.014, 6329.500, 6341.000,   6352.500, 6594.986, 6978.500, 7231.000, 7341.500, 7860.004,   7973.000, 8340.500, 8675.750, 11555.000, 11976.500 )  quantiles <- c(0.01, 0.025, seq(0.05, 0.95, 0.05), 0.975, 0.99)  true_value <- 8062  bias_quantile(predictions, quantiles, true_value = true_value) #> [1] -0.8"},{"path":"https://epiforecasts.io/scoringutils/reference/bias_range.html","id":null,"dir":"Reference","previous_headings":"","what":"Determines Bias of Quantile Forecasts based on the range of the prediction intervals — bias_range","title":"Determines Bias of Quantile Forecasts based on the range of the prediction intervals — bias_range","text":"Determines bias quantile forecasts based range prediction intervals. increasing number quantiles measure converges sample based bias version integer continuous forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_range.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determines Bias of Quantile Forecasts based on the range of the prediction intervals — bias_range","text":"","code":"bias_range(lower, upper, range, true_value)"},{"path":"https://epiforecasts.io/scoringutils/reference/bias_range.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determines Bias of Quantile Forecasts based on the range of the prediction intervals — bias_range","text":"lower vector length corresponding number central prediction intervals holds predictions lower bounds prediction interval upper vector length corresponding number central prediction intervals holds predictions upper bounds prediction interval range vector corresponding size information width central prediction interval true_value single true value","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_range.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determines Bias of Quantile Forecasts based on the range of the prediction intervals — bias_range","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_range.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determines Bias of Quantile Forecasts based on the range of the prediction intervals — bias_range","text":"quantile forecasts, bias measured $$ B_t = (1 - 2 \\cdot \\max \\{| q_{t,} \\Q_t \\land q_{t,} \\leq x_t\\}) \\mathbf{1}( x_t \\leq q_{t, 0.5}) \\\\ + (1 - 2 \\cdot \\min \\{| q_{t,} \\Q_t \\land q_{t,} \\geq x_t\\})  \\mathbf{1}( x_t \\geq q_{t, 0.5}), $$ \\(Q_t\\) set quantiles form predictive distribution time \\(t\\). represent belief true value \\(x_t\\) . consistency, define \\(Q_t\\) always includes element \\(q_{t, 0} = - \\infty\\) \\(q_{t,1} = \\infty\\). \\(\\mathbf{1}()\\) indicator function \\(1\\) condition satisfied $0$ otherwise. clearer terms, \\(B_t\\) defined maximum percentile rank corresponding quantile still true value, true value smaller median predictive distribution. true value median predictive distribution, $B_t$ minimum percentile rank corresponding quantile still larger true value. true value exactly median, terms cancel \\(B_t\\) zero. large enough number quantiles, percentile rank equal proportion predictive samples observed true value, metric coincides one continuous forecasts. Bias can assume values -1 1 0 ideally.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/reference/bias_range.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Determines Bias of Quantile Forecasts based on the range of the prediction intervals — bias_range","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_range.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determines Bias of Quantile Forecasts based on the range of the prediction intervals — bias_range","text":"","code":"lower <- c(   6341.000, 6329.500, 6087.014, 5703.500,   5451.000, 5340.500, 4821.996, 4709.000,   4341.500, 4006.250, 1127.000, 705.500 )  upper <- c(   6341.000, 6352.500, 6594.986, 6978.500,   7231.000, 7341.500, 7860.004, 7973.000,   8340.500, 8675.750, 11555.000, 11976.500 )  range <- c(0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 98)  true_value <- 8062  bias_range(   lower = lower, upper = upper,   range = range, true_value = true_value ) #> [1] -0.8"},{"path":"https://epiforecasts.io/scoringutils/reference/bias_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determines bias of forecasts — bias_sample","title":"Determines bias of forecasts — bias_sample","text":"Determines bias predictive Monte-Carlo samples. function automatically recognises, whether forecasts continuous integer valued adapts Bias function accordingly.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determines bias of forecasts — bias_sample","text":"","code":"bias_sample(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/bias_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determines bias of forecasts — bias_sample","text":"true_values vector true observed values size n predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determines bias of forecasts — bias_sample","text":"vector length n biases predictive samples respect true values.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determines bias of forecasts — bias_sample","text":"continuous forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - 2 * (P_t (x_t)) $$ \\(P_t\\) empirical cumulative distribution function prediction true value \\(x_t\\). Computationally, \\(P_t (x_t)\\) just calculated fraction predictive samples \\(x_t\\) smaller \\(x_t\\). integer valued forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - (P_t (x_t) + P_t (x_t + 1)) $$ adjust integer nature forecasts. cases, Bias can assume values -1 1 0 ideally.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determines bias of forecasts — bias_sample","text":"integer valued Bias function discussed Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15 Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, et al. (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLOS Computational Biology 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_sample.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Determines bias of forecasts — bias_sample","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determines bias of forecasts — bias_sample","text":"","code":"## integer valued forecasts true_values <- rpois(30, lambda = 1:30) predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) bias_sample(true_values, predictions) #>  [1] -0.065 -0.930 -0.100 -0.815  0.275  0.585  0.510  0.340  0.130  0.165 #> [11] -0.785 -0.820 -0.160  0.670 -0.970 -0.460 -0.890 -0.915  0.960 -0.090 #> [21] -0.255  0.395 -0.785 -0.270 -0.245 -0.590  1.000 -0.955 -0.180  0.730  ## continuous forecasts true_values <- rnorm(30, mean = 1:30) predictions <- replicate(200, rnorm(30, mean = 1:30)) bias_sample(true_values, predictions) #>  [1]  0.06 -0.92 -0.96 -0.22 -0.41  0.97 -0.23 -0.43 -0.60  0.21 -0.45  0.21 #> [13] -0.87  0.97  0.05  0.99  0.68 -0.50  0.96 -0.82 -0.30 -0.74 -0.86 -0.45 #> [25] -0.66 -0.49  0.30 -0.31  1.00  0.62"},{"path":"https://epiforecasts.io/scoringutils/reference/brier_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Brier Score — brier_score","title":"Brier Score — brier_score","text":"Computes Brier Score probabilistic forecasts binary outcomes.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/brier_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Brier Score — brier_score","text":"","code":"brier_score(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/brier_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Brier Score — brier_score","text":"true_values vector true observed values size n values equal either 0 1 predictions vector predicted probability true_value = 1.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/brier_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Brier Score — brier_score","text":"numeric value Brier Score, .e. mean squared error given probability forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/brier_score.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Brier Score — brier_score","text":"Brier score proper score rule assesses accuracy probabilistic binary predictions. outcomes can either 0 1, predictions must probability true outcome 1. Brier Score computed mean squared error probabilistic prediction true outcome. $$   \\textrm{Brier\\_Score} = \\frac{1}{N} \\sum_{t = 1}^{n} (\\textrm{prediction}_t -   \\textrm{outcome}_t)^2 $$","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/brier_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Brier Score — brier_score","text":"","code":"true_values <- sample(c(0, 1), size = 30, replace = TRUE) predictions <- runif(n = 30, min = 0, max = 1)  brier_score(true_values, predictions) #>  [1] 0.3523912288 0.1652343826 0.7890216269 0.0009722258 0.0762192058 #>  [6] 0.5930475789 0.2965580809 0.1823641411 0.0383435004 0.9586519512 #> [11] 0.0313536327 0.0001852220 0.9999462993 0.0055385235 0.2591137215 #> [16] 0.3501003067 0.4612365644 0.2210000000 0.6576505469 0.1554480141 #> [21] 0.6381849505 0.0464214581 0.0014798787 0.0853600302 0.9918920937 #> [26] 0.7998561744 0.1744389547 0.0871353702 0.3980443052 0.1664929109"},{"path":"https://epiforecasts.io/scoringutils/reference/check_equal_length.html","id":null,"dir":"Reference","previous_headings":"","what":"Check Length — check_equal_length","title":"Check Length — check_equal_length","text":"Check whether variables length","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_equal_length.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check Length — check_equal_length","text":"","code":"check_equal_length(..., one_allowed = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/reference/check_equal_length.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check Length — check_equal_length","text":"... variables check one_allowed logical, allow arguments length one can recycled","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_equal_length.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check Length — check_equal_length","text":"function returns NULL, throws error variable lengths differ","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Check forecasts — check_forecasts","title":"Check forecasts — check_forecasts","text":"Function check input data running score(). data come one three different formats: format binary predictions (see example_binary) sample-based format discrete continuous predictions (see example_continuous example_integer) quantile-based format (see example_quantile)","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check forecasts — check_forecasts","text":"","code":"check_forecasts(data)"},{"path":"https://epiforecasts.io/scoringutils/reference/check_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check forecasts — check_forecasts","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check forecasts — check_forecasts","text":"list elements give information scoringutils thinks trying potential issues. target_type type prediction target inferred input: 'binary', values true_value either 0 1 values prediction 0 1, 'discrete' true values integers. 'continuous' . prediction_type inferred type prediction. 'quantile', column called 'quantile', else 'discrete' values prediction integer, else 'continuous. forecast_unit unit single forecast, .e. grouping uniquely defines single forecast. assumed present columns apart following protected columns: c(\"prediction\", \"true_value\", \"sample\", \"quantile\",\"range\", \"boundary\"). important remove unnecessary columns scoring. rows_per_forecast data.frame shows many rows (usually quantiles samples available per forecast. forecast model several entries, forecasts differing numbers quantiles / samples. unique_values data.frame shows many unique values present per model column data. directly show missing values, rather maximum number unique values across whole data. warnings vector warnings. can ignored know . errors vector issues cause error running score(). messages verbal explanation information provided .","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/reference/check_forecasts.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Check forecasts — check_forecasts","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Check forecasts — check_forecasts","text":"","code":"check <- check_forecasts(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. print(check) #> Your forecasts seem to be for a target of the following type: #> $target_type #> [1] \"integer\" #>  #> and in the following format: #> $prediction_type #> [1] \"quantile\" #>  #> The unit of a single forecast is defined by: #> $forecast_unit #> [1] \"location\"        \"target_end_date\" \"target_type\"     \"location_name\"   #> [5] \"forecast_date\"   \"model\"           \"horizon\"         #>  #> Cleaned data, rows with NA values in prediction or true_value removed: #> $cleaned_data #>        location target_end_date target_type true_value location_name #>     1:       DE      2021-05-08       Cases     106987       Germany #>     2:       DE      2021-05-08       Cases     106987       Germany #>     3:       DE      2021-05-08       Cases     106987       Germany #>     4:       DE      2021-05-08       Cases     106987       Germany #>     5:       DE      2021-05-08       Cases     106987       Germany #>    ---                                                               #> 20397:       IT      2021-07-24      Deaths         78         Italy #> 20398:       IT      2021-07-24      Deaths         78         Italy #> 20399:       IT      2021-07-24      Deaths         78         Italy #> 20400:       IT      2021-07-24      Deaths         78         Italy #> 20401:       IT      2021-07-24      Deaths         78         Italy #>        forecast_date quantile prediction                 model horizon #>     1:    2021-05-03    0.010      82466 EuroCOVIDhub-ensemble       1 #>     2:    2021-05-03    0.025      86669 EuroCOVIDhub-ensemble       1 #>     3:    2021-05-03    0.050      90285 EuroCOVIDhub-ensemble       1 #>     4:    2021-05-03    0.100      95341 EuroCOVIDhub-ensemble       1 #>     5:    2021-05-03    0.150      99171 EuroCOVIDhub-ensemble       1 #>    ---                                                                 #> 20397:    2021-07-12    0.850        352  epiforecasts-EpiNow2       2 #> 20398:    2021-07-12    0.900        397  epiforecasts-EpiNow2       2 #> 20399:    2021-07-12    0.950        499  epiforecasts-EpiNow2       2 #> 20400:    2021-07-12    0.975        611  epiforecasts-EpiNow2       2 #> 20401:    2021-07-12    0.990        719  epiforecasts-EpiNow2       2 #>  #> Number of unique values per column per model: #> $unique_values #>                    model location target_end_date target_type true_value #> 1: EuroCOVIDhub-ensemble        4              12           2         96 #> 2: EuroCOVIDhub-baseline        4              12           2         96 #> 3:  epiforecasts-EpiNow2        4              12           2         95 #> 4:       UMass-MechBayes        4              12           1         48 #>    location_name forecast_date quantile prediction horizon #> 1:             4            11       23       3969       3 #> 2:             4            11       23       3733       3 #> 3:             4            11       23       3903       3 #> 4:             4            11       23       1058       3 #>  #> $messages #> [1] \"144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.\" #>  check_forecasts(example_binary) #> The following messages were produced when checking inputs: #> 1.  144 values for `true_value` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #> 2.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #> Your forecasts seem to be for a target of the following type: #> $target_type #> [1] \"binary\" #>  #> and in the following format: #> $prediction_type #> [1] \"continuous\" #>  #> The unit of a single forecast is defined by: #> $forecast_unit #> [1] \"location\"        \"location_name\"   \"target_end_date\" \"target_type\"     #> [5] \"forecast_date\"   \"model\"           \"horizon\"         #>  #> Cleaned data, rows with NA values in prediction or true_value removed: #> $cleaned_data #>      location location_name target_end_date target_type forecast_date #>   1:       DE       Germany      2021-05-08       Cases    2021-05-03 #>   2:       DE       Germany      2021-05-08       Cases    2021-05-03 #>   3:       DE       Germany      2021-05-08       Cases    2021-05-03 #>   4:       DE       Germany      2021-05-08      Deaths    2021-05-03 #>   5:       DE       Germany      2021-05-08      Deaths    2021-05-03 #>  ---                                                                  #> 883:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 884:       IT         Italy      2021-07-24      Deaths    2021-07-05 #> 885:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 886:       IT         Italy      2021-07-24      Deaths    2021-07-05 #> 887:       IT         Italy      2021-07-24      Deaths    2021-07-12 #>                      model horizon prediction true_value #>   1: EuroCOVIDhub-ensemble       1      0.375          0 #>   2: EuroCOVIDhub-baseline       1      0.475          0 #>   3:  epiforecasts-EpiNow2       1      0.425          0 #>   4: EuroCOVIDhub-ensemble       1      0.425          0 #>   5: EuroCOVIDhub-baseline       1      0.500          1 #>  ---                                                     #> 883: EuroCOVIDhub-baseline       2      0.250          0 #> 884:       UMass-MechBayes       3      0.475          0 #> 885:       UMass-MechBayes       2      0.450          0 #> 886:  epiforecasts-EpiNow2       3      0.375          0 #> 887:  epiforecasts-EpiNow2       2      0.300          0 #>  #> Number of unique values per column per model: #> $unique_values #>                    model location location_name target_end_date target_type #> 1: EuroCOVIDhub-ensemble        4             4              12           2 #> 2: EuroCOVIDhub-baseline        4             4              12           2 #> 3:  epiforecasts-EpiNow2        4             4              12           2 #> 4:       UMass-MechBayes        4             4              12           1 #>    forecast_date horizon prediction true_value #> 1:            11       3         13          2 #> 2:            11       3         20          2 #> 3:            11       3         13          2 #> 4:            11       3         10          2 #>  #> $messages #> [1] \"144 values for `true_value` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.\" #> [2] \"144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.\" #>"},{"path":"https://epiforecasts.io/scoringutils/reference/check_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Check whether the desired metrics are available in scoringutils — check_metrics","title":"Check whether the desired metrics are available in scoringutils — check_metrics","text":"Helper function check whether desired metrics available. input NULL, metrics returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check whether the desired metrics are available in scoringutils — check_metrics","text":"","code":"check_metrics(metrics)"},{"path":"https://epiforecasts.io/scoringutils/reference/check_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check whether the desired metrics are available in scoringutils — check_metrics","text":"metrics character vector desired metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check whether the desired metrics are available in scoringutils — check_metrics","text":"character vector metrics can used downstream computation","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_not_null.html","id":null,"dir":"Reference","previous_headings":"","what":"Check Variable is not NULL — check_not_null","title":"Check Variable is not NULL — check_not_null","text":"Check whether certain variable NULL return name variable function call variable missing. function helper function called within functions","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_not_null.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check Variable is not NULL — check_not_null","text":"","code":"check_not_null(...)"},{"path":"https://epiforecasts.io/scoringutils/reference/check_not_null.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check Variable is not NULL — check_not_null","text":"... variables check","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_not_null.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check Variable is not NULL — check_not_null","text":"function returns NULL, throws error variable missing.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_predictions.html","id":null,"dir":"Reference","previous_headings":"","what":"Check Prediction Input For Lower-level Scoring Functions — check_predictions","title":"Check Prediction Input For Lower-level Scoring Functions — check_predictions","text":"Helper function check inputs lower-level score functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_predictions.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check Prediction Input For Lower-level Scoring Functions — check_predictions","text":"","code":"check_predictions(   predictions,   true_values = NULL,   type = c(\"continuous\", \"integer\", \"binary\"),   class = c(\"vector\", \"matrix\") )"},{"path":"https://epiforecasts.io/scoringutils/reference/check_predictions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check Prediction Input For Lower-level Scoring Functions — check_predictions","text":"predictions object predictions. Depending whether class = vector class = \"matrix\" can either vector length n (corresponding length true_values) nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples true_values vector true observed values size n type character, one \"continuous\" (default), \"integer\" \"binary\" defines type forecast class character, either \"vector\" (default) \"matrix\" determines class input correspond ","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_quantiles.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that quantiles are valid — check_quantiles","title":"Check that quantiles are valid — check_quantiles","text":"Helper function check input quantiles valid. Quantiles must range specified, increase monotonically, contain duplicates. used bias_range() bias_quantile() provide informative errors users.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_quantiles.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that quantiles are valid — check_quantiles","text":"","code":"check_quantiles(quantiles, name = \"quantiles\", range = c(0, 1))"},{"path":"https://epiforecasts.io/scoringutils/reference/check_quantiles.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that quantiles are valid — check_quantiles","text":"quantiles Numeric vector quantiles check name Character name use error messages range Numeric vector giving allowed range","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_quantiles.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that quantiles are valid — check_quantiles","text":"None. Function errors quantiles invalid.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_summary_params.html","id":null,"dir":"Reference","previous_headings":"","what":"Check input parameters for summarise_scores() — check_summary_params","title":"Check input parameters for summarise_scores() — check_summary_params","text":"helper function check input parameters score().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_summary_params.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check input parameters for summarise_scores() — check_summary_params","text":"","code":"check_summary_params(scores, by, relative_skill, baseline, metric)"},{"path":"https://epiforecasts.io/scoringutils/reference/check_summary_params.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check input parameters for summarise_scores() — check_summary_params","text":"scores data.table scores produced score(). character vector column names summarise scores . Default NULL, meaning summary takes place summarising samples quantiles (case quantile-based forecasts), one score per forecast defined unit single forecast (rather one score every sample quantile). unit single forecast determined columns present input data correspond metric produced score(), indicate indicate grouping forecasts (example may one forecast per day, location model). Adding additional, unrelated, columns may alter results unpredictable way. relative_skill logical, whether compute relative performance models based pairwise comparisons. TRUE (default FALSE), column called 'model' must present input data. information computation relative skill, see pairwise_comparison(). Relative skill calculated aggregation level specified . baseline character string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. metric Deprecated 1.1.0. Use relative_skill_metric instead.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_true_values.html","id":null,"dir":"Reference","previous_headings":"","what":"Check Observed Value Input For Lower-level Scoring Functions — check_true_values","title":"Check Observed Value Input For Lower-level Scoring Functions — check_true_values","text":"Helper function check inputs lower-level score functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_true_values.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check Observed Value Input For Lower-level Scoring Functions — check_true_values","text":"","code":"check_true_values(true_values, type = c(\"continuous\", \"integer\", \"binary\"))"},{"path":"https://epiforecasts.io/scoringutils/reference/check_true_values.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check Observed Value Input For Lower-level Scoring Functions — check_true_values","text":"true_values vector true observed values size n type character, one \"continuous\" (default), \"integer\" \"binary\" defines type forecast","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/collapse_messages.html","id":null,"dir":"Reference","previous_headings":"","what":"Collapse several messages to one — collapse_messages","title":"Collapse several messages to one — collapse_messages","text":"Internal helper function facilitate generating messages warnings check_forecasts()","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/collapse_messages.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Collapse several messages to one — collapse_messages","text":"","code":"collapse_messages(type = \"messages\", messages)"},{"path":"https://epiforecasts.io/scoringutils/reference/collapse_messages.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Collapse several messages to one — collapse_messages","text":"type character, either \"messages\", \"warnings\" \"errors\" messages messages warnings collapse","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/collapse_messages.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Collapse several messages to one — collapse_messages","text":"string message warning","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/compare_two_models.html","id":null,"dir":"Reference","previous_headings":"","what":"Compare Two Models Based on Subset of Common Forecasts — compare_two_models","title":"Compare Two Models Based on Subset of Common Forecasts — compare_two_models","text":"function compares two models based subset forecasts models made prediction. gets called pairwise_comparison_one_group(), handles comparison multiple models single set forecasts (subsets forecasts distinguished). pairwise_comparison_one_group() turn gets called pairwise_comparison() can handle pairwise comparisons set forecasts multiple subsets, e.g. pairwise comparisons one set forecasts, done separately two different forecast targets.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/compare_two_models.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compare Two Models Based on Subset of Common Forecasts — compare_two_models","text":"","code":"compare_two_models(   scores,   name_model1,   name_model2,   metric,   one_sided = FALSE,   test_type = c(\"non_parametric\", \"permutation\"),   n_permutations = 999 )"},{"path":"https://epiforecasts.io/scoringutils/reference/compare_two_models.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compare Two Models Based on Subset of Common Forecasts — compare_two_models","text":"scores data.table scores produced score(). name_model1 character, name first model name_model2 character, name model compare metric character vector length one metric comparison . default \"auto\", meaning either \"interval_score\", \"crps\", \"brier_score\" selected available. See available_metrics() available metrics. one_sided Boolean, default FALSE, whether two conduct one-sided instead two-sided test determine significance pairwise comparison. test_type character, either \"non_parametric\" (default) \"permutation\". determines kind test shall conducted determine p-values. n_permutations numeric, number permutations permutation test. Default 999.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/compare_two_models.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compare Two Models Based on Subset of Common Forecasts — compare_two_models","text":"Johannes Bracher, johannes.bracher@kit.edu Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/correlation.html","id":null,"dir":"Reference","previous_headings":"","what":"Correlation Between Metrics — correlation","title":"Correlation Between Metrics — correlation","text":"Calculate correlation different metrics data.frame scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/correlation.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Correlation Between Metrics — correlation","text":"","code":"correlation(scores, metrics = NULL)"},{"path":"https://epiforecasts.io/scoringutils/reference/correlation.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Correlation Between Metrics — correlation","text":"scores data.table scores produced score(). metrics character vector metrics show. set NULL (default), metrics present scores shown","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/correlation.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Correlation Between Metrics — correlation","text":"data.table correlations different metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/correlation.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Correlation Between Metrics — correlation","text":"","code":"scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. correlation(scores) #> Warning: There is a column called 'quantile' in the scores. Usually, you should call 'summarise_scores()' to summarise over quantiles and obtain one score per forecast before calculating correlations. You can ignore this warning if you know what you're doing. #>    interval_score dispersion underprediction overprediction coverage #> 1:           1.00       0.41            0.31           0.93    -0.23 #> 2:           0.41       1.00            0.10           0.27    -0.05 #> 3:           0.31       0.10            1.00          -0.03    -0.23 #> 4:           0.93       0.27           -0.03           1.00    -0.16 #> 5:          -0.23      -0.05           -0.23          -0.16     1.00 #> 6:          -0.23      -0.08           -0.20          -0.17     0.78 #> 7:           0.10       0.10           -0.32           0.21     0.03 #> 8:           0.97       0.47            0.30           0.89    -0.21 #>    coverage_deviation  bias ae_median             metric #> 1:              -0.23  0.10      0.97     interval_score #> 2:              -0.08  0.10      0.47         dispersion #> 3:              -0.20 -0.32      0.30    underprediction #> 4:              -0.17  0.21      0.89     overprediction #> 5:               0.78  0.03     -0.21           coverage #> 6:               1.00  0.04     -0.26 coverage_deviation #> 7:               0.04  1.00      0.10               bias #> 8:              -0.26  0.10      1.00          ae_median"},{"path":"https://epiforecasts.io/scoringutils/reference/crps_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Ranked Probability Score — crps_sample","title":"Ranked Probability Score — crps_sample","text":"Wrapper around crps_sample() function scoringRules package. Can used continuous well integer valued forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/crps_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Ranked Probability Score — crps_sample","text":"","code":"crps_sample(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/crps_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Ranked Probability Score — crps_sample","text":"true_values vector true observed values size n predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/crps_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Ranked Probability Score — crps_sample","text":"vector scoring values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/crps_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Ranked Probability Score — crps_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/crps_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Ranked Probability Score — crps_sample","text":"","code":"true_values <- rpois(30, lambda = 1:30) predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) crps_sample(true_values, predictions) #>  [1] 0.493375 0.289675 1.255600 0.650175 0.590175 1.180775 0.634925 0.707700 #>  [9] 0.771975 1.169275 0.812675 0.900500 3.160875 2.941025 1.229875 0.919850 #> [17] 2.975750 1.115300 1.921425 2.843275 2.219425 5.387575 1.101775 1.375850 #> [25] 1.481175 1.413175 1.464050 4.910000 1.536100 1.311625"},{"path":"https://epiforecasts.io/scoringutils/reference/delete_columns.html","id":null,"dir":"Reference","previous_headings":"","what":"Delete Columns From a Data.table — delete_columns","title":"Delete Columns From a Data.table — delete_columns","text":"take vector column names delete columns present data.table","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/delete_columns.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Delete Columns From a Data.table — delete_columns","text":"","code":"delete_columns(df, cols_to_delete, make_unique = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/reference/delete_columns.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Delete Columns From a Data.table — delete_columns","text":"df data.table data.frame columns shall deleted cols_to_delete character vector names columns deleted make_unique whether make data set unique removing columns","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/delete_columns.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Delete Columns From a Data.table — delete_columns","text":"data.table","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/dss_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Dawid-Sebastiani Score — dss_sample","title":"Dawid-Sebastiani Score — dss_sample","text":"Wrapper around dss_sample() function scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/dss_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Dawid-Sebastiani Score — dss_sample","text":"","code":"dss_sample(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/dss_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Dawid-Sebastiani Score — dss_sample","text":"true_values vector true observed values size n predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/dss_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Dawid-Sebastiani Score — dss_sample","text":"vector scoring values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/dss_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Dawid-Sebastiani Score — dss_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/dss_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Dawid-Sebastiani Score — dss_sample","text":"","code":"true_values <- rpois(30, lambda = 1:30) predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) dss_sample(true_values, predictions) #>  [1] 0.9390834 0.7371880 2.6558613 2.4016219 1.4872945 3.4290742 3.0316810 #>  [8] 4.0471224 2.3517762 3.2996424 2.4827979 3.0100034 4.9783075 3.2694590 #> [15] 3.9500410 3.6489525 3.3176212 3.4205666 3.0294219 3.3162573 3.2252580 #> [22] 3.0564948 3.4780737 4.0662328 4.7203740 4.2505824 3.9260245 4.0784721 #> [29] 3.3347969 4.6343248"},{"path":"https://epiforecasts.io/scoringutils/reference/example_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Binary Forecast Example Data — example_binary","title":"Binary Forecast Example Data — example_binary","text":"data set binary predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Binary Forecast Example Data — example_binary","text":"","code":"example_binary"},{"path":"https://epiforecasts.io/scoringutils/reference/example_binary.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Binary Forecast Example Data — example_binary","text":"data frame 346 rows 10 columns: location country prediction made location_name name country prediction made target_end_date date prediction made target_type target predicted (cases deaths) true_value true observed values forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks prediction predicted value","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_binary.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Binary Forecast Example Data — example_binary","text":"https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/ # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Binary Forecast Example Data — example_binary","text":"Predictions data set constructed based continuous example data looking number samples mean prediction. outcome constructed whether actually observed value mean prediction. understood sound statistical practice, rather practical way create example data set. data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_continuous.html","id":null,"dir":"Reference","previous_headings":"","what":"Continuous Forecast Example Data — example_continuous","title":"Continuous Forecast Example Data — example_continuous","text":"data set continuous predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_continuous.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Continuous Forecast Example Data — example_continuous","text":"","code":"example_continuous"},{"path":"https://epiforecasts.io/scoringutils/reference/example_continuous.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Continuous Forecast Example Data — example_continuous","text":"data frame 13,429 rows 10 columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) true_value true observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks prediction predicted value sample id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_continuous.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Continuous Forecast Example Data — example_continuous","text":"https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/ # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_continuous.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Continuous Forecast Example Data — example_continuous","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_integer.html","id":null,"dir":"Reference","previous_headings":"","what":"Integer Forecast Example Data — example_integer","title":"Integer Forecast Example Data — example_integer","text":"data set integer predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_integer.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Integer Forecast Example Data — example_integer","text":"","code":"example_integer"},{"path":"https://epiforecasts.io/scoringutils/reference/example_integer.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Integer Forecast Example Data — example_integer","text":"data frame 13,429 rows 10 columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) true_value true observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks prediction predicted value sample id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_integer.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Integer Forecast Example Data — example_integer","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Point Forecast Example Data — example_point","title":"Point Forecast Example Data — example_point","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub. data set like quantile example data, median replaced point forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Point Forecast Example Data — example_point","text":"","code":"example_point"},{"path":"https://epiforecasts.io/scoringutils/reference/example_point.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Point Forecast Example Data — example_point","text":"data frame location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) true_value true observed values location_name name country prediction made forecast_date date prediction made quantile quantile corresponding prediction prediction predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_point.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Point Forecast Example Data — example_point","text":"https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/ # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_point.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Point Forecast Example Data — example_point","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile Example Data — example_quantile","title":"Quantile Example Data — example_quantile","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile Example Data — example_quantile","text":"","code":"example_quantile"},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Quantile Example Data — example_quantile","text":"data frame location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) true_value true observed values location_name name country prediction made forecast_date date prediction made quantile quantile corresponding prediction prediction predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Quantile Example Data — example_quantile","text":"https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/ # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Quantile Example Data — example_quantile","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile_forecasts_only.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile Example Data - Forecasts only — example_quantile_forecasts_only","title":"Quantile Example Data - Forecasts only — example_quantile_forecasts_only","text":"data set quantile predictions COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile_forecasts_only.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile Example Data - Forecasts only — example_quantile_forecasts_only","text":"","code":"example_quantile_forecasts_only"},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile_forecasts_only.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Quantile Example Data - Forecasts only — example_quantile_forecasts_only","text":"data frame 7,581 rows 9 columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) forecast_date date prediction made quantile quantile corresponding prediction prediction predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile_forecasts_only.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Quantile Example Data - Forecasts only — example_quantile_forecasts_only","text":"https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/ # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile_forecasts_only.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Quantile Example Data - Forecasts only — example_quantile_forecasts_only","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_truth_only.html","id":null,"dir":"Reference","previous_headings":"","what":"Truth data only — example_truth_only","title":"Truth data only — example_truth_only","text":"data set truth values COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_truth_only.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Truth data only — example_truth_only","text":"","code":"example_truth_only"},{"path":"https://epiforecasts.io/scoringutils/reference/example_truth_only.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Truth data only — example_truth_only","text":"data frame 140 rows 5 columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) true_value true observed values location_name name country prediction made","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_truth_only.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Truth data only — example_truth_only","text":"https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/ # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_truth_only.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Truth data only — example_truth_only","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/find_duplicates.html","id":null,"dir":"Reference","previous_headings":"","what":"Find duplicate forecasts — find_duplicates","title":"Find duplicate forecasts — find_duplicates","text":"Helper function identify duplicate forecasts, .e. instances one forecast prediction target.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/find_duplicates.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Find duplicate forecasts — find_duplicates","text":"","code":"find_duplicates(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/reference/find_duplicates.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Find duplicate forecasts — find_duplicates","text":"data data.frame used score() forecast_unit character vector column names define unit single forecast. missing function tries infer unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/find_duplicates.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Find duplicate forecasts — find_duplicates","text":"data.frame rows duplicate forecast found","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/find_duplicates.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Find duplicate forecasts — find_duplicates","text":"","code":"example <- rbind(example_quantile, example_quantile[1000:1010]) find_duplicates(example) #>     location target_end_date target_type true_value location_name forecast_date #>  1:       DE      2021-05-22      Deaths       1285       Germany    2021-05-17 #>  2:       DE      2021-05-22      Deaths       1285       Germany    2021-05-17 #>  3:       DE      2021-05-22      Deaths       1285       Germany    2021-05-17 #>  4:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #>  5:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #>  6:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #>  7:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #>  8:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #>  9:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 10:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 11:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 12:       DE      2021-05-22      Deaths       1285       Germany    2021-05-17 #> 13:       DE      2021-05-22      Deaths       1285       Germany    2021-05-17 #> 14:       DE      2021-05-22      Deaths       1285       Germany    2021-05-17 #> 15:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 16:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 17:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 18:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 19:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 20:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 21:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 22:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #>     location target_end_date target_type true_value location_name forecast_date #>     quantile prediction                 model horizon #>  1:    0.950       1464  epiforecasts-EpiNow2       1 #>  2:    0.975       1642  epiforecasts-EpiNow2       1 #>  3:    0.990       1951  epiforecasts-EpiNow2       1 #>  4:    0.010      28999 EuroCOVIDhub-ensemble       3 #>  5:    0.025      32612 EuroCOVIDhub-ensemble       3 #>  6:    0.050      36068 EuroCOVIDhub-ensemble       3 #>  7:    0.100      41484 EuroCOVIDhub-ensemble       3 #>  8:    0.150      47110 EuroCOVIDhub-ensemble       3 #>  9:    0.200      50929 EuroCOVIDhub-ensemble       3 #> 10:    0.250      54561 EuroCOVIDhub-ensemble       3 #> 11:    0.300      57739 EuroCOVIDhub-ensemble       3 #> 12:    0.950       1464  epiforecasts-EpiNow2       1 #> 13:    0.975       1642  epiforecasts-EpiNow2       1 #> 14:    0.990       1951  epiforecasts-EpiNow2       1 #> 15:    0.010      28999 EuroCOVIDhub-ensemble       3 #> 16:    0.025      32612 EuroCOVIDhub-ensemble       3 #> 17:    0.050      36068 EuroCOVIDhub-ensemble       3 #> 18:    0.100      41484 EuroCOVIDhub-ensemble       3 #> 19:    0.150      47110 EuroCOVIDhub-ensemble       3 #> 20:    0.200      50929 EuroCOVIDhub-ensemble       3 #> 21:    0.250      54561 EuroCOVIDhub-ensemble       3 #> 22:    0.300      57739 EuroCOVIDhub-ensemble       3 #>     quantile prediction                 model horizon"},{"path":"https://epiforecasts.io/scoringutils/reference/geom_mean_helper.html","id":null,"dir":"Reference","previous_headings":"","what":"Calculate Geometric Mean — geom_mean_helper","title":"Calculate Geometric Mean — geom_mean_helper","text":"Calculate Geometric Mean","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/geom_mean_helper.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Calculate Geometric Mean — geom_mean_helper","text":"","code":"geom_mean_helper(x)"},{"path":"https://epiforecasts.io/scoringutils/reference/geom_mean_helper.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Calculate Geometric Mean — geom_mean_helper","text":"x numeric vector values calculate geometric mean","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/geom_mean_helper.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Calculate Geometric Mean — geom_mean_helper","text":"geometric mean values x","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Get unit of a single forecast — get_forecast_unit","title":"Get unit of a single forecast — get_forecast_unit","text":"Helper function get unit single forecast, .e. column names define single forecast made ","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get unit of a single forecast — get_forecast_unit","text":"","code":"get_forecast_unit(data)"},{"path":"https://epiforecasts.io/scoringutils/reference/get_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get unit of a single forecast — get_forecast_unit","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get unit of a single forecast — get_forecast_unit","text":"character vector column names define unit single forecast","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_prediction_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Get prediction type of a forecast — get_prediction_type","title":"Get prediction type of a forecast — get_prediction_type","text":"Internal helper function get prediction type forecast. inferred based properties values prediction column.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_prediction_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get prediction type of a forecast — get_prediction_type","text":"","code":"get_prediction_type(data)"},{"path":"https://epiforecasts.io/scoringutils/reference/get_prediction_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get prediction type of a forecast — get_prediction_type","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_prediction_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get prediction type of a forecast — get_prediction_type","text":"Character vector length one either \"quantile\", \"integer\", \"continuous\".","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_protected_columns.html","id":null,"dir":"Reference","previous_headings":"","what":"Get protected columns from a data frame — get_protected_columns","title":"Get protected columns from a data frame — get_protected_columns","text":"Helper function get names columns data frame protected columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_protected_columns.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get protected columns from a data frame — get_protected_columns","text":"","code":"get_protected_columns(data)"},{"path":"https://epiforecasts.io/scoringutils/reference/get_protected_columns.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get protected columns from a data frame — get_protected_columns","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_protected_columns.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get protected columns from a data frame — get_protected_columns","text":"character vector names protected columns data","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_target_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Get type of the target true values of a forecast — get_target_type","title":"Get type of the target true values of a forecast — get_target_type","text":"Internal helper function get type target true values forecast. inferred based columns present data.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_target_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get type of the target true values of a forecast — get_target_type","text":"","code":"get_target_type(data)"},{"path":"https://epiforecasts.io/scoringutils/reference/get_target_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get type of the target true values of a forecast — get_target_type","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_target_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get type of the target true values of a forecast — get_target_type","text":"Character vector length one either \"binary\", \"integer\", \"continuous\"","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/infer_rel_skill_metric.html","id":null,"dir":"Reference","previous_headings":"","what":"Infer metric for pairwise comparisons — infer_rel_skill_metric","title":"Infer metric for pairwise comparisons — infer_rel_skill_metric","text":"Helper function infer metric pairwise comparisons shall made. function simply checks names available columns chooses widely used metric.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/infer_rel_skill_metric.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Infer metric for pairwise comparisons — infer_rel_skill_metric","text":"","code":"infer_rel_skill_metric(scores)"},{"path":"https://epiforecasts.io/scoringutils/reference/infer_rel_skill_metric.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Infer metric for pairwise comparisons — infer_rel_skill_metric","text":"scores data.table scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/interval_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval Score — interval_score","title":"Interval Score — interval_score","text":"Proper Scoring Rule score quantile predictions, following Gneiting Raftery (2007). Smaller values better. score computed $$ \\textrm{score} = (\\textrm{upper} - \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{lower}  - \\textrm{true\\_value}) * \\mathbf{1}(\\textrm{true\\_value} < \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{true\\_value} - \\textrm{upper}) * \\mathbf{1}(\\textrm{true\\_value} > \\textrm{upper}) $$ \\(\\mathbf{1}()\\) indicator function indicates much outside prediction interval. \\(\\alpha\\) decimal value indicates much outside prediction interval. improve usability, user asked provide interval range percentage terms, .e. interval_range = 90 (percent) 90 percent prediction interval. Correspondingly, user provide 5% 95% quantiles (corresponding alpha 0.1). specific distribution assumed, range symmetric (.e use 0.1 quantile lower bound 0.7 quantile upper). Non-symmetric quantiles can scored using function quantile_score().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/interval_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval Score — interval_score","text":"","code":"interval_score(   true_values,   lower,   upper,   interval_range,   weigh = TRUE,   separate_results = FALSE )"},{"path":"https://epiforecasts.io/scoringutils/reference/interval_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval Score — interval_score","text":"true_values vector true observed values size n lower vector size n prediction lower quantile given range upper vector size n prediction upper quantile given range interval_range range prediction intervals. .e. forecasting 0.05 0.95 quantile, interval_range 90. Can either single number vector size n, range changes different forecasts scored. corresponds (100-alpha)/100 Gneiting Raftery (2007). Internally, range transformed alpha. weigh TRUE, weigh score alpha / 2, can averaged interval score , limit, corresponds CRPS. Alpha decimal value  represents much outside central prediction interval (e.g. 90 percent central prediction interval, alpha 0.1) Default: TRUE. separate_results TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/interval_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval Score — interval_score","text":"vector scoring values, list separate entries separate_results TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/interval_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Interval Score — interval_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/interval_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval Score — interval_score","text":"","code":"true_values <- rnorm(30, mean = 1:30) interval_range <- rep(90, 30) alpha <- (100 - interval_range) / 100 lower <- qnorm(alpha / 2, rnorm(30, mean = 1:30)) upper <- qnorm((1 - alpha / 2), rnorm(30, mean = 1:30))  interval_score(   true_values = true_values,   lower = lower,   upper = upper,   interval_range = interval_range ) #>  [1] 0.27293349 0.18546570 0.43039104 1.04487964 0.09440684 2.16892458 #>  [7] 0.15922210 0.15676948 0.23634107 0.17408376 0.64563459 0.14810861 #> [13] 0.23310837 0.18772632 0.15344188 0.17462532 0.26430655 0.14241786 #> [19] 0.22042963 0.15942879 0.19539303 0.24145151 0.10768440 0.32457468 #> [25] 0.18910149 0.53891555 0.08214458 0.10659996 0.20495251 0.20553410  # gives a warning, as the interval_range should likely be 50 instead of 0.5 interval_score(true_value = 4, upper = 2, lower = 8, interval_range = 0.5) #> Warning: Found interval ranges between 0 and 1. Are you sure that's right? An interval range of 0.5 e.g. implies a (49.75%, 50.25%) prediction interval. If you want to score a (25%, 75%) prediction interval, set interval_range = 50. #> This warning is displayed once per session. #> [1] 3.015  # example with missing values and separate results interval_score(   true_values = c(true_values, NA),   lower = c(lower, NA),   upper = c(NA, upper),   separate_results = TRUE,   interval_range = 90 ) #> $interval_score #>  [1]         NA 1.46364084 0.33481206 0.93192886 0.18168710 2.08230042 #>  [7] 0.08550798 0.07930845 0.61179960 0.15602139 2.59530642 0.15752202 #> [13] 0.19052900 0.11386365 0.08943640 0.14771378 0.25058509 1.20513877 #> [19] 0.20827582 0.11900102 0.08399175 0.20612850 1.56251164 1.52590516 #> [25] 0.14197488 1.27746481 0.04878732 0.51578138 0.39172486 0.19794848 #> [31]         NA #>  #> $dispersion #>  [1]           NA  0.117563356  0.008662780  0.015151200  0.181687097 #>  [6] -0.061200790  0.085507980  0.079308453  0.156692099  0.156021392 #> [11]  0.138894171  0.157522019  0.190528997  0.113863654  0.089436398 #> [16]  0.147713782  0.250585090  0.010752790  0.208275816  0.119001016 #> [21]  0.083991752  0.206128504  0.003812329  0.096152881  0.141974876 #> [26]  0.081811218  0.048787325  0.022467765  0.113553512  0.197948479 #> [31]           NA #>  #> $underprediction #>  [1]        NA 1.3460775 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #>  [8] 0.0000000 0.4551075 0.0000000 2.4564123 0.0000000 0.0000000 0.0000000 #> [15] 0.0000000 0.0000000 0.0000000 1.1943860 0.0000000 0.0000000 0.0000000 #> [22] 0.0000000 1.5586993 1.4297523 0.0000000 1.1956536 0.0000000 0.4933136 #> [29] 0.2781713 0.0000000        NA #>  #> $overprediction #>  [1] 0.0000000 0.0000000 0.3261493 0.9167777 0.0000000 2.1435012 0.0000000 #>  [8] 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #> [15] 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #> [22] 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #> [29] 0.0000000 0.0000000        NA #>"},{"path":"https://epiforecasts.io/scoringutils/reference/is_scoringutils_check.html","id":null,"dir":"Reference","previous_headings":"","what":"Check whether object has been checked with check_forecasts() — is_scoringutils_check","title":"Check whether object has been checked with check_forecasts() — is_scoringutils_check","text":"Helper function determine whether object checked passed check_forecasts().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/is_scoringutils_check.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check whether object has been checked with check_forecasts() — is_scoringutils_check","text":"","code":"is_scoringutils_check(data)"},{"path":"https://epiforecasts.io/scoringutils/reference/is_scoringutils_check.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check whether object has been checked with check_forecasts() — is_scoringutils_check","text":"data object class scoringutils_check() produced check_forecasts().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/is_scoringutils_check.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check whether object has been checked with check_forecasts() — is_scoringutils_check","text":"Logical, either TRUE FALSE","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/log_shift.html","id":null,"dir":"Reference","previous_headings":"","what":"Log transformation with an additive shift — log_shift","title":"Log transformation with an additive shift — log_shift","text":"Function shifts value offset applies natural logarithm .","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/log_shift.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Log transformation with an additive shift — log_shift","text":"","code":"log_shift(x, offset = 0, base = exp(1))"},{"path":"https://epiforecasts.io/scoringutils/reference/log_shift.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Log transformation with an additive shift — log_shift","text":"x vector input values transformed offset number add input value taking natural logarithm base positive complex number: base respect logarithms computed. Defaults e = exp(1).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/log_shift.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Log transformation with an additive shift — log_shift","text":"numeric vector transformed values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/log_shift.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Log transformation with an additive shift — log_shift","text":"output computed log(x + offset)","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/log_shift.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Log transformation with an additive shift — log_shift","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722  https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/log_shift.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Log transformation with an additive shift — log_shift","text":"","code":"log_shift(1:10) #>  [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #>  [8] 2.0794415 2.1972246 2.3025851 log_shift(0:9, offset = 1) #>  [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #>  [8] 2.0794415 2.1972246 2.3025851  transform_forecasts(   example_quantile[true_value > 0, ],   fun = log_shift,   offset = 1  ) #>        location target_end_date target_type   true_value location_name #>     1:       DE      2021-01-02       Cases 1.273000e+05       Germany #>     2:       DE      2021-01-02      Deaths 4.534000e+03       Germany #>     3:       DE      2021-01-09       Cases 1.549220e+05       Germany #>     4:       DE      2021-01-09      Deaths 6.117000e+03       Germany #>     5:       DE      2021-01-16       Cases 1.101830e+05       Germany #>    ---                                                                 #> 40672:       IT      2021-07-24      Deaths 4.369448e+00         Italy #> 40673:       IT      2021-07-24      Deaths 4.369448e+00         Italy #> 40674:       IT      2021-07-24      Deaths 4.369448e+00         Italy #> 40675:       IT      2021-07-24      Deaths 4.369448e+00         Italy #> 40676:       IT      2021-07-24      Deaths 4.369448e+00         Italy #>        forecast_date quantile prediction                model horizon   scale #>     1:          <NA>       NA         NA                 <NA>      NA natural #>     2:          <NA>       NA         NA                 <NA>      NA natural #>     3:          <NA>       NA         NA                 <NA>      NA natural #>     4:          <NA>       NA         NA                 <NA>      NA natural #>     5:          <NA>       NA         NA                 <NA>      NA natural #>    ---                                                                        #> 40672:    2021-07-12    0.850   5.866468 epiforecasts-EpiNow2       2     log #> 40673:    2021-07-12    0.900   5.986452 epiforecasts-EpiNow2       2     log #> 40674:    2021-07-12    0.950   6.214608 epiforecasts-EpiNow2       2     log #> 40675:    2021-07-12    0.975   6.416732 epiforecasts-EpiNow2       2     log #> 40676:    2021-07-12    0.990   6.579251 epiforecasts-EpiNow2       2     log"},{"path":"https://epiforecasts.io/scoringutils/reference/logs_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Log Score for Binary outcomes — logs_binary","title":"Log Score for Binary outcomes — logs_binary","text":"Computes Log Score probabilistic forecasts binary outcomes.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/logs_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Log Score for Binary outcomes — logs_binary","text":"","code":"logs_binary(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/logs_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Log Score for Binary outcomes — logs_binary","text":"true_values vector true observed values size n values equal either 0 1 predictions vector predicted probability true_value = 1.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/logs_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Log Score for Binary outcomes — logs_binary","text":"numeric value Log Score, .e. mean squared error given probability forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/logs_binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Log Score for Binary outcomes — logs_binary","text":"Log Score proper score rule suited assessing accuracy probabilistic binary predictions. outcomes can either 0 1, predictions must probability true outcome 1. Log Score computed negative logarithm probability assigned true outcome. Reporting negative logarithm means smaller values better.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/logs_binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Log Score for Binary outcomes — logs_binary","text":"","code":"true_values <- sample(c(0, 1), size = 30, replace = TRUE) predictions <- runif(n = 30, min = 0, max = 1) logs_binary(true_values, predictions) #>  [1] 1.99297273 0.76611191 0.76338707 0.17870758 0.73110535 0.79260410 #>  [7] 1.13429992 0.93289482 0.42156539 0.22229161 0.99467703 0.33112339 #> [13] 3.99983158 1.31923934 0.03721416 0.33524961 0.48426520 0.10720729 #> [19] 1.62850953 2.89970529 0.53315687 0.69754142 0.62716305 1.99952220 #> [25] 0.27048504 0.02102428 0.57074522 0.62028884 0.03238712 0.27333754"},{"path":"https://epiforecasts.io/scoringutils/reference/logs_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Logarithmic score — logs_sample","title":"Logarithmic score — logs_sample","text":"Wrapper around logs_sample() function scoringRules package. Used score continuous predictions. Log Score theory also applicable integer forecasts, problem lies implementation: Log Score needs kernel density estimation, well defined integer-valued Monte Carlo Samples. Log Score can used specific integer valued probability distributions. See scoringRules package details.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/logs_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Logarithmic score — logs_sample","text":"","code":"logs_sample(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/logs_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Logarithmic score — logs_sample","text":"true_values vector true observed values size n predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/logs_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Logarithmic score — logs_sample","text":"vector scoring values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/logs_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Logarithmic score — logs_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/logs_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Logarithmic score — logs_sample","text":"","code":"true_values <- rpois(30, lambda = 1:30) predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) logs_sample(true_values, predictions) #>  [1] 1.485173 1.614623 2.004157 1.706181 1.921952 2.173025 2.016472 1.934295 #>  [9] 2.151720 3.131112 2.865392 2.238183 2.602272 2.888706 5.822223 2.711567 #> [17] 2.434798 2.523354 2.591544 2.425989 2.645081 2.617121 2.386618 2.613739 #> [25] 2.511558 5.206675 2.752121 2.737813 2.629800 3.482330"},{"path":"https://epiforecasts.io/scoringutils/reference/mad_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determine dispersion of a probabilistic forecast — mad_sample","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Determine dispersion probabilistic forecast","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/mad_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"mad_sample(predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/mad_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/mad_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"vector dispersion values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/mad_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Sharpness ability model generate predictions within narrow range dispersion lack thereof. data-independent measure, purely feature forecasts . Dispersion predictive samples corresponding one single true value measured normalised median absolute deviation median predictive samples. details, see mad() explanations given Funk et al. (2019)","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/mad_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, Edmunds WJ (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLoS Comput Biol 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/mad_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) mad_sample(predictions) #>  [1] 1.4826 1.4826 1.4826 1.4826 1.4826 2.9652 2.9652 2.9652 2.9652 2.9652 #> [11] 2.9652 2.9652 2.9652 4.4478 4.4478 4.4478 4.4478 4.4478 4.4478 4.4478 #> [21] 4.4478 4.4478 4.4478 5.9304 4.4478 4.4478 5.1891 5.9304 4.4478 4.4478"},{"path":"https://epiforecasts.io/scoringutils/reference/make_NA.html","id":null,"dir":"Reference","previous_headings":"","what":"Make Rows NA in Data for Plotting — make_NA","title":"Make Rows NA in Data for Plotting — make_NA","text":"Filters data turns values NA data gets passed plot_predictions(). reason allows 'filter' prediction truth data separately. value NA removed subsequent call plot_predictions().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/make_NA.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Make Rows NA in Data for Plotting — make_NA","text":"","code":"make_NA(data = NULL, what = c(\"truth\", \"forecast\", \"both\"), ...)  make_na(data = NULL, what = c(\"truth\", \"forecast\", \"both\"), ...)"},{"path":"https://epiforecasts.io/scoringutils/reference/make_NA.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Make Rows NA in Data for Plotting — make_NA","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary). character vector determines values turned NA. = \"truth\", values column 'true_value' turned NA. = \"forecast\", values column 'prediction' turned NA. = \"\", values column turned NA. ... logical statements used filter data","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/make_NA.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Make Rows NA in Data for Plotting — make_NA","text":"data.table","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/make_NA.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Make Rows NA in Data for Plotting — make_NA","text":"","code":"make_NA (     example_continuous,     what = \"truth\",     target_end_date >= \"2021-07-22\",     target_end_date < \"2021-05-01\"   ) #>        location location_name target_end_date target_type forecast_date #>     1:       DE       Germany      2021-01-02       Cases          <NA> #>     2:       DE       Germany      2021-01-02      Deaths          <NA> #>     3:       DE       Germany      2021-01-09       Cases          <NA> #>     4:       DE       Germany      2021-01-09      Deaths          <NA> #>     5:       DE       Germany      2021-01-16       Cases          <NA> #>    ---                                                                  #> 35620:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 35621:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 35622:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 35623:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 35624:       IT         Italy      2021-07-24      Deaths    2021-07-12 #>                       model horizon prediction sample true_value #>     1:                 <NA>      NA         NA     NA         NA #>     2:                 <NA>      NA         NA     NA         NA #>     3:                 <NA>      NA         NA     NA         NA #>     4:                 <NA>      NA         NA     NA         NA #>     5:                 <NA>      NA         NA     NA         NA #>    ---                                                           #> 35620: epiforecasts-EpiNow2       2  159.84534     36         NA #> 35621: epiforecasts-EpiNow2       2  128.21214     37         NA #> 35622: epiforecasts-EpiNow2       2  190.52560     38         NA #> 35623: epiforecasts-EpiNow2       2  141.06659     39         NA #> 35624: epiforecasts-EpiNow2       2   24.43419     40         NA"},{"path":"https://epiforecasts.io/scoringutils/reference/merge_pred_and_obs.html","id":null,"dir":"Reference","previous_headings":"","what":"Merge Forecast Data And Observations — merge_pred_and_obs","title":"Merge Forecast Data And Observations — merge_pred_and_obs","text":"function less provides wrapper around merge aims handle merging well additional columns present one data sets. doubt, probably merge data sets manually.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/merge_pred_and_obs.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Merge Forecast Data And Observations — merge_pred_and_obs","text":"","code":"merge_pred_and_obs(   forecasts,   observations,   join = c(\"left\", \"full\", \"right\"),   by = NULL )"},{"path":"https://epiforecasts.io/scoringutils/reference/merge_pred_and_obs.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Merge Forecast Data And Observations — merge_pred_and_obs","text":"forecasts data.frame forecast data (can passed score()). observations data.frame observations join character, one c(\"left\", \"full\", \"right\"). Determines type join. Usually, left join appropriate, sometimes may want full join keep dates forecast, ground truth data. character vector denotes columns merge. value column observations removed.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/merge_pred_and_obs.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Merge Forecast Data And Observations — merge_pred_and_obs","text":"data.frame forecasts observations","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/merge_pred_and_obs.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Merge Forecast Data And Observations — merge_pred_and_obs","text":"","code":"forecasts <- example_quantile_forecasts_only observations <- example_truth_only merge_pred_and_obs(forecasts, observations) #>        location target_end_date target_type true_value location_name #>     1:       DE      2021-01-02       Cases     127300       Germany #>     2:       DE      2021-01-02      Deaths       4534       Germany #>     3:       DE      2021-01-09       Cases     154922       Germany #>     4:       DE      2021-01-09      Deaths       6117       Germany #>     5:       DE      2021-01-16       Cases     110183       Germany #>    ---                                                               #> 20541:       IT      2021-07-24      Deaths         78         Italy #> 20542:       IT      2021-07-24      Deaths         78         Italy #> 20543:       IT      2021-07-24      Deaths         78         Italy #> 20544:       IT      2021-07-24      Deaths         78         Italy #> 20545:       IT      2021-07-24      Deaths         78         Italy #>        forecast_date quantile prediction                model horizon #>     1:          <NA>       NA         NA                 <NA>      NA #>     2:          <NA>       NA         NA                 <NA>      NA #>     3:          <NA>       NA         NA                 <NA>      NA #>     4:          <NA>       NA         NA                 <NA>      NA #>     5:          <NA>       NA         NA                 <NA>      NA #>    ---                                                                #> 20541:    2021-07-12    0.850        352 epiforecasts-EpiNow2       2 #> 20542:    2021-07-12    0.900        397 epiforecasts-EpiNow2       2 #> 20543:    2021-07-12    0.950        499 epiforecasts-EpiNow2       2 #> 20544:    2021-07-12    0.975        611 epiforecasts-EpiNow2       2 #> 20545:    2021-07-12    0.990        719 epiforecasts-EpiNow2       2"},{"path":"https://epiforecasts.io/scoringutils/reference/metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Summary information for selected metrics — metrics","title":"Summary information for selected metrics — metrics","text":"data set summary information selected metrics implemented scoringutils","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Summary information for selected metrics — metrics","text":"","code":"metrics"},{"path":"https://epiforecasts.io/scoringutils/reference/metrics.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Summary information for selected metrics — metrics","text":"object class data.table (inherits data.frame) 22 rows 8 columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/metrics.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Summary information for selected metrics — metrics","text":"data created using script create-metric-tables.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison.html","id":null,"dir":"Reference","previous_headings":"","what":"Do Pairwise Comparisons of Scores — pairwise_comparison","title":"Do Pairwise Comparisons of Scores — pairwise_comparison","text":"Compute relative scores different models making pairwise comparisons. Pairwise comparisons sort pairwise tournament combinations two models compared based overlapping set available forecasts common models. Internally, ratio mean scores models computed. relative score model geometric mean mean score ratios involve model. baseline provided, baseline excluded relative scores individual models (therefore differ slightly relative scores without baseline) relative scores scaled (.e. divided ) relative score baseline model. Usually, function input unsummarised scores produced score(). Note function internally infers unit single forecast determining columns input correspond metrics computed score(). Adding unrelated columns change results unpredictable way. code pairwise comparisons inspired implementation Johannes Bracher. implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Do Pairwise Comparisons of Scores — pairwise_comparison","text":"","code":"pairwise_comparison(   scores,   by = \"model\",   metric = \"auto\",   baseline = NULL,   ... )"},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Do Pairwise Comparisons of Scores — pairwise_comparison","text":"scores data.table scores produced score(). character vector names columns present input data.frame. determines pairwise comparisons computed. get relative skill score every grouping level determined . , example, = c(\"model\", \"location\"). get separate relative skill score every model every location. Internally, data.frame split according (removing \"model\" splitting) pairwise comparisons computed separately split data.frames. metric character vector length one metric comparison . default \"auto\", meaning either \"interval_score\", \"crps\", \"brier_score\" selected available. See available_metrics() available metrics. baseline character vector length one denotes baseline model compare models. ... additional arguments comparison two models. See compare_two_models() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Do Pairwise Comparisons of Scores — pairwise_comparison","text":"ggplot2 object coloured table summarised scores","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Do Pairwise Comparisons of Scores — pairwise_comparison","text":"Nikos Bosse nikosbosse@gmail.com Johannes Bracher, johannes.bracher@kit.edu","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Do Pairwise Comparisons of Scores — pairwise_comparison","text":"","code":"data.table::setDTthreads(1) # only needed to avoid issues on CRAN  scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. pairwise <- pairwise_comparison(scores, by = \"target_type\")  library(ggplot2) plot_pairwise_comparison(pairwise, type = \"mean_scores_ratio\") +   facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison_one_group.html","id":null,"dir":"Reference","previous_headings":"","what":"Do Pairwise Comparison for one Set of Forecasts — pairwise_comparison_one_group","title":"Do Pairwise Comparison for one Set of Forecasts — pairwise_comparison_one_group","text":"function pairwise comparison one set forecasts, multiple models involved. gets called pairwise_comparison(). pairwise_comparison() splits data arbitrary subgroups specified user (e.g. pairwise comparison done separately different forecast targets) actual pairwise comparison subgroup managed pairwise_comparison_one_group(). order actually comparison two models subset common forecasts calls compare_two_models().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison_one_group.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Do Pairwise Comparison for one Set of Forecasts — pairwise_comparison_one_group","text":"","code":"pairwise_comparison_one_group(scores, metric, baseline, by, ...)"},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison_one_group.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Do Pairwise Comparison for one Set of Forecasts — pairwise_comparison_one_group","text":"scores data.table scores produced score(). metric character vector length one metric comparison . default \"auto\", meaning either \"interval_score\", \"crps\", \"brier_score\" selected available. See available_metrics() available metrics. baseline character vector length one denotes baseline model compare models. character vector names columns present input data.frame. determines pairwise comparisons computed. get relative skill score every grouping level determined . , example, = c(\"model\", \"location\"). get separate relative skill score every model every location. Internally, data.frame split according (removing \"model\" splitting) pairwise comparisons computed separately split data.frames. ... additional arguments comparison two models. See compare_two_models() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/permutation_test.html","id":null,"dir":"Reference","previous_headings":"","what":"Simple permutation test — permutation_test","title":"Simple permutation test — permutation_test","text":"#' implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/permutation_test.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Simple permutation test — permutation_test","text":"","code":"permutation_test(   scores1,   scores2,   n_permutation = 999,   one_sided = FALSE,   comparison_mode = c(\"difference\", \"ratio\") )"},{"path":"https://epiforecasts.io/scoringutils/reference/permutation_test.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Simple permutation test — permutation_test","text":"p-value permutation test","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability Integral Transformation (data.frame Format) — pit","title":"Probability Integral Transformation (data.frame Format) — pit","text":"Wrapper around pit() use data.frames","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability Integral Transformation (data.frame Format) — pit","text":"","code":"pit(data, by, n_replicates = 100)"},{"path":"https://epiforecasts.io/scoringutils/reference/pit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability Integral Transformation (data.frame Format) — pit","text":"data data.frame following columns: true_value, prediction, sample. Character vector columns according PIT values shall grouped. e.g. columns 'model' 'location' data want PIT histogram every model location, specify = c(\"model\", \"location\"). n_replicates number draws randomised PIT integer predictions.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability Integral Transformation (data.frame Format) — pit","text":"data.table PIT values according grouping specified ","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Probability Integral Transformation (data.frame Format) — pit","text":"see pit()","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability Integral Transformation (data.frame Format) — pit","text":"Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability Integral Transformation (data.frame Format) — pit","text":"","code":"result <- pit(example_continuous, by = \"model\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. plot_pit(result)   # example with quantile data result <- pit(example_quantile, by = \"model\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. plot_pit(result)"},{"path":"https://epiforecasts.io/scoringutils/reference/pit_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability Integral Transformation (sample-based version) — pit_sample","title":"Probability Integral Transformation (sample-based version) — pit_sample","text":"Uses Probability Integral Transformation (PIT) (randomised PIT integer forecasts) assess calibration predictive Monte Carlo samples. Returns p-values resulting Anderson-Darling test uniformity (randomised) PIT well PIT histogram specified.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability Integral Transformation (sample-based version) — pit_sample","text":"","code":"pit_sample(true_values, predictions, n_replicates = 100)"},{"path":"https://epiforecasts.io/scoringutils/reference/pit_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability Integral Transformation (sample-based version) — pit_sample","text":"true_values vector true observed values size n predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n. n_replicates number draws randomised PIT integer predictions.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability Integral Transformation (sample-based version) — pit_sample","text":"vector PIT-values. continuous forecasts, vector correspond length true_values. integer forecasts, randomised PIT returned length length(true_values) * n_replicates","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Probability Integral Transformation (sample-based version) — pit_sample","text":"Calibration reliability forecasts ability model correctly identify uncertainty making predictions. model perfect calibration, observed data time point look came predictive probability distribution time. Equivalently, one can inspect probability integral transform predictive distribution time t, $$ u_t = F_t (x_t) $$ \\(x_t\\) observed data point time \\(t \\textrm{ } t_1, …, t_n\\), n number forecasts, \\(F_t\\) (continuous) predictive cumulative probability distribution time t. true probability distribution outcomes time t \\(G_t\\) forecasts \\(F_t\\) said ideal \\(F_t = G_t\\) times t. case, probabilities \\(u_t\\) distributed uniformly. case discrete outcomes incidence counts, PIT longer uniform even forecasts ideal. case randomised PIT can used instead: $$ u_t = P_t(k_t) + v * (P_t(k_t) - P_t(k_t - 1) ) $$ \\(k_t\\) observed count, \\(P_t(x)\\) predictive cumulative probability observing incidence k time t, \\(P_t (-1) = 0\\) definition v standard uniform independent k. \\(P_t\\) true cumulative probability distribution, \\(u_t\\) standard uniform. function checks whether integer continuous forecasts provided. applies (randomised) probability integral tests values \\(u_t\\) uniformity using Anderson-Darling test. rule thumb, evidence suggest forecasting model miscalibrated p-value found greater threshold p >= 0.1, evidence miscalibrated 0.01 < p < 0.1, good evidence miscalibrated p <= 0.01. However, AD-p-values may overly strict actual usefulness may questionable. context noted, though, uniformity PIT necessary sufficient condition calibration.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability Integral Transformation (sample-based version) — pit_sample","text":"Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/reference/pit_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability Integral Transformation (sample-based version) — pit_sample","text":"","code":"data.table::setDTthreads(1) # only needed to avoid issues on CRAN  ## continuous predictions true_values <- rnorm(20, mean = 1:20) predictions <- replicate(100, rnorm(n = 20, mean = 1:20)) pit <- pit_sample(true_values, predictions) plot_pit(pit)   ## integer predictions true_values <- rpois(50, lambda = 1:50) predictions <- replicate(2000, rpois(n = 50, lambda = 1:50)) pit <- pit_sample(true_values, predictions, n_replicates = 30) plot_pit(pit)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_avail_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Visualise Where Forecasts Are Available — plot_avail_forecasts","title":"Visualise Where Forecasts Are Available — plot_avail_forecasts","text":"Visualise Forecasts Available","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_avail_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Visualise Where Forecasts Are Available — plot_avail_forecasts","text":"","code":"plot_avail_forecasts(   avail_forecasts,   y = \"model\",   x = \"forecast_date\",   make_x_factor = TRUE,   show_numbers = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_avail_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Visualise Where Forecasts Are Available — plot_avail_forecasts","text":"avail_forecasts data.frame column called Number forecasts produced avail_forecasts() y character vector length one denotes name column appear y-axis plot. Default \"model\". x character vector length one denotes name column appear x-axis plot. Default \"forecast_date\". make_x_factor logical (default TRUE). Whether convert variable x-axis factor. effect e.g. dates shown x-axis. show_numbers logical (default TRUE) indicates whether show actual count numbers plot","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_avail_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Visualise Where Forecasts Are Available — plot_avail_forecasts","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_avail_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Visualise Where Forecasts Are Available — plot_avail_forecasts","text":"","code":"library(ggplot2) avail_forecasts <- avail_forecasts(   example_quantile, by = c(\"model\", \"target_type\", \"target_end_date\") ) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. plot_avail_forecasts(  avail_forecasts, x = \"target_end_date\", show_numbers = FALSE ) +  facet_wrap(\"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_correlation.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot Correlation Between Metrics — plot_correlation","title":"Plot Correlation Between Metrics — plot_correlation","text":"Plots heatmap correlations different metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_correlation.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot Correlation Between Metrics — plot_correlation","text":"","code":"plot_correlation(correlations)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_correlation.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot Correlation Between Metrics — plot_correlation","text":"correlations data.table correlations scores produced correlation().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_correlation.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot Correlation Between Metrics — plot_correlation","text":"ggplot2 object showing coloured matrix correlations metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_correlation.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot Correlation Between Metrics — plot_correlation","text":"","code":"scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. correlations <- correlation(  summarise_scores(scores) ) plot_correlation(correlations)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_heatmap.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a Heatmap of a Scoring Metric — plot_heatmap","title":"Create a Heatmap of a Scoring Metric — plot_heatmap","text":"function can used create heatmap one metric across different groups, e.g. interval score obtained several forecasting models different locations.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_heatmap.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a Heatmap of a Scoring Metric — plot_heatmap","text":"","code":"plot_heatmap(scores, y = \"model\", x, metric)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_heatmap.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a Heatmap of a Scoring Metric — plot_heatmap","text":"scores data.frame scores based quantile forecasts produced score(). y variable scores want show y-Axis. default \"model\" x variable scores want show x-Axis. something like \"horizon\", \"location\" metric metric determines value colour shown tiles heatmap","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_heatmap.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a Heatmap of a Scoring Metric — plot_heatmap","text":"ggplot2 object showing heatmap desired metric","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_heatmap.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create a Heatmap of a Scoring Metric — plot_heatmap","text":"","code":"scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\", \"range\"))  plot_heatmap(scores, x = \"target_type\", metric = \"bias\")"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_interval_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot Interval Coverage — plot_interval_coverage","title":"Plot Interval Coverage — plot_interval_coverage","text":"Plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_interval_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot Interval Coverage — plot_interval_coverage","text":"","code":"plot_interval_coverage(scores, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_interval_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot Interval Coverage — plot_interval_coverage","text":"scores data.frame scores based quantile forecasts produced score() summarise_scores(). Note \"range\" must included argument running summarise_scores() colour According variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_interval_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot Interval Coverage — plot_interval_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_interval_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot Interval Coverage — plot_interval_coverage","text":"","code":"data.table::setDTthreads(1) # only needed to avoid issues on CRAN scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"range\")) plot_interval_coverage(scores)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pairwise_comparison.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot Heatmap of Pairwise Comparisons — plot_pairwise_comparison","title":"Plot Heatmap of Pairwise Comparisons — plot_pairwise_comparison","text":"Creates heatmap ratios pvalues pairwise comparison models","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pairwise_comparison.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot Heatmap of Pairwise Comparisons — plot_pairwise_comparison","text":"","code":"plot_pairwise_comparison(   comparison_result,   type = c(\"mean_scores_ratio\", \"pval\") )"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pairwise_comparison.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot Heatmap of Pairwise Comparisons — plot_pairwise_comparison","text":"comparison_result data.frame produced pairwise_comparison() type character vector length one either \"mean_scores_ratio\" \"pval\". denotes whether visualise ratio p-value pairwise comparison. Default \"mean_scores_ratio\".","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pairwise_comparison.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot Heatmap of Pairwise Comparisons — plot_pairwise_comparison","text":"","code":"library(ggplot2) scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. pairwise <- pairwise_comparison(scores, by = \"target_type\") plot_pairwise_comparison(pairwise, type = \"mean_scores_ratio\") +   facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pit.html","id":null,"dir":"Reference","previous_headings":"","what":"PIT Histogram — plot_pit","title":"PIT Histogram — plot_pit","text":"Make simple histogram probability integral transformed values visually check whether uniform distribution seems likely.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"PIT Histogram — plot_pit","text":"","code":"plot_pit(pit, num_bins = \"auto\", breaks = NULL)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"PIT Histogram — plot_pit","text":"pit either vector PIT values size n, data.frame produced pit() num_bins number bins PIT histogram, default \"auto\". num_bins == \"auto\", plot_pit() either display 10 bins, display bin available quantile case passed data quantile-based format. can control number bins supplying number. fine sample-based pit histograms, may fail quantile-based formats. case preferred supply explicit breaks points using breaks argument. breaks numeric vector break points bins PIT histogram. preferred creating PIT histogram based quantile-based data. Default NULL breaks determined num_bins.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"PIT Histogram — plot_pit","text":"vector scoring values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"PIT Histogram — plot_pit","text":"","code":"data.table::setDTthreads(1) # only needed to avoid issues on CRAN  # PIT histogram in vector based format true_values <- rnorm(30, mean = 1:30) predictions <- replicate(200, rnorm(n = 30, mean = 1:30)) pit <- pit_sample(true_values, predictions) plot_pit(pit)   # quantile-based pit pit <- pit(example_quantile,by = \"model\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. plot_pit(pit, breaks = seq(0.1, 1, 0.1))   # sample-based pit pit <- pit(example_integer,by = \"model\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. plot_pit(pit)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_predictions.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot Predictions vs True Values — plot_predictions","title":"Plot Predictions vs True Values — plot_predictions","text":"Make plot observed predicted values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_predictions.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot Predictions vs True Values — plot_predictions","text":"","code":"plot_predictions(data, by = NULL, x = \"date\", range = c(0, 50, 90))"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_predictions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot Predictions vs True Values — plot_predictions","text":"data data.frame follows specifications outlined score(). customise plotting, can filter data using function make_NA(). character vector column names denote categories plot stratified. example want facetted plot, character vector columns used facetting (note facetting still needs done outside function call) x character vector length one denotes name variable range numeric vector indicating interval ranges plot. 0 included range, median prediction shown.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_predictions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot Predictions vs True Values — plot_predictions","text":"ggplot object plot true vs predicted values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_predictions.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot Predictions vs True Values — plot_predictions","text":"","code":"library(ggplot2) library(magrittr)  example_continuous %>%   make_NA (     what = \"truth\",     target_end_date >= \"2021-07-22\",     target_end_date < \"2021-05-01\"   ) %>%   make_NA (     what = \"forecast\",     model != \"EuroCOVIDhub-ensemble\",     forecast_date != \"2021-06-07\"   ) %>%   plot_predictions (     x = \"target_end_date\",     by = c(\"target_type\", \"location\"),     range = c(0, 50, 90, 95)   ) +   facet_wrap(~ location + target_type, scales = \"free_y\") +   aes(fill = model, color = model)   example_continuous %>%   make_NA (     what = \"truth\",     target_end_date >= \"2021-07-22\",     target_end_date < \"2021-05-01\"   ) %>%   make_NA (     what = \"forecast\",     forecast_date != \"2021-06-07\"   ) %>%   plot_predictions (     x = \"target_end_date\",     by = c(\"target_type\", \"location\"),     range = c(0)   ) +   facet_wrap(~ location + target_type, scales = \"free_y\") +   aes(fill = model, color = model)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_quantile_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot Quantile Coverage — plot_quantile_coverage","title":"Plot Quantile Coverage — plot_quantile_coverage","text":"Plot quantile coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_quantile_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot Quantile Coverage — plot_quantile_coverage","text":"","code":"plot_quantile_coverage(scores, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_quantile_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot Quantile Coverage — plot_quantile_coverage","text":"scores data.frame scores based quantile forecasts produced score() summarise_scores(). Note \"range\" must included argument running summarise_scores() colour According variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_quantile_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot Quantile Coverage — plot_quantile_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_quantile_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot Quantile Coverage — plot_quantile_coverage","text":"","code":"scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"quantile\")) plot_quantile_coverage(scores)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_ranges.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot Metrics by Range of the Prediction Interval — plot_ranges","title":"Plot Metrics by Range of the Prediction Interval — plot_ranges","text":"Visualise metrics range, e.g. interested different interval ranges contribute overall interval score, sharpness / dispersion changes range.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_ranges.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot Metrics by Range of the Prediction Interval — plot_ranges","text":"","code":"plot_ranges(scores, y = \"interval_score\", x = \"model\", colour = \"range\")"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_ranges.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot Metrics by Range of the Prediction Interval — plot_ranges","text":"scores data.frame scores based quantile forecasts produced score() summarise_scores(). Note \"range\" must included argument running summarise_scores() y variable scores want show y-Axis. something like \"interval_score\" (default) \"dispersion\" x variable scores want show x-Axis. Usually \"model\" colour Character vector length one used determine variable colouring dots. Default \"range\".","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_ranges.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot Metrics by Range of the Prediction Interval — plot_ranges","text":"ggplot2 object showing contributions three components weighted interval score","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_ranges.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot Metrics by Range of the Prediction Interval — plot_ranges","text":"","code":"library(ggplot2) scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\", \"range\"))  plot_ranges(scores, x = \"model\") +   facet_wrap(~target_type, scales = \"free\")   # visualise dispersion instead of interval score plot_ranges(scores, y = \"dispersion\", x = \"model\") +   facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_score_table.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot Coloured Score Table — plot_score_table","title":"Plot Coloured Score Table — plot_score_table","text":"Plots coloured table summarised scores obtained using score().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_score_table.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot Coloured Score Table — plot_score_table","text":"","code":"plot_score_table(scores, y = \"model\", by = NULL, metrics = NULL)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_score_table.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot Coloured Score Table — plot_score_table","text":"scores data.table scores produced score(). y variable shown y-axis. Instead single character string, can also specify vector column names, e.g. y = c(\"model\", \"location\"). column names concatenated create unique row identifier (e.g. \"model1_location1\"). character vector determines colour shading plot gets computed. default (NULL), shading determined per metric, can provide additional column names (see examples). metrics character vector metrics show. set NULL (default), metrics present scores shown.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_score_table.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot Coloured Score Table — plot_score_table","text":"ggplot2 object coloured table summarised scores","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_score_table.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot Coloured Score Table — plot_score_table","text":"","code":"library(ggplot2) library(magrittr) # pipe operator data.table::setDTthreads(1) # only needed to avoid issues on CRAN  scores <- score(example_quantile) %>%   summarise_scores(by = c(\"model\", \"target_type\")) %>%   summarise_scores(fun = signif, digits = 2) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.  plot_score_table(scores, y = \"model\", by = \"target_type\") +   facet_wrap(~target_type, ncol = 1)   # can also put target description on the y-axis plot_score_table(scores,                  y = c(\"model\", \"target_type\"),                  by = \"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_wis.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot Contributions to the Weighted Interval Score — plot_wis","title":"Plot Contributions to the Weighted Interval Score — plot_wis","text":"Visualise components weighted interval score: penalties -prediction, -prediction high dispersion (lack sharpness)","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_wis.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot Contributions to the Weighted Interval Score — plot_wis","text":"","code":"plot_wis(scores, x = \"model\", relative_contributions = FALSE, flip = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_wis.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot Contributions to the Weighted Interval Score — plot_wis","text":"scores data.frame scores based quantile forecasts produced score() summarised using summarise_scores() x variable scores want show x-Axis. Usually \"model\". relative_contributions show relative contributions instead absolute contributions. Default FALSE functionality available yet. flip boolean (default FALSE), whether flip axes.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_wis.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot Contributions to the Weighted Interval Score — plot_wis","text":"ggplot2 object showing contributions three components weighted interval score","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_wis.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Plot Contributions to the Weighted Interval Score — plot_wis","text":"Bracher J, Ray E, Gneiting T, Reich, N (2020) Evaluating epidemic forecasts interval format. https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_wis.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot Contributions to the Weighted Interval Score — plot_wis","text":"","code":"library(ggplot2) scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\"))  plot_wis(scores,   x = \"model\",   relative_contributions = TRUE ) +   facet_wrap(~target_type)  plot_wis(scores,   x = \"model\",   relative_contributions = FALSE ) +   facet_wrap(~target_type, scales = \"free_x\")"},{"path":"https://epiforecasts.io/scoringutils/reference/prediction_is_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Check if predictions are quantile forecasts — prediction_is_quantile","title":"Check if predictions are quantile forecasts — prediction_is_quantile","text":"Internal helper function check data frame contains quantile forecast predictions. determined checking \"quantile\" column present.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/prediction_is_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check if predictions are quantile forecasts — prediction_is_quantile","text":"","code":"prediction_is_quantile(data)"},{"path":"https://epiforecasts.io/scoringutils/reference/prediction_is_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check if predictions are quantile forecasts — prediction_is_quantile","text":"data Data frame containing forecast predictions","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/prediction_is_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check if predictions are quantile forecasts — prediction_is_quantile","text":"Logical indicating whether predictions quantile forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/print.scoringutils_check.html","id":null,"dir":"Reference","previous_headings":"","what":"Print output from check_forecasts() — print.scoringutils_check","title":"Print output from check_forecasts() — print.scoringutils_check","text":"Helper function prints output generated check_forecasts()","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/print.scoringutils_check.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Print output from check_forecasts() — print.scoringutils_check","text":"","code":"# S3 method for scoringutils_check print(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/reference/print.scoringutils_check.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Print output from check_forecasts() — print.scoringutils_check","text":"x object class 'scoringutils_check' produced check_forecasts() ... additional arguments (used )","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/print.scoringutils_check.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Print output from check_forecasts() — print.scoringutils_check","text":"","code":"check <- check_forecasts(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. print(check) #> Your forecasts seem to be for a target of the following type: #> $target_type #> [1] \"integer\" #>  #> and in the following format: #> $prediction_type #> [1] \"quantile\" #>  #> The unit of a single forecast is defined by: #> $forecast_unit #> [1] \"location\"        \"target_end_date\" \"target_type\"     \"location_name\"   #> [5] \"forecast_date\"   \"model\"           \"horizon\"         #>  #> Cleaned data, rows with NA values in prediction or true_value removed: #> $cleaned_data #>        location target_end_date target_type true_value location_name #>     1:       DE      2021-05-08       Cases     106987       Germany #>     2:       DE      2021-05-08       Cases     106987       Germany #>     3:       DE      2021-05-08       Cases     106987       Germany #>     4:       DE      2021-05-08       Cases     106987       Germany #>     5:       DE      2021-05-08       Cases     106987       Germany #>    ---                                                               #> 20397:       IT      2021-07-24      Deaths         78         Italy #> 20398:       IT      2021-07-24      Deaths         78         Italy #> 20399:       IT      2021-07-24      Deaths         78         Italy #> 20400:       IT      2021-07-24      Deaths         78         Italy #> 20401:       IT      2021-07-24      Deaths         78         Italy #>        forecast_date quantile prediction                 model horizon #>     1:    2021-05-03    0.010      82466 EuroCOVIDhub-ensemble       1 #>     2:    2021-05-03    0.025      86669 EuroCOVIDhub-ensemble       1 #>     3:    2021-05-03    0.050      90285 EuroCOVIDhub-ensemble       1 #>     4:    2021-05-03    0.100      95341 EuroCOVIDhub-ensemble       1 #>     5:    2021-05-03    0.150      99171 EuroCOVIDhub-ensemble       1 #>    ---                                                                 #> 20397:    2021-07-12    0.850        352  epiforecasts-EpiNow2       2 #> 20398:    2021-07-12    0.900        397  epiforecasts-EpiNow2       2 #> 20399:    2021-07-12    0.950        499  epiforecasts-EpiNow2       2 #> 20400:    2021-07-12    0.975        611  epiforecasts-EpiNow2       2 #> 20401:    2021-07-12    0.990        719  epiforecasts-EpiNow2       2 #>  #> Number of unique values per column per model: #> $unique_values #>                    model location target_end_date target_type true_value #> 1: EuroCOVIDhub-ensemble        4              12           2         96 #> 2: EuroCOVIDhub-baseline        4              12           2         96 #> 3:  epiforecasts-EpiNow2        4              12           2         95 #> 4:       UMass-MechBayes        4              12           1         48 #>    location_name forecast_date quantile prediction horizon #> 1:             4            11       23       3969       3 #> 2:             4            11       23       3733       3 #> 3:             4            11       23       3903       3 #> 4:             4            11       23       1058       3 #>  #> $messages #> [1] \"144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.\" #>"},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile Score — quantile_score","title":"Quantile Score — quantile_score","text":"Proper Scoring Rule score quantile predictions. Smaller values better. quantile score closely related Interval score (see interval_score()) quantile equivalent works single quantiles instead central prediction intervals.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile Score — quantile_score","text":"","code":"quantile_score(true_values, predictions, quantiles, weigh = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Quantile Score — quantile_score","text":"true_values vector true observed values size n predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n. quantiles vector size n quantile values corresponding predictions. weigh TRUE, weigh score alpha / 2, can averaged interval score , limit, corresponds CRPS. Alpha value corresponds (alpha/2) (1 - alpha/2) quantiles provided computed quantile. Alpha decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1). Default: TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Quantile Score — quantile_score","text":"vector scoring values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Quantile Score — quantile_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Quantile Score — quantile_score","text":"","code":"true_values <- rnorm(10, mean = 1:10) alpha <- 0.5  lower <- qnorm(alpha / 2, rnorm(10, mean = 1:10)) upper <- qnorm((1 - alpha / 2), rnorm(10, mean = 1:10))  qs_lower <- quantile_score(true_values,   predictions = lower,   quantiles = alpha / 2 ) qs_upper <- quantile_score(true_values,   predictions = upper,   quantiles = 1 - alpha / 2 ) interval_score <- (qs_lower + qs_upper) / 2"},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_to_range_long.html","id":null,"dir":"Reference","previous_headings":"","what":"Change Data from a Plain Quantile Format to a Long Range Format — quantile_to_range_long","title":"Change Data from a Plain Quantile Format to a Long Range Format — quantile_to_range_long","text":"Transform data format uses quantiles one uses interval ranges denote quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_to_range_long.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Change Data from a Plain Quantile Format to a Long Range Format — quantile_to_range_long","text":"","code":"quantile_to_range_long(data, keep_quantile_col = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_to_range_long.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Change Data from a Plain Quantile Format to a Long Range Format — quantile_to_range_long","text":"data data.frame quantile format keep_quantile_col keep quantile column final output transformation (default FALSE)","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_to_range_long.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Change Data from a Plain Quantile Format to a Long Range Format — quantile_to_range_long","text":"data.frame long interval range format","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/range_long_to_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Change Data from a Range Format to a Quantile Format — range_long_to_quantile","title":"Change Data from a Range Format to a Quantile Format — range_long_to_quantile","text":"Transform data format uses interval ranges denote quantiles format uses quantiles .","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/range_long_to_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Change Data from a Range Format to a Quantile Format — range_long_to_quantile","text":"","code":"range_long_to_quantile(data, keep_range_col = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/reference/range_long_to_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Change Data from a Range Format to a Quantile Format — range_long_to_quantile","text":"data data.frame following specifications score()) quantile forecasts. keep_range_col keep range boundary columns transformation (default FALSE)","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/range_long_to_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Change Data from a Range Format to a Quantile Format — range_long_to_quantile","text":"data.frame plain quantile format","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Change Data from a Sample Based Format to a Quantile Format — sample_to_quantile","title":"Change Data from a Sample Based Format to a Quantile Format — sample_to_quantile","text":"Transform data format based predictive samples format based plain quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Change Data from a Sample Based Format to a Quantile Format — sample_to_quantile","text":"","code":"sample_to_quantile(data, quantiles = c(0.05, 0.25, 0.5, 0.75, 0.95), type = 7)"},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Change Data from a Sample Based Format to a Quantile Format — sample_to_quantile","text":"data data.frame samples quantiles numeric vector quantiles extract type type argument passed quantile function. information, see quantile()","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Change Data from a Sample Based Format to a Quantile Format — sample_to_quantile","text":"data.frame long interval range format","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Change Data from a Sample Based Format to a Quantile Format — sample_to_quantile","text":"","code":"sample_to_quantile(example_integer) #>       location location_name target_end_date target_type forecast_date #>    1:       DE       Germany      2021-01-02       Cases          <NA> #>    2:       DE       Germany      2021-01-02       Cases          <NA> #>    3:       DE       Germany      2021-01-02       Cases          <NA> #>    4:       DE       Germany      2021-01-02       Cases          <NA> #>    5:       DE       Germany      2021-01-02       Cases          <NA> #>   ---                                                                  #> 5151:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 5152:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 5153:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 5154:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 5155:       IT         Italy      2021-07-24      Deaths    2021-07-12 #>                      model horizon true_value quantile prediction #>    1:                 <NA>      NA     127300     0.05         NA #>    2:                 <NA>      NA     127300     0.25         NA #>    3:                 <NA>      NA     127300     0.50         NA #>    4:                 <NA>      NA     127300     0.75         NA #>    5:                 <NA>      NA     127300     0.95         NA #>   ---                                                             #> 5151: epiforecasts-EpiNow2       2         78     0.05      44.90 #> 5152: epiforecasts-EpiNow2       2         78     0.25      89.25 #> 5153: epiforecasts-EpiNow2       2         78     0.50     151.50 #> 5154: epiforecasts-EpiNow2       2         78     0.75     208.00 #> 5155: epiforecasts-EpiNow2       2         78     0.95     469.10"},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_range_long.html","id":null,"dir":"Reference","previous_headings":"","what":"Change Data from a Sample Based Format to a Long Interval Range Format — sample_to_range_long","title":"Change Data from a Sample Based Format to a Long Interval Range Format — sample_to_range_long","text":"Transform data format based predictive samples format based interval ranges","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_range_long.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Change Data from a Sample Based Format to a Long Interval Range Format — sample_to_range_long","text":"","code":"sample_to_range_long(   data,   range = c(0, 50, 90),   type = 7,   keep_quantile_col = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_range_long.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Change Data from a Sample Based Format to a Long Interval Range Format — sample_to_range_long","text":"data data.frame samples range numeric vector interval ranges extract (e.g. c(0, 50, 90)) type type argument passed quantile function. information, see quantile() keep_quantile_col keep quantile column, default TRUE","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_range_long.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Change Data from a Sample Based Format to a Long Interval Range Format — sample_to_range_long","text":"data.frame long interval range format","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score.html","id":null,"dir":"Reference","previous_headings":"","what":"Evaluate forecasts — score","title":"Evaluate forecasts — score","text":"function allows automatic scoring forecasts using range metrics. users workhorse scoring forecasts wraps lower level functions package functions. However, functions also available wish make use independently. range forecasts formats supported, including quantile-based, sample-based, binary forecasts. Prior scoring, users may wish make use check_forecasts() ensure input data supported format though also run internally score(). Examples format also provided (see documentation data check_forecasts()). format set required columns (see ). Additional columns may present indicate grouping forecasts. example, forecasts made different models various locations different time points, several weeks future. important, columns present relevant order group forecasts. combination different columns uniquely define unit single forecast, meaning single forecast defined values columns. Adding additional unrelated columns may alter results. obtain quick overview currently supported evaluation metrics, look metrics data included package. column metrics$Name gives overview available metric names can computed. interested unsupported metric please open feature request consider contributing pull request. additional help examples, check Getting Started Vignette well paper Evaluating Forecasts scoringutils R.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Evaluate forecasts — score","text":"","code":"score(data, metrics = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/reference/score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Evaluate forecasts — score","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary). metrics metrics want output. NULL (default), available metrics computed. list available metrics see available_metrics(),  check metrics data set. ... additional parameters passed score_quantile() (internal function used scoring forecasts quantile-based format).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Evaluate forecasts — score","text":"data.table unsummarised scores. one score per quantile sample, usually desired, almost always run summarise_scores() unsummarised scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Evaluate forecasts — score","text":"Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, Edmunds WJ (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLoS Comput Biol 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Evaluate forecasts — score","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Evaluate forecasts — score","text":"","code":"library(magrittr) # pipe operator data.table::setDTthreads(1) # only needed to avoid issues on CRAN  check_forecasts(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #> Your forecasts seem to be for a target of the following type: #> $target_type #> [1] \"integer\" #>  #> and in the following format: #> $prediction_type #> [1] \"quantile\" #>  #> The unit of a single forecast is defined by: #> $forecast_unit #> [1] \"location\"        \"target_end_date\" \"target_type\"     \"location_name\"   #> [5] \"forecast_date\"   \"model\"           \"horizon\"         #>  #> Cleaned data, rows with NA values in prediction or true_value removed: #> $cleaned_data #>        location target_end_date target_type true_value location_name #>     1:       DE      2021-05-08       Cases     106987       Germany #>     2:       DE      2021-05-08       Cases     106987       Germany #>     3:       DE      2021-05-08       Cases     106987       Germany #>     4:       DE      2021-05-08       Cases     106987       Germany #>     5:       DE      2021-05-08       Cases     106987       Germany #>    ---                                                               #> 20397:       IT      2021-07-24      Deaths         78         Italy #> 20398:       IT      2021-07-24      Deaths         78         Italy #> 20399:       IT      2021-07-24      Deaths         78         Italy #> 20400:       IT      2021-07-24      Deaths         78         Italy #> 20401:       IT      2021-07-24      Deaths         78         Italy #>        forecast_date quantile prediction                 model horizon #>     1:    2021-05-03    0.010      82466 EuroCOVIDhub-ensemble       1 #>     2:    2021-05-03    0.025      86669 EuroCOVIDhub-ensemble       1 #>     3:    2021-05-03    0.050      90285 EuroCOVIDhub-ensemble       1 #>     4:    2021-05-03    0.100      95341 EuroCOVIDhub-ensemble       1 #>     5:    2021-05-03    0.150      99171 EuroCOVIDhub-ensemble       1 #>    ---                                                                 #> 20397:    2021-07-12    0.850        352  epiforecasts-EpiNow2       2 #> 20398:    2021-07-12    0.900        397  epiforecasts-EpiNow2       2 #> 20399:    2021-07-12    0.950        499  epiforecasts-EpiNow2       2 #> 20400:    2021-07-12    0.975        611  epiforecasts-EpiNow2       2 #> 20401:    2021-07-12    0.990        719  epiforecasts-EpiNow2       2 #>  #> Number of unique values per column per model: #> $unique_values #>                    model location target_end_date target_type true_value #> 1: EuroCOVIDhub-ensemble        4              12           2         96 #> 2: EuroCOVIDhub-baseline        4              12           2         96 #> 3:  epiforecasts-EpiNow2        4              12           2         95 #> 4:       UMass-MechBayes        4              12           1         48 #>    location_name forecast_date quantile prediction horizon #> 1:             4            11       23       3969       3 #> 2:             4            11       23       3733       3 #> 3:             4            11       23       3903       3 #> 4:             4            11       23       1058       3 #>  #> $messages #> [1] \"144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.\" #>  score(example_quantile) %>%   add_coverage(by = c(\"model\", \"target_type\")) %>%   summarise_scores(by = c(\"model\", \"target_type\")) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model target_type interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline       Cases    28483.57465 4102.50094    10284.972826 #> 2: EuroCOVIDhub-baseline      Deaths      159.40387   91.40625        2.098505 #> 3: EuroCOVIDhub-ensemble       Cases    17943.82383 3663.52458     4237.177310 #> 4: EuroCOVIDhub-ensemble      Deaths       41.42249   30.18099        4.103261 #> 5:       UMass-MechBayes      Deaths       52.65195   26.87239       16.800951 #> 6:  epiforecasts-EpiNow2       Cases    20831.55662 5664.37795     3260.355639 #> 7:  epiforecasts-EpiNow2      Deaths       66.64282   31.85692       15.893314 #>    overprediction coverage_deviation        bias   ae_median coverage_50 #> 1:   14096.100883        -0.11211957  0.09796875 38473.60156   0.3281250 #> 2:      65.899117         0.11614130  0.33906250   233.25781   0.6640625 #> 3:   10043.121943        -0.09785326 -0.05640625 24101.07031   0.3906250 #> 4:       7.138247         0.19528533  0.07265625    53.13281   0.8750000 #> 5:       8.978601        -0.02312500 -0.02234375    78.47656   0.4609375 #> 6:   11906.823030        -0.06660326 -0.07890625 27923.81250   0.4687500 #> 7:      18.892583        -0.04287176 -0.00512605   104.74790   0.4201681 #>    coverage_90 #> 1:   0.8203125 #> 2:   1.0000000 #> 3:   0.8046875 #> 4:   1.0000000 #> 5:   0.8750000 #> 6:   0.7890625 #> 7:   0.9075630  # set forecast unit manually (to avoid issues with scoringutils trying to # determine the forecast unit automatically), check forecasts before scoring example_quantile %>%   set_forecast_unit(     c(\"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\")   ) %>%   check_forecasts() %>%   score() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>        location target_end_date target_type horizon                 model range #>     1:       DE      2021-05-08       Cases       1 EuroCOVIDhub-baseline     0 #>     2:       DE      2021-05-08       Cases       1 EuroCOVIDhub-baseline    10 #>     3:       DE      2021-05-08       Cases       1 EuroCOVIDhub-baseline    10 #>     4:       DE      2021-05-08       Cases       1 EuroCOVIDhub-baseline    20 #>     5:       DE      2021-05-08       Cases       1 EuroCOVIDhub-baseline    20 #>    ---                                                                          #> 20397:       IT      2021-07-24      Deaths       3  epiforecasts-EpiNow2    90 #> 20398:       IT      2021-07-24      Deaths       3  epiforecasts-EpiNow2    95 #> 20399:       IT      2021-07-24      Deaths       3  epiforecasts-EpiNow2    95 #> 20400:       IT      2021-07-24      Deaths       3  epiforecasts-EpiNow2    98 #> 20401:       IT      2021-07-24      Deaths       3  epiforecasts-EpiNow2    98 #>        interval_score dispersion underprediction overprediction coverage #>     1:       25620.00       0.00               0          25620        0 #>     2:       25599.50     184.50               0          25415        0 #>     3:       25599.50     184.50               0          25415        0 #>     4:       25481.00     556.00               0          24925        0 #>     5:       25481.00     556.00               0          24925        0 #>    ---                                                                   #> 20397:          15.85      15.85               0              0        1 #> 20398:          11.75      11.75               0              0        1 #> 20399:          11.75      11.75               0              0        1 #> 20400:           6.52       6.52               0              0        1 #> 20401:           6.52       6.52               0              0        1 #>        coverage_deviation bias quantile ae_median quantile_coverage #>     1:               0.00 0.95    0.500     25620              TRUE #>     2:              -0.10 0.95    0.450     25620              TRUE #>     3:              -0.10 0.95    0.550     25620              TRUE #>     4:              -0.20 0.95    0.400     25620              TRUE #>     5:              -0.20 0.95    0.600     25620              TRUE #>    ---                                                              #> 20397:               0.10 0.50    0.950        26              TRUE #> 20398:               0.05 0.50    0.025        26             FALSE #> 20399:               0.05 0.50    0.975        26              TRUE #> 20400:               0.02 0.50    0.010        26             FALSE #> 20401:               0.02 0.50    0.990        26              TRUE  # forecast formats with different metrics if (FALSE) { score(example_binary) score(example_quantile) score(example_integer) score(example_continuous) }  # score point forecasts (marked by 'NA' in the quantile column) score(example_point) %>%   summarise_scores(by = \"model\", na.rm = TRUE) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #>                    model interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline     14092.7647 2192.26967      4996.92898 #> 2: EuroCOVIDhub-ensemble      8852.4196 1930.80064      2005.56357 #> 3:  epiforecasts-EpiNow2     10659.5125 3084.85850      1604.96135 #> 4:       UMass-MechBayes        51.4781   28.09387        15.32315 #>    overprediction coverage_deviation        bias    ae_point     se_point #> 1:     6903.56605        0.002102273  0.21851562 19353.42969 2.883446e+09 #> 2:     4916.05540        0.050752841  0.00812500 12077.10156 1.945118e+09 #> 3:     5969.69268       -0.057861612 -0.04376518 14521.10526 2.680928e+09 #> 4:        8.06108       -0.024886364 -0.02312500    78.47656 1.170976e+04 #>    ae_median #> 1:       NaN #> 2:       NaN #> 3:       NaN #> 4:       NaN"},{"path":"https://epiforecasts.io/scoringutils/reference/score_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Evaluate forecasts in a Binary Format — score_binary","title":"Evaluate forecasts in a Binary Format — score_binary","text":"Evaluate forecasts Binary Format","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Evaluate forecasts in a Binary Format — score_binary","text":"","code":"score_binary(data, forecast_unit, metrics)"},{"path":"https://epiforecasts.io/scoringutils/reference/score_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Evaluate forecasts in a Binary Format — score_binary","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary). forecast_unit character vector column names define unit single forecast, .e. forecast made combination values forecast_unit. metrics metrics want output. NULL (default), available metrics computed. list available metrics see available_metrics(),  check metrics data set.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Evaluate forecasts in a Binary Format — score_binary","text":"data.table appropriate scores. information see score().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_binary.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Evaluate forecasts in a Binary Format — score_binary","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Evaluate forecasts in a Quantile-Based Format — score_quantile","title":"Evaluate forecasts in a Quantile-Based Format — score_quantile","text":"Evaluate forecasts Quantile-Based Format","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Evaluate forecasts in a Quantile-Based Format — score_quantile","text":"","code":"score_quantile(   data,   forecast_unit,   metrics,   weigh = TRUE,   count_median_twice = FALSE,   separate_results = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/reference/score_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Evaluate forecasts in a Quantile-Based Format — score_quantile","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary). forecast_unit character vector column names define unit single forecast, .e. forecast made combination values forecast_unit metrics metrics want output. NULL (default), available metrics computed. list available metrics see available_metrics(),  check metrics data set. weigh TRUE, weigh score alpha / 2, can averaged interval score , limit, corresponds CRPS. Alpha decimal value  represents much outside central prediction interval (e.g. 90 percent central prediction interval, alpha 0.1) Default: TRUE. count_median_twice logical controls whether count median twice summarising (default FALSE). Counting median twice conceptually treat 0\\ median lower well upper bound. alternative treat median single quantile forecast instead interval. interval score better understood average quantile scores. separate_results TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Evaluate forecasts in a Quantile-Based Format — score_quantile","text":"data.table appropriate scores. information see score()","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_quantile.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Evaluate forecasts in a Quantile-Based Format — score_quantile","text":"Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, Edmunds WJ (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLoS Comput Biol 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_quantile.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Evaluate forecasts in a Quantile-Based Format — score_quantile","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Evaluate forecasts in a Sample-Based Format (Integer or Continuous) — score_sample","title":"Evaluate forecasts in a Sample-Based Format (Integer or Continuous) — score_sample","text":"Evaluate forecasts Sample-Based Format (Integer Continuous)","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Evaluate forecasts in a Sample-Based Format (Integer or Continuous) — score_sample","text":"","code":"score_sample(data, forecast_unit, metrics, prediction_type)"},{"path":"https://epiforecasts.io/scoringutils/reference/score_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Evaluate forecasts in a Sample-Based Format (Integer or Continuous) — score_sample","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary). forecast_unit character vector column names define unit single forecast, .e. forecast made combination values forecast_unit metrics metrics want output. NULL (default), available metrics computed. list available metrics see available_metrics(),  check metrics data set. prediction_type character, either \"continuous\" \"integer\"","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Evaluate forecasts in a Sample-Based Format (Integer or Continuous) — score_sample","text":"data.table appropriate scores. information see score()","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Evaluate forecasts in a Sample-Based Format (Integer or Continuous) — score_sample","text":"Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, Edmunds WJ (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLoS Comput Biol 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_sample.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Evaluate forecasts in a Sample-Based Format (Integer or Continuous) — score_sample","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/scoringutils-package.html","id":null,"dir":"Reference","previous_headings":"","what":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","title":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","text":"Provides collection metrics proper scoring rules (Tilmann Gneiting & Adrian E Raftery (2007) doi:10.1198/016214506000001437 , Jordan, ., Krüger, F., & Lerch, S. (2019) doi:10.18637/jss.v090.i12 ) within consistent framework evaluation, comparison visualisation forecasts. addition proper scoring rules, functions provided assess bias, sharpness calibration (Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) doi:10.1371/journal.pcbi.1006785 ) forecasts. Several types predictions (e.g. binary, discrete, continuous) may come different formats (e.g. forecasts represented predictive samples quantiles predictive distribution) can evaluated. Scoring metrics can used either convenient data.frame format, can applied individual functions vector / matrix format. functionality implemented focus performance robustly tested. Find information scoringutils accompanying paper (Bosse et al., 2022) arXiv:2205.07090v1. Provides collection metrics proper scoring rules (Tilmann Gneiting & Adrian E Raftery (2007) doi:10.1198/016214506000001437 , Jordan, ., Krüger, F., & Lerch, S. (2019) doi:10.18637/jss.v090.i12 ) within consistent framework evaluation, comparison visualisation forecasts. addition proper scoring rules, functions provided assess bias, sharpness calibration (Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) doi:10.1371/journal.pcbi.1006785 ) forecasts. Several types predictions (e.g. binary, discrete, continuous) may come different formats (e.g. forecasts represented predictive samples quantiles predictive distribution) can evaluated. Scoring metrics can used either convenient data.frame format, can applied individual functions vector / matrix format. functionality implemented focus performance robustly tested. Find information scoringutils accompanying paper (Bosse et al., 2022) arXiv:2205.07090v1.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/reference/scoringutils-package.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","text":"Maintainer: Nikos Bosse nikosbosse@gmail.com (ORCID) Authors: Sam Abbott contact@samabbott.co.uk (ORCID) Hugo Gruson hugo.gruson@lshtm.ac.uk (ORCID) Sebastian Funk sebastian.funk@lshtm.ac.uk contributors: Johannes Bracher johannes.bracher@kit.edu (ORCID) [contributor]","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/se_mean_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Squared Error of the Mean (Sample-based Version) — se_mean_sample","title":"Squared Error of the Mean (Sample-based Version) — se_mean_sample","text":"Squared error mean calculated $$   \\textrm{mean}(\\textrm{true\\_value} - \\textrm{prediction})^2 $$","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/se_mean_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Squared Error of the Mean (Sample-based Version) — se_mean_sample","text":"","code":"se_mean_sample(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/se_mean_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Squared Error of the Mean (Sample-based Version) — se_mean_sample","text":"true_values vector true observed values size n predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/se_mean_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Squared Error of the Mean (Sample-based Version) — se_mean_sample","text":"vector scoring values","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/reference/se_mean_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Squared Error of the Mean (Sample-based Version) — se_mean_sample","text":"","code":"true_values <- rnorm(30, mean = 1:30) predicted_values <- rnorm(30, mean = 1:30) se_mean_sample(true_values, predicted_values) #>  [1] 4.860613e-01 1.744323e+00 1.360432e-01 3.302945e-01 3.751945e-01 #>  [6] 3.798420e-01 3.073065e+00 2.825546e+00 1.037927e+00 1.759771e+00 #> [11] 9.024846e-01 2.191300e+00 1.669601e+01 2.655281e-02 2.879499e-02 #> [16] 1.966896e+00 8.706269e-04 6.036063e-01 1.630220e+00 2.489198e-01 #> [21] 6.122734e-01 1.001614e+01 8.163432e-01 2.706157e-01 2.772148e+00 #> [26] 3.985060e-01 2.621120e+00 3.539384e+00 2.818409e-01 1.450891e+00"},{"path":"https://epiforecasts.io/scoringutils/reference/set_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Set unit of a single forecast manually — set_forecast_unit","title":"Set unit of a single forecast manually — set_forecast_unit","text":"Helper function set unit single forecast (.e. combination columns uniquely define single forecast) manually. simple function keeps columns specified forecast_unit (plus additional protected columns, e.g. true values, predictions quantile levels) removes duplicate rows. done manually, scoringutils attempts determine unit single forecast automatically simply assuming column names relevant determine forecast unit. may lead unexpected behaviour, setting forecast unit explicitly can help make code easier debug easier read. used part workflow, set_forecast_unit() can directly piped check_forecasts() check everything order.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/set_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Set unit of a single forecast manually — set_forecast_unit","text":"","code":"set_forecast_unit(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/reference/set_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Set unit of a single forecast manually — set_forecast_unit","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary). forecast_unit Character vector names columns uniquely identify single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/set_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Set unit of a single forecast manually — set_forecast_unit","text":"data.table columns kept relevant scoring denote unit single forecast specified user.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/set_forecast_unit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Set unit of a single forecast manually — set_forecast_unit","text":"","code":"set_forecast_unit(   example_quantile,   c(\"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\") ) #>        true_value quantile prediction location target_end_date target_type #>     1:     127300       NA         NA       DE      2021-01-02       Cases #>     2:       4534       NA         NA       DE      2021-01-02      Deaths #>     3:     154922       NA         NA       DE      2021-01-09       Cases #>     4:       6117       NA         NA       DE      2021-01-09      Deaths #>     5:     110183       NA         NA       DE      2021-01-16       Cases #>    ---                                                                     #> 20541:         78    0.850        352       IT      2021-07-24      Deaths #> 20542:         78    0.900        397       IT      2021-07-24      Deaths #> 20543:         78    0.950        499       IT      2021-07-24      Deaths #> 20544:         78    0.975        611       IT      2021-07-24      Deaths #> 20545:         78    0.990        719       IT      2021-07-24      Deaths #>        horizon                model #>     1:      NA                 <NA> #>     2:      NA                 <NA> #>     3:      NA                 <NA> #>     4:      NA                 <NA> #>     5:      NA                 <NA> #>    ---                              #> 20541:       2 epiforecasts-EpiNow2 #> 20542:       2 epiforecasts-EpiNow2 #> 20543:       2 epiforecasts-EpiNow2 #> 20544:       2 epiforecasts-EpiNow2 #> 20545:       2 epiforecasts-EpiNow2"},{"path":"https://epiforecasts.io/scoringutils/reference/squared_error.html","id":null,"dir":"Reference","previous_headings":"","what":"Squared Error — squared_error","title":"Squared Error — squared_error","text":"Squared Error SE calculated $$   (\\textrm{true\\_values} - \\textrm{predicted\\_values})^2 $$","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/squared_error.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Squared Error — squared_error","text":"","code":"squared_error(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/squared_error.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Squared Error — squared_error","text":"true_values vector true observed values size n predictions vector predicted values size n","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/squared_error.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Squared Error — squared_error","text":"vector scoring values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/squared_error.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Squared Error — squared_error","text":"","code":"true_values <- rnorm(30, mean = 1:30) predicted_values <- rnorm(30, mean = 1:30) squared_error(true_values, predicted_values) #>  [1]  0.51162387  1.78113912  0.79705667  1.83329995  0.30668430  0.36035257 #>  [7] 10.35323665  0.78037062  1.98496901  0.03794845  0.13034237  3.26910428 #> [13]  2.84648268  1.00105946  0.75844437  0.46828924  7.78161096  0.63237370 #> [19]  0.06595388  1.17345957  1.54562777  4.11031694  0.06182022  1.01685836 #> [25]  0.61568034  7.35156133  6.54269939  0.12839408  0.25734304  2.38597297"},{"path":"https://epiforecasts.io/scoringutils/reference/summarise_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Summarise scores as produced by score() — summarise_scores","title":"Summarise scores as produced by score() — summarise_scores","text":"Summarise scores produced score()","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/summarise_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Summarise scores as produced by score() — summarise_scores","text":"","code":"summarise_scores(   scores,   by = NULL,   across = NULL,   fun = mean,   relative_skill = FALSE,   relative_skill_metric = \"auto\",   metric = deprecated(),   baseline = NULL,   ... )  summarize_scores(   scores,   by = NULL,   across = NULL,   fun = mean,   relative_skill = FALSE,   relative_skill_metric = \"auto\",   metric = deprecated(),   baseline = NULL,   ... )"},{"path":"https://epiforecasts.io/scoringutils/reference/summarise_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Summarise scores as produced by score() — summarise_scores","text":"scores data.table scores produced score(). character vector column names summarise scores . Default NULL, meaning summary takes place summarising samples quantiles (case quantile-based forecasts), one score per forecast defined unit single forecast (rather one score every sample quantile). unit single forecast determined columns present input data correspond metric produced score(), indicate indicate grouping forecasts (example may one forecast per day, location model). Adding additional, unrelated, columns may alter results unpredictable way. across character vector column names vector variables define unit single forecast (see ) summarise scores across (meaning specified columns dropped). alternative specifying directly. NULL (default), used inferred internally also specified.  one across  may used time. fun function used summarising scores. Default mean. relative_skill logical, whether compute relative performance models based pairwise comparisons. TRUE (default FALSE), column called 'model' must present input data. information computation relative skill, see pairwise_comparison(). Relative skill calculated aggregation level specified . relative_skill_metric character name metric relative skill shall computed. equal 'auto' (default), either interval score, CRPS Brier score (depending available input data) metric Deprecated 1.1.0. Use relative_skill_metric instead. baseline character string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. ... additional parameters can passed summary function provided fun. information see documentation respective function.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/summarise_scores.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Summarise scores as produced by score() — summarise_scores","text":"","code":"data.table::setDTthreads(1) # only needed to avoid issues on CRAN library(magrittr) # pipe operator  scores <- score(example_continuous) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. summarise_scores(scores) #>      location location_name target_end_date target_type forecast_date #>   1:       DE       Germany      2021-05-08       Cases    2021-05-03 #>   2:       DE       Germany      2021-05-08       Cases    2021-05-03 #>   3:       DE       Germany      2021-05-08       Cases    2021-05-03 #>   4:       DE       Germany      2021-05-08      Deaths    2021-05-03 #>   5:       DE       Germany      2021-05-08      Deaths    2021-05-03 #>  ---                                                                  #> 883:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 884:       IT         Italy      2021-07-24      Deaths    2021-07-05 #> 885:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 886:       IT         Italy      2021-07-24      Deaths    2021-07-05 #> 887:       IT         Italy      2021-07-24      Deaths    2021-07-12 #>                      model horizon         mad  bias       dss         crps #>   1: EuroCOVIDhub-ensemble       1 17641.24334  0.55 20.601386  7482.975177 #>   2: EuroCOVIDhub-baseline       1 19341.68942  0.95 22.443370 20371.250988 #>   3:  epiforecasts-EpiNow2       1 32348.79978  0.80 22.310568 24810.424753 #>   4: EuroCOVIDhub-ensemble       1   267.13585  0.20 11.112388    67.510511 #>   5: EuroCOVIDhub-baseline       1   397.09371 -0.05 11.541040    86.462930 #>  ---                                                                        #> 883: EuroCOVIDhub-baseline       2   168.03093  0.45 12.913561    66.515150 #> 884:       UMass-MechBayes       3    24.76457  0.10  6.353679     6.616381 #> 885:       UMass-MechBayes       2    39.96542  0.80  8.765716    29.446723 #> 886:  epiforecasts-EpiNow2       3   107.93293  0.10 10.283832    28.863742 #> 887:  epiforecasts-EpiNow2       2    93.54245  0.65 10.563729    49.998906 #>      log_score    ae_median      se_mean #>   1: 10.989568  9413.839675 2.303980e+08 #>   2: 11.941135 29378.807513 9.128538e+08 #>   3: 12.007007 36512.810659 1.603609e+09 #>   4:  6.516954    77.077318 8.593954e+03 #>   5:  6.913879    27.429626 2.123067e+03 #>  ---                                     #> 883:  6.067282    58.606302 6.065218e+04 #> 884:  4.242600     5.071742 6.043161e+01 #> 885:  5.321013    42.271396 2.153122e+03 #> 886:  5.709216     7.760566 3.212150e+03 #> 887:  5.736238    73.670518 1.263027e+04   # summarise over samples or quantiles to get one score per forecast scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. summarise_scores(scores) #>      location target_end_date target_type location_name forecast_date #>   1:       DE      2021-05-08       Cases       Germany    2021-05-03 #>   2:       DE      2021-05-08       Cases       Germany    2021-05-03 #>   3:       DE      2021-05-08       Cases       Germany    2021-05-03 #>   4:       DE      2021-05-08      Deaths       Germany    2021-05-03 #>   5:       DE      2021-05-08      Deaths       Germany    2021-05-03 #>  ---                                                                  #> 883:       IT      2021-07-24      Deaths         Italy    2021-07-05 #> 884:       IT      2021-07-24      Deaths         Italy    2021-07-12 #> 885:       IT      2021-07-24      Deaths         Italy    2021-07-12 #> 886:       IT      2021-07-24      Deaths         Italy    2021-07-12 #> 887:       IT      2021-07-24      Deaths         Italy    2021-07-12 #>                      model horizon interval_score  dispersion underprediction #>   1: EuroCOVIDhub-baseline       1    16925.04696 1649.220870       0.0000000 #>   2: EuroCOVIDhub-ensemble       1     7990.85478 5440.985217       0.0000000 #>   3:  epiforecasts-EpiNow2       1    25395.96087 8173.700000       0.0000000 #>   4: EuroCOVIDhub-baseline       1       46.79304   44.662609       0.0000000 #>   5: EuroCOVIDhub-ensemble       1       53.88000   53.271304       0.6086957 #>  ---                                                                          #> 883:  epiforecasts-EpiNow2       3       19.76261   14.284348       0.0000000 #> 884: EuroCOVIDhub-baseline       2       80.33696   76.728261       0.0000000 #> 885: EuroCOVIDhub-ensemble       2       18.65870   13.354348       0.0000000 #> 886:       UMass-MechBayes       2       25.58174    7.755652       0.0000000 #> 887:  epiforecasts-EpiNow2       2       66.16174   25.553043       0.0000000 #>      overprediction coverage_deviation  bias ae_median #>   1:   15275.826087        -0.38521739  0.95     25620 #>   2:    2549.869565         0.04956522  0.50     12271 #>   3:   17222.260870        -0.29826087  0.90     44192 #>   4:       2.130435         0.22347826  0.30        15 #>   5:       0.000000         0.39739130 -0.10        14 #>  ---                                                   #> 883:       5.478261         0.04956522  0.50        26 #> 884:       3.608696         0.31043478  0.20        53 #> 885:       5.304348         0.13652174  0.40        30 #> 886:      17.826087        -0.21130435  0.80        46 #> 887:      40.608696        -0.29826087  0.90       108  # get scores by model summarise_scores(scores,by = \"model\") #>                    model interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline    14321.48926 2096.95360      5143.53567 #> 2: EuroCOVIDhub-ensemble     8992.62316 1846.85278      2120.64029 #> 3:  epiforecasts-EpiNow2    10827.40786 2950.73422      1697.23411 #> 4:       UMass-MechBayes       52.65195   26.87239        16.80095 #>    overprediction coverage_deviation        bias   ae_median #> 1:    7081.000000         0.00201087  0.21851562 19353.42969 #> 2:    5025.130095         0.04871603  0.00812500 12077.10156 #> 3:    6179.439535        -0.05516986 -0.04336032 14521.10526 #> 4:       8.978601        -0.02312500 -0.02234375    78.47656  # get scores by model and target type summarise_scores(scores, by = c(\"model\", \"target_type\")) #>                    model target_type interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline       Cases    28483.57465 4102.50094    10284.972826 #> 2: EuroCOVIDhub-ensemble       Cases    17943.82383 3663.52458     4237.177310 #> 3:  epiforecasts-EpiNow2       Cases    20831.55662 5664.37795     3260.355639 #> 4: EuroCOVIDhub-baseline      Deaths      159.40387   91.40625        2.098505 #> 5: EuroCOVIDhub-ensemble      Deaths       41.42249   30.18099        4.103261 #> 6:       UMass-MechBayes      Deaths       52.65195   26.87239       16.800951 #> 7:  epiforecasts-EpiNow2      Deaths       66.64282   31.85692       15.893314 #>    overprediction coverage_deviation        bias   ae_median #> 1:   14096.100883        -0.11211957  0.09796875 38473.60156 #> 2:   10043.121943        -0.09785326 -0.05640625 24101.07031 #> 3:   11906.823030        -0.06660326 -0.07890625 27923.81250 #> 4:      65.899117         0.11614130  0.33906250   233.25781 #> 5:       7.138247         0.19528533  0.07265625    53.13281 #> 6:       8.978601        -0.02312500 -0.02234375    78.47656 #> 7:      18.892583        -0.04287176 -0.00512605   104.74790  # Get scores summarised across horizon, forecast date, and target end date summarise_scores(  scores, across = c(\"horizon\", \"forecast_date\", \"target_end_date\") ) #>     location target_type  location_name                 model interval_score #>  1:       DE       Cases        Germany EuroCOVIDhub-baseline    14506.65500 #>  2:       DE       Cases        Germany EuroCOVIDhub-ensemble     6286.66495 #>  3:       DE       Cases        Germany  epiforecasts-EpiNow2    11684.72865 #>  4:       DE      Deaths        Germany EuroCOVIDhub-baseline      155.91235 #>  5:       DE      Deaths        Germany EuroCOVIDhub-ensemble       44.46077 #>  6:       DE      Deaths        Germany       UMass-MechBayes       68.91583 #>  7:       DE      Deaths        Germany  epiforecasts-EpiNow2       93.33921 #>  8:       FR       Cases         France EuroCOVIDhub-baseline    54147.68308 #>  9:       FR       Cases         France EuroCOVIDhub-ensemble    44537.04769 #> 10:       FR       Cases         France  epiforecasts-EpiNow2    50141.70361 #> 11:       FR      Deaths         France EuroCOVIDhub-baseline      187.63842 #> 12:       FR      Deaths         France EuroCOVIDhub-ensemble       56.70677 #> 13:       FR      Deaths         France       UMass-MechBayes       74.17250 #> 14:       FR      Deaths         France  epiforecasts-EpiNow2       96.69760 #> 15:       GB       Cases United Kingdom EuroCOVIDhub-baseline    36032.52912 #> 16:       GB       Cases United Kingdom EuroCOVIDhub-ensemble    16010.55516 #> 17:       GB       Cases United Kingdom  epiforecasts-EpiNow2    18303.21825 #> 18:       GB      Deaths United Kingdom EuroCOVIDhub-baseline       95.17583 #> 19:       GB      Deaths United Kingdom EuroCOVIDhub-ensemble       20.04649 #> 20:       GB      Deaths United Kingdom       UMass-MechBayes       33.45330 #> 21:       GB      Deaths United Kingdom  epiforecasts-EpiNow2       25.77830 #> 22:       IT       Cases          Italy EuroCOVIDhub-baseline     9247.43141 #> 23:       IT       Cases          Italy EuroCOVIDhub-ensemble     4941.02753 #> 24:       IT       Cases          Italy  epiforecasts-EpiNow2     3196.57595 #> 25:       IT      Deaths          Italy EuroCOVIDhub-baseline      198.88887 #> 26:       IT      Deaths          Italy EuroCOVIDhub-ensemble       44.47594 #> 27:       IT      Deaths          Italy       UMass-MechBayes       34.06615 #> 28:       IT      Deaths          Italy  epiforecasts-EpiNow2       59.20908 #>     location target_type  location_name                 model interval_score #>      dispersion underprediction overprediction coverage_deviation        bias #>  1:  2923.63054    1.147921e+02   1.146823e+04        -0.08630435  0.49062500 #>  2:  2286.83071    4.768886e+02   3.522946e+03        -0.06184783  0.17375000 #>  3:  3382.48817    9.200027e+02   7.382238e+03        -0.14608696  0.13375000 #>  4:    78.26697    0.000000e+00   7.764538e+01         0.02510870  0.52812500 #>  5:    35.27327    4.533967e+00   4.653533e+00         0.23842391 -0.06875000 #>  6:    31.75822    3.715761e+01   0.000000e+00        -0.07271739 -0.63312500 #>  7:    44.37726    3.803397e+01   1.092799e+01        -0.06184783 -0.45937500 #>  8:  6294.16678    8.578344e+03   3.927517e+04        -0.13521739  0.16250000 #>  9:  4834.10747    6.054179e+03   3.364876e+04        -0.13521739  0.02281250 #> 10:  7852.48758    7.281398e+03   3.500782e+04        -0.06456522 -0.05718750 #> 11:   101.19685    0.000000e+00   8.644158e+01         0.07945652  0.46562500 #> 12:    41.62796    7.942935e+00   7.135870e+00         0.20445652  0.06562500 #> 13:    40.59505    2.091984e+01   1.265761e+01         0.05635870  0.08812500 #> 14:    60.31008    1.061815e+01   2.576938e+01         0.02310019 -0.07173913 #> 15:  4300.45982    3.166997e+04   6.210326e+01        -0.18141304 -0.63156250 #> 16:  6016.72500    8.891671e+03   1.102159e+03        -0.09445652 -0.50937500 #> 17: 10391.44923    3.617322e+03   4.294447e+03         0.01695652 -0.28125000 #> 18:    86.38914    8.266304e+00   5.203804e-01         0.29956522 -0.10625000 #> 19:    16.20003    1.241848e+00   2.604620e+00         0.19358696  0.15937500 #> 20:    14.34596    4.891304e-02   1.905842e+01        -0.12978261  0.64468750 #> 21:    13.71580    1.086957e-02   1.205163e+01        -0.01701087  0.54875000 #> 22:  2891.74663    7.767894e+02   5.578895e+03        -0.04554348  0.37031250 #> 23:  1516.43514    1.525970e+03   1.898622e+03        -0.09989130  0.08718750 #> 24:  1031.08682    1.222700e+03   9.427894e+02        -0.07271739 -0.11093750 #> 25:    99.77202    1.277174e-01   9.898913e+01         0.06043478  0.46875000 #> 26:    27.62268    2.694293e+00   1.415897e+01         0.14467391  0.13437500 #> 27:    20.79034    9.077446e+00   4.198370e+00         0.05364130 -0.18906250 #> 28:    17.02701    1.342663e+01   2.875543e+01        -0.09717391 -0.05687500 #>      dispersion underprediction overprediction coverage_deviation        bias #>       ae_median #>  1: 21109.81250 #>  2:  9949.12500 #>  3: 18315.09375 #>  4:   261.03125 #>  5:    51.93750 #>  6:   124.50000 #>  7:   154.62500 #>  8: 67726.87500 #>  9: 52432.93750 #> 10: 60549.84375 #> 11:   293.59375 #> 12:    72.12500 #> 13:    88.53125 #> 14:   143.00000 #> 15: 49934.03125 #> 16: 26422.84375 #> 17: 27901.06250 #> 18:    59.03125 #> 19:    22.96875 #> 20:    55.34375 #> 21:    40.62500 #> 22: 15123.68750 #> 23:  7599.37500 #> 24:  4929.25000 #> 25:   319.37500 #> 26:    65.50000 #> 27:    45.53125 #> 28:    91.50000 #>       ae_median  # get standard deviation summarise_scores(scores, by = \"model\", fun = sd) #>                    model interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline      43157.488  2682.7722      18417.2012 #> 2: EuroCOVIDhub-ensemble      38788.740  3863.3379       7179.5954 #> 3:  epiforecasts-EpiNow2      42478.208  6698.7470       7629.9180 #> 4:       UMass-MechBayes         42.813    25.2045         30.8519 #>    overprediction coverage_deviation      bias   ae_median #> 1:     38723.6131          0.2589550 0.5628605 50186.94967 #> 2:     36978.7730          0.2726646 0.5707267 42500.79359 #> 3:     38832.5598          0.2838865 0.6632842 49800.66917 #> 4:        18.3498          0.2705116 0.6457779    74.79904  # round digits summarise_scores(scores,by = \"model\") %>%   summarise_scores(fun = signif, digits = 2) #>                    model interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline          14000       2100            5100 #> 2: EuroCOVIDhub-ensemble           9000       1800            2100 #> 3:  epiforecasts-EpiNow2          11000       3000            1700 #> 4:       UMass-MechBayes             53         27              17 #>    overprediction coverage_deviation    bias ae_median #> 1:           7100              0.002  0.2200     19000 #> 2:           5000              0.049  0.0081     12000 #> 3:           6200             -0.055 -0.0430     15000 #> 4:              9             -0.023 -0.0220        78  # get quantiles of scores # make sure to aggregate over ranges first summarise_scores(scores,   by = \"model\", fun = quantile,   probs = c(0.25, 0.5, 0.75) ) #>                     model interval_score dispersion underprediction #>  1: EuroCOVIDhub-baseline      130.23793   87.81804      0.00000000 #>  2: EuroCOVIDhub-baseline      934.41870  163.28739      0.00000000 #>  3: EuroCOVIDhub-baseline    10795.61967 3476.52750      4.42391304 #>  4: EuroCOVIDhub-ensemble       28.33707   23.67087      0.00000000 #>  5: EuroCOVIDhub-ensemble      300.30739  168.95543      0.00000000 #>  6: EuroCOVIDhub-ensemble     5532.28565 2030.41870     44.10869565 #>  7:  epiforecasts-EpiNow2       50.94935   16.90326      0.00000000 #>  8:  epiforecasts-EpiNow2      442.81696  152.12565      0.08695652 #>  9:  epiforecasts-EpiNow2     5902.85957 2033.24370    155.76086957 #> 10:       UMass-MechBayes       18.49435    7.73837      0.00000000 #> 11:       UMass-MechBayes       46.05413   17.15413      0.02173913 #> 12:       UMass-MechBayes       73.27087   38.04620     18.41304348 #>     overprediction coverage_deviation  bias ae_median #>  1:      0.0000000         -0.1243478 -0.20    166.00 #>  2:     54.1304348         -0.0373913  0.40    763.50 #>  3:    496.7065217          0.2234783  0.60  17709.00 #>  4:      0.0000000         -0.1243478 -0.40     27.00 #>  5:      0.2826087          0.1365217  0.10    277.00 #>  6:     43.5217391          0.2452174  0.40   8100.00 #>  7:      0.0000000         -0.2982609 -0.70     68.00 #>  8:      0.0000000         -0.0373913 -0.10    404.00 #>  9:     45.6956522          0.1365217  0.60   7177.00 #> 10:      0.0000000         -0.2113043 -0.60     24.75 #> 11:      0.0000000         -0.0373913 -0.05     52.00 #> 12:     10.2500000          0.2234783  0.60    106.00  # get ranges # summarise_scores(scores, by = \"range\")"},{"path":"https://epiforecasts.io/scoringutils/reference/theme_scoringutils.html","id":null,"dir":"Reference","previous_headings":"","what":"Scoringutils ggplot2 theme — theme_scoringutils","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"theme ggplot2 plots used scoringutils","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/theme_scoringutils.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"","code":"theme_scoringutils()"},{"path":"https://epiforecasts.io/scoringutils/reference/theme_scoringutils.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"ggplot2 theme","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/transform_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Transform forecasts and observed values — transform_forecasts","title":"Transform forecasts and observed values — transform_forecasts","text":"Function transform forecasts true values scoring.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/transform_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Transform forecasts and observed values — transform_forecasts","text":"","code":"transform_forecasts(data, fun = log_shift, append = TRUE, label = \"log\", ...)"},{"path":"https://epiforecasts.io/scoringutils/reference/transform_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Transform forecasts and observed values — transform_forecasts","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary). fun function used transform true values predictions. default function log_shift(), custom function essentially log(), additional arguments (offset) allows add offset applying logarithm. often helpful natural log transformation defined zero. common, pragmatic solution, add small offset data applying log transformation. work often used offset 1 precise value depend application. append Logical, defaults TRUE. Whether append transformed version data currently existing data (TRUE). selected, data gets transformed appended existing data frame, making possible use outcome directly score(). additional column, 'scale', gets created denotes rows untransformed ('scale' value \"natural\") transformed ('scale' value passed argument label). label string newly created 'scale' column denote newly transformed values. relevant append = TRUE. ... Additional parameters pass function supplied. default option log_shift() offset argument.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/transform_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Transform forecasts and observed values — transform_forecasts","text":"data.table either transformed version data, one untransformed transformed data. includes original data well transformation original data. one additional column, `scale', present set \"natural\" untransformed forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/transform_forecasts.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Transform forecasts and observed values — transform_forecasts","text":"reasons, depending circumstances, might desirable (check linked reference info). epidemiology, example, may useful log-transform incidence counts evaluating forecasts using scores weighted interval score (WIS) continuous ranked probability score (CRPS). Log-transforming forecasts observations changes interpretation score measure absolute distance forecast observation score evaluates forecast exponential growth rate. Another motivation can apply variance-stabilising transformation standardise incidence counts population. Note want apply transformation, important transform forecasts observations apply score. Applying transformation score risks losing propriety proper scoring rule.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/transform_forecasts.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Transform forecasts and observed values — transform_forecasts","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722  https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/transform_forecasts.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Transform forecasts and observed values — transform_forecasts","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/transform_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Transform forecasts and observed values — transform_forecasts","text":"","code":"library(magrittr) # pipe operator  # transform forecasts using the natural logarithm # negative values need to be handled (here by replacing them with 0) example_quantile %>%   .[, true_value := ifelse(true_value < 0, 0, true_value)] %>% # Here we use the default function log_shift() which is essentially the same # as log(), but has an additional arguments (offset) that allows you add an # offset before applying the logarithm.   transform_forecasts(append = FALSE) %>%   head() #> Warning: Detected zeros in input values.Try specifying offset = 1 (or any other offset). #> Warning: Detected zeros in input values.Try specifying offset = 1 (or any other offset). #>    location target_end_date target_type true_value location_name forecast_date #> 1:       DE      2021-01-02       Cases  11.754302       Germany          <NA> #> 2:       DE      2021-01-02      Deaths   8.419360       Germany          <NA> #> 3:       DE      2021-01-09       Cases  11.950677       Germany          <NA> #> 4:       DE      2021-01-09      Deaths   8.718827       Germany          <NA> #> 5:       DE      2021-01-16       Cases  11.609898       Germany          <NA> #> 6:       DE      2021-01-16      Deaths   8.677099       Germany          <NA> #>    quantile prediction model horizon #> 1:       NA         NA  <NA>      NA #> 2:       NA         NA  <NA>      NA #> 3:       NA         NA  <NA>      NA #> 4:       NA         NA  <NA>      NA #> 5:       NA         NA  <NA>      NA #> 6:       NA         NA  <NA>      NA  # alternatively, integrating the truncation in the transformation function: example_quantile %>%  transform_forecasts(    fun = function(x) {log_shift(pmax(0, x))}, append = FALSE  ) %>%  head() #> Warning: Detected zeros in input values.Try specifying offset = 1 (or any other offset). #> Warning: Detected zeros in input values.Try specifying offset = 1 (or any other offset). #>    location target_end_date target_type true_value location_name forecast_date #> 1:       DE      2021-01-02       Cases  11.754302       Germany          <NA> #> 2:       DE      2021-01-02      Deaths   8.419360       Germany          <NA> #> 3:       DE      2021-01-09       Cases  11.950677       Germany          <NA> #> 4:       DE      2021-01-09      Deaths   8.718827       Germany          <NA> #> 5:       DE      2021-01-16       Cases  11.609898       Germany          <NA> #> 6:       DE      2021-01-16      Deaths   8.677099       Germany          <NA> #>    quantile prediction model horizon #> 1:       NA         NA  <NA>      NA #> 2:       NA         NA  <NA>      NA #> 3:       NA         NA  <NA>      NA #> 4:       NA         NA  <NA>      NA #> 5:       NA         NA  <NA>      NA #> 6:       NA         NA  <NA>      NA  # specifying an offset for the log transformation removes the # warning caused by zeros in the data example_quantile %>%   .[, true_value := ifelse(true_value < 0, 0, true_value)] %>%   transform_forecasts(offset = 1, append = FALSE) %>%   head() #>    location target_end_date target_type true_value location_name forecast_date #> 1:       DE      2021-01-02       Cases  11.754310       Germany          <NA> #> 2:       DE      2021-01-02      Deaths   8.419580       Germany          <NA> #> 3:       DE      2021-01-09       Cases  11.950683       Germany          <NA> #> 4:       DE      2021-01-09      Deaths   8.718991       Germany          <NA> #> 5:       DE      2021-01-16       Cases  11.609907       Germany          <NA> #> 6:       DE      2021-01-16      Deaths   8.677269       Germany          <NA> #>    quantile prediction model horizon #> 1:       NA         NA  <NA>      NA #> 2:       NA         NA  <NA>      NA #> 3:       NA         NA  <NA>      NA #> 4:       NA         NA  <NA>      NA #> 5:       NA         NA  <NA>      NA #> 6:       NA         NA  <NA>      NA  # adding square root transformed forecasts to the original ones example_quantile %>%   .[, true_value := ifelse(true_value < 0, 0, true_value)] %>%   transform_forecasts(fun = sqrt, label = \"sqrt\") %>%   score() %>%   summarise_scores(by = c(\"model\", \"scale\")) #> The following messages were produced when checking inputs: #> 1.  288 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model   scale interval_score   dispersion underprediction #> 1: EuroCOVIDhub-baseline natural   11124.930667 2096.9535954    5143.5356658 #> 2: EuroCOVIDhub-baseline    sqrt      27.742316    7.7296761       9.5936380 #> 3: EuroCOVIDhub-ensemble natural    5796.064569 1846.8527819    2120.6402853 #> 4: EuroCOVIDhub-ensemble    sqrt      14.974344    4.2878323       5.1827454 #> 5:  epiforecasts-EpiNow2 natural    7514.375476 2950.7342158    1697.2341137 #> 6:  epiforecasts-EpiNow2    sqrt      17.704899    5.4112770       5.7235785 #> 7:       UMass-MechBayes natural      52.651946   26.8723947      16.8009511 #> 8:       UMass-MechBayes    sqrt       1.328653    0.5993586       0.4019195 #>    overprediction coverage_deviation        bias    ae_median #> 1:   3884.4414062        0.003369565  0.21816406 16156.871094 #> 2:     10.4190016        0.003369565  0.21816406    39.185406 #> 3:   1828.5715014        0.048716033  0.00812500  8880.542969 #> 4:      5.5037665        0.048716033  0.00812500    22.458900 #> 5:   2866.4071466       -0.055169864 -0.04336032 11208.072874 #> 6:      6.5700431       -0.055169864 -0.04336032    25.585018 #> 7:      8.9786005       -0.023125000 -0.02234375    78.476562 #> 8:      0.3273746       -0.023125000 -0.02234375     2.069103  # adding multiple transformations example_quantile %>%   .[, true_value := ifelse(true_value < 0, 0, true_value)] %>%   transform_forecasts(fun = log_shift, offset = 1) %>%   transform_forecasts(fun = sqrt, label = \"sqrt\") %>%   head() #>    location target_end_date target_type true_value location_name forecast_date #> 1:       DE      2021-01-02       Cases     127300       Germany          <NA> #> 2:       DE      2021-01-02      Deaths       4534       Germany          <NA> #> 3:       DE      2021-01-09       Cases     154922       Germany          <NA> #> 4:       DE      2021-01-09      Deaths       6117       Germany          <NA> #> 5:       DE      2021-01-16       Cases     110183       Germany          <NA> #> 6:       DE      2021-01-16      Deaths       5867       Germany          <NA> #>    quantile prediction model horizon   scale #> 1:       NA         NA  <NA>      NA natural #> 2:       NA         NA  <NA>      NA natural #> 3:       NA         NA  <NA>      NA natural #> 4:       NA         NA  <NA>      NA natural #> 5:       NA         NA  <NA>      NA natural #> 6:       NA         NA  <NA>      NA natural"},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-1-2-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.1","text":"gets rid preferably package _pkgdown.yml. theme toggle light dark theme didn’t work properly updates gh pages deploy action v4 also cleans files triggered introduces gh action automatically render Readme Readme.Rmd removes links vignettes renamed","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"scoringutils-120","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.0","title":"scoringutils 1.2.0","text":"major release contains range new features bug fixes introduced minor releases since 1.1.0. important changes : Documentation updated reflect changes since version 1.1.0, including new transform workflow functions. New set_forecast_unit() function allows manual setting forecast unit. summarise_scores() gains new across argument summarizing across variables. New transform_forecasts() log_shift() functions allow forecast transformations. See documentation transform_forecasts() details example use case. Input checks test coverage improved bias functions. Bug fix get_prediction_type() integer matrix input. Links scoringutils paper citation updates. Warning added interval_score() small interval ranges. Linting updates improvements. Thanks @nikosbosse, @seabbs, @sbfnk code review contributions. Thanks @bisaloo suggestion use linting GitHub Action triggers changes, @adrian-lison suggestion add warning interval_score() interval range 0 1.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-1-2-0","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.0","text":"documentation updated reflect recent changes since scoringutils 1.1.0. particular, usage functions set_forecast_unit(), check_forecasts() transform_forecasts() now documented Vignettes. introduction functions enhances overall workflow help make code readable. functions designed used together pipe operator. example, one can now use something like following: Documentation transform_forecasts() also extended. functions allows user easily add transformations forecasts, suggested paper “Scoring epidemiological forecasts transformed scales”. epidemiological context, example, may make sense apply natural logarithm first scoring forecasts, order obtain scores reflect well models able predict exponential growth rates, rather absolute values. Users can now something like following score transformed version data addition original one: use log_shift() function apply logarithmic transformation forecasts. function introduced scoringutils 1.1.2 helper function acts just like log(), additional argument offset can add number every prediction observed value applying log transformation.","code":"example_quantile |>    set_forecast_unit(c(\"model\", \"location\", \"forecast_date\", \"horizon\", \"target_type\")) |>    check_forecasts() |>    score() data <- example_quantile[true_value > 0, ] data |>   transform_forecasts(fun = log_shift, offset = 1) |>    score() |>    summarise_scores(by = c(\"model\", \"scale\"))"},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-1-2-0","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.2.0","text":"Made check_forecasts() score() pipeable (see issue #290). means users can now directly use output check_forecasts() input score(). score() otherwise runs check_forecasts() internally anyway simply makes step explicit helps writing clearer code.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"scoringutils-117","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.7","title":"scoringutils 1.1.7","text":"Release @seabbs #305. Reviewed @nikosbosse @sbfnk.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"breaking-changes-1-1-7","dir":"Changelog","previous_headings":"","what":"Breaking changes","title":"scoringutils 1.1.7","text":"prediction_type argument get_forecast_unit() changed dropped. Instead new internal function prediction_is_quantile() used detect quantile variable present. Whilst internal function may impact users accessible via `find_duplicates().","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-1-1-7","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.7","text":"Made imputation median bias_range() bias_quantile() obvious user may cause unexpected behaviour. Simplified bias_range() uses bias_quantile() internally. Added additional input checks bias_range(), bias_quantile(), check_predictions() make sure input valid. Improve coverage unit tests bias_range(), bias_quantile(), bias_sample(). Updated pairwise comparison unit tests use realistic data.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"bug-fixes-1-1-7","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.1.7","text":"Fixed bug get_prediction_type() led unable correctly detect integer (instead categorising continuous) forecasts input matrix. issue impacted bias_sample() also score() used integer forecasts resulting lower bias scores expected.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-1-1-6","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.6","text":"Added new argument, across, summarise_scores(). argument allows user summarise scores across different forecast units alternative specifying . See documentation summarise_scores() details example use case.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-1-1-5","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.5","text":"Added new function, set_forecast_unit() allows user set forecast unit manually. function removes columns relevant uniquely identifying single forecast. done manually, scoringutils attempts determine unit single automatically simply assuming column names relevant determine forecast unit. can lead unexpected behaviour, setting forecast unit explicitly can help make code easier debug easier read (see issue #268). used part workflow, set_forecast_unit() can directly piped check_forecasts() check everything order.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-1-1-4","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.4","text":"Added links scoringutils paper Evaluating Forecasts scoringutils R package. Updated citation formatting comply newer standards.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-1-1-3","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.3","text":"Added warning interval_score() interval range 0 1. Thanks @adrian-lison (see #277) suggestion.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-1-1-3-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.3","text":"Switched linting GitHub Action triggers changes. Inspired @bisaloo recent contribution epinowcast package. Updated package linters extensive. Inspired @bisaloo recent contribution epinowcast package. Resolved flagged linting issues across package.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-1-1-2","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.2","text":"Added new function, transform_forecasts() make easy transform forecasts scoring , suggested Bosse et al. (2023), https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1. Added function, log_shift() implements default transformation function. function allows add offset applying logarithm.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"scoringutils-111","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.1","title":"scoringutils 1.1.1","text":"Added small change interval_score() explicitly converts logical vector numeric one. happen implicitly anyway, now done explicitly order avoid issues may come input vector type doesn’t allow implicit conversion.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"scoringutils-110","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.0","title":"scoringutils 1.1.0","text":"CRAN release: 2023-01-30 minor update package bug fixes minor changes.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-1-1-0","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.0","text":"Removed attach message warned breaking changes 1.0.0. Renamed metric argument summarise_scores() relative_skill_metric. argument now deprecated removed future version package. Please use new argument instead. Updated documentation score() related functions make soft requirement model column input data explicit. Updated documentation score(), pairwise_comparison() summarise_scores() make clearer unit single forecast required computations Simplified function plot_pairwise_comparison() now supports plotting mean score ratios p-values removed hybrid option print time.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"bug-fixes-1-1-0","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.1.0","text":"Missing baseline forecasts pairwise_comparison() now trigger explicit informative error message. requirements table getting started vignette now correct. Added support optional sample column using quantile forecast format. Previously resulted error.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"scoringutils-100","dir":"Changelog","previous_headings":"","what":"scoringutils 1.0.0","title":"scoringutils 1.0.0","text":"CRAN release: 2022-05-13 Major update package package functions lots breaking changes.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-1-0-0","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.0.0","text":"New updated Readme vignette. proposed scoring workflow reworked. Functions changed can easily piped simplified arguments outputs.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"new-functions-and-function-changes-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"New functions and function changes","title":"scoringutils 1.0.0","text":"function eval_forecasts() replaced function score() much reduced set function arguments. Functionality summarise scores add relative skill scores moved function summarise_scores() New function check_forecasts() analyse input data scoring New function correlation() compute correlations different metrics New function add_coverage() add coverage specific central prediction intervals. New function avail_forecasts() allows visualise number available forecasts. New function find_duplicates() find duplicate forecasts cause error. plotting functions renamed begin plot_. Arguments simplified. function pit() now works based data.frames. old pit function renamed pit_sample(). PIT p-values removed entirely. function plot_pit() now works directly input produced pit() Many data-handling functions removed input types score() restricted sample-based, quantile-based binary forecasts. function brier_score() now returns brier scores, rather taking mean returning output. crps(), dss() logs() renamed crps_sample(), dss_sample(), logs_sample()","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"bug-fixes-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"Bug fixes","title":"scoringutils 1.0.0","text":"Testing expanded Minor bugs fixed, example bug sample_to_quantile() function (https://github.com/epiforecasts/scoringutils/pull/223)","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-data-updated-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"Package data updated","title":"scoringutils 1.0.0","text":"Package data now based forecasts submitted European Forecast Hub (https://covid19forecasthub.eu/). example data files renamed begin example_. new data set, summary_metrics included contains summary metrics implemented scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"other-breaking-changes-1-0-0","dir":"Changelog","previous_headings":"","what":"Other breaking changes","title":"scoringutils 1.0.0","text":"‘sharpness’ component weighted interval score renamed dispersion. done make clear component represents maintain consistency used places.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-0-1-8","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.8","text":"Added function check_forecasts() runs basic checks input data provides feedback.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"scoringutils-0172","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.7.2","title":"scoringutils 0.1.7.2","text":"CRAN release: 2021-07-21","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-0-1-7-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.7.2","text":"Minor bug fixes (previously, ‘interval_score’ needed among selected metrics). data.tables now returned table[] rather table, don’t called twice display contents.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"scoringutils-017","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.7","title":"scoringutils 0.1.7","text":"CRAN release: 2021-07-14","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-0-1-7","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.7","text":"Added function, pairwise_comparison() runs pairwise comparisons models output eval_forecasts() Added functionality compute relative skill within eval_forecasts(). Added function visualise pairwise comparisons.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-0-1-7","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.7","text":"WIS definition change introduced version 0.1.5 partly corrected difference weighting introduced summarising scores different interval ranges. “sharpness” renamed ‘mad’ output [score()] sample-based forecasts.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-0-1","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.","text":"eval_forecasts() can now handle separate forecast truth data set input. eval_forecasts() now supports scoring point forecasts along side quantiles quantile-based format. Currently metric used absolute error.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-0-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.","text":"Many functions, especially eval_forecasts() got major rewrite. functionality unchanged, code now easier maintain data-handling functions got renamed, old names supported well now.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-0-1-5","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.5","text":"Changed default definition weighted interval score. Previously, median prediction counted twice, counted . want go back old behaviour, can call interval_score function argument count_median_twice = FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"scoringutils-014","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.4","title":"scoringutils 0.1.4","text":"CRAN release: 2020-11-17","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-0-1-4","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.4","text":"Added basic plotting functionality visualise scores. can now easily obtain diagnostic plots based scores produced score. correlation_plot() shows correlation metrics. plot_ranges() shows contribution different prediction intervals chosen metric. plot_heatmap() visualises scores heatmap. plot_score_table() shows coloured summary table scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-0-1-4","dir":"Changelog","previous_headings":"","what":"package updates","title":"scoringutils 0.1.4","text":"Renamed “calibration” “coverage”. Renamed “true_values” “true_value” data.frames. Renamed “predictions” “prediction” data.frames. Renamed “is_overprediction” “overprediction”. Renamed “is_underprediction” “underprediction”.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"potentially-breaking-changes-0-1-3","dir":"Changelog","previous_headings":"","what":"(Potentially) Breaking changes","title":"scoringutils 0.1.3","text":"argument score now slightly changed meaning. now denotes lowest possible grouping unit, .e. unit one observation needs specified explicitly. default now NULL. reason change metrics need scoring observation level consistent implementation principle. pit function receives grouping now summarise_by. similar spirit, summarise_by specified explicitly e.g. doesn’t assume anymore want ‘range’ included. interval score, weigh = TRUE now default option. Renamed true_values true_value predictions prediction.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-0-1-3","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.3","text":"Updated quantile evaluation metrics score. Bias well calibration now take quantiles account. Included option summarise scores according summarise_by argument score() summary can return mean, standard deviation well arbitrary set quantiles. score() can now return pit histograms. Switched ggplot2 plotting.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"potentially-breaking-changes-0-1-2","dir":"Changelog","previous_headings":"","what":"(Potentially) Breaking changes","title":"scoringutils 0.1.2","text":"scores score consistently renamed lower case. Interval_score now interval_score, CRPS now crps etc.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-0-1-2","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.2","text":"Included support grouping scores according vector column names score(). Included support passing arguments lower-level functions score() Included support three new metrics score quantiles score(): bias, sharpness calibration","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-0-1-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.2","text":"Example data now horizon column illustrate use grouping. Documentation updated explain listed changes.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-0-1-1","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.1","text":"Included support long well wide input formats quantile forecasts scored score().","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-0-1-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.1","text":"Updated documentation score(). Added badges README.","code":""}]
+[{"path":[]},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"our-pledge","dir":"","previous_headings":"","what":"Our Pledge","title":"Contributor Covenant Code of Conduct","text":"members, contributors, leaders pledge make participation community harassment-free experience everyone, regardless age, body size, visible invisible disability, ethnicity, sex characteristics, gender identity expression, level experience, education, socio-economic status, nationality, personal appearance, race, religion, sexual identity orientation. pledge act interact ways contribute open, welcoming, diverse, inclusive, healthy community.","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"our-standards","dir":"","previous_headings":"","what":"Our Standards","title":"Contributor Covenant Code of Conduct","text":"Examples behavior contributes positive environment community include: Demonstrating empathy kindness toward people respectful differing opinions, viewpoints, experiences Giving gracefully accepting constructive feedback Accepting responsibility apologizing affected mistakes, learning experience Focusing best just us individuals, overall community Examples unacceptable behavior include: use sexualized language imagery, sexual attention advances kind Trolling, insulting derogatory comments, personal political attacks Public private harassment Publishing others’ private information, physical email address, without explicit permission conduct reasonably considered inappropriate professional setting","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"enforcement-responsibilities","dir":"","previous_headings":"","what":"Enforcement Responsibilities","title":"Contributor Covenant Code of Conduct","text":"Community leaders responsible clarifying enforcing standards acceptable behavior take appropriate fair corrective action response behavior deem inappropriate, threatening, offensive, harmful. Community leaders right responsibility remove, edit, reject comments, commits, code, wiki edits, issues, contributions aligned Code Conduct, communicate reasons moderation decisions appropriate.","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"scope","dir":"","previous_headings":"","what":"Scope","title":"Contributor Covenant Code of Conduct","text":"Code Conduct applies within community spaces, also applies individual officially representing community public spaces. Examples representing community include using official e-mail address, posting via official social media account, acting appointed representative online offline event.","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"enforcement","dir":"","previous_headings":"","what":"Enforcement","title":"Contributor Covenant Code of Conduct","text":"Instances abusive, harassing, otherwise unacceptable behavior may reported community leaders responsible enforcement Sebastian.Funk@lshtm.ac.uk. complaints reviewed investigated promptly fairly. community leaders obligated respect privacy security reporter incident.","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"enforcement-guidelines","dir":"","previous_headings":"","what":"Enforcement Guidelines","title":"Contributor Covenant Code of Conduct","text":"Community leaders follow Community Impact Guidelines determining consequences action deem violation Code Conduct:","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"id_1-correction","dir":"","previous_headings":"Enforcement Guidelines","what":"1. Correction","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Use inappropriate language behavior deemed unprofessional unwelcome community. Consequence: private, written warning community leaders, providing clarity around nature violation explanation behavior inappropriate. public apology may requested.","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"id_2-warning","dir":"","previous_headings":"Enforcement Guidelines","what":"2. Warning","title":"Contributor Covenant Code of Conduct","text":"Community Impact: violation single incident series actions. Consequence: warning consequences continued behavior. interaction people involved, including unsolicited interaction enforcing Code Conduct, specified period time. includes avoiding interactions community spaces well external channels like social media. Violating terms may lead temporary permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"id_3-temporary-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"3. Temporary Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: serious violation community standards, including sustained inappropriate behavior. Consequence: temporary ban sort interaction public communication community specified period time. public private interaction people involved, including unsolicited interaction enforcing Code Conduct, allowed period. Violating terms may lead permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"id_4-permanent-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"4. Permanent Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Demonstrating pattern violation community standards, including sustained inappropriate behavior, harassment individual, aggression toward disparagement classes individuals. Consequence: permanent ban sort public interaction within community.","code":""},{"path":"https://epiforecasts.io/scoringutils/CODE_OF_CONDUCT.html","id":"attribution","dir":"","previous_headings":"","what":"Attribution","title":"Contributor Covenant Code of Conduct","text":"Code Conduct adapted Contributor Covenant, version 2.0, available https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. Community Impact Guidelines inspired Mozilla’s code conduct enforcement ladder. answers common questions code conduct, see FAQ https://www.contributor-covenant.org/faq. Translations available https://www.contributor-covenant.org/translations.","code":""},{"path":"https://epiforecasts.io/scoringutils/CONTRIBUTING.html","id":null,"dir":"","previous_headings":"","what":"Contributing to scoringutils","title":"Contributing to scoringutils","text":"outlines propose change scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/CONTRIBUTING.html","id":"fixing-typos","dir":"","previous_headings":"","what":"Fixing typos","title":"Contributing to scoringutils","text":"can fix typos, spelling mistakes, grammatical errors documentation directly using GitHub web interface, long changes made source file. generally means ’ll need edit roxygen2 comments .R, .Rd file. can find .R file generates .Rd reading comment first line.","code":""},{"path":"https://epiforecasts.io/scoringutils/CONTRIBUTING.html","id":"bigger-changes","dir":"","previous_headings":"","what":"Bigger changes","title":"Contributing to scoringutils","text":"want make bigger change, ’s good idea first file issue make sure someone team agrees ’s needed. ’ve found bug, please file issue illustrates bug minimal reprex (also help write unit test, needed).","code":""},{"path":"https://epiforecasts.io/scoringutils/CONTRIBUTING.html","id":"pull-request-process","dir":"","previous_headings":"Bigger changes","what":"Pull request process","title":"Contributing to scoringutils","text":"Fork package clone onto computer. haven’t done , recommend using usethis::create_from_github(\"epiforecasts/scoringutils\", fork = TRUE). Install development dependences devtools::install_dev_deps(), make sure package passes R CMD check running devtools::check(). R CMD check doesn’t pass cleanly, ’s good idea ask help continuing. Create Git branch pull request (PR). recommend using usethis::pr_init(\"brief-description--change\"). Make changes, commit git, create PR running usethis::pr_push(), following prompts browser. title PR briefly describe change. body PR contain Fixes #issue-number. user-facing changes, add bullet top NEWS.md (.e. just first header). Follow style described https://style.tidyverse.org/news.html.","code":""},{"path":"https://epiforecasts.io/scoringutils/CONTRIBUTING.html","id":"code-style","dir":"","previous_headings":"Bigger changes","what":"Code style","title":"Contributing to scoringutils","text":"New code follow tidyverse style guide. can use styler package apply styles, please don’t restyle code nothing PR. use roxygen2, Markdown syntax, documentation. use testthat unit tests. Contributions test cases included easier accept.","code":""},{"path":"https://epiforecasts.io/scoringutils/CONTRIBUTING.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Contributing to scoringutils","text":"Please note scoringutils project released Contributor Code Conduct. contributing project agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/LICENSE.html","id":null,"dir":"","previous_headings":"","what":"MIT License","title":"MIT License","text":"Copyright (c) 2020 EpiForecasts Permission hereby granted, free charge, person obtaining copy software associated documentation files (“Software”), deal Software without restriction, including without limitation rights use, copy, modify, merge, publish, distribute, sublicense, /sell copies Software, permit persons Software furnished , subject following conditions: copyright notice permission notice shall included copies substantial portions Software. SOFTWARE PROVIDED “”, WITHOUT WARRANTY KIND, EXPRESS IMPLIED, INCLUDING LIMITED WARRANTIES MERCHANTABILITY, FITNESS PARTICULAR PURPOSE NONINFRINGEMENT. EVENT SHALL AUTHORS COPYRIGHT HOLDERS LIABLE CLAIM, DAMAGES LIABILITY, WHETHER ACTION CONTRACT, TORT OTHERWISE, ARISING , CONNECTION SOFTWARE USE DEALINGS SOFTWARE.","code":""},{"path":"https://epiforecasts.io/scoringutils/articles/metric-details.html","id":"applicability-of-different-metrics-for-different-types-of-forecasts-and-formats","dir":"Articles","previous_headings":"","what":"Applicability of different metrics for different types of forecasts and formats","title":"Details on the metrics implemented in `scoringutils`","text":"table gives overview metric can applied gives brief description. Note table shows metrics implemented scoringutils. example, scoring sample-based discrete continuous distributions implemented scoringutils, closed-form solutions often exist (e.g. scoringRules package).","code":""},{"path":"https://epiforecasts.io/scoringutils/articles/metric-details.html","id":"detailed-explanation-of-the-metrics-implemented-in-scoringutils","dir":"Articles","previous_headings":"","what":"Detailed explanation of the metrics implemented in scoringutils","title":"Details on the metrics implemented in `scoringutils`","text":"crps proper scoring rule generalises absolute error probabilistic forecasts. measures ‘distance’ predictive distribution observed data-generating distribution. CRPS given \\[\\text{CRPS}(F, y) = \\int_{-\\infty}^\\infty \\left( F(x) - 1(x \\geq y) \\right)^2 dx,\\] y true observed value F CDF predictive distribution. Often alternative representation used: \\[ \\text{CRPS}(F, y) = \\frac{1}{2} \\mathbb{E}_{F} |X - X'| - \\mathbb{E}_P |X - y|,\\] \\(X\\) \\(X'\\) independent realisations predictive distributions \\(F\\) finite first moment \\(y\\) true value. represenation can simply replace \\(X\\) \\(X'\\) samples sum possible combinations obtain CRPS. integer-valued forecasts, RPS given \\[ \\text{RPS}(F, y) = \\sum_{x = 0}^\\infty (F(x) - 1(x \\geq y))^2. \\] Log score proper scoring rule simply compuated log predictive density evaluated true observed value. given \\[ \\text{log score} = \\log f(y), \\] \\(f\\) predictive density function y true value. integer-valued forecasts, log score can computed \\[ \\text{log score} = \\log p_y, \\] \\(p_y\\) probability assigned outcome p forecast F. (weighted) interval score proper scoring rule quantile forecasts converges crps increasing number intervals. score can decomposed sharpness (uncertainty) component penalties - underprediction. single interval, score computed \\[IS_\\alpha(F,y) = (u-l) + \\frac{2}{\\alpha} \\cdot (l-y) \\cdot \\mathbf{1}(y \\leq l) + \\frac{2}{\\alpha} \\cdot (y-u) \\cdot \\mathbf{1}(y \\geq u), \\] \\(\\mathbf{1}()\\) indicator function, \\(y\\) true value, \\(l\\) \\(u\\) \\(\\frac{\\alpha}{2}\\) \\(1 - \\frac{\\alpha}{2}\\) quantiles predictive distribution \\(F\\), .e. lower upper bound single prediction interval. set \\(K\\) prediction intervals median \\(m\\), score computed weighted sum, \\[WIS = \\frac{1}{K + 0.5} \\cdot \\left(w_0 \\cdot |y - m| + \\sum_{k = 1}^{K} w_k \\cdot IS_{\\alpha}(F, y)\\right),\\] \\(w_k\\) weight every interval. Usually, \\(w_k = \\frac{\\alpha_k}{2}\\) \\(w_0 = 0.5\\). Dawid-Sebastiani-Score proper scoring rule proposed relies first moments predictive distribution therefore easy compute. given \\[\\text{dss}(F, y) = \\left( \\frac{y - \\mu}{\\sigma} \\right)^2 + 2 \\cdot \\log \\sigma,\\] \\(F\\) predictive distribution mean \\(\\mu\\) standard deviation \\(\\sigma\\) \\(y\\) true observed value. Proper scoring rule binary forecasts. Brier score computed \\[\\text{Brier Score} = \\frac{1}{N} \\sum_{n = 1}^{N} (f_n - y_n),\\] \\(f_n\\), \\(n = 1, \\dots, N\\) predicted probablities corresponding events, \\(y_n \\(0, 1)\\) equal one. Interval coverage measures proportion observed values fall given prediction interval range. Interval coverage single prediction interval range can calculated \\[IC_{\\alpha} = \\text{nominal coverage} - \\text{empirical coverage},\\] nominal coverage \\(1 - \\alpha\\) empirical coverage proportion true values actually covered \\(1 - \\alpha\\) prediction intervals. summarise interval coverage different multiple interval ranges, can compute coverage deviation defined mean interval coverage \\(K\\) interval ranges \\(\\alpha_k\\) \\(k = 1, \\dots, K\\): \\[\\text{Coverage deviation} = \\frac{1}{K} \\sum_{k = 1}^{K} \\text{IC}_{\\alpha_k}\\] Quantile coverage given quantile level proportion true values smaller predictions corresponding quantile level. probability integral transform (PIT, Dawid 1984) represents succinct way visualise deviations predictive distribution \\(F\\) true data-generating distribution \\(G\\). idea transform observed values agreement forecasts data can examined observing whether transformed values follow uniform distribution. PIT given \\[u = F (y),\\] \\(u\\) transformed variable \\(F(y)\\) predictive distribution \\(F\\) evaluated true observed value \\(y\\). \\(F = G\\), \\(u\\) follows uniform distribution. integer outcomes, PIT longer uniform even forecasts ideal. Instead, randomised PIT can used: \\[u = P(y) + v \\cdot (P(y) - P(y - 1) ),\\] \\(y\\) observed value \\(P()\\) cumulative probability assigned values smaller equal \\(y\\) (\\(P(-1) = 0\\) definition, \\(v\\) standard uniform variable independent \\(y\\). \\(P\\) equal true data-generating distribution function, \\(u\\) standard uniform. also propose non-randomised version PIT count data used alternatively. Sharpness ability produce narrow forecasts feature forecasts depend observations. Sharpness therefore interest conditional calibration: precise forecast useful clearly wrong. suggested Funk et al. (2019), measure sharpness continuous integer forecasts represented predictive samples normalised median absolute deviation median (MADN) ), .e. \\[ S(F) = \\frac{1}{0.675} \\cdot \\text{median}(|x - \\text{median(x)}|), \\] \\(x\\) vector predictive samples \\(\\frac{1}{0.675}\\) normalising constant. predictive distribution \\(F\\) CDF normal distribution, sharpness equal standard deviation \\(F\\). Bias measure tendency forecaster - underpredict. continuous forecasts, bias given \\[B(F, y) = 1 - 2 \\cdot (F (y)), \\] \\(F\\) CDF predictive distribution \\(y\\) observed value. integer-valued forecasts, maximum minimum bias respect data point \\(y\\) \\(\\pm (1 - p(y))\\), \\(p(y)\\) predicted probability mass data point. Bias can calculated \\[B(P, y) = 1 - (P(y) + P(y + 1)), \\] \\(P(y)\\) cumulative probability assigned outcomes smaller equal \\(y\\), .e. cumulative probability mass function corresponding \\(p(y)\\). quantile forecasts, Bias can calculated maximum percentile rank prediction smaller \\(y\\), true value smaller median predictive distribution. true value median predictive distribution, bias minimum percentile rank corresponding quantile still larger true value. true value exactly median, bias zero. large enough number quantiles, percentile rank equal proportion predictive samples observed true value, metric coincides one continuous forecasts. mean score ratio used compare two models overlapping set forecast targets models made prediction. mean score ratio calculated mean score achieved first model mean score achieved second model. precisely, two models \\(, j\\), determine set overlapping forecasts, denoted \\(\\mathcal{}_{ij}\\) compute mean score ratio \\(\\theta_{ij}\\) \\[\\theta_{ij} =\\frac{\\text{mean score model } \\text{ } \\mathcal{}_{ij}}{\\text{mean score model } j \\text{ } \\mathcal{}_{ij}}.\\] mean score ratio can principle computed arbitrary score. Relative skill scores can used obtain ranking models based pairwise comparisons models. compute relative skill \\(\\theta_i\\) model \\(\\), take geometric mean mean score ratios involve model \\(\\), .e. \\[ \\theta_{} = \\left(\\prod_{m = 1}^M \\theta_{im}\\right)^{1/M}, \\] M number models.","code":""},{"path":"https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html","id":"bias","dir":"Articles","previous_headings":"","what":"Bias","title":"Scoring forecasts directly","text":"function bias determines bias predictive Monte-Carlo samples, automatically recognising whether forecasts continuous integer valued. continuous forecasts, Bias measured \\[B_t (P_t, x_t) = 1 - 2 \\cdot (P_t (x_t))\\] \\(P_t\\) empirical cumulative distribution function prediction true value \\(x_t\\). Computationally, \\(P_t (x_t)\\) just calculated fraction predictive samples \\(x_t\\) smaller \\(x_t\\). integer valued forecasts, Bias measured \\[B_t (P_t, x_t) = 1 - (P_t (x_t) + P_t (x_t + 1))\\] adjust integer nature forecasts. cases, Bias can assume values -1 1 0 ideally.","code":"## integer valued forecasts true_values <- rpois(30, lambda = 1:30) predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) bias_sample(true_values, predictions) #>  [1] -0.150 -0.925 -0.775 -0.700 -0.030  0.795  0.485 -0.720 -0.805  0.910 #> [11]  0.790 -0.855  0.375 -0.850 -0.560 -0.815 -0.625  0.110 -0.605  0.775 #> [21] -0.690  0.105  0.940 -0.975  0.520  0.600 -0.535 -0.590 -0.745 -0.965  ## continuous forecasts true_values <- rnorm(30, mean = 1:30) predictions <- replicate(200, rnorm(30, mean = 1:30)) bias_sample(true_values, predictions) #>  [1] -0.33 -0.61  0.89  0.60 -0.48  0.16 -0.10 -0.16 -0.34  0.10 -0.37  0.68 #> [13]  0.61 -0.44  0.15  0.52 -0.92  0.34  0.58  0.95 -1.00  0.51  0.73  0.28 #> [25]  0.10  0.61 -0.55  0.37  0.39  0.28"},{"path":"https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html","id":"sharpness","dir":"Articles","previous_headings":"","what":"Sharpness","title":"Scoring forecasts directly","text":"Sharpness ability model generate predictions within narrow range. data-independent measure, purely feature forecasts . Sharpness / dispersion predictive samples corresponding one single true value measured normalised median absolute deviation median predictive samples. details, see ?stats::mad","code":"predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) mad_sample(predictions) #>  [1] 1.4826 1.4826 1.4826 1.4826 2.9652 2.9652 2.9652 2.9652 2.9652 2.9652 #> [11] 2.9652 2.9652 4.4478 4.4478 3.7065 4.4478 4.4478 4.4478 4.4478 4.4478 #> [21] 4.4478 4.4478 4.4478 4.4478 4.4478 5.9304 5.1891 5.9304 5.1891 5.9304"},{"path":"https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html","id":"calibration","dir":"Articles","previous_headings":"","what":"Calibration","title":"Scoring forecasts directly","text":"Calibration reliability forecasts ability model correctly identify uncertainty making predictions. model perfect calibration, observed data time point look came predictive probability distribution time. Equivalently, one can inspect probability integral transform predictive distribution time t, \\[u_t = F_t (x_t)\\] \\(x_t\\) observed data point time \\(t \\text{ } t_1, …, t_n\\), n number forecasts, \\(F_t\\) (continuous) predictive cumulative probability distribution time t. true probability distribution outcomes time t \\(G_t\\) forecasts \\(F_t\\) said ideal \\(F_t = G_t\\) times \\(t\\). case, probabilities ut distributed uniformly. case discrete outcomes incidence counts, PIT longer uniform even forecasts ideal. case randomised PIT can used instead: \\[u_t = P_t(k_t) + v \\cdot (P_t(k_t) - P_t(k_t - 1) )\\] \\(k_t\\) observed count, \\(P_t(x)\\) predictive cumulative probability observing incidence \\(k\\) time \\(t\\), \\(P_t (-1) = 0\\) definition \\(v\\) standard uniform independent \\(k\\). \\(P_t\\) true cumulative probability distribution, \\(u_t\\) standard uniform. function checks whether integer continuous forecasts provided. applies (randomised) probability integral tests values \\(u_t\\) uniformity using Anderson-Darling test. rule thumb, evidence suggest forecasting model miscalibrated p-value found greater threshold \\(p >= 0.1\\), evidence miscalibrated \\(0.01 < p < 0.1\\), good evidence miscalibrated \\(p <= 0.01\\). context noted, though, uniformity PIT necessary sufficient condition calibration. also noted test works given sufficient samples, otherwise Null hypothesis often rejected outright.","code":""},{"path":"https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html","id":"continuous-ranked-probability-score-crps","dir":"Articles","previous_headings":"","what":"Continuous Ranked Probability Score (CRPS)","title":"Scoring forecasts directly","text":"Wrapper around crps_sample() function scoringRules package. information look manuals scoringRules package. function can used continuous well integer valued forecasts. Smaller values better.","code":"true_values <- rpois(30, lambda = 1:30) predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) crps_sample(true_values, predictions) #>  [1] 0.437750 0.526425 2.028150 0.411950 1.418025 0.819125 1.008050 2.439350 #>  [9] 2.645050 4.083900 1.774925 2.544100 5.016000 2.483075 4.007525 1.068925 #> [17] 1.160800 1.071375 1.087625 1.020400 1.285150 1.290725 1.248575 1.311500 #> [25] 1.587425 3.729225 1.389350 3.174075 1.651250 1.446800"},{"path":"https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html","id":"dawid-sebastiani-score-dss","dir":"Articles","previous_headings":"","what":"Dawid-Sebastiani Score (DSS)","title":"Scoring forecasts directly","text":"Wrapper around dss_sample() function scoringRules package. information look manuals scoringRules package. function can used continuous well integer valued forecasts. Smaller values better.","code":"true_values <- rpois(30, lambda = 1:30) predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) dss_sample(true_values, predictions) #>  [1] 1.201671 1.106247 1.393634 1.788146 8.381764 3.481015 2.065385 2.506923 #>  [9] 2.791431 2.442296 3.159900 3.216813 2.617877 2.976601 4.631072 9.775002 #> [17] 2.751900 3.458061 2.986383 5.149527 3.007225 3.173718 4.868428 4.623816 #> [25] 3.797667 4.396779 3.288402 4.326138 4.169233 4.985792"},{"path":"https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html","id":"log-score","dir":"Articles","previous_headings":"","what":"Log Score","title":"Scoring forecasts directly","text":"Wrapper around logs_sample() function scoringRules package. information look manuals scoringRules package. function used integer valued forecasts. Log Scores principle possible integer valued forecasts require kernel density estimate well defined discrete values. Smaller values better.","code":"true_values <- rnorm(30, mean = 1:30) predictions <- replicate(200, rnorm(n = 30, mean = 1:30)) logs_sample(true_values, predictions) #>  [1] 1.9280996 0.9180784 0.9872729 1.3282525 1.1176960 1.0086863 1.2985107 #>  [8] 1.6362565 1.1220392 0.9922387 1.1432151 1.3851892 1.9439170 1.3101964 #> [15] 3.2849860 1.2641101 0.9449585 1.4246469 1.2998487 1.2791942 1.5384806 #> [22] 0.8795043 0.9449376 1.6840263 2.0839656 2.9628825 1.8347612 1.5779114 #> [29] 1.3147235 4.9312163"},{"path":"https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html","id":"brier-score","dir":"Articles","previous_headings":"","what":"Brier Score","title":"Scoring forecasts directly","text":"Brier score proper score rule assesses accuracy probabilistic binary predictions. outcomes can either 0 1, predictions must probability true outcome 1. Brier Score computed mean squared error probabilistic prediction true outcome. \\[\\text{Brier_Score} = \\frac{1}{N} \\sum_{t = 1}^{n} (\\text{prediction}_t - \\text{outcome}_t)^2\\]","code":"true_values <- sample(c(0, 1), size = 30, replace = TRUE) predictions <- runif(n = 30, min = 0, max = 1)  brier_score(true_values, predictions) #>  [1] 0.603223037 0.029673510 0.407015505 0.506307687 0.005642171 0.140049312 #>  [7] 0.041655720 0.712188179 0.962009950 0.082081483 0.324530596 0.094023639 #> [13] 0.076481431 0.061101161 0.358976157 0.553464690 0.360497195 0.002520284 #> [19] 0.169877178 0.100306497 0.602803465 0.129698620 0.005134905 0.002643595 #> [25] 0.601742141 0.023382466 0.255938699 0.420297424 0.290987315 0.876199227"},{"path":"https://epiforecasts.io/scoringutils/articles/scoring-forecasts-directly.html","id":"interval-score","dir":"Articles","previous_headings":"Brier Score","what":"Interval Score","title":"Scoring forecasts directly","text":"Interval Score Proper Scoring Rule score quantile predictions, following Gneiting Raftery (2007). Smaller values better. score computed \\[ \\text{score} = (\\text{upper} - \\text{lower}) + \\\\ \\frac{2}{\\alpha} \\cdot (\\text{lower} - \\text{true_value}) \\cdot 1(\\text{true_values} < \\text{lower}) + \\\\ \\frac{2}{\\alpha} \\cdot (\\text{true_value} - \\text{upper}) \\cdot 1(\\text{true_value} > \\text{upper})\\] \\(1()\\) indicator function \\(\\alpha\\) decimal value indicates much outside prediction interval. improve usability, user asked provide interval range percentage terms, .e. interval_range = 90 (percent) 90 percent prediction interval. Correspondingly, user provide 5% 95% quantiles (corresponding alpha 0.1). specific distribution assumed, range symmetric (.e can’t use 0.1 quantile lower bound 0.7 quantile upper). Setting weigh = TRUE weigh score \\(\\frac{\\alpha}{2}\\) Interval Score converges CRPS increasing number quantiles.","code":"true_values <- rnorm(30, mean = 1:30) interval_range <- 90 alpha <- (100 - interval_range) / 100 lower <- qnorm(alpha / 2, rnorm(30, mean = 1:30)) upper <- qnorm((1 - alpha / 2), rnorm(30, mean = 1:30))  interval_score(   true_values = true_values,   lower = lower,   upper = upper,   interval_range = interval_range ) #>  [1] 0.11903087 0.57353725 0.15732685 0.10296708 0.29979639 0.14926685 #>  [7] 0.92400696 0.07768853 0.79109767 0.18002388 1.42849397 0.21166370 #> [13] 0.66613211 0.24871027 0.20020090 1.25006751 0.23292766 0.17809847 #> [19] 0.22016549 0.08261552 0.19754827 0.21248512 0.22215709 0.12071060 #> [25] 0.02870520 0.22496183 0.15415855 0.14431259 0.22507579 0.26498555"},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"input-formats","dir":"Articles","previous_headings":"","what":"Input formats","title":"Getting started","text":"time, score() function able entire evaluation . need pass data.frame appropriate columns. columns required depends format forecasts come . forecast format can either based quantiles (see example_quantile expected format), based predictive samples (see example_continuous example_integer expected format case) binary format. following table gives overview (pairwise comparisons explained detail ): Additional columns may present indicate grouping forecasts. example, forecasts made different models various locations different time points, several weeks future. scoringutils automatically tries determine unit single forecast, .e. combination existing columns able uniquely identify single forecast. uses existing columns , can sometimes lead issues. therefore recommend using function set_forecast_unit() determine forecast unit manually. function simply drops unneeded columns, making sure necessary, ‘protected columns’ like “prediction” “true_value” retained.","code":"colnames(example_quantile) #>  [1] \"location\"        \"target_end_date\" \"target_type\"     \"true_value\"      #>  [5] \"location_name\"   \"forecast_date\"   \"quantile\"        \"prediction\"      #>  [9] \"model\"           \"horizon\"  set_forecast_unit(   example_quantile,   c(\"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\") ) %>%   colnames() #> [1] \"true_value\"      \"quantile\"        \"prediction\"      \"location\"        #> [5] \"target_end_date\" \"target_type\"     \"horizon\"         \"model\""},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"checking-the-input-data","dir":"Articles","previous_headings":"","what":"Checking the input data","title":"Getting started","text":"function check_forecasts() can used check input data. gives summary scoringutils thinks trying achieve. infers type prediction target, prediction format, unit single forecasts, gives overview number unique values per column (helpful spotting missing data) returns warnings errors. unsure input data look like, look example_quantile, example_integer, example_continuous example_binary data sets provided package. output check_forecasts() can later directly used input score() (otherwise, score() just run check_forecasts() anyway internally).","code":"head(example_quantile) #>    location target_end_date target_type true_value location_name forecast_date #> 1:       DE      2021-01-02       Cases     127300       Germany          <NA> #> 2:       DE      2021-01-02      Deaths       4534       Germany          <NA> #> 3:       DE      2021-01-09       Cases     154922       Germany          <NA> #> 4:       DE      2021-01-09      Deaths       6117       Germany          <NA> #> 5:       DE      2021-01-16       Cases     110183       Germany          <NA> #> 6:       DE      2021-01-16      Deaths       5867       Germany          <NA> #>    quantile prediction model horizon #> 1:       NA         NA  <NA>      NA #> 2:       NA         NA  <NA>      NA #> 3:       NA         NA  <NA>      NA #> 4:       NA         NA  <NA>      NA #> 5:       NA         NA  <NA>      NA #> 6:       NA         NA  <NA>      NA check_forecasts(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #> Your forecasts seem to be for a target of the following type: #> $target_type #> [1] \"integer\" #>  #> and in the following format: #> $prediction_type #> [1] \"quantile\" #>  #> The unit of a single forecast is defined by: #> $forecast_unit #> [1] \"location\"        \"target_end_date\" \"target_type\"     \"location_name\"   #> [5] \"forecast_date\"   \"model\"           \"horizon\"         #>  #> Cleaned data, rows with NA values in prediction or true_value removed: #> $cleaned_data #>        location target_end_date target_type true_value location_name #>     1:       DE      2021-05-08       Cases     106987       Germany #>     2:       DE      2021-05-08       Cases     106987       Germany #>     3:       DE      2021-05-08       Cases     106987       Germany #>     4:       DE      2021-05-08       Cases     106987       Germany #>     5:       DE      2021-05-08       Cases     106987       Germany #>    ---                                                               #> 20397:       IT      2021-07-24      Deaths         78         Italy #> 20398:       IT      2021-07-24      Deaths         78         Italy #> 20399:       IT      2021-07-24      Deaths         78         Italy #> 20400:       IT      2021-07-24      Deaths         78         Italy #> 20401:       IT      2021-07-24      Deaths         78         Italy #>        forecast_date quantile prediction                 model horizon #>     1:    2021-05-03    0.010      82466 EuroCOVIDhub-ensemble       1 #>     2:    2021-05-03    0.025      86669 EuroCOVIDhub-ensemble       1 #>     3:    2021-05-03    0.050      90285 EuroCOVIDhub-ensemble       1 #>     4:    2021-05-03    0.100      95341 EuroCOVIDhub-ensemble       1 #>     5:    2021-05-03    0.150      99171 EuroCOVIDhub-ensemble       1 #>    ---                                                                 #> 20397:    2021-07-12    0.850        352  epiforecasts-EpiNow2       2 #> 20398:    2021-07-12    0.900        397  epiforecasts-EpiNow2       2 #> 20399:    2021-07-12    0.950        499  epiforecasts-EpiNow2       2 #> 20400:    2021-07-12    0.975        611  epiforecasts-EpiNow2       2 #> 20401:    2021-07-12    0.990        719  epiforecasts-EpiNow2       2 #>  #> Number of unique values per column per model: #> $unique_values #>                    model location target_end_date target_type true_value #> 1: EuroCOVIDhub-ensemble        4              12           2         96 #> 2: EuroCOVIDhub-baseline        4              12           2         96 #> 3:  epiforecasts-EpiNow2        4              12           2         95 #> 4:       UMass-MechBayes        4              12           1         48 #>    location_name forecast_date quantile prediction horizon #> 1:             4            11       23       3969       3 #> 2:             4            11       23       3733       3 #> 3:             4            11       23       3903       3 #> 4:             4            11       23       1058       3 #>  #> $messages #> [1] \"144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.\""},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"showing-available-forecasts","dir":"Articles","previous_headings":"","what":"Showing available forecasts","title":"Getting started","text":"function avail_forecasts() may also helpful determine forecasts available. Using argument can specify level summary. example, see many forecasts per model target_type, can run see ‘epiforecasts-EpiNow2’ missing forecasts deaths forecast target UMass-MechBayes case forecasts. information can also visualised using plot_avail_forecasts() function:  can also visualise forecasts directly using plot_predictions() function:","code":"avail_forecasts(example_quantile, by = c(\"model\", \"target_type\")) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model target_type Number forecasts #> 1: EuroCOVIDhub-ensemble       Cases              128 #> 2: EuroCOVIDhub-baseline       Cases              128 #> 3:  epiforecasts-EpiNow2       Cases              128 #> 4: EuroCOVIDhub-ensemble      Deaths              128 #> 5: EuroCOVIDhub-baseline      Deaths              128 #> 6:       UMass-MechBayes      Deaths              128 #> 7:  epiforecasts-EpiNow2      Deaths              119 example_quantile %>%   avail_forecasts(by = c(\"model\", \"forecast_date\", \"target_type\")) %>%   plot_avail_forecasts() +   facet_wrap(~ target_type) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. example_quantile %>%   make_NA(     what = \"truth\",     target_end_date >= \"2021-07-15\",     target_end_date < \"2021-05-22\"   ) %>%   make_NA(     what = \"forecast\",     model != \"EuroCOVIDhub-ensemble\",     forecast_date != \"2021-06-28\"   ) %>%   plot_predictions(     x = \"target_end_date\",     by = c(\"target_type\", \"location\")   ) +   facet_wrap(target_type ~ location, ncol = 4, scales = \"free\")"},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"scoring-and-summarising-forecasts","dir":"Articles","previous_headings":"","what":"Scoring and summarising forecasts","title":"Getting started","text":"Forecasts can easily scored using score() function. clarity, suggest setting forecast unit explicitly may also want call check_forecasts() explicitly. Note example columns contain duplicated information regards forecast unit, e.g. “location” “location_name”, dropped. drop essential information, example, “target_type” column, ’ll get error informing us forecasts aren’t uniquely identified . function find_duplicates() may help investigate issue. filtering median forecasts EuroCOVIDhub-ensemble, can see indeed two forecasts every date, location horizon. function score() returns unsumarised scores, cases user wants. returns single score per forecast (determined forecast unit). forecasts quantile format, returns one score per quantile. second function, summarise_scores() takes care summarising scores level specified user. argument can used define level summary. default, = NULL summary unit assumed unit single forecast. continuous forecasts, means nothing happens isn’t specified. quantile based forecasts, = NULL, scores summarised across quantiles instead one score per forecast_unit quantile get one score per forecast unit. argument can specify unit summary want. can also call sumarise_scores() multiple tines, e.g round outputs specifying e.g. signif() summary function.","code":"scores <- example_quantile %>%   set_forecast_unit(     c(\"location\", \"target_end_date\", \"target_type\", \"location_name\",       \"forecast_date\", \"model\", \"horizon\")   ) %>%   check_forecasts() %>%   score() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. head(scores) #>    location target_end_date target_type location_name forecast_date #> 1:       DE      2021-05-08       Cases       Germany    2021-05-03 #> 2:       DE      2021-05-08       Cases       Germany    2021-05-03 #> 3:       DE      2021-05-08       Cases       Germany    2021-05-03 #> 4:       DE      2021-05-08       Cases       Germany    2021-05-03 #> 5:       DE      2021-05-08       Cases       Germany    2021-05-03 #> 6:       DE      2021-05-08       Cases       Germany    2021-05-03 #>                    model horizon range interval_score dispersion #> 1: EuroCOVIDhub-baseline       1     0        25620.0        0.0 #> 2: EuroCOVIDhub-baseline       1    10        25599.5      184.5 #> 3: EuroCOVIDhub-baseline       1    10        25599.5      184.5 #> 4: EuroCOVIDhub-baseline       1    20        25481.0      556.0 #> 5: EuroCOVIDhub-baseline       1    20        25481.0      556.0 #> 6: EuroCOVIDhub-baseline       1    30        25270.2      816.2 #>    underprediction overprediction coverage coverage_deviation bias quantile #> 1:               0          25620        0                0.0 0.95     0.50 #> 2:               0          25415        0               -0.1 0.95     0.45 #> 3:               0          25415        0               -0.1 0.95     0.55 #> 4:               0          24925        0               -0.2 0.95     0.40 #> 5:               0          24925        0               -0.2 0.95     0.60 #> 6:               0          24454        0               -0.3 0.95     0.35 #>    ae_median quantile_coverage #> 1:     25620              TRUE #> 2:     25620              TRUE #> 3:     25620              TRUE #> 4:     25620              TRUE #> 5:     25620              TRUE #> 6:     25620              TRUE example_quantile %>%   set_forecast_unit(     c(\"location\", \"target_end_date\", \"target_type\",       \"forecast_date\", \"model\", \"horizon\")   ) %>%   check_forecasts() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #> Your forecasts seem to be for a target of the following type: #> $target_type #> [1] \"integer\" #>  #> and in the following format: #> $prediction_type #> [1] \"quantile\" #>  #> The unit of a single forecast is defined by: #> $forecast_unit #> [1] \"location\"        \"target_end_date\" \"target_type\"     \"forecast_date\"   #> [5] \"model\"           \"horizon\"         #>  #> Cleaned data, rows with NA values in prediction or true_value removed: #> $cleaned_data #>        true_value quantile prediction location target_end_date target_type #>     1:     106987    0.010      82466       DE      2021-05-08       Cases #>     2:     106987    0.025      86669       DE      2021-05-08       Cases #>     3:     106987    0.050      90285       DE      2021-05-08       Cases #>     4:     106987    0.100      95341       DE      2021-05-08       Cases #>     5:     106987    0.150      99171       DE      2021-05-08       Cases #>    ---                                                                     #> 20397:         78    0.850        352       IT      2021-07-24      Deaths #> 20398:         78    0.900        397       IT      2021-07-24      Deaths #> 20399:         78    0.950        499       IT      2021-07-24      Deaths #> 20400:         78    0.975        611       IT      2021-07-24      Deaths #> 20401:         78    0.990        719       IT      2021-07-24      Deaths #>        forecast_date                 model horizon #>     1:    2021-05-03 EuroCOVIDhub-ensemble       1 #>     2:    2021-05-03 EuroCOVIDhub-ensemble       1 #>     3:    2021-05-03 EuroCOVIDhub-ensemble       1 #>     4:    2021-05-03 EuroCOVIDhub-ensemble       1 #>     5:    2021-05-03 EuroCOVIDhub-ensemble       1 #>    ---                                             #> 20397:    2021-07-12  epiforecasts-EpiNow2       2 #> 20398:    2021-07-12  epiforecasts-EpiNow2       2 #> 20399:    2021-07-12  epiforecasts-EpiNow2       2 #> 20400:    2021-07-12  epiforecasts-EpiNow2       2 #> 20401:    2021-07-12  epiforecasts-EpiNow2       2 #>  #> Number of unique values per column per model: #> $unique_values #>                    model true_value quantile prediction location #> 1: EuroCOVIDhub-ensemble         96       23       3969        4 #> 2: EuroCOVIDhub-baseline         96       23       3733        4 #> 3:  epiforecasts-EpiNow2         95       23       3903        4 #> 4:       UMass-MechBayes         48       23       1058        4 #>    target_end_date target_type forecast_date horizon #> 1:              12           2            11       3 #> 2:              12           2            11       3 #> 3:              12           2            11       3 #> 4:              12           1            11       3 #>  #> $messages #> [1] \"144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.\" example_quantile %>%   set_forecast_unit(     c(\"location\", \"target_end_date\",       \"forecast_date\", \"model\", \"horizon\")   ) %>%   check_forecasts() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #> Warning in check_forecasts(.): The following warnings were produced when checking inputs: #> 1.  Some forecasts have different numbers of rows  #> 2.  (e.g. quantiles or samples).  #> 3.  scoringutils found:  #> 4.  46, 23 #> 5.  . This may be a problem (it can potentially distort scores,  #> 6.  making it more difficult to compare them),  #> 7.  so make sure this is intended. #> Error in check_forecasts(.): The following errors were produced when checking inputs: #> 1.  There are instances with more than one forecast for the same target. This can't be right and needs to be resolved. Maybe you need to check the unit of a single forecast and add missing columns? Use the  function find_duplicates() to identify duplicate rows. duplicates <- example_quantile %>%   set_forecast_unit(     c(\"location\", \"target_end_date\",       \"forecast_date\", \"model\", \"horizon\")   ) %>%   find_duplicates()  duplicates[quantile == 0.5 & model == \"EuroCOVIDhub-ensemble\", ] %>%   head() #>    true_value quantile prediction location target_end_date forecast_date #> 1:     106987      0.5     119258       DE      2021-05-08    2021-05-03 #> 2:       1582      0.5       1568       DE      2021-05-08    2021-05-03 #> 3:      64985      0.5     110716       DE      2021-05-15    2021-05-03 #> 4:      64985      0.5      92649       DE      2021-05-15    2021-05-10 #> 5:       1311      0.5       1521       DE      2021-05-15    2021-05-03 #> 6:       1311      0.5       1422       DE      2021-05-15    2021-05-10 #>                    model horizon #> 1: EuroCOVIDhub-ensemble       1 #> 2: EuroCOVIDhub-ensemble       1 #> 3: EuroCOVIDhub-ensemble       2 #> 4: EuroCOVIDhub-ensemble       1 #> 5: EuroCOVIDhub-ensemble       2 #> 6: EuroCOVIDhub-ensemble       1 scores <- score(example_continuous) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. all(scores == summarise_scores(scores), na.rm = TRUE) #> [1] TRUE scores <- example_quantile %>%   set_forecast_unit(     c(\"location\", \"target_end_date\", \"target_type\",       \"forecast_date\", \"model\", \"horizon\")   ) %>%   check_forecasts() %>%   score() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.  head(scores) #>    location target_end_date target_type forecast_date                 model #> 1:       DE      2021-05-08       Cases    2021-05-03 EuroCOVIDhub-baseline #> 2:       DE      2021-05-08       Cases    2021-05-03 EuroCOVIDhub-baseline #> 3:       DE      2021-05-08       Cases    2021-05-03 EuroCOVIDhub-baseline #> 4:       DE      2021-05-08       Cases    2021-05-03 EuroCOVIDhub-baseline #> 5:       DE      2021-05-08       Cases    2021-05-03 EuroCOVIDhub-baseline #> 6:       DE      2021-05-08       Cases    2021-05-03 EuroCOVIDhub-baseline #>    horizon range interval_score dispersion underprediction overprediction #> 1:       1     0        25620.0        0.0               0          25620 #> 2:       1    10        25599.5      184.5               0          25415 #> 3:       1    10        25599.5      184.5               0          25415 #> 4:       1    20        25481.0      556.0               0          24925 #> 5:       1    20        25481.0      556.0               0          24925 #> 6:       1    30        25270.2      816.2               0          24454 #>    coverage coverage_deviation bias quantile ae_median quantile_coverage #> 1:        0                0.0 0.95     0.50     25620              TRUE #> 2:        0               -0.1 0.95     0.45     25620              TRUE #> 3:        0               -0.1 0.95     0.55     25620              TRUE #> 4:        0               -0.2 0.95     0.40     25620              TRUE #> 5:        0               -0.2 0.95     0.60     25620              TRUE #> 6:        0               -0.3 0.95     0.35     25620              TRUE  scores %>%   summarise_scores() %>%   head() #>    location target_end_date target_type forecast_date                 model #> 1:       DE      2021-05-08       Cases    2021-05-03 EuroCOVIDhub-baseline #> 2:       DE      2021-05-08       Cases    2021-05-03 EuroCOVIDhub-ensemble #> 3:       DE      2021-05-08       Cases    2021-05-03  epiforecasts-EpiNow2 #> 4:       DE      2021-05-08      Deaths    2021-05-03 EuroCOVIDhub-baseline #> 5:       DE      2021-05-08      Deaths    2021-05-03 EuroCOVIDhub-ensemble #> 6:       DE      2021-05-08      Deaths    2021-05-03       UMass-MechBayes #>    horizon interval_score dispersion underprediction overprediction #> 1:       1    16925.04696 1649.22087       0.0000000   15275.826087 #> 2:       1     7990.85478 5440.98522       0.0000000    2549.869565 #> 3:       1    25395.96087 8173.70000       0.0000000   17222.260870 #> 4:       1       46.79304   44.66261       0.0000000       2.130435 #> 5:       1       53.88000   53.27130       0.6086957       0.000000 #> 6:       1      116.12174   77.07826      39.0434783       0.000000 #>    coverage_deviation  bias ae_median #> 1:        -0.38521739  0.95     25620 #> 2:         0.04956522  0.50     12271 #> 3:        -0.29826087  0.90     44192 #> 4:         0.22347826  0.30        15 #> 5:         0.39739130 -0.10        14 #> 6:         0.04956522 -0.50       208 scores %>%   summarise_scores(by = c(\"model\", \"target_type\")) %>%   summarise_scores(fun = signif, digits = 2) %>%   kable()"},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"scoring-point-forecasts","dir":"Articles","previous_headings":"Scoring and summarising forecasts","what":"Scoring point forecasts","title":"Getting started","text":"Point forecasts can scored making use quantile-based format, value NA \"point\" quantile column. Point forecasts can scored alongside quantile-based forecasts. point forecasts values NA metrics designed probabilistic forecasts, important use na.rm = TRUE summarising.","code":"suppressMessages(score(example_point)) %>%   summarise_scores(by = \"model\", na.rm = TRUE) #>                    model interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline     14092.7647 2192.26967      4996.92898 #> 2: EuroCOVIDhub-ensemble      8852.4196 1930.80064      2005.56357 #> 3:  epiforecasts-EpiNow2     10659.5125 3084.85850      1604.96135 #> 4:       UMass-MechBayes        51.4781   28.09387        15.32315 #>    overprediction coverage_deviation        bias    ae_point     se_point #> 1:     6903.56605        0.002102273  0.21851562 19353.42969 2.883446e+09 #> 2:     4916.05540        0.050752841  0.00812500 12077.10156 1.945118e+09 #> 3:     5969.69268       -0.057861612 -0.04376518 14521.10526 2.680928e+09 #> 4:        8.06108       -0.024886364 -0.02312500    78.47656 1.170976e+04 #>    ae_median #> 1:       NaN #> 2:       NaN #> 3:       NaN #> 4:       NaN"},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"adding-empirical-coverage","dir":"Articles","previous_headings":"Scoring and summarising forecasts","what":"Adding empirical coverage","title":"Getting started","text":"quantile-based forecasts often interested specific coverage-levels, example, percentage true values fell 50% 90% prediction intervals. can add information using function add_coverage(). function also requires argument defines level grouping percentage true values covered certain prediction intervals computed.","code":"score(example_quantile) %>%   add_coverage(ranges = c(50, 90), by = c(\"model\", \"target_type\")) %>%   summarise_scores(by = c(\"model\", \"target_type\")) %>%   summarise_scores(fun = signif, digits = 2) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model target_type interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline       Cases          28000       4100         10000.0 #> 2: EuroCOVIDhub-baseline      Deaths            160         91             2.1 #> 3: EuroCOVIDhub-ensemble       Cases          18000       3700          4200.0 #> 4: EuroCOVIDhub-ensemble      Deaths             41         30             4.1 #> 5:       UMass-MechBayes      Deaths             53         27            17.0 #> 6:  epiforecasts-EpiNow2       Cases          21000       5700          3300.0 #> 7:  epiforecasts-EpiNow2      Deaths             67         32            16.0 #>    overprediction coverage_deviation    bias ae_median coverage_50 coverage_90 #> 1:        14000.0             -0.110  0.0980     38000        0.33        0.82 #> 2:           66.0              0.120  0.3400       230        0.66        1.00 #> 3:        10000.0             -0.098 -0.0560     24000        0.39        0.80 #> 4:            7.1              0.200  0.0730        53        0.88        1.00 #> 5:            9.0             -0.023 -0.0220        78        0.46        0.88 #> 6:        12000.0             -0.067 -0.0790     28000        0.47        0.79 #> 7:           19.0             -0.043 -0.0051       100        0.42        0.91"},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"adding-relative-scores","dir":"Articles","previous_headings":"Scoring and summarising forecasts","what":"Adding relative scores","title":"Getting started","text":"order better compare models can use relative scores computed based pairwise comparisons (see details ). Relative scores can added evaluation using function summarise_scores(). requires column called ‘model’ present. Pairwise comparisons computed according grouping specified : essentially, data.frame scores gets split different data.frames according values specified relative scores computed every individual group separately. baseline argument allows us specify baseline can used scale relative scores (scores divided baseline relative score). example, obtain relative scores separately different forecast targets, can run","code":"score(example_quantile) %>%   summarise_scores(by = c(\"model\", \"target_type\"),                    relative_skill = TRUE,                    baseline = \"EuroCOVIDhub-ensemble\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model target_type interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline       Cases    28483.57465 4102.50094    10284.972826 #> 2: EuroCOVIDhub-baseline      Deaths      159.40387   91.40625        2.098505 #> 3: EuroCOVIDhub-ensemble       Cases    17943.82383 3663.52458     4237.177310 #> 4: EuroCOVIDhub-ensemble      Deaths       41.42249   30.18099        4.103261 #> 5:       UMass-MechBayes      Deaths       52.65195   26.87239       16.800951 #> 6:  epiforecasts-EpiNow2       Cases    20831.55662 5664.37795     3260.355639 #> 7:  epiforecasts-EpiNow2      Deaths       66.64282   31.85692       15.893314 #>    overprediction coverage_deviation        bias   ae_median relative_skill #> 1:   14096.100883        -0.11211957  0.09796875 38473.60156      1.2947445 #> 2:      65.899117         0.11614130  0.33906250   233.25781      2.2958723 #> 3:   10043.121943        -0.09785326 -0.05640625 24101.07031      0.8156514 #> 4:       7.138247         0.19528533  0.07265625    53.13281      0.5966310 #> 5:       8.978601        -0.02312500 -0.02234375    78.47656      0.7475873 #> 6:   11906.823030        -0.06660326 -0.07890625 27923.81250      0.9469157 #> 7:      18.892583        -0.04287176 -0.00512605   104.74790      0.9765276 #>    scaled_rel_skill #> 1:         1.587375 #> 2:         3.848060 #> 3:         1.000000 #> 4:         1.000000 #> 5:         1.253014 #> 6:         1.160932 #> 7:         1.636736"},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"coloured-table","dir":"Articles","previous_headings":"Visualising scores","what":"Coloured table","title":"Getting started","text":"simple coloured table can produced based scores:","code":"score(example_integer) %>%   summarise_scores(by = c(\"model\", \"target_type\"), na.rm = TRUE) %>%   summarise_scores(fun = signif, digits = 2) %>%   plot_score_table(by = \"target_type\") +   facet_wrap(~ target_type, nrow = 1) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected."},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"score-heatmap","dir":"Articles","previous_headings":"Visualising scores","what":"Score heatmap","title":"Getting started","text":"can also summarise one particular metric across different categories using simple heatmap:","code":"score(example_continuous) %>%   summarise_scores(by = c(\"model\", \"location\", \"target_type\")) %>%   plot_heatmap(x = \"location\", metric = \"bias\") +   facet_wrap(~ target_type) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected."},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"weighted-interval-score-components","dir":"Articles","previous_headings":"Visualising scores","what":"Weighted interval score components","title":"Getting started","text":"weighted interval score can split three components: -prediction, -prediction dispersion. can visualised separately following way:","code":"score(example_quantile) %>%   summarise_scores(by = c(\"target_type\", \"model\")) %>%   plot_wis() +   facet_wrap(~ target_type, scales = \"free\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected."},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"calibration","dir":"Articles","previous_headings":"","what":"Calibration","title":"Getting started","text":"Calibration measure statistical consistency forecasts observed values. common way assessing calibration (precisely: probabilistic calibration) PIT histograms. probability integral transform (PIT) equal cumulative distribution function forecast evaluated true observed value. Ideally, pit values uniformly distributed transformation. can compute pit values : visualise results :  Similarly quantile-based forecasts:  Another way look calibration interval coverage quantile coverage. Interval coverage percentage true values fall inside given central prediction interval. Quantile coverage percentage observed values fall given quantile level. order plot interval coverage, need include “range” argument summarise_scores(). green area plot marks conservative behaviour, .e. empirical coverage greater nominally need (e.g. 55% true values covered 50% central prediction intervals.)  visualise quantile coverage, need include “quantile” . , green area corresponds conservative forecasts, central prediction intervals cover needed.","code":"example_continuous %>%   pit(by = \"model\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                      model pit_value #>   1: EuroCOVIDhub-baseline     0.025 #>   2: EuroCOVIDhub-baseline     0.525 #>   3: EuroCOVIDhub-baseline     0.000 #>   4: EuroCOVIDhub-baseline     0.000 #>   5: EuroCOVIDhub-baseline     0.200 #>  ---                                 #> 883:       UMass-MechBayes     0.950 #> 884:       UMass-MechBayes     0.500 #> 885:       UMass-MechBayes     0.100 #> 886:       UMass-MechBayes     0.450 #> 887:       UMass-MechBayes     0.100 example_continuous %>%   pit(by = c(\"model\", \"target_type\")) %>%   plot_pit() +   facet_grid(model ~ target_type) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. example_quantile[quantile %in% seq(0.1, 0.9, 0.1), ] %>%   pit(by = c(\"model\", \"target_type\")) %>%   plot_pit() +   facet_grid(model ~ target_type) example_quantile %>%   score() %>%   summarise_scores(by = c(\"model\", \"range\")) %>%   plot_interval_coverage() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. example_quantile %>%   score() %>%   summarise_scores(by = c(\"model\", \"quantile\")) %>%   plot_quantile_coverage() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected."},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"pairwise-comparisons","dir":"Articles","previous_headings":"","what":"Pairwise comparisons","title":"Getting started","text":"Relative scores different models can computed using pairwise comparisons, sort pairwise tournament combinations two models compared based overlapping set available forecasts common models. Internally, ratio mean scores models computed. relative score model geometric mean mean score ratios involve model. baseline provided, baseline excluded relative scores individual models (therefore differ slightly relative scores without baseline) relative scores scaled (.e. divided ) relative score baseline model. scoringutils, pairwise comparisons can made two ways: standalone function pairwise_comparison() within summarise_scores() simply adds relative scores existing set scores. using pairwise_comparison() function, can also visualise pairwise comparisons showing mean score ratios models. default, smaller values better model care showing y axis left, model compared shown x-axis bottom. example , EuroCOVIDhub-ensemble performs best (values smaller 1), EuroCOVIDhub-baseline performs worst (values larger 1). cases, UMass-MechBayes model course excluded case forecasts available therefore set overlapping forecasts empty.","code":"example_quantile %>%   score() %>%   pairwise_comparison(by = \"model\", baseline = \"EuroCOVIDhub-baseline\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                     model       compare_against mean_scores_ratio         pval #>  1: EuroCOVIDhub-baseline  epiforecasts-EpiNow2         1.3703452 9.164893e-18 #>  2: EuroCOVIDhub-baseline EuroCOVIDhub-ensemble         1.5925819 2.608666e-32 #>  3: EuroCOVIDhub-baseline       UMass-MechBayes         3.0275019 2.627464e-20 #>  4: EuroCOVIDhub-baseline EuroCOVIDhub-baseline         1.0000000 1.000000e+00 #>  5: EuroCOVIDhub-ensemble       UMass-MechBayes         0.7867229 1.244731e-04 #>  6: EuroCOVIDhub-ensemble  epiforecasts-EpiNow2         0.8606607 1.881520e-02 #>  7: EuroCOVIDhub-ensemble EuroCOVIDhub-baseline         0.6279112 2.608666e-32 #>  8: EuroCOVIDhub-ensemble EuroCOVIDhub-ensemble         1.0000000 1.000000e+00 #>  9:       UMass-MechBayes EuroCOVIDhub-ensemble         1.2710955 1.244731e-04 #> 10:       UMass-MechBayes  epiforecasts-EpiNow2         0.7439673 7.253878e-03 #> 11:       UMass-MechBayes EuroCOVIDhub-baseline         0.3303053 2.627464e-20 #> 12:       UMass-MechBayes       UMass-MechBayes         1.0000000 1.000000e+00 #> 13:  epiforecasts-EpiNow2       UMass-MechBayes         1.3441452 7.253878e-03 #> 14:  epiforecasts-EpiNow2 EuroCOVIDhub-ensemble         1.1618981 1.881520e-02 #> 15:  epiforecasts-EpiNow2 EuroCOVIDhub-baseline         0.7297431 9.164893e-18 #> 16:  epiforecasts-EpiNow2  epiforecasts-EpiNow2         1.0000000 1.000000e+00 #>         adj_pval relative_skill scaled_rel_skill #>  1: 3.665957e-17      1.6032604        1.0000000 #>  2: 1.565200e-31      1.6032604        1.0000000 #>  3: 1.313732e-19      1.6032604        1.0000000 #>  4: 1.000000e+00      1.6032604        1.0000000 #>  5: 3.734192e-04      0.8074916        0.5036559 #>  6: 1.881520e-02      0.8074916        0.5036559 #>  7: 1.565200e-31      0.8074916        0.5036559 #>  8: 1.000000e+00      0.8074916        0.5036559 #>  9: 3.734192e-04      0.7475873        0.4662919 #> 10: 1.450776e-02      0.7475873        0.4662919 #> 11: 1.313732e-19      0.7475873        0.4662919 #> 12: 1.000000e+00      0.7475873        0.4662919 #> 13: 1.450776e-02      1.0332277        0.6444541 #> 14: 1.881520e-02      1.0332277        0.6444541 #> 15: 3.665957e-17      1.0332277        0.6444541 #> 16: 1.000000e+00      1.0332277        0.6444541 example_quantile %>%   score() %>%   summarise_scores(     by = \"model\", relative_skill = TRUE, baseline = \"EuroCOVIDhub-baseline\"   ) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline    14321.48926 2096.95360      5143.53567 #> 2: EuroCOVIDhub-ensemble     8992.62316 1846.85278      2120.64029 #> 3:       UMass-MechBayes       52.65195   26.87239        16.80095 #> 4:  epiforecasts-EpiNow2    10827.40786 2950.73422      1697.23411 #>    overprediction coverage_deviation        bias   ae_median relative_skill #> 1:    7081.000000         0.00201087  0.21851562 19353.42969      1.6032604 #> 2:    5025.130095         0.04871603  0.00812500 12077.10156      0.8074916 #> 3:       8.978601        -0.02312500 -0.02234375    78.47656      0.7475873 #> 4:    6179.439535        -0.05516986 -0.04336032 14521.10526      1.0332277 #>    scaled_rel_skill #> 1:        1.0000000 #> 2:        0.5036559 #> 3:        0.4662919 #> 4:        0.6444541 example_quantile %>%   score() %>%   pairwise_comparison(by = c(\"model\", \"target_type\")) %>%   plot_pairwise_comparison() +   facet_wrap(~ target_type) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected."},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"correlation-between-scores","dir":"Articles","previous_headings":"Additional analyses and visualisations","what":"Correlation between scores","title":"Getting started","text":"may sometimes interesting see different scores correlate . can examine using function correlation(). dealing quantile-based forecasts, important call summarise_scorees() correlation() summarise quantiles computing correlations. Visualising correlations:","code":"example_quantile %>%   score() %>%   summarise_scores() %>%   correlation() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>    interval_score dispersion underprediction overprediction coverage_deviation #> 1:           1.00       0.46            0.28           0.94              -0.34 #> 2:           0.46       1.00            0.15           0.32              -0.12 #> 3:           0.28       0.15            1.00          -0.03              -0.33 #> 4:           0.94       0.32           -0.03           1.00              -0.25 #> 5:          -0.34      -0.12           -0.33          -0.25               1.00 #> 6:           0.11       0.11           -0.35           0.22               0.06 #> 7:           0.99       0.54            0.34           0.90              -0.38 #>     bias ae_median             metric #> 1:  0.11      0.99     interval_score #> 2:  0.11      0.54         dispersion #> 3: -0.35      0.34    underprediction #> 4:  0.22      0.90     overprediction #> 5:  0.06     -0.38 coverage_deviation #> 6:  1.00      0.10               bias #> 7:  0.10      1.00          ae_median example_quantile %>%   score() %>%   summarise_scores() %>%   correlation() %>%   plot_correlation() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected."},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"scores-by-interval-ranges","dir":"Articles","previous_headings":"Additional analyses and visualisations","what":"Scores by interval ranges","title":"Getting started","text":"like see different forecast interval ranges contribute average scores, can visualise scores interval range:","code":"example_quantile %>%   score() %>%   summarise_scores(by = c(\"model\", \"range\", \"target_type\")) %>%   plot_ranges() +   facet_wrap(~ target_type, scales = \"free\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected."},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"converting-to-quantile-based-forecasts","dir":"Articles","previous_headings":"","what":"Converting to quantile-based forecasts","title":"Getting started","text":"Different metrics available different forecasting formats. cases, may example forecasts sample-based format, wish make use functionality available quantile-based forecasts. example, may want use decomposition weighted interval score, may like compute interval coverage values. can convert sample-based forecasts quantile-based format using function sample_to_quantile(). , however, one caveat: Quantiles calculated based predictive samples, may introduce bias number available samples small.","code":"example_integer %>%   sample_to_quantile(     quantiles = c(0.01, 0.025, seq(0.05, 0.95, 0.05), 0.975, 0.99)   ) %>%   score() %>%   add_coverage(by = c(\"model\", \"target_type\")) #> The following messages were produced when checking inputs: #> 1.  3312 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                        model target_type location location_name target_end_date #>     1: EuroCOVIDhub-baseline       Cases       DE       Germany      2021-05-08 #>     2: EuroCOVIDhub-baseline       Cases       DE       Germany      2021-05-08 #>     3: EuroCOVIDhub-baseline       Cases       DE       Germany      2021-05-08 #>     4: EuroCOVIDhub-baseline       Cases       DE       Germany      2021-05-08 #>     5: EuroCOVIDhub-baseline       Cases       DE       Germany      2021-05-08 #>    ---                                                                          #> 20397:  epiforecasts-EpiNow2      Deaths       IT         Italy      2021-07-24 #> 20398:  epiforecasts-EpiNow2      Deaths       IT         Italy      2021-07-24 #> 20399:  epiforecasts-EpiNow2      Deaths       IT         Italy      2021-07-24 #> 20400:  epiforecasts-EpiNow2      Deaths       IT         Italy      2021-07-24 #> 20401:  epiforecasts-EpiNow2      Deaths       IT         Italy      2021-07-24 #>        forecast_date horizon range interval_score dispersion underprediction #>     1:    2021-05-03       1     0    29379.00000    0.00000               0 #>     2:    2021-05-03       1    10    27857.47500 3012.07500               0 #>     3:    2021-05-03       1    10    27857.47500 3012.07500               0 #>     4:    2021-05-03       1    20    26443.96000 4109.76000               0 #>     5:    2021-05-03       1    20    26443.96000 4109.76000               0 #>    ---                                                                       #> 20397:    2021-07-12       2    90       21.21000   21.21000               0 #> 20398:    2021-07-12       2    95       15.00563   15.00563               0 #> 20399:    2021-07-12       2    95       15.00563   15.00563               0 #> 20400:    2021-07-12       2    98        6.28890    6.28890               0 #> 20401:    2021-07-12       2    98        6.28890    6.28890               0 #>        overprediction coverage coverage_deviation bias quantile ae_median #>     1:        29379.0        0               0.00 0.98    0.500   29379.0 #>     2:        24845.4        0              -0.10 0.98    0.450   29379.0 #>     3:        24845.4        0              -0.10 0.98    0.550   29379.0 #>     4:        22334.2        0              -0.20 0.98    0.400   29379.0 #>     5:        22334.2        0              -0.20 0.98    0.600   29379.0 #>    ---                                                                    #> 20397:            0.0        1               0.10 0.70    0.950      73.5 #> 20398:            0.0        1               0.05 0.70    0.025      73.5 #> 20399:            0.0        1               0.05 0.70    0.975      73.5 #> 20400:            0.0        1               0.02 0.70    0.010      73.5 #> 20401:            0.0        1               0.02 0.70    0.990      73.5 #>        quantile_coverage coverage_50 coverage_90 #>     1:              TRUE   0.3828125   0.7265625 #>     2:              TRUE   0.3828125   0.7265625 #>     3:              TRUE   0.3828125   0.7265625 #>     4:              TRUE   0.3828125   0.7265625 #>     5:              TRUE   0.3828125   0.7265625 #>    ---                                           #> 20397:              TRUE   0.5462185   0.9243697 #> 20398:             FALSE   0.5462185   0.9243697 #> 20399:              TRUE   0.5462185   0.9243697 #> 20400:             FALSE   0.5462185   0.9243697 #> 20401:              TRUE   0.5462185   0.9243697"},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"available-metrics","dir":"Articles","previous_headings":"","what":"Available metrics","title":"Getting started","text":"overview available metrics can found metrics data set included package.","code":"metrics #>                                           Metric               Name #>  1:                               Absolute error           ae_point #>  2:                               Absolute error          ae_median #>  3:                                Squared error           se_point #>  4:                                Squared error            se_mean #>  5: (Continuous) ranked probability score (CRPS)               crps #>  6:                                    Log score          log_score #>  7:              (Weighted) interval score (WIS)     interval_score #>  8:                 Dawid-Sebastiani score (DSS)                dss #>  9:                             Brier score (BS)        brier_score #> 10:                            Interval coverage           coverage #> 11:                           Coverage deviation coverage_deviation #> 12:                            Quantile coverage  quantile_coverage #> 13:                                   Dispersion         dispersion #> 14:       Median Absolute Deviation (Dispersion)                mad #> 15:                      Under-, Over-prediction    underprediction #> 16:                      Under-, Over-prediction     overprediction #> 17:         Probability integral transform (PIT)               crps #> 18:                                   Dispersion         dispersion #> 19:                                         Bias               bias #> 20:                             Mean score ratio  mean_scores_ratio #> 21:                               Relative skill     relative_skill #> 22:                        Scaled relative skill   scaled_rel_skill #>                                           Metric               Name #>                                   Functions Discrete Continuous Binary Quantile #>  1: score(), ae_point()), ae_median_sample(        +          +      -        + #>  2: score(), ae_point()), ae_median_sample(        +          +      -        + #>  3:   score(), se_point(), se_mean_sample()        +          +      -        + #>  4:   score(), se_point(), se_mean_sample()        +          +      -        + #>  5:                       score(), ae_point        +          +      -        - #>  6:   score(), logs_sample(), logs_binary()        -          +      +        - #>  7:               score(), interval_score()        +          +      -        + #>  8:                   score(), dss_sample()        +          +      -        - #>  9:                  score(), brier_score()        -          -      +        - #> 10:                                 score()        -          -      -        + #> 11:                                 score()        -          -      -        + #> 12:                                 score()        +          +      -        - #> 13:               score(), interval_score()        -          -      -        + #> 14:                   score(), mad_sample()        +          +      -        - #> 15:               score(), interval_score()        -          -      -        + #> 16:               score(), interval_score()        -          -      -        + #> 17:                          score(), pit()        +          +      -        + #> 18:               score(), interval_score()        -          -      -        + #> 19: score(), bias_sample(), bias_quantile()        +          +      -        + #> 20:                   pairwise_comparison()        ~          ~      ~        ~ #> 21:          score(), pairwise_comparison()        ~          ~      ~        ~ #> 22:          score(), pairwise_comparison()        ~          ~      ~        ~ #>                                   Functions Discrete Continuous Binary Quantile #>                                                                                                                                                                                                           Info #>  1:                                                                                                                                               Suitable for scoring the median of a predictive distribution #>  2:                                                                                                                                               Suitable for scoring the median of a predictive distribution #>  3:                                                                                                                                                Suitable for scoring the mean of a predictive distribution. #>  4:                                                                                                                                                Suitable for scoring the mean of a predictive distribution. #>  5:                           Proper scoring rule (smaller is better), takes entire predictive distribution into account (global), penalises over- and under-confidence similarly, stable handling of outliers #>  6:                                           Proper scoring rule, smaller is better, only evaluates predictive density at observed value (local), penalises over-confidence severely, susceptible to outliers #>  7:                                                              Proper scoring rule, smaller is better, similar properties to CRPS and converges to CRPS for an increasing number of equally spaced intervals #>  8:                           Proper scoring rule, smaller is better, evaluates forecast based on mean and sd of predictive distribution (global), susceptible to outliers, penalises over-confidence severely #>  9:                                                                                    Proper scoring rule, smaller is better, equals CRPS for binary outcomes, penalises over- and under-confidence similarly #> 10:                                                 Proportion of observations falling inside a given central prediction interval (= 'empirical interval coverage'). Used to assess probabilistic calibration. #> 11:                                                                                               Average difference between empirical and nominal interval coverage (coverage that should have been realised) #> 12:                                                                                         Proportion of observations below a given quantile of the predictive CDF. Used to assess probabilistic calibration. #> 13:                                                                                                                                       Dispersion component of WIS, measures width of predictive intervals. #> 14:                                                                                                                    Measure for dispersion of a forecast: median of the absolute deviations from the median #> 15:                                                                                                                                            Absolute amount of over-or under-prediction (components of WIS) #> 16:                                                                                                                                            Absolute amount of over-or under-prediction (components of WIS) #> 17:                                                                                   PIT transform is the CDF of the predictive distribution evaluated at the observed values. PIT values should be uniform.  #> 18:                                                                                                                                       Dispersion component of WIS, measures width of predictive intervals. #> 19:                                                                                       Measure of relative tendency to over- or under-predict (aspect of calibration), bounded between -1 and 1 (ideally 0) #> 20:                                                                                                             Compares performance of two models. Properties depend on the metric chosen for the comparison. #> 21:                                                         Ranks models based on pairwise comparisons, useful in the context of missing forecasts. Properties depend on the metric chosen for the comparison. #> 22: Ranks models based on pairwise comparisons, useful in the context of missing forecasts. Scaled (i.e. divided) by the score of a baseline model. Properties depend on the metric chosen for the comparison. #>                                                                                                                                                                                                           Info"},{"path":"https://epiforecasts.io/scoringutils/articles/scoringutils.html","id":"lower-level-functions","dir":"Articles","previous_headings":"","what":"Lower level functions","title":"Getting started","text":"metrics available lower level functions extensively documented. look help files understand better.","code":""},{"path":"https://epiforecasts.io/scoringutils/authors.html","id":null,"dir":"","previous_headings":"","what":"Authors","title":"Authors and Citation","text":"Nikos Bosse. Author, maintainer. Sam Abbott. Author. Hugo Gruson. Author. Johannes Bracher. Contributor. Sebastian Funk. Author.","code":""},{"path":"https://epiforecasts.io/scoringutils/authors.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Authors and Citation","text":"Nikos . Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van Leeuwen, Sam Abbott (2022). Evaluating Forecasts scoringutils R, arXiv. DOI: 10.48550/ARXIV.2205.07090 Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating Probabilistic Forecasts scoringRules. Journal Statistical Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12","code":"@Article{,   title = {Evaluating Forecasts with scoringutils in R},   author = {Nikos I. Bosse and Hugo Gruson and Anne Cori and Edwin {van Leeuwen} and Sebastian Funk and Sam Abbott},   journal = {arXiv},   url = {https://arxiv.org/abs/2205.07090},   year = {2022},   doi = {10.48550/ARXIV.2205.07090}, } @Article{,   title = {Evaluating Probabilistic Forecasts with {scoringRules}},   author = {Alexander Jordan and Fabian Kr\\\"uger and Sebastian Lerch},   journal = {Journal of Statistical Software},   year = {2019},   volume = {90},   number = {12},   pages = {1--37},   doi = {10.18637/jss.v090.i12}, }"},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"scoringutils-utilities-for-scoring-and-assessing-predictions","dir":"","previous_headings":"","what":"Utilities for Scoring and Assessing Predictions","title":"Utilities for Scoring and Assessing Predictions","text":"scoringutils package provides collection metrics proper scoring rules aims make simple score probabilistic forecasts true observed values. can find additional information examples papers Evaluating Forecasts scoringutils R Scoring epidemiological forecasts transformed scales well Vignettes (Getting started, Details metrics implemented Scoring forecasts directly). scoringutils package offers convenient automated forecast evaluation function score(). function operates data.frames (uses data.table internally speed efficiency) can easily integrated workflow based dplyr data.table. also provides experienced users set reliable lower-level scoring metrics operating vectors/matrices can build upon applications. addition implements wide range flexible plots designed cover many use cases. available scoringutils depends functionality scoringRules provides comprehensive collection proper scoring rules predictive probability distributions represented sample parametric distributions. forecast types, quantile forecasts, scoringutils also implements additional metrics evaluating forecasts. top providing interface proper scoring rules implemented scoringRules natively, scoringutils also offers utilities summarising visualising forecasts scores, obtain relative scores models may useful non-overlapping forecasts forecasts across scales. Predictions can handled various formats: scoringutils can handle probabilistic forecasts either sample based quantile based format. detail expected input formats please see . True values can integer, continuous binary, appropriate scores value types selected automatically.","code":""},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"installation","dir":"","previous_headings":"","what":"Installation","title":"Utilities for Scoring and Assessing Predictions","text":"Install CRAN version package using: Install stable development version package : Install unstable development GitHub using following,","code":"install.packages(\"scoringutils\") install.packages(\"scoringutils\", repos = \"https://epiforecasts.r-universe.dev\") remotes::install_github(\"epiforecasts/scoringutils\", dependencies = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"quick-start","dir":"","previous_headings":"","what":"Quick start","title":"Utilities for Scoring and Assessing Predictions","text":"quick start guide explore functionality scoringutils package using quantile forecasts ECDC forecasting hub example. detailed documentation please see package vignettes, individual function documentation.","code":""},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"plotting-forecasts","dir":"","previous_headings":"Quick start","what":"Plotting forecasts","title":"Utilities for Scoring and Assessing Predictions","text":"first step evaluating forecasts visualise . purposes example make use plot_predictions() filter available forecasts single model, forecast date.","code":"example_quantile %>%   make_NA(what = \"truth\",            target_end_date >= \"2021-07-15\",            target_end_date < \"2021-05-22\"   ) %>%   make_NA(what = \"forecast\",           model != \"EuroCOVIDhub-ensemble\",            forecast_date != \"2021-06-28\"   ) %>%   plot_predictions(     x = \"target_end_date\",     by = c(\"target_type\", \"location\")   ) +   facet_wrap(target_type ~ location, ncol = 4, scales = \"free\")"},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"scoring-forecasts","dir":"","previous_headings":"Quick start","what":"Scoring forecasts","title":"Utilities for Scoring and Assessing Predictions","text":"Forecasts can easily quickly scored using score() function. score() automatically tries determine forecast_unit, .e. set columns uniquely defines single forecast, taking column names data account. However, recommended set forecast unit manually using set_forecast_unit() may help avoid errors, especially scoringutils used automated pipelines. function set_forecast_unit() simply drop unneeded columns. verify everything order, function check_forecasts() used. result check can passed directly score(). score() returns unsummarised scores, cases user wants. make use additional functions scoringutils add empirical coverage-levels (add_coverage()), scores relative baseline model (chosen EuroCOVIDhub-ensemble model). See getting started vignette details. Finally summarise scores model target type. scoringutils contains additional functionality transform forecasts, summarise scores different levels, visualise , explore forecasts . See package vignettes function documentation information. may want score forecasts based transformations original data order obtain complete evaluation (see paper information). can done using function transform_forecasts(). following example, truncate values 0 use function log_shift() add 1 values applying natural logarithm.","code":"example_quantile %>%   set_forecast_unit(c(\"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\")) %>%   check_forecasts() %>%   score() %>%   add_coverage(ranges = c(50, 90), by = c(\"model\", \"target_type\")) %>%   summarise_scores(     by = c(\"model\", \"target_type\"),     relative_skill = TRUE,     baseline = \"EuroCOVIDhub-ensemble\"   ) %>%   summarise_scores(     fun = signif,      digits = 2   ) %>%   kable() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. example_quantile %>%  .[, true_value := ifelse(true_value < 0, 0, true_value)] %>%   transform_forecasts(append = TRUE, fun = log_shift, offset = 1) %>%   score %>%   summarise_scores(by = c(\"model\", \"target_type\", \"scale\")) %>%   head() #> The following messages were produced when checking inputs: #> 1.  288 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model target_type   scale interval_score   dispersion #> 1: EuroCOVIDhub-baseline       Cases     log   1.169972e+00    0.4373146 #> 2: EuroCOVIDhub-baseline       Cases natural   2.209046e+04 4102.5009443 #> 3: EuroCOVIDhub-ensemble       Cases     log   5.500974e-01    0.1011850 #> 4: EuroCOVIDhub-ensemble       Cases natural   1.155071e+04 3663.5245788 #> 5:  epiforecasts-EpiNow2       Cases     log   6.005778e-01    0.1066329 #> 6:  epiforecasts-EpiNow2       Cases natural   1.443844e+04 5664.3779484 #>    underprediction overprediction coverage_deviation        bias    ae_median #> 1:    3.521964e-01      0.3804607        -0.10940217  0.09726562 1.185905e+00 #> 2:    1.028497e+04   7702.9836957        -0.10940217  0.09726562 3.208048e+04 #> 3:    1.356563e-01      0.3132561        -0.09785326 -0.05640625 7.410484e-01 #> 4:    4.237177e+03   3650.0047554        -0.09785326 -0.05640625 1.770795e+04 #> 5:    1.858699e-01      0.3080750        -0.06660326 -0.07890625 7.656591e-01 #> 6:    3.260356e+03   5513.7058424        -0.06660326 -0.07890625 2.153070e+04"},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Utilities for Scoring and Assessing Predictions","text":"using scoringutils work please consider citing using output citation(\"scoringutils\"):","code":"#> To cite scoringutils in publications use the following. If you use the #> CRPS, DSS, or Log Score, please also cite scoringRules. #>  #>   Nikos I. Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van #>   Leeuwen, and Sam Abbott (2022). Evaluating Forecasts with #>   scoringutils in R, arXiv. DOI: 10.48550/ARXIV.2205.07090 #>  #> To cite scoringRules in publications use: #>  #>   Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating #>   Probabilistic Forecasts with scoringRules. Journal of Statistical #>   Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12 #>  #> To see these entries in BibTeX format, use 'print(<citation>, #> bibtex=TRUE)', 'toBibtex(.)', or set #> 'options(citation.bibtex.max=999)'."},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"how-to-make-a-bug-report-or-feature-request","dir":"","previous_headings":"","what":"How to make a bug report or feature request","title":"Utilities for Scoring and Assessing Predictions","text":"Please briefly describe problem output expect issue. question, please don’t open issue. Instead, ask Q page.","code":""},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"contributing","dir":"","previous_headings":"","what":"Contributing","title":"Utilities for Scoring and Assessing Predictions","text":"welcome contributions new contributors! particularly appreciate help priority problems issues. Please check add issues, /add pull request.","code":""},{"path":"https://epiforecasts.io/scoringutils/index.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Utilities for Scoring and Assessing Predictions","text":"Please note scoringutils project released Contributor Code Conduct. contributing project, agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/abs_error.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute Error — abs_error","title":"Absolute Error — abs_error","text":"Calculate absolute error $$   \\textrm{abs}(\\textrm{true\\_value} - \\textrm{median\\_prediction}) $$","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/abs_error.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute Error — abs_error","text":"","code":"abs_error(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/abs_error.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute Error — abs_error","text":"true_values vector true observed values size n predictions numeric vector predictions, corresponding quantiles second vector, quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/abs_error.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute Error — abs_error","text":"vector absolute error","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/reference/abs_error.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute Error — abs_error","text":"","code":"true_values <- rnorm(30, mean = 1:30) predicted_values <- rnorm(30, mean = 1:30) abs_error(true_values, predicted_values) #>  [1] 0.04335247 0.78169750 1.72566866 1.65897287 0.92520395 0.57798934 #>  [7] 1.40227305 1.53346288 0.04651595 0.72696856 0.61086690 1.16778933 #> [13] 2.44073495 1.64170045 1.83422664 0.53999974 1.18998257 0.37440980 #> [19] 0.95626491 1.46325686 1.21805693 0.56912591 1.17229418 1.80345311 #> [25] 2.66248107 0.84947828 0.24647184 1.04848255 1.28635859 1.81194099"},{"path":"https://epiforecasts.io/scoringutils/reference/add_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Add coverage of central prediction intervals — add_coverage","title":"Add coverage of central prediction intervals — add_coverage","text":"Adds column coverage central prediction intervals unsummarised scores produced score()","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/add_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Add coverage of central prediction intervals — add_coverage","text":"","code":"add_coverage(scores, by, ranges = c(50, 90))"},{"path":"https://epiforecasts.io/scoringutils/reference/add_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Add coverage of central prediction intervals — add_coverage","text":"scores data.table scores produced score(). character vector column names add coverage . ranges numeric vector ranges central prediction intervals coverage values shall added.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/add_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Add coverage of central prediction intervals — add_coverage","text":"data.table unsummarised scores columns added coverage central prediction intervals. overall data.table still unsummarised, note coverage columns level summary present according value specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/add_coverage.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Add coverage of central prediction intervals — add_coverage","text":"coverage values added computed according values specified . , example, = \"model\", one coverage value every model add_coverage() compute coverage every model across values present columns define unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/add_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Add coverage of central prediction intervals — add_coverage","text":"","code":"library(magrittr) # pipe operator score(example_quantile) %>%   add_coverage(by = c(\"model\", \"target_type\")) %>%   summarise_scores(by = c(\"model\", \"target_type\")) %>%   summarise_scores(fun = signif, digits = 2) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model target_type interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline       Cases          28000       4100         10000.0 #> 2: EuroCOVIDhub-baseline      Deaths            160         91             2.1 #> 3: EuroCOVIDhub-ensemble       Cases          18000       3700          4200.0 #> 4: EuroCOVIDhub-ensemble      Deaths             41         30             4.1 #> 5:       UMass-MechBayes      Deaths             53         27            17.0 #> 6:  epiforecasts-EpiNow2       Cases          21000       5700          3300.0 #> 7:  epiforecasts-EpiNow2      Deaths             67         32            16.0 #>    overprediction coverage_deviation    bias ae_median coverage_50 coverage_90 #> 1:        14000.0             -0.110  0.0980     38000        0.33        0.82 #> 2:           66.0              0.120  0.3400       230        0.66        1.00 #> 3:        10000.0             -0.098 -0.0560     24000        0.39        0.80 #> 4:            7.1              0.200  0.0730        53        0.88        1.00 #> 5:            9.0             -0.023 -0.0220        78        0.46        0.88 #> 6:        12000.0             -0.067 -0.0790     28000        0.47        0.79 #> 7:           19.0             -0.043 -0.0051       100        0.42        0.91"},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute Error of the Median (Quantile-based Version) — ae_median_quantile","title":"Absolute Error of the Median (Quantile-based Version) — ae_median_quantile","text":"Absolute error median calculated $$   \\textrm{abs}(\\textrm{true\\_value} - \\textrm{prediction}) $$ function created internal use within score(), can also used standalone function.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute Error of the Median (Quantile-based Version) — ae_median_quantile","text":"","code":"ae_median_quantile(true_values, predictions, quantiles = NULL)"},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute Error of the Median (Quantile-based Version) — ae_median_quantile","text":"true_values vector true observed values size n predictions numeric vector predictions, corresponding quantiles second vector, quantiles. quantiles numeric vector denotes quantile values predictions. predictions quantiles == 0.5 kept. quantiles NULL, predictions true_values used (abs_error())","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute Error of the Median (Quantile-based Version) — ae_median_quantile","text":"vector scoring values","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute Error of the Median (Quantile-based Version) — ae_median_quantile","text":"","code":"true_values <- rnorm(30, mean = 1:30) predicted_values <- rnorm(30, mean = 1:30) ae_median_quantile(true_values, predicted_values, quantiles = 0.5) #>  [1] 0.009167113 0.872604126 0.515186522 0.626420244 2.133607786 0.580075604 #>  [7] 2.085917221 0.728393181 0.641991758 0.679059189 1.520541579 0.182937000 #> [13] 0.071280772 3.176288601 1.428557684 0.907705662 0.062685133 0.426616895 #> [19] 0.744986362 2.283214165 2.277201261 2.651972106 1.856070426 3.913091404 #> [25] 1.790528259 1.671142980 0.050199270 0.433765063 1.154432409 0.070320886"},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute Error of the Median (Sample-based Version) — ae_median_sample","title":"Absolute Error of the Median (Sample-based Version) — ae_median_sample","text":"Absolute error median calculated $$%   \\textrm{abs}(\\textrm{true\\_value} - \\textrm{median\\_prediction}) $$","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute Error of the Median (Sample-based Version) — ae_median_sample","text":"","code":"ae_median_sample(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute Error of the Median (Sample-based Version) — ae_median_sample","text":"true_values vector true observed values size n predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute Error of the Median (Sample-based Version) — ae_median_sample","text":"vector scoring values","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/reference/ae_median_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute Error of the Median (Sample-based Version) — ae_median_sample","text":"","code":"true_values <- rnorm(30, mean = 1:30) predicted_values <- rnorm(30, mean = 1:30) ae_median_sample(true_values, predicted_values) #>  [1] 2.0964594 1.7387376 0.8076599 0.1504663 0.5778161 0.9841827 0.4550418 #>  [8] 0.6935833 2.6801398 0.9680780 0.4389062 0.1298220 2.6308444 1.9791294 #> [15] 1.2582431 0.3692456 1.0107132 0.3056360 0.8101142 0.1494411 0.6472083 #> [22] 0.4068241 1.2441764 2.6587995 1.2021722 0.7939360 2.0019957 1.2597231 #> [29] 1.8978191 1.5750900"},{"path":"https://epiforecasts.io/scoringutils/reference/avail_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Display Number of Forecasts Available — avail_forecasts","title":"Display Number of Forecasts Available — avail_forecasts","text":"Given data set forecasts, count number available forecasts arbitrary grouping (e.g. number forecasts per model, number forecasts per model location). useful determine whether missing forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/avail_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Display Number of Forecasts Available — avail_forecasts","text":"","code":"avail_forecasts(data, by = NULL, collapse = c(\"quantile\", \"sample\"))"},{"path":"https://epiforecasts.io/scoringutils/reference/avail_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Display Number of Forecasts Available — avail_forecasts","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary). character vector NULL (default) denotes categories number forecasts counted. default (= NULL) unit single forecast (.e. available columns (apart \"protected\" columns 'prediction' 'true value') plus \"quantile\" \"sample\" present). collapse character vector (default c(\"quantile\", \"sample\") names categories number rows collapsed one counting. example, single forecast usually represented set several quantiles samples collapsing one makes sure single forecast gets counted .","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/avail_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Display Number of Forecasts Available — avail_forecasts","text":"data.table columns specified additional column number forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/avail_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Display Number of Forecasts Available — avail_forecasts","text":"","code":"data.table::setDTthreads(1) # only needed to avoid issues on CRAN  avail_forecasts(example_quantile,   collapse = c(\"quantile\"),   by = c(\"model\", \"target_type\") ) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model target_type Number forecasts #> 1: EuroCOVIDhub-ensemble       Cases              128 #> 2: EuroCOVIDhub-baseline       Cases              128 #> 3:  epiforecasts-EpiNow2       Cases              128 #> 4: EuroCOVIDhub-ensemble      Deaths              128 #> 5: EuroCOVIDhub-baseline      Deaths              128 #> 6:       UMass-MechBayes      Deaths              128 #> 7:  epiforecasts-EpiNow2      Deaths              119"},{"path":"https://epiforecasts.io/scoringutils/reference/available_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Available metrics in scoringutils — available_metrics","title":"Available metrics in scoringutils — available_metrics","text":"Available metrics scoringutils","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/available_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Available metrics in scoringutils — available_metrics","text":"","code":"available_metrics()"},{"path":"https://epiforecasts.io/scoringutils/reference/available_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Available metrics in scoringutils — available_metrics","text":"vector name available metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Determines Bias of Quantile Forecasts — bias_quantile","title":"Determines Bias of Quantile Forecasts — bias_quantile","text":"Determines bias quantile forecasts. increasing number quantiles measure converges sample based bias version integer continuous forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determines Bias of Quantile Forecasts — bias_quantile","text":"","code":"bias_quantile(predictions, quantiles, true_value)"},{"path":"https://epiforecasts.io/scoringutils/reference/bias_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determines Bias of Quantile Forecasts — bias_quantile","text":"predictions vector length corresponding number quantiles holds predictions quantiles vector corresponding size quantiles predictions made. contain median (0.5) median imputed mean two innermost quantiles. true_value single true value","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determines Bias of Quantile Forecasts — bias_quantile","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determines Bias of Quantile Forecasts — bias_quantile","text":"quantile forecasts, bias measured $$ B_t = (1 - 2 \\cdot \\max \\{| q_{t,} \\Q_t \\land q_{t,} \\leq x_t\\})  \\mathbf{1}( x_t \\leq q_{t, 0.5}) \\\\ + (1 - 2 \\cdot \\min \\{| q_{t,} \\Q_t \\land q_{t,} \\geq x_t\\})  1( x_t \\geq q_{t, 0.5}),$$ \\(Q_t\\) set quantiles form predictive distribution time \\(t\\). represent belief true value $x_t$ . consistency, define \\(Q_t\\) always includes element \\(q_{t, 0} = - \\infty\\) \\(q_{t,1} = \\infty\\). \\(1()\\) indicator function \\(1\\) condition satisfied $0$ otherwise. clearer terms, \\(B_t\\) defined maximum percentile rank corresponding quantile still true value, true value smaller median predictive distribution. true value median predictive distribution, $B_t$ minimum percentile rank corresponding quantile still larger true value. true value exactly median, terms cancel \\(B_t\\) zero. large enough number quantiles, percentile rank equal proportion predictive samples observed true value, metric coincides one continuous forecasts. Bias can assume values -1 1 0 ideally (.e. unbiased).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_quantile.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Determines Bias of Quantile Forecasts — bias_quantile","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determines Bias of Quantile Forecasts — bias_quantile","text":"","code":"predictions <- c(   705.500, 1127.000, 4006.250, 4341.500, 4709.000, 4821.996,   5340.500, 5451.000, 5703.500, 6087.014, 6329.500, 6341.000,   6352.500, 6594.986, 6978.500, 7231.000, 7341.500, 7860.004,   7973.000, 8340.500, 8675.750, 11555.000, 11976.500 )  quantiles <- c(0.01, 0.025, seq(0.05, 0.95, 0.05), 0.975, 0.99)  true_value <- 8062  bias_quantile(predictions, quantiles, true_value = true_value) #> [1] -0.8"},{"path":"https://epiforecasts.io/scoringutils/reference/bias_range.html","id":null,"dir":"Reference","previous_headings":"","what":"Determines Bias of Quantile Forecasts based on the range of the prediction intervals — bias_range","title":"Determines Bias of Quantile Forecasts based on the range of the prediction intervals — bias_range","text":"Determines bias quantile forecasts based range prediction intervals. increasing number quantiles measure converges sample based bias version integer continuous forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_range.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determines Bias of Quantile Forecasts based on the range of the prediction intervals — bias_range","text":"","code":"bias_range(lower, upper, range, true_value)"},{"path":"https://epiforecasts.io/scoringutils/reference/bias_range.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determines Bias of Quantile Forecasts based on the range of the prediction intervals — bias_range","text":"lower vector length corresponding number central prediction intervals holds predictions lower bounds prediction interval upper vector length corresponding number central prediction intervals holds predictions upper bounds prediction interval range vector corresponding size information width central prediction interval true_value single true value","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_range.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determines Bias of Quantile Forecasts based on the range of the prediction intervals — bias_range","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_range.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determines Bias of Quantile Forecasts based on the range of the prediction intervals — bias_range","text":"quantile forecasts, bias measured $$ B_t = (1 - 2 \\cdot \\max \\{| q_{t,} \\Q_t \\land q_{t,} \\leq x_t\\}) \\mathbf{1}( x_t \\leq q_{t, 0.5}) \\\\ + (1 - 2 \\cdot \\min \\{| q_{t,} \\Q_t \\land q_{t,} \\geq x_t\\})  \\mathbf{1}( x_t \\geq q_{t, 0.5}), $$ \\(Q_t\\) set quantiles form predictive distribution time \\(t\\). represent belief true value \\(x_t\\) . consistency, define \\(Q_t\\) always includes element \\(q_{t, 0} = - \\infty\\) \\(q_{t,1} = \\infty\\). \\(\\mathbf{1}()\\) indicator function \\(1\\) condition satisfied $0$ otherwise. clearer terms, \\(B_t\\) defined maximum percentile rank corresponding quantile still true value, true value smaller median predictive distribution. true value median predictive distribution, $B_t$ minimum percentile rank corresponding quantile still larger true value. true value exactly median, terms cancel \\(B_t\\) zero. large enough number quantiles, percentile rank equal proportion predictive samples observed true value, metric coincides one continuous forecasts. Bias can assume values -1 1 0 ideally.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/reference/bias_range.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Determines Bias of Quantile Forecasts based on the range of the prediction intervals — bias_range","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_range.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determines Bias of Quantile Forecasts based on the range of the prediction intervals — bias_range","text":"","code":"lower <- c(   6341.000, 6329.500, 6087.014, 5703.500,   5451.000, 5340.500, 4821.996, 4709.000,   4341.500, 4006.250, 1127.000, 705.500 )  upper <- c(   6341.000, 6352.500, 6594.986, 6978.500,   7231.000, 7341.500, 7860.004, 7973.000,   8340.500, 8675.750, 11555.000, 11976.500 )  range <- c(0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 98)  true_value <- 8062  bias_range(   lower = lower, upper = upper,   range = range, true_value = true_value ) #> [1] -0.8"},{"path":"https://epiforecasts.io/scoringutils/reference/bias_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determines bias of forecasts — bias_sample","title":"Determines bias of forecasts — bias_sample","text":"Determines bias predictive Monte-Carlo samples. function automatically recognises, whether forecasts continuous integer valued adapts Bias function accordingly.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determines bias of forecasts — bias_sample","text":"","code":"bias_sample(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/bias_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determines bias of forecasts — bias_sample","text":"true_values vector true observed values size n predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determines bias of forecasts — bias_sample","text":"vector length n biases predictive samples respect true values.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determines bias of forecasts — bias_sample","text":"continuous forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - 2 * (P_t (x_t)) $$ \\(P_t\\) empirical cumulative distribution function prediction true value \\(x_t\\). Computationally, \\(P_t (x_t)\\) just calculated fraction predictive samples \\(x_t\\) smaller \\(x_t\\). integer valued forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - (P_t (x_t) + P_t (x_t + 1)) $$ adjust integer nature forecasts. cases, Bias can assume values -1 1 0 ideally.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determines bias of forecasts — bias_sample","text":"integer valued Bias function discussed Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15 Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, et al. (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLOS Computational Biology 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_sample.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Determines bias of forecasts — bias_sample","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/bias_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determines bias of forecasts — bias_sample","text":"","code":"## integer valued forecasts true_values <- rpois(30, lambda = 1:30) predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) bias_sample(true_values, predictions) #>  [1]  0.665  0.520  0.465  0.895 -0.985 -0.465 -0.550 -0.370  0.670 -0.890 #> [11] -0.565 -0.880  0.755  0.190  0.030 -0.415 -0.725  0.940 -0.650  0.535 #> [21]  0.630  0.575 -0.275 -0.240 -0.170 -0.470  0.495  0.530 -0.095  0.305  ## continuous forecasts true_values <- rnorm(30, mean = 1:30) predictions <- replicate(200, rnorm(30, mean = 1:30)) bias_sample(true_values, predictions) #>  [1] -0.02 -0.76  0.63  0.92  0.49 -0.23 -0.67 -0.84  0.21 -0.19  0.34  0.85 #> [13]  0.67 -0.98  0.81  0.52 -0.54 -0.47  0.34 -0.19  0.67  0.73 -0.45  0.53 #> [25] -0.97 -0.52 -0.40 -0.50 -0.72 -0.97"},{"path":"https://epiforecasts.io/scoringutils/reference/brier_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Brier Score — brier_score","title":"Brier Score — brier_score","text":"Computes Brier Score probabilistic forecasts binary outcomes.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/brier_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Brier Score — brier_score","text":"","code":"brier_score(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/brier_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Brier Score — brier_score","text":"true_values vector true observed values size n values equal either 0 1 predictions vector predicted probability true_value = 1.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/brier_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Brier Score — brier_score","text":"numeric value Brier Score, .e. mean squared error given probability forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/brier_score.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Brier Score — brier_score","text":"Brier score proper score rule assesses accuracy probabilistic binary predictions. outcomes can either 0 1, predictions must probability true outcome 1. Brier Score computed mean squared error probabilistic prediction true outcome. $$   \\textrm{Brier\\_Score} = \\frac{1}{N} \\sum_{t = 1}^{n} (\\textrm{prediction}_t -   \\textrm{outcome}_t)^2 $$","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/brier_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Brier Score — brier_score","text":"","code":"true_values <- sample(c(0, 1), size = 30, replace = TRUE) predictions <- runif(n = 30, min = 0, max = 1)  brier_score(true_values, predictions) #>  [1] 0.0112829959 0.0752915165 0.2847032519 0.0199814359 0.5776206858 #>  [6] 0.9898446829 0.0994000260 0.0004272894 0.9905717892 0.1014083421 #> [11] 0.0454723878 0.6849139082 0.0140767038 0.0888843918 0.6781037162 #> [16] 0.8549518110 0.3778401233 0.0602922087 0.6247052148 0.4125147605 #> [21] 0.6405651209 0.0049223738 0.3352601066 0.8346801463 0.4579342835 #> [26] 0.0143894841 0.2619389016 0.1100379825 0.5566204308 0.5573266538"},{"path":"https://epiforecasts.io/scoringutils/reference/check_equal_length.html","id":null,"dir":"Reference","previous_headings":"","what":"Check Length — check_equal_length","title":"Check Length — check_equal_length","text":"Check whether variables length","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_equal_length.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check Length — check_equal_length","text":"","code":"check_equal_length(..., one_allowed = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/reference/check_equal_length.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check Length — check_equal_length","text":"... variables check one_allowed logical, allow arguments length one can recycled","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_equal_length.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check Length — check_equal_length","text":"function returns NULL, throws error variable lengths differ","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Check forecasts — check_forecasts","title":"Check forecasts — check_forecasts","text":"Function check input data running score(). data come one three different formats: format binary predictions (see example_binary) sample-based format discrete continuous predictions (see example_continuous example_integer) quantile-based format (see example_quantile)","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check forecasts — check_forecasts","text":"","code":"check_forecasts(data)"},{"path":"https://epiforecasts.io/scoringutils/reference/check_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check forecasts — check_forecasts","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check forecasts — check_forecasts","text":"list elements give information scoringutils thinks trying potential issues. target_type type prediction target inferred input: 'binary', values true_value either 0 1 values prediction 0 1, 'discrete' true values integers. 'continuous' . prediction_type inferred type prediction. 'quantile', column called 'quantile', else 'discrete' values prediction integer, else 'continuous. forecast_unit unit single forecast, .e. grouping uniquely defines single forecast. assumed present columns apart following protected columns: c(\"prediction\", \"true_value\", \"sample\", \"quantile\",\"range\", \"boundary\"). important remove unnecessary columns scoring. rows_per_forecast data.frame shows many rows (usually quantiles samples available per forecast. forecast model several entries, forecasts differing numbers quantiles / samples. unique_values data.frame shows many unique values present per model column data. directly show missing values, rather maximum number unique values across whole data. warnings vector warnings. can ignored know . errors vector issues cause error running score(). messages verbal explanation information provided .","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/reference/check_forecasts.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Check forecasts — check_forecasts","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Check forecasts — check_forecasts","text":"","code":"check <- check_forecasts(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. print(check) #> Your forecasts seem to be for a target of the following type: #> $target_type #> [1] \"integer\" #>  #> and in the following format: #> $prediction_type #> [1] \"quantile\" #>  #> The unit of a single forecast is defined by: #> $forecast_unit #> [1] \"location\"        \"target_end_date\" \"target_type\"     \"location_name\"   #> [5] \"forecast_date\"   \"model\"           \"horizon\"         #>  #> Cleaned data, rows with NA values in prediction or true_value removed: #> $cleaned_data #>        location target_end_date target_type true_value location_name #>     1:       DE      2021-05-08       Cases     106987       Germany #>     2:       DE      2021-05-08       Cases     106987       Germany #>     3:       DE      2021-05-08       Cases     106987       Germany #>     4:       DE      2021-05-08       Cases     106987       Germany #>     5:       DE      2021-05-08       Cases     106987       Germany #>    ---                                                               #> 20397:       IT      2021-07-24      Deaths         78         Italy #> 20398:       IT      2021-07-24      Deaths         78         Italy #> 20399:       IT      2021-07-24      Deaths         78         Italy #> 20400:       IT      2021-07-24      Deaths         78         Italy #> 20401:       IT      2021-07-24      Deaths         78         Italy #>        forecast_date quantile prediction                 model horizon #>     1:    2021-05-03    0.010      82466 EuroCOVIDhub-ensemble       1 #>     2:    2021-05-03    0.025      86669 EuroCOVIDhub-ensemble       1 #>     3:    2021-05-03    0.050      90285 EuroCOVIDhub-ensemble       1 #>     4:    2021-05-03    0.100      95341 EuroCOVIDhub-ensemble       1 #>     5:    2021-05-03    0.150      99171 EuroCOVIDhub-ensemble       1 #>    ---                                                                 #> 20397:    2021-07-12    0.850        352  epiforecasts-EpiNow2       2 #> 20398:    2021-07-12    0.900        397  epiforecasts-EpiNow2       2 #> 20399:    2021-07-12    0.950        499  epiforecasts-EpiNow2       2 #> 20400:    2021-07-12    0.975        611  epiforecasts-EpiNow2       2 #> 20401:    2021-07-12    0.990        719  epiforecasts-EpiNow2       2 #>  #> Number of unique values per column per model: #> $unique_values #>                    model location target_end_date target_type true_value #> 1: EuroCOVIDhub-ensemble        4              12           2         96 #> 2: EuroCOVIDhub-baseline        4              12           2         96 #> 3:  epiforecasts-EpiNow2        4              12           2         95 #> 4:       UMass-MechBayes        4              12           1         48 #>    location_name forecast_date quantile prediction horizon #> 1:             4            11       23       3969       3 #> 2:             4            11       23       3733       3 #> 3:             4            11       23       3903       3 #> 4:             4            11       23       1058       3 #>  #> $messages #> [1] \"144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.\" #>  check_forecasts(example_binary) #> The following messages were produced when checking inputs: #> 1.  144 values for `true_value` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #> 2.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #> Your forecasts seem to be for a target of the following type: #> $target_type #> [1] \"binary\" #>  #> and in the following format: #> $prediction_type #> [1] \"continuous\" #>  #> The unit of a single forecast is defined by: #> $forecast_unit #> [1] \"location\"        \"location_name\"   \"target_end_date\" \"target_type\"     #> [5] \"forecast_date\"   \"model\"           \"horizon\"         #>  #> Cleaned data, rows with NA values in prediction or true_value removed: #> $cleaned_data #>      location location_name target_end_date target_type forecast_date #>   1:       DE       Germany      2021-05-08       Cases    2021-05-03 #>   2:       DE       Germany      2021-05-08       Cases    2021-05-03 #>   3:       DE       Germany      2021-05-08       Cases    2021-05-03 #>   4:       DE       Germany      2021-05-08      Deaths    2021-05-03 #>   5:       DE       Germany      2021-05-08      Deaths    2021-05-03 #>  ---                                                                  #> 883:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 884:       IT         Italy      2021-07-24      Deaths    2021-07-05 #> 885:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 886:       IT         Italy      2021-07-24      Deaths    2021-07-05 #> 887:       IT         Italy      2021-07-24      Deaths    2021-07-12 #>                      model horizon prediction true_value #>   1: EuroCOVIDhub-ensemble       1      0.375          0 #>   2: EuroCOVIDhub-baseline       1      0.475          0 #>   3:  epiforecasts-EpiNow2       1      0.425          0 #>   4: EuroCOVIDhub-ensemble       1      0.425          0 #>   5: EuroCOVIDhub-baseline       1      0.500          1 #>  ---                                                     #> 883: EuroCOVIDhub-baseline       2      0.250          0 #> 884:       UMass-MechBayes       3      0.475          0 #> 885:       UMass-MechBayes       2      0.450          0 #> 886:  epiforecasts-EpiNow2       3      0.375          0 #> 887:  epiforecasts-EpiNow2       2      0.300          0 #>  #> Number of unique values per column per model: #> $unique_values #>                    model location location_name target_end_date target_type #> 1: EuroCOVIDhub-ensemble        4             4              12           2 #> 2: EuroCOVIDhub-baseline        4             4              12           2 #> 3:  epiforecasts-EpiNow2        4             4              12           2 #> 4:       UMass-MechBayes        4             4              12           1 #>    forecast_date horizon prediction true_value #> 1:            11       3         13          2 #> 2:            11       3         20          2 #> 3:            11       3         13          2 #> 4:            11       3         10          2 #>  #> $messages #> [1] \"144 values for `true_value` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.\" #> [2] \"144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.\" #>"},{"path":"https://epiforecasts.io/scoringutils/reference/check_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Check whether the desired metrics are available in scoringutils — check_metrics","title":"Check whether the desired metrics are available in scoringutils — check_metrics","text":"Helper function check whether desired metrics available. input NULL, metrics returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check whether the desired metrics are available in scoringutils — check_metrics","text":"","code":"check_metrics(metrics)"},{"path":"https://epiforecasts.io/scoringutils/reference/check_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check whether the desired metrics are available in scoringutils — check_metrics","text":"metrics character vector desired metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check whether the desired metrics are available in scoringutils — check_metrics","text":"character vector metrics can used downstream computation","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_not_null.html","id":null,"dir":"Reference","previous_headings":"","what":"Check Variable is not NULL — check_not_null","title":"Check Variable is not NULL — check_not_null","text":"Check whether certain variable NULL return name variable function call variable missing. function helper function called within functions","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_not_null.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check Variable is not NULL — check_not_null","text":"","code":"check_not_null(...)"},{"path":"https://epiforecasts.io/scoringutils/reference/check_not_null.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check Variable is not NULL — check_not_null","text":"... variables check","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_not_null.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check Variable is not NULL — check_not_null","text":"function returns NULL, throws error variable missing.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_predictions.html","id":null,"dir":"Reference","previous_headings":"","what":"Check Prediction Input For Lower-level Scoring Functions — check_predictions","title":"Check Prediction Input For Lower-level Scoring Functions — check_predictions","text":"Helper function check inputs lower-level score functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_predictions.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check Prediction Input For Lower-level Scoring Functions — check_predictions","text":"","code":"check_predictions(   predictions,   true_values = NULL,   type = c(\"continuous\", \"integer\", \"binary\"),   class = c(\"vector\", \"matrix\") )"},{"path":"https://epiforecasts.io/scoringutils/reference/check_predictions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check Prediction Input For Lower-level Scoring Functions — check_predictions","text":"predictions object predictions. Depending whether class = vector class = \"matrix\" can either vector length n (corresponding length true_values) nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples true_values vector true observed values size n type character, one \"continuous\" (default), \"integer\" \"binary\" defines type forecast class character, either \"vector\" (default) \"matrix\" determines class input correspond ","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_quantiles.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that quantiles are valid — check_quantiles","title":"Check that quantiles are valid — check_quantiles","text":"Helper function check input quantiles valid. Quantiles must range specified, increase monotonically, contain duplicates. used bias_range() bias_quantile() provide informative errors users.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_quantiles.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that quantiles are valid — check_quantiles","text":"","code":"check_quantiles(quantiles, name = \"quantiles\", range = c(0, 1))"},{"path":"https://epiforecasts.io/scoringutils/reference/check_quantiles.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that quantiles are valid — check_quantiles","text":"quantiles Numeric vector quantiles check name Character name use error messages range Numeric vector giving allowed range","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_quantiles.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that quantiles are valid — check_quantiles","text":"None. Function errors quantiles invalid.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_summary_params.html","id":null,"dir":"Reference","previous_headings":"","what":"Check input parameters for summarise_scores() — check_summary_params","title":"Check input parameters for summarise_scores() — check_summary_params","text":"helper function check input parameters score().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_summary_params.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check input parameters for summarise_scores() — check_summary_params","text":"","code":"check_summary_params(scores, by, relative_skill, baseline, metric)"},{"path":"https://epiforecasts.io/scoringutils/reference/check_summary_params.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check input parameters for summarise_scores() — check_summary_params","text":"scores data.table scores produced score(). character vector column names summarise scores . Default NULL, meaning summary takes place summarising samples quantiles (case quantile-based forecasts), one score per forecast defined unit single forecast (rather one score every sample quantile). unit single forecast determined columns present input data correspond metric produced score(), indicate indicate grouping forecasts (example may one forecast per day, location model). Adding additional, unrelated, columns may alter results unpredictable way. relative_skill logical, whether compute relative performance models based pairwise comparisons. TRUE (default FALSE), column called 'model' must present input data. information computation relative skill, see pairwise_comparison(). Relative skill calculated aggregation level specified . baseline character string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. metric Deprecated 1.1.0. Use relative_skill_metric instead.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_true_values.html","id":null,"dir":"Reference","previous_headings":"","what":"Check Observed Value Input For Lower-level Scoring Functions — check_true_values","title":"Check Observed Value Input For Lower-level Scoring Functions — check_true_values","text":"Helper function check inputs lower-level score functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/check_true_values.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check Observed Value Input For Lower-level Scoring Functions — check_true_values","text":"","code":"check_true_values(true_values, type = c(\"continuous\", \"integer\", \"binary\"))"},{"path":"https://epiforecasts.io/scoringutils/reference/check_true_values.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check Observed Value Input For Lower-level Scoring Functions — check_true_values","text":"true_values vector true observed values size n type character, one \"continuous\" (default), \"integer\" \"binary\" defines type forecast","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/collapse_messages.html","id":null,"dir":"Reference","previous_headings":"","what":"Collapse several messages to one — collapse_messages","title":"Collapse several messages to one — collapse_messages","text":"Internal helper function facilitate generating messages warnings check_forecasts()","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/collapse_messages.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Collapse several messages to one — collapse_messages","text":"","code":"collapse_messages(type = \"messages\", messages)"},{"path":"https://epiforecasts.io/scoringutils/reference/collapse_messages.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Collapse several messages to one — collapse_messages","text":"type character, either \"messages\", \"warnings\" \"errors\" messages messages warnings collapse","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/collapse_messages.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Collapse several messages to one — collapse_messages","text":"string message warning","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/compare_two_models.html","id":null,"dir":"Reference","previous_headings":"","what":"Compare Two Models Based on Subset of Common Forecasts — compare_two_models","title":"Compare Two Models Based on Subset of Common Forecasts — compare_two_models","text":"function compares two models based subset forecasts models made prediction. gets called pairwise_comparison_one_group(), handles comparison multiple models single set forecasts (subsets forecasts distinguished). pairwise_comparison_one_group() turn gets called pairwise_comparison() can handle pairwise comparisons set forecasts multiple subsets, e.g. pairwise comparisons one set forecasts, done separately two different forecast targets.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/compare_two_models.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compare Two Models Based on Subset of Common Forecasts — compare_two_models","text":"","code":"compare_two_models(   scores,   name_model1,   name_model2,   metric,   one_sided = FALSE,   test_type = c(\"non_parametric\", \"permutation\"),   n_permutations = 999 )"},{"path":"https://epiforecasts.io/scoringutils/reference/compare_two_models.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compare Two Models Based on Subset of Common Forecasts — compare_two_models","text":"scores data.table scores produced score(). name_model1 character, name first model name_model2 character, name model compare metric character vector length one metric comparison . default \"auto\", meaning either \"interval_score\", \"crps\", \"brier_score\" selected available. See available_metrics() available metrics. one_sided Boolean, default FALSE, whether two conduct one-sided instead two-sided test determine significance pairwise comparison. test_type character, either \"non_parametric\" (default) \"permutation\". determines kind test shall conducted determine p-values. n_permutations numeric, number permutations permutation test. Default 999.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/compare_two_models.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compare Two Models Based on Subset of Common Forecasts — compare_two_models","text":"Johannes Bracher, johannes.bracher@kit.edu Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/correlation.html","id":null,"dir":"Reference","previous_headings":"","what":"Correlation Between Metrics — correlation","title":"Correlation Between Metrics — correlation","text":"Calculate correlation different metrics data.frame scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/correlation.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Correlation Between Metrics — correlation","text":"","code":"correlation(scores, metrics = NULL)"},{"path":"https://epiforecasts.io/scoringutils/reference/correlation.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Correlation Between Metrics — correlation","text":"scores data.table scores produced score(). metrics character vector metrics show. set NULL (default), metrics present scores shown","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/correlation.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Correlation Between Metrics — correlation","text":"data.table correlations different metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/correlation.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Correlation Between Metrics — correlation","text":"","code":"scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. correlation(scores) #> Warning: There is a column called 'quantile' in the scores. Usually, you should call 'summarise_scores()' to summarise over quantiles and obtain one score per forecast before calculating correlations. You can ignore this warning if you know what you're doing. #>    interval_score dispersion underprediction overprediction coverage #> 1:           1.00       0.41            0.31           0.93    -0.23 #> 2:           0.41       1.00            0.10           0.27    -0.05 #> 3:           0.31       0.10            1.00          -0.03    -0.23 #> 4:           0.93       0.27           -0.03           1.00    -0.16 #> 5:          -0.23      -0.05           -0.23          -0.16     1.00 #> 6:          -0.23      -0.08           -0.20          -0.17     0.78 #> 7:           0.10       0.10           -0.32           0.21     0.03 #> 8:           0.97       0.47            0.30           0.89    -0.21 #>    coverage_deviation  bias ae_median             metric #> 1:              -0.23  0.10      0.97     interval_score #> 2:              -0.08  0.10      0.47         dispersion #> 3:              -0.20 -0.32      0.30    underprediction #> 4:              -0.17  0.21      0.89     overprediction #> 5:               0.78  0.03     -0.21           coverage #> 6:               1.00  0.04     -0.26 coverage_deviation #> 7:               0.04  1.00      0.10               bias #> 8:              -0.26  0.10      1.00          ae_median"},{"path":"https://epiforecasts.io/scoringutils/reference/crps_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Ranked Probability Score — crps_sample","title":"Ranked Probability Score — crps_sample","text":"Wrapper around crps_sample() function scoringRules package. Can used continuous well integer valued forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/crps_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Ranked Probability Score — crps_sample","text":"","code":"crps_sample(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/crps_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Ranked Probability Score — crps_sample","text":"true_values vector true observed values size n predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/crps_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Ranked Probability Score — crps_sample","text":"vector scoring values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/crps_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Ranked Probability Score — crps_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/crps_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Ranked Probability Score — crps_sample","text":"","code":"true_values <- rpois(30, lambda = 1:30) predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) crps_sample(true_values, predictions) #>  [1] 0.506600 0.599175 0.532050 0.693600 0.898050 0.740175 0.695800 2.560775 #>  [9] 0.698925 1.744675 2.023825 6.090100 1.026500 0.795300 3.337050 1.323525 #> [17] 3.239475 1.667400 4.251350 1.313575 9.930200 4.993325 2.040375 2.070175 #> [25] 2.055775 1.302100 8.186025 1.244550 1.396025 3.978250"},{"path":"https://epiforecasts.io/scoringutils/reference/delete_columns.html","id":null,"dir":"Reference","previous_headings":"","what":"Delete Columns From a Data.table — delete_columns","title":"Delete Columns From a Data.table — delete_columns","text":"take vector column names delete columns present data.table","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/delete_columns.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Delete Columns From a Data.table — delete_columns","text":"","code":"delete_columns(df, cols_to_delete, make_unique = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/reference/delete_columns.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Delete Columns From a Data.table — delete_columns","text":"df data.table data.frame columns shall deleted cols_to_delete character vector names columns deleted make_unique whether make data set unique removing columns","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/delete_columns.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Delete Columns From a Data.table — delete_columns","text":"data.table","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/dss_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Dawid-Sebastiani Score — dss_sample","title":"Dawid-Sebastiani Score — dss_sample","text":"Wrapper around dss_sample() function scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/dss_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Dawid-Sebastiani Score — dss_sample","text":"","code":"dss_sample(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/dss_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Dawid-Sebastiani Score — dss_sample","text":"true_values vector true observed values size n predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/dss_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Dawid-Sebastiani Score — dss_sample","text":"vector scoring values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/dss_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Dawid-Sebastiani Score — dss_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/dss_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Dawid-Sebastiani Score — dss_sample","text":"","code":"true_values <- rpois(30, lambda = 1:30) predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) dss_sample(true_values, predictions) #>  [1] 0.7529455 1.1681697 1.4945488 1.4134274 3.6974837 4.6144495 4.0673072 #>  [8] 4.0583601 2.9199115 2.6286791 4.9543650 3.3044757 5.5078382 4.1154664 #> [15] 2.5331940 4.4050080 3.4696983 3.2481334 6.4119310 5.3720219 3.4350516 #> [22] 4.8295404 3.0192077 3.4335919 4.0517961 5.3975668 3.4011170 4.0102301 #> [29] 3.2499874 4.7312445"},{"path":"https://epiforecasts.io/scoringutils/reference/example_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Binary Forecast Example Data — example_binary","title":"Binary Forecast Example Data — example_binary","text":"data set binary predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Binary Forecast Example Data — example_binary","text":"","code":"example_binary"},{"path":"https://epiforecasts.io/scoringutils/reference/example_binary.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Binary Forecast Example Data — example_binary","text":"data frame 346 rows 10 columns: location country prediction made location_name name country prediction made target_end_date date prediction made target_type target predicted (cases deaths) true_value true observed values forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks prediction predicted value","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_binary.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Binary Forecast Example Data — example_binary","text":"https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/ # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Binary Forecast Example Data — example_binary","text":"Predictions data set constructed based continuous example data looking number samples mean prediction. outcome constructed whether actually observed value mean prediction. understood sound statistical practice, rather practical way create example data set. data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_continuous.html","id":null,"dir":"Reference","previous_headings":"","what":"Continuous Forecast Example Data — example_continuous","title":"Continuous Forecast Example Data — example_continuous","text":"data set continuous predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_continuous.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Continuous Forecast Example Data — example_continuous","text":"","code":"example_continuous"},{"path":"https://epiforecasts.io/scoringutils/reference/example_continuous.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Continuous Forecast Example Data — example_continuous","text":"data frame 13,429 rows 10 columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) true_value true observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks prediction predicted value sample id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_continuous.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Continuous Forecast Example Data — example_continuous","text":"https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/ # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_continuous.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Continuous Forecast Example Data — example_continuous","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_integer.html","id":null,"dir":"Reference","previous_headings":"","what":"Integer Forecast Example Data — example_integer","title":"Integer Forecast Example Data — example_integer","text":"data set integer predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_integer.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Integer Forecast Example Data — example_integer","text":"","code":"example_integer"},{"path":"https://epiforecasts.io/scoringutils/reference/example_integer.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Integer Forecast Example Data — example_integer","text":"data frame 13,429 rows 10 columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) true_value true observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks prediction predicted value sample id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_integer.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Integer Forecast Example Data — example_integer","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Point Forecast Example Data — example_point","title":"Point Forecast Example Data — example_point","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub. data set like quantile example data, median replaced point forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Point Forecast Example Data — example_point","text":"","code":"example_point"},{"path":"https://epiforecasts.io/scoringutils/reference/example_point.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Point Forecast Example Data — example_point","text":"data frame location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) true_value true observed values location_name name country prediction made forecast_date date prediction made quantile quantile corresponding prediction prediction predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_point.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Point Forecast Example Data — example_point","text":"https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/ # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_point.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Point Forecast Example Data — example_point","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile Example Data — example_quantile","title":"Quantile Example Data — example_quantile","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile Example Data — example_quantile","text":"","code":"example_quantile"},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Quantile Example Data — example_quantile","text":"data frame location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) true_value true observed values location_name name country prediction made forecast_date date prediction made quantile quantile corresponding prediction prediction predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Quantile Example Data — example_quantile","text":"https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/ # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Quantile Example Data — example_quantile","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile_forecasts_only.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile Example Data - Forecasts only — example_quantile_forecasts_only","title":"Quantile Example Data - Forecasts only — example_quantile_forecasts_only","text":"data set quantile predictions COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile_forecasts_only.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile Example Data - Forecasts only — example_quantile_forecasts_only","text":"","code":"example_quantile_forecasts_only"},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile_forecasts_only.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Quantile Example Data - Forecasts only — example_quantile_forecasts_only","text":"data frame 7,581 rows 9 columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) forecast_date date prediction made quantile quantile corresponding prediction prediction predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile_forecasts_only.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Quantile Example Data - Forecasts only — example_quantile_forecasts_only","text":"https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/ # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_quantile_forecasts_only.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Quantile Example Data - Forecasts only — example_quantile_forecasts_only","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_truth_only.html","id":null,"dir":"Reference","previous_headings":"","what":"Truth data only — example_truth_only","title":"Truth data only — example_truth_only","text":"data set truth values COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_truth_only.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Truth data only — example_truth_only","text":"","code":"example_truth_only"},{"path":"https://epiforecasts.io/scoringutils/reference/example_truth_only.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Truth data only — example_truth_only","text":"data frame 140 rows 5 columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) true_value true observed values location_name name country prediction made","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_truth_only.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Truth data only — example_truth_only","text":"https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/ # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/example_truth_only.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Truth data only — example_truth_only","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/find_duplicates.html","id":null,"dir":"Reference","previous_headings":"","what":"Find duplicate forecasts — find_duplicates","title":"Find duplicate forecasts — find_duplicates","text":"Helper function identify duplicate forecasts, .e. instances one forecast prediction target.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/find_duplicates.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Find duplicate forecasts — find_duplicates","text":"","code":"find_duplicates(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/reference/find_duplicates.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Find duplicate forecasts — find_duplicates","text":"data data.frame used score() forecast_unit character vector column names define unit single forecast. missing function tries infer unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/find_duplicates.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Find duplicate forecasts — find_duplicates","text":"data.frame rows duplicate forecast found","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/find_duplicates.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Find duplicate forecasts — find_duplicates","text":"","code":"example <- rbind(example_quantile, example_quantile[1000:1010]) find_duplicates(example) #>     location target_end_date target_type true_value location_name forecast_date #>  1:       DE      2021-05-22      Deaths       1285       Germany    2021-05-17 #>  2:       DE      2021-05-22      Deaths       1285       Germany    2021-05-17 #>  3:       DE      2021-05-22      Deaths       1285       Germany    2021-05-17 #>  4:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #>  5:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #>  6:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #>  7:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #>  8:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #>  9:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 10:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 11:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 12:       DE      2021-05-22      Deaths       1285       Germany    2021-05-17 #> 13:       DE      2021-05-22      Deaths       1285       Germany    2021-05-17 #> 14:       DE      2021-05-22      Deaths       1285       Germany    2021-05-17 #> 15:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 16:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 17:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 18:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 19:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 20:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 21:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #> 22:       DE      2021-05-29       Cases      31653       Germany    2021-05-10 #>     location target_end_date target_type true_value location_name forecast_date #>     quantile prediction                 model horizon #>  1:    0.950       1464  epiforecasts-EpiNow2       1 #>  2:    0.975       1642  epiforecasts-EpiNow2       1 #>  3:    0.990       1951  epiforecasts-EpiNow2       1 #>  4:    0.010      28999 EuroCOVIDhub-ensemble       3 #>  5:    0.025      32612 EuroCOVIDhub-ensemble       3 #>  6:    0.050      36068 EuroCOVIDhub-ensemble       3 #>  7:    0.100      41484 EuroCOVIDhub-ensemble       3 #>  8:    0.150      47110 EuroCOVIDhub-ensemble       3 #>  9:    0.200      50929 EuroCOVIDhub-ensemble       3 #> 10:    0.250      54561 EuroCOVIDhub-ensemble       3 #> 11:    0.300      57739 EuroCOVIDhub-ensemble       3 #> 12:    0.950       1464  epiforecasts-EpiNow2       1 #> 13:    0.975       1642  epiforecasts-EpiNow2       1 #> 14:    0.990       1951  epiforecasts-EpiNow2       1 #> 15:    0.010      28999 EuroCOVIDhub-ensemble       3 #> 16:    0.025      32612 EuroCOVIDhub-ensemble       3 #> 17:    0.050      36068 EuroCOVIDhub-ensemble       3 #> 18:    0.100      41484 EuroCOVIDhub-ensemble       3 #> 19:    0.150      47110 EuroCOVIDhub-ensemble       3 #> 20:    0.200      50929 EuroCOVIDhub-ensemble       3 #> 21:    0.250      54561 EuroCOVIDhub-ensemble       3 #> 22:    0.300      57739 EuroCOVIDhub-ensemble       3 #>     quantile prediction                 model horizon"},{"path":"https://epiforecasts.io/scoringutils/reference/geom_mean_helper.html","id":null,"dir":"Reference","previous_headings":"","what":"Calculate Geometric Mean — geom_mean_helper","title":"Calculate Geometric Mean — geom_mean_helper","text":"Calculate Geometric Mean","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/geom_mean_helper.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Calculate Geometric Mean — geom_mean_helper","text":"","code":"geom_mean_helper(x)"},{"path":"https://epiforecasts.io/scoringutils/reference/geom_mean_helper.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Calculate Geometric Mean — geom_mean_helper","text":"x numeric vector values calculate geometric mean","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/geom_mean_helper.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Calculate Geometric Mean — geom_mean_helper","text":"geometric mean values x","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Get unit of a single forecast — get_forecast_unit","title":"Get unit of a single forecast — get_forecast_unit","text":"Helper function get unit single forecast, .e. column names define single forecast made ","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get unit of a single forecast — get_forecast_unit","text":"","code":"get_forecast_unit(data)"},{"path":"https://epiforecasts.io/scoringutils/reference/get_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get unit of a single forecast — get_forecast_unit","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get unit of a single forecast — get_forecast_unit","text":"character vector column names define unit single forecast","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_prediction_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Get prediction type of a forecast — get_prediction_type","title":"Get prediction type of a forecast — get_prediction_type","text":"Internal helper function get prediction type forecast. inferred based properties values prediction column.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_prediction_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get prediction type of a forecast — get_prediction_type","text":"","code":"get_prediction_type(data)"},{"path":"https://epiforecasts.io/scoringutils/reference/get_prediction_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get prediction type of a forecast — get_prediction_type","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_prediction_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get prediction type of a forecast — get_prediction_type","text":"Character vector length one either \"quantile\", \"integer\", \"continuous\".","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_protected_columns.html","id":null,"dir":"Reference","previous_headings":"","what":"Get protected columns from a data frame — get_protected_columns","title":"Get protected columns from a data frame — get_protected_columns","text":"Helper function get names columns data frame protected columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_protected_columns.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get protected columns from a data frame — get_protected_columns","text":"","code":"get_protected_columns(data)"},{"path":"https://epiforecasts.io/scoringutils/reference/get_protected_columns.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get protected columns from a data frame — get_protected_columns","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_protected_columns.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get protected columns from a data frame — get_protected_columns","text":"character vector names protected columns data","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_target_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Get type of the target true values of a forecast — get_target_type","title":"Get type of the target true values of a forecast — get_target_type","text":"Internal helper function get type target true values forecast. inferred based columns present data.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_target_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get type of the target true values of a forecast — get_target_type","text":"","code":"get_target_type(data)"},{"path":"https://epiforecasts.io/scoringutils/reference/get_target_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get type of the target true values of a forecast — get_target_type","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/get_target_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get type of the target true values of a forecast — get_target_type","text":"Character vector length one either \"binary\", \"integer\", \"continuous\"","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/infer_rel_skill_metric.html","id":null,"dir":"Reference","previous_headings":"","what":"Infer metric for pairwise comparisons — infer_rel_skill_metric","title":"Infer metric for pairwise comparisons — infer_rel_skill_metric","text":"Helper function infer metric pairwise comparisons shall made. function simply checks names available columns chooses widely used metric.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/infer_rel_skill_metric.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Infer metric for pairwise comparisons — infer_rel_skill_metric","text":"","code":"infer_rel_skill_metric(scores)"},{"path":"https://epiforecasts.io/scoringutils/reference/infer_rel_skill_metric.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Infer metric for pairwise comparisons — infer_rel_skill_metric","text":"scores data.table scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/interval_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval Score — interval_score","title":"Interval Score — interval_score","text":"Proper Scoring Rule score quantile predictions, following Gneiting Raftery (2007). Smaller values better. score computed $$ \\textrm{score} = (\\textrm{upper} - \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{lower}  - \\textrm{true\\_value}) * \\mathbf{1}(\\textrm{true\\_value} < \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{true\\_value} - \\textrm{upper}) * \\mathbf{1}(\\textrm{true\\_value} > \\textrm{upper}) $$ \\(\\mathbf{1}()\\) indicator function indicates much outside prediction interval. \\(\\alpha\\) decimal value indicates much outside prediction interval. improve usability, user asked provide interval range percentage terms, .e. interval_range = 90 (percent) 90 percent prediction interval. Correspondingly, user provide 5% 95% quantiles (corresponding alpha 0.1). specific distribution assumed, range symmetric (.e use 0.1 quantile lower bound 0.7 quantile upper). Non-symmetric quantiles can scored using function quantile_score().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/interval_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval Score — interval_score","text":"","code":"interval_score(   true_values,   lower,   upper,   interval_range,   weigh = TRUE,   separate_results = FALSE )"},{"path":"https://epiforecasts.io/scoringutils/reference/interval_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval Score — interval_score","text":"true_values vector true observed values size n lower vector size n prediction lower quantile given range upper vector size n prediction upper quantile given range interval_range range prediction intervals. .e. forecasting 0.05 0.95 quantile, interval_range 90. Can either single number vector size n, range changes different forecasts scored. corresponds (100-alpha)/100 Gneiting Raftery (2007). Internally, range transformed alpha. weigh TRUE, weigh score alpha / 2, can averaged interval score , limit, corresponds CRPS. Alpha decimal value  represents much outside central prediction interval (e.g. 90 percent central prediction interval, alpha 0.1) Default: TRUE. separate_results TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/interval_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval Score — interval_score","text":"vector scoring values, list separate entries separate_results TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/interval_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Interval Score — interval_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/interval_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval Score — interval_score","text":"","code":"true_values <- rnorm(30, mean = 1:30) interval_range <- rep(90, 30) alpha <- (100 - interval_range) / 100 lower <- qnorm(alpha / 2, rnorm(30, mean = 1:30)) upper <- qnorm((1 - alpha / 2), rnorm(30, mean = 1:30))  interval_score(   true_values = true_values,   lower = lower,   upper = upper,   interval_range = interval_range ) #>  [1] 0.1822568 0.1970791 0.2332590 0.1173598 0.2386262 0.1792888 0.1862348 #>  [8] 0.7123322 0.2117145 0.1722823 0.4906667 0.1277423 0.2021887 0.2265155 #> [15] 0.1313495 0.1780059 0.1348840 0.1519159 0.1521136 0.1632998 1.5697556 #> [22] 0.2974954 0.1727772 0.1965859 0.3131081 0.2503699 0.9567075 0.2227665 #> [29] 1.0525656 0.2037366  # gives a warning, as the interval_range should likely be 50 instead of 0.5 interval_score(true_value = 4, upper = 2, lower = 8, interval_range = 0.5) #> Warning: Found interval ranges between 0 and 1. Are you sure that's right? An interval range of 0.5 e.g. implies a (49.75%, 50.25%) prediction interval. If you want to score a (25%, 75%) prediction interval, set interval_range = 50. #> This warning is displayed once per session. #> [1] 3.015  # example with missing values and separate results interval_score(   true_values = c(true_values, NA),   lower = c(lower, NA),   upper = c(NA, upper),   separate_results = TRUE,   interval_range = 90 ) #> $interval_score #>  [1]         NA 0.13799207 0.18007754 0.05725046 0.18893065 0.14852296 #>  [7] 0.16965832 1.05394105 0.33253174 0.11388981 0.43369921 0.10145401 #> [13] 0.12607438 0.12456986 0.18931956 0.08312870 0.15224190 0.93500917 #> [19] 0.18240163 0.34947402 1.54103261 0.13225473 0.19374337 0.76565650 #> [25] 0.18129546 0.30883740 0.13999485 1.40240250 1.03164032 0.13596727 #> [31]         NA #>  #> $dispersion #>  [1]          NA  0.13799207  0.18007754  0.03178123  0.18893065  0.14852296 #>  [7]  0.16965832  0.07206667  0.12086215  0.11388981  0.06023895  0.10145401 #> [13]  0.12607438  0.12456986  0.18931956  0.08312870  0.15224190  0.01614648 #> [19]  0.18240163  0.08154878 -0.05042919  0.13225473  0.19374337  0.10115010 #> [25]  0.18129546  0.30883740  0.13999485 -0.07381066  0.11466466  0.13596727 #> [31]          NA #>  #> $underprediction #>  [1]         NA 0.00000000 0.00000000 0.02546923 0.00000000 0.00000000 #>  [7] 0.00000000 0.98187438 0.21166959 0.00000000 0.00000000 0.00000000 #> [13] 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.91886269 #> [19] 0.00000000 0.26792524 0.00000000 0.00000000 0.00000000 0.66450640 #> [25] 0.00000000 0.00000000 0.00000000 1.43564123 0.00000000 0.00000000 #> [31]         NA #>  #> $overprediction #>  [1] 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 #>  [7] 0.00000000 0.00000000 0.00000000 0.00000000 0.37346026 0.00000000 #> [13] 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 #> [19] 0.00000000 0.00000000 1.59146180 0.00000000 0.00000000 0.00000000 #> [25] 0.00000000 0.00000000 0.00000000 0.04057193 0.91697566 0.00000000 #> [31]         NA #>"},{"path":"https://epiforecasts.io/scoringutils/reference/is_scoringutils_check.html","id":null,"dir":"Reference","previous_headings":"","what":"Check whether object has been checked with check_forecasts() — is_scoringutils_check","title":"Check whether object has been checked with check_forecasts() — is_scoringutils_check","text":"Helper function determine whether object checked passed check_forecasts().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/is_scoringutils_check.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check whether object has been checked with check_forecasts() — is_scoringutils_check","text":"","code":"is_scoringutils_check(data)"},{"path":"https://epiforecasts.io/scoringutils/reference/is_scoringutils_check.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check whether object has been checked with check_forecasts() — is_scoringutils_check","text":"data object class scoringutils_check() produced check_forecasts().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/is_scoringutils_check.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check whether object has been checked with check_forecasts() — is_scoringutils_check","text":"Logical, either TRUE FALSE","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/log_shift.html","id":null,"dir":"Reference","previous_headings":"","what":"Log transformation with an additive shift — log_shift","title":"Log transformation with an additive shift — log_shift","text":"Function shifts value offset applies natural logarithm .","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/log_shift.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Log transformation with an additive shift — log_shift","text":"","code":"log_shift(x, offset = 0, base = exp(1))"},{"path":"https://epiforecasts.io/scoringutils/reference/log_shift.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Log transformation with an additive shift — log_shift","text":"x vector input values transformed offset number add input value taking natural logarithm base positive complex number: base respect logarithms computed. Defaults e = exp(1).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/log_shift.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Log transformation with an additive shift — log_shift","text":"numeric vector transformed values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/log_shift.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Log transformation with an additive shift — log_shift","text":"output computed log(x + offset)","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/log_shift.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Log transformation with an additive shift — log_shift","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722  https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/log_shift.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Log transformation with an additive shift — log_shift","text":"","code":"log_shift(1:10) #>  [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #>  [8] 2.0794415 2.1972246 2.3025851 log_shift(0:9, offset = 1) #>  [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #>  [8] 2.0794415 2.1972246 2.3025851  transform_forecasts(   example_quantile[true_value > 0, ],   fun = log_shift,   offset = 1  ) #>        location target_end_date target_type   true_value location_name #>     1:       DE      2021-01-02       Cases 1.273000e+05       Germany #>     2:       DE      2021-01-02      Deaths 4.534000e+03       Germany #>     3:       DE      2021-01-09       Cases 1.549220e+05       Germany #>     4:       DE      2021-01-09      Deaths 6.117000e+03       Germany #>     5:       DE      2021-01-16       Cases 1.101830e+05       Germany #>    ---                                                                 #> 40672:       IT      2021-07-24      Deaths 4.369448e+00         Italy #> 40673:       IT      2021-07-24      Deaths 4.369448e+00         Italy #> 40674:       IT      2021-07-24      Deaths 4.369448e+00         Italy #> 40675:       IT      2021-07-24      Deaths 4.369448e+00         Italy #> 40676:       IT      2021-07-24      Deaths 4.369448e+00         Italy #>        forecast_date quantile prediction                model horizon   scale #>     1:          <NA>       NA         NA                 <NA>      NA natural #>     2:          <NA>       NA         NA                 <NA>      NA natural #>     3:          <NA>       NA         NA                 <NA>      NA natural #>     4:          <NA>       NA         NA                 <NA>      NA natural #>     5:          <NA>       NA         NA                 <NA>      NA natural #>    ---                                                                        #> 40672:    2021-07-12    0.850   5.866468 epiforecasts-EpiNow2       2     log #> 40673:    2021-07-12    0.900   5.986452 epiforecasts-EpiNow2       2     log #> 40674:    2021-07-12    0.950   6.214608 epiforecasts-EpiNow2       2     log #> 40675:    2021-07-12    0.975   6.416732 epiforecasts-EpiNow2       2     log #> 40676:    2021-07-12    0.990   6.579251 epiforecasts-EpiNow2       2     log"},{"path":"https://epiforecasts.io/scoringutils/reference/logs_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Log Score for Binary outcomes — logs_binary","title":"Log Score for Binary outcomes — logs_binary","text":"Computes Log Score probabilistic forecasts binary outcomes.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/logs_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Log Score for Binary outcomes — logs_binary","text":"","code":"logs_binary(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/logs_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Log Score for Binary outcomes — logs_binary","text":"true_values vector true observed values size n values equal either 0 1 predictions vector predicted probability true_value = 1.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/logs_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Log Score for Binary outcomes — logs_binary","text":"numeric value Log Score, .e. mean squared error given probability forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/logs_binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Log Score for Binary outcomes — logs_binary","text":"Log Score proper score rule suited assessing accuracy probabilistic binary predictions. outcomes can either 0 1, predictions must probability true outcome 1. Log Score computed negative logarithm probability assigned true outcome. Reporting negative logarithm means smaller values better.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/logs_binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Log Score for Binary outcomes — logs_binary","text":"","code":"true_values <- sample(c(0, 1), size = 30, replace = TRUE) predictions <- runif(n = 30, min = 0, max = 1) logs_binary(true_values, predictions) #>  [1] 1.20906030 0.02033411 0.05340961 0.76117242 0.17654643 0.71277523 #>  [7] 0.28237863 0.63359548 0.37188147 2.42112348 0.51437877 0.67967567 #> [13] 0.34696851 0.08778954 0.30022236 0.68583582 0.74310165 0.47061757 #> [19] 0.31336920 0.05072400 0.02918284 0.40280624 1.50419322 0.30777101 #> [25] 0.95504696 1.46372405 2.76565118 0.95719771 1.52262728 1.69986720"},{"path":"https://epiforecasts.io/scoringutils/reference/logs_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Logarithmic score — logs_sample","title":"Logarithmic score — logs_sample","text":"Wrapper around logs_sample() function scoringRules package. Used score continuous predictions. Log Score theory also applicable integer forecasts, problem lies implementation: Log Score needs kernel density estimation, well defined integer-valued Monte Carlo Samples. Log Score can used specific integer valued probability distributions. See scoringRules package details.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/logs_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Logarithmic score — logs_sample","text":"","code":"logs_sample(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/logs_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Logarithmic score — logs_sample","text":"true_values vector true observed values size n predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/logs_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Logarithmic score — logs_sample","text":"vector scoring values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/logs_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Logarithmic score — logs_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/logs_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Logarithmic score — logs_sample","text":"","code":"true_values <- rpois(30, lambda = 1:30) predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) logs_sample(true_values, predictions) #>  [1] 0.6171748 2.0233417 1.8573057 2.3973257 1.7390020 1.8890290 1.9355473 #>  [8] 1.9645747 3.8215958 3.0070751 2.5518583 2.6304564 2.5590551 2.9976456 #> [15] 2.6427894 2.8578589 2.2871401 2.4836108 2.7957332 3.5362551 3.1394826 #> [22] 2.6714621 3.2088482 3.6395443 3.1061173 2.6878009 2.5935118 3.3132310 #> [29] 2.6611293 3.8780586"},{"path":"https://epiforecasts.io/scoringutils/reference/mad_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determine dispersion of a probabilistic forecast — mad_sample","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Determine dispersion probabilistic forecast","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/mad_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"mad_sample(predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/mad_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/mad_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"vector dispersion values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/mad_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Sharpness ability model generate predictions within narrow range dispersion lack thereof. data-independent measure, purely feature forecasts . Dispersion predictive samples corresponding one single true value measured normalised median absolute deviation median predictive samples. details, see mad() explanations given Funk et al. (2019)","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/mad_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, Edmunds WJ (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLoS Comput Biol 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/mad_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) mad_sample(predictions) #>  [1] 1.4826 1.4826 1.4826 1.4826 1.4826 1.4826 2.9652 2.9652 2.9652 2.9652 #> [11] 2.9652 2.9652 2.9652 2.9652 4.4478 4.4478 4.4478 2.9652 4.4478 4.4478 #> [21] 4.4478 4.4478 4.4478 5.9304 4.4478 4.4478 4.4478 5.9304 4.4478 5.9304"},{"path":"https://epiforecasts.io/scoringutils/reference/make_NA.html","id":null,"dir":"Reference","previous_headings":"","what":"Make Rows NA in Data for Plotting — make_NA","title":"Make Rows NA in Data for Plotting — make_NA","text":"Filters data turns values NA data gets passed plot_predictions(). reason allows 'filter' prediction truth data separately. value NA removed subsequent call plot_predictions().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/make_NA.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Make Rows NA in Data for Plotting — make_NA","text":"","code":"make_NA(data = NULL, what = c(\"truth\", \"forecast\", \"both\"), ...)  make_na(data = NULL, what = c(\"truth\", \"forecast\", \"both\"), ...)"},{"path":"https://epiforecasts.io/scoringutils/reference/make_NA.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Make Rows NA in Data for Plotting — make_NA","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary). character vector determines values turned NA. = \"truth\", values column 'true_value' turned NA. = \"forecast\", values column 'prediction' turned NA. = \"\", values column turned NA. ... logical statements used filter data","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/make_NA.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Make Rows NA in Data for Plotting — make_NA","text":"data.table","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/make_NA.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Make Rows NA in Data for Plotting — make_NA","text":"","code":"make_NA (     example_continuous,     what = \"truth\",     target_end_date >= \"2021-07-22\",     target_end_date < \"2021-05-01\"   ) #>        location location_name target_end_date target_type forecast_date #>     1:       DE       Germany      2021-01-02       Cases          <NA> #>     2:       DE       Germany      2021-01-02      Deaths          <NA> #>     3:       DE       Germany      2021-01-09       Cases          <NA> #>     4:       DE       Germany      2021-01-09      Deaths          <NA> #>     5:       DE       Germany      2021-01-16       Cases          <NA> #>    ---                                                                  #> 35620:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 35621:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 35622:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 35623:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 35624:       IT         Italy      2021-07-24      Deaths    2021-07-12 #>                       model horizon prediction sample true_value #>     1:                 <NA>      NA         NA     NA         NA #>     2:                 <NA>      NA         NA     NA         NA #>     3:                 <NA>      NA         NA     NA         NA #>     4:                 <NA>      NA         NA     NA         NA #>     5:                 <NA>      NA         NA     NA         NA #>    ---                                                           #> 35620: epiforecasts-EpiNow2       2  159.84534     36         NA #> 35621: epiforecasts-EpiNow2       2  128.21214     37         NA #> 35622: epiforecasts-EpiNow2       2  190.52560     38         NA #> 35623: epiforecasts-EpiNow2       2  141.06659     39         NA #> 35624: epiforecasts-EpiNow2       2   24.43419     40         NA"},{"path":"https://epiforecasts.io/scoringutils/reference/merge_pred_and_obs.html","id":null,"dir":"Reference","previous_headings":"","what":"Merge Forecast Data And Observations — merge_pred_and_obs","title":"Merge Forecast Data And Observations — merge_pred_and_obs","text":"function less provides wrapper around merge aims handle merging well additional columns present one data sets. doubt, probably merge data sets manually.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/merge_pred_and_obs.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Merge Forecast Data And Observations — merge_pred_and_obs","text":"","code":"merge_pred_and_obs(   forecasts,   observations,   join = c(\"left\", \"full\", \"right\"),   by = NULL )"},{"path":"https://epiforecasts.io/scoringutils/reference/merge_pred_and_obs.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Merge Forecast Data And Observations — merge_pred_and_obs","text":"forecasts data.frame forecast data (can passed score()). observations data.frame observations join character, one c(\"left\", \"full\", \"right\"). Determines type join. Usually, left join appropriate, sometimes may want full join keep dates forecast, ground truth data. character vector denotes columns merge. value column observations removed.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/merge_pred_and_obs.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Merge Forecast Data And Observations — merge_pred_and_obs","text":"data.frame forecasts observations","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/merge_pred_and_obs.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Merge Forecast Data And Observations — merge_pred_and_obs","text":"","code":"forecasts <- example_quantile_forecasts_only observations <- example_truth_only merge_pred_and_obs(forecasts, observations) #>        location target_end_date target_type true_value location_name #>     1:       DE      2021-01-02       Cases     127300       Germany #>     2:       DE      2021-01-02      Deaths       4534       Germany #>     3:       DE      2021-01-09       Cases     154922       Germany #>     4:       DE      2021-01-09      Deaths       6117       Germany #>     5:       DE      2021-01-16       Cases     110183       Germany #>    ---                                                               #> 20541:       IT      2021-07-24      Deaths         78         Italy #> 20542:       IT      2021-07-24      Deaths         78         Italy #> 20543:       IT      2021-07-24      Deaths         78         Italy #> 20544:       IT      2021-07-24      Deaths         78         Italy #> 20545:       IT      2021-07-24      Deaths         78         Italy #>        forecast_date quantile prediction                model horizon #>     1:          <NA>       NA         NA                 <NA>      NA #>     2:          <NA>       NA         NA                 <NA>      NA #>     3:          <NA>       NA         NA                 <NA>      NA #>     4:          <NA>       NA         NA                 <NA>      NA #>     5:          <NA>       NA         NA                 <NA>      NA #>    ---                                                                #> 20541:    2021-07-12    0.850        352 epiforecasts-EpiNow2       2 #> 20542:    2021-07-12    0.900        397 epiforecasts-EpiNow2       2 #> 20543:    2021-07-12    0.950        499 epiforecasts-EpiNow2       2 #> 20544:    2021-07-12    0.975        611 epiforecasts-EpiNow2       2 #> 20545:    2021-07-12    0.990        719 epiforecasts-EpiNow2       2"},{"path":"https://epiforecasts.io/scoringutils/reference/metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Summary information for selected metrics — metrics","title":"Summary information for selected metrics — metrics","text":"data set summary information selected metrics implemented scoringutils","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Summary information for selected metrics — metrics","text":"","code":"metrics"},{"path":"https://epiforecasts.io/scoringutils/reference/metrics.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Summary information for selected metrics — metrics","text":"object class data.table (inherits data.frame) 22 rows 8 columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/metrics.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Summary information for selected metrics — metrics","text":"data created using script create-metric-tables.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison.html","id":null,"dir":"Reference","previous_headings":"","what":"Do Pairwise Comparisons of Scores — pairwise_comparison","title":"Do Pairwise Comparisons of Scores — pairwise_comparison","text":"Compute relative scores different models making pairwise comparisons. Pairwise comparisons sort pairwise tournament combinations two models compared based overlapping set available forecasts common models. Internally, ratio mean scores models computed. relative score model geometric mean mean score ratios involve model. baseline provided, baseline excluded relative scores individual models (therefore differ slightly relative scores without baseline) relative scores scaled (.e. divided ) relative score baseline model. Usually, function input unsummarised scores produced score(). Note function internally infers unit single forecast determining columns input correspond metrics computed score(). Adding unrelated columns change results unpredictable way. code pairwise comparisons inspired implementation Johannes Bracher. implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Do Pairwise Comparisons of Scores — pairwise_comparison","text":"","code":"pairwise_comparison(   scores,   by = \"model\",   metric = \"auto\",   baseline = NULL,   ... )"},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Do Pairwise Comparisons of Scores — pairwise_comparison","text":"scores data.table scores produced score(). character vector names columns present input data.frame. determines pairwise comparisons computed. get relative skill score every grouping level determined . , example, = c(\"model\", \"location\"). get separate relative skill score every model every location. Internally, data.frame split according (removing \"model\" splitting) pairwise comparisons computed separately split data.frames. metric character vector length one metric comparison . default \"auto\", meaning either \"interval_score\", \"crps\", \"brier_score\" selected available. See available_metrics() available metrics. baseline character vector length one denotes baseline model compare models. ... additional arguments comparison two models. See compare_two_models() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Do Pairwise Comparisons of Scores — pairwise_comparison","text":"ggplot2 object coloured table summarised scores","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Do Pairwise Comparisons of Scores — pairwise_comparison","text":"Nikos Bosse nikosbosse@gmail.com Johannes Bracher, johannes.bracher@kit.edu","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Do Pairwise Comparisons of Scores — pairwise_comparison","text":"","code":"data.table::setDTthreads(1) # only needed to avoid issues on CRAN  scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. pairwise <- pairwise_comparison(scores, by = \"target_type\")  library(ggplot2) plot_pairwise_comparison(pairwise, type = \"mean_scores_ratio\") +   facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison_one_group.html","id":null,"dir":"Reference","previous_headings":"","what":"Do Pairwise Comparison for one Set of Forecasts — pairwise_comparison_one_group","title":"Do Pairwise Comparison for one Set of Forecasts — pairwise_comparison_one_group","text":"function pairwise comparison one set forecasts, multiple models involved. gets called pairwise_comparison(). pairwise_comparison() splits data arbitrary subgroups specified user (e.g. pairwise comparison done separately different forecast targets) actual pairwise comparison subgroup managed pairwise_comparison_one_group(). order actually comparison two models subset common forecasts calls compare_two_models().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison_one_group.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Do Pairwise Comparison for one Set of Forecasts — pairwise_comparison_one_group","text":"","code":"pairwise_comparison_one_group(scores, metric, baseline, by, ...)"},{"path":"https://epiforecasts.io/scoringutils/reference/pairwise_comparison_one_group.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Do Pairwise Comparison for one Set of Forecasts — pairwise_comparison_one_group","text":"scores data.table scores produced score(). metric character vector length one metric comparison . default \"auto\", meaning either \"interval_score\", \"crps\", \"brier_score\" selected available. See available_metrics() available metrics. baseline character vector length one denotes baseline model compare models. character vector names columns present input data.frame. determines pairwise comparisons computed. get relative skill score every grouping level determined . , example, = c(\"model\", \"location\"). get separate relative skill score every model every location. Internally, data.frame split according (removing \"model\" splitting) pairwise comparisons computed separately split data.frames. ... additional arguments comparison two models. See compare_two_models() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/permutation_test.html","id":null,"dir":"Reference","previous_headings":"","what":"Simple permutation test — permutation_test","title":"Simple permutation test — permutation_test","text":"#' implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/permutation_test.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Simple permutation test — permutation_test","text":"","code":"permutation_test(   scores1,   scores2,   n_permutation = 999,   one_sided = FALSE,   comparison_mode = c(\"difference\", \"ratio\") )"},{"path":"https://epiforecasts.io/scoringutils/reference/permutation_test.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Simple permutation test — permutation_test","text":"p-value permutation test","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability Integral Transformation (data.frame Format) — pit","title":"Probability Integral Transformation (data.frame Format) — pit","text":"Wrapper around pit() use data.frames","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability Integral Transformation (data.frame Format) — pit","text":"","code":"pit(data, by, n_replicates = 100)"},{"path":"https://epiforecasts.io/scoringutils/reference/pit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability Integral Transformation (data.frame Format) — pit","text":"data data.frame following columns: true_value, prediction, sample. Character vector columns according PIT values shall grouped. e.g. columns 'model' 'location' data want PIT histogram every model location, specify = c(\"model\", \"location\"). n_replicates number draws randomised PIT integer predictions.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability Integral Transformation (data.frame Format) — pit","text":"data.table PIT values according grouping specified ","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Probability Integral Transformation (data.frame Format) — pit","text":"see pit()","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability Integral Transformation (data.frame Format) — pit","text":"Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability Integral Transformation (data.frame Format) — pit","text":"","code":"result <- pit(example_continuous, by = \"model\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. plot_pit(result)   # example with quantile data result <- pit(example_quantile, by = \"model\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. plot_pit(result)"},{"path":"https://epiforecasts.io/scoringutils/reference/pit_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability Integral Transformation (sample-based version) — pit_sample","title":"Probability Integral Transformation (sample-based version) — pit_sample","text":"Uses Probability Integral Transformation (PIT) (randomised PIT integer forecasts) assess calibration predictive Monte Carlo samples. Returns p-values resulting Anderson-Darling test uniformity (randomised) PIT well PIT histogram specified.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability Integral Transformation (sample-based version) — pit_sample","text":"","code":"pit_sample(true_values, predictions, n_replicates = 100)"},{"path":"https://epiforecasts.io/scoringutils/reference/pit_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability Integral Transformation (sample-based version) — pit_sample","text":"true_values vector true observed values size n predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n. n_replicates number draws randomised PIT integer predictions.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability Integral Transformation (sample-based version) — pit_sample","text":"vector PIT-values. continuous forecasts, vector correspond length true_values. integer forecasts, randomised PIT returned length length(true_values) * n_replicates","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Probability Integral Transformation (sample-based version) — pit_sample","text":"Calibration reliability forecasts ability model correctly identify uncertainty making predictions. model perfect calibration, observed data time point look came predictive probability distribution time. Equivalently, one can inspect probability integral transform predictive distribution time t, $$ u_t = F_t (x_t) $$ \\(x_t\\) observed data point time \\(t \\textrm{ } t_1, …, t_n\\), n number forecasts, \\(F_t\\) (continuous) predictive cumulative probability distribution time t. true probability distribution outcomes time t \\(G_t\\) forecasts \\(F_t\\) said ideal \\(F_t = G_t\\) times t. case, probabilities \\(u_t\\) distributed uniformly. case discrete outcomes incidence counts, PIT longer uniform even forecasts ideal. case randomised PIT can used instead: $$ u_t = P_t(k_t) + v * (P_t(k_t) - P_t(k_t - 1) ) $$ \\(k_t\\) observed count, \\(P_t(x)\\) predictive cumulative probability observing incidence k time t, \\(P_t (-1) = 0\\) definition v standard uniform independent k. \\(P_t\\) true cumulative probability distribution, \\(u_t\\) standard uniform. function checks whether integer continuous forecasts provided. applies (randomised) probability integral tests values \\(u_t\\) uniformity using Anderson-Darling test. rule thumb, evidence suggest forecasting model miscalibrated p-value found greater threshold p >= 0.1, evidence miscalibrated 0.01 < p < 0.1, good evidence miscalibrated p <= 0.01. However, AD-p-values may overly strict actual usefulness may questionable. context noted, though, uniformity PIT necessary sufficient condition calibration.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/pit_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability Integral Transformation (sample-based version) — pit_sample","text":"Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/reference/pit_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability Integral Transformation (sample-based version) — pit_sample","text":"","code":"data.table::setDTthreads(1) # only needed to avoid issues on CRAN  ## continuous predictions true_values <- rnorm(20, mean = 1:20) predictions <- replicate(100, rnorm(n = 20, mean = 1:20)) pit <- pit_sample(true_values, predictions) plot_pit(pit)   ## integer predictions true_values <- rpois(50, lambda = 1:50) predictions <- replicate(2000, rpois(n = 50, lambda = 1:50)) pit <- pit_sample(true_values, predictions, n_replicates = 30) plot_pit(pit)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_avail_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Visualise Where Forecasts Are Available — plot_avail_forecasts","title":"Visualise Where Forecasts Are Available — plot_avail_forecasts","text":"Visualise Forecasts Available","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_avail_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Visualise Where Forecasts Are Available — plot_avail_forecasts","text":"","code":"plot_avail_forecasts(   avail_forecasts,   y = \"model\",   x = \"forecast_date\",   make_x_factor = TRUE,   show_numbers = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_avail_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Visualise Where Forecasts Are Available — plot_avail_forecasts","text":"avail_forecasts data.frame column called Number forecasts produced avail_forecasts() y character vector length one denotes name column appear y-axis plot. Default \"model\". x character vector length one denotes name column appear x-axis plot. Default \"forecast_date\". make_x_factor logical (default TRUE). Whether convert variable x-axis factor. effect e.g. dates shown x-axis. show_numbers logical (default TRUE) indicates whether show actual count numbers plot","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_avail_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Visualise Where Forecasts Are Available — plot_avail_forecasts","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_avail_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Visualise Where Forecasts Are Available — plot_avail_forecasts","text":"","code":"library(ggplot2) avail_forecasts <- avail_forecasts(   example_quantile, by = c(\"model\", \"target_type\", \"target_end_date\") ) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. plot_avail_forecasts(  avail_forecasts, x = \"target_end_date\", show_numbers = FALSE ) +  facet_wrap(\"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_correlation.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot Correlation Between Metrics — plot_correlation","title":"Plot Correlation Between Metrics — plot_correlation","text":"Plots heatmap correlations different metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_correlation.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot Correlation Between Metrics — plot_correlation","text":"","code":"plot_correlation(correlations)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_correlation.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot Correlation Between Metrics — plot_correlation","text":"correlations data.table correlations scores produced correlation().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_correlation.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot Correlation Between Metrics — plot_correlation","text":"ggplot2 object showing coloured matrix correlations metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_correlation.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot Correlation Between Metrics — plot_correlation","text":"","code":"scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. correlations <- correlation(  summarise_scores(scores) ) plot_correlation(correlations)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_heatmap.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a Heatmap of a Scoring Metric — plot_heatmap","title":"Create a Heatmap of a Scoring Metric — plot_heatmap","text":"function can used create heatmap one metric across different groups, e.g. interval score obtained several forecasting models different locations.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_heatmap.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a Heatmap of a Scoring Metric — plot_heatmap","text":"","code":"plot_heatmap(scores, y = \"model\", x, metric)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_heatmap.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a Heatmap of a Scoring Metric — plot_heatmap","text":"scores data.frame scores based quantile forecasts produced score(). y variable scores want show y-Axis. default \"model\" x variable scores want show x-Axis. something like \"horizon\", \"location\" metric metric determines value colour shown tiles heatmap","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_heatmap.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a Heatmap of a Scoring Metric — plot_heatmap","text":"ggplot2 object showing heatmap desired metric","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_heatmap.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create a Heatmap of a Scoring Metric — plot_heatmap","text":"","code":"scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\", \"range\"))  plot_heatmap(scores, x = \"target_type\", metric = \"bias\")"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_interval_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot Interval Coverage — plot_interval_coverage","title":"Plot Interval Coverage — plot_interval_coverage","text":"Plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_interval_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot Interval Coverage — plot_interval_coverage","text":"","code":"plot_interval_coverage(scores, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_interval_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot Interval Coverage — plot_interval_coverage","text":"scores data.frame scores based quantile forecasts produced score() summarise_scores(). Note \"range\" must included argument running summarise_scores() colour According variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_interval_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot Interval Coverage — plot_interval_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_interval_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot Interval Coverage — plot_interval_coverage","text":"","code":"data.table::setDTthreads(1) # only needed to avoid issues on CRAN scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"range\")) plot_interval_coverage(scores)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pairwise_comparison.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot Heatmap of Pairwise Comparisons — plot_pairwise_comparison","title":"Plot Heatmap of Pairwise Comparisons — plot_pairwise_comparison","text":"Creates heatmap ratios pvalues pairwise comparison models","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pairwise_comparison.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot Heatmap of Pairwise Comparisons — plot_pairwise_comparison","text":"","code":"plot_pairwise_comparison(   comparison_result,   type = c(\"mean_scores_ratio\", \"pval\") )"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pairwise_comparison.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot Heatmap of Pairwise Comparisons — plot_pairwise_comparison","text":"comparison_result data.frame produced pairwise_comparison() type character vector length one either \"mean_scores_ratio\" \"pval\". denotes whether visualise ratio p-value pairwise comparison. Default \"mean_scores_ratio\".","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pairwise_comparison.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot Heatmap of Pairwise Comparisons — plot_pairwise_comparison","text":"","code":"library(ggplot2) scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. pairwise <- pairwise_comparison(scores, by = \"target_type\") plot_pairwise_comparison(pairwise, type = \"mean_scores_ratio\") +   facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pit.html","id":null,"dir":"Reference","previous_headings":"","what":"PIT Histogram — plot_pit","title":"PIT Histogram — plot_pit","text":"Make simple histogram probability integral transformed values visually check whether uniform distribution seems likely.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"PIT Histogram — plot_pit","text":"","code":"plot_pit(pit, num_bins = \"auto\", breaks = NULL)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"PIT Histogram — plot_pit","text":"pit either vector PIT values size n, data.frame produced pit() num_bins number bins PIT histogram, default \"auto\". num_bins == \"auto\", plot_pit() either display 10 bins, display bin available quantile case passed data quantile-based format. can control number bins supplying number. fine sample-based pit histograms, may fail quantile-based formats. case preferred supply explicit breaks points using breaks argument. breaks numeric vector break points bins PIT histogram. preferred creating PIT histogram based quantile-based data. Default NULL breaks determined num_bins.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"PIT Histogram — plot_pit","text":"vector scoring values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_pit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"PIT Histogram — plot_pit","text":"","code":"data.table::setDTthreads(1) # only needed to avoid issues on CRAN  # PIT histogram in vector based format true_values <- rnorm(30, mean = 1:30) predictions <- replicate(200, rnorm(n = 30, mean = 1:30)) pit <- pit_sample(true_values, predictions) plot_pit(pit)   # quantile-based pit pit <- pit(example_quantile,by = \"model\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. plot_pit(pit, breaks = seq(0.1, 1, 0.1))   # sample-based pit pit <- pit(example_integer,by = \"model\") #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. plot_pit(pit)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_predictions.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot Predictions vs True Values — plot_predictions","title":"Plot Predictions vs True Values — plot_predictions","text":"Make plot observed predicted values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_predictions.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot Predictions vs True Values — plot_predictions","text":"","code":"plot_predictions(data, by = NULL, x = \"date\", range = c(0, 50, 90))"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_predictions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot Predictions vs True Values — plot_predictions","text":"data data.frame follows specifications outlined score(). customise plotting, can filter data using function make_NA(). character vector column names denote categories plot stratified. example want facetted plot, character vector columns used facetting (note facetting still needs done outside function call) x character vector length one denotes name variable range numeric vector indicating interval ranges plot. 0 included range, median prediction shown.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_predictions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot Predictions vs True Values — plot_predictions","text":"ggplot object plot true vs predicted values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_predictions.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot Predictions vs True Values — plot_predictions","text":"","code":"library(ggplot2) library(magrittr)  example_continuous %>%   make_NA (     what = \"truth\",     target_end_date >= \"2021-07-22\",     target_end_date < \"2021-05-01\"   ) %>%   make_NA (     what = \"forecast\",     model != \"EuroCOVIDhub-ensemble\",     forecast_date != \"2021-06-07\"   ) %>%   plot_predictions (     x = \"target_end_date\",     by = c(\"target_type\", \"location\"),     range = c(0, 50, 90, 95)   ) +   facet_wrap(~ location + target_type, scales = \"free_y\") +   aes(fill = model, color = model)   example_continuous %>%   make_NA (     what = \"truth\",     target_end_date >= \"2021-07-22\",     target_end_date < \"2021-05-01\"   ) %>%   make_NA (     what = \"forecast\",     forecast_date != \"2021-06-07\"   ) %>%   plot_predictions (     x = \"target_end_date\",     by = c(\"target_type\", \"location\"),     range = c(0)   ) +   facet_wrap(~ location + target_type, scales = \"free_y\") +   aes(fill = model, color = model)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_quantile_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot Quantile Coverage — plot_quantile_coverage","title":"Plot Quantile Coverage — plot_quantile_coverage","text":"Plot quantile coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_quantile_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot Quantile Coverage — plot_quantile_coverage","text":"","code":"plot_quantile_coverage(scores, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_quantile_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot Quantile Coverage — plot_quantile_coverage","text":"scores data.frame scores based quantile forecasts produced score() summarise_scores(). Note \"range\" must included argument running summarise_scores() colour According variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_quantile_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot Quantile Coverage — plot_quantile_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_quantile_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot Quantile Coverage — plot_quantile_coverage","text":"","code":"scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"quantile\")) plot_quantile_coverage(scores)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_ranges.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot Metrics by Range of the Prediction Interval — plot_ranges","title":"Plot Metrics by Range of the Prediction Interval — plot_ranges","text":"Visualise metrics range, e.g. interested different interval ranges contribute overall interval score, sharpness / dispersion changes range.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_ranges.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot Metrics by Range of the Prediction Interval — plot_ranges","text":"","code":"plot_ranges(scores, y = \"interval_score\", x = \"model\", colour = \"range\")"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_ranges.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot Metrics by Range of the Prediction Interval — plot_ranges","text":"scores data.frame scores based quantile forecasts produced score() summarise_scores(). Note \"range\" must included argument running summarise_scores() y variable scores want show y-Axis. something like \"interval_score\" (default) \"dispersion\" x variable scores want show x-Axis. Usually \"model\" colour Character vector length one used determine variable colouring dots. Default \"range\".","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_ranges.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot Metrics by Range of the Prediction Interval — plot_ranges","text":"ggplot2 object showing contributions three components weighted interval score","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_ranges.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot Metrics by Range of the Prediction Interval — plot_ranges","text":"","code":"library(ggplot2) scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\", \"range\"))  plot_ranges(scores, x = \"model\") +   facet_wrap(~target_type, scales = \"free\")   # visualise dispersion instead of interval score plot_ranges(scores, y = \"dispersion\", x = \"model\") +   facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_score_table.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot Coloured Score Table — plot_score_table","title":"Plot Coloured Score Table — plot_score_table","text":"Plots coloured table summarised scores obtained using score().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_score_table.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot Coloured Score Table — plot_score_table","text":"","code":"plot_score_table(scores, y = \"model\", by = NULL, metrics = NULL)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_score_table.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot Coloured Score Table — plot_score_table","text":"scores data.table scores produced score(). y variable shown y-axis. Instead single character string, can also specify vector column names, e.g. y = c(\"model\", \"location\"). column names concatenated create unique row identifier (e.g. \"model1_location1\"). character vector determines colour shading plot gets computed. default (NULL), shading determined per metric, can provide additional column names (see examples). metrics character vector metrics show. set NULL (default), metrics present scores shown.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_score_table.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot Coloured Score Table — plot_score_table","text":"ggplot2 object coloured table summarised scores","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_score_table.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot Coloured Score Table — plot_score_table","text":"","code":"library(ggplot2) library(magrittr) # pipe operator data.table::setDTthreads(1) # only needed to avoid issues on CRAN  scores <- score(example_quantile) %>%   summarise_scores(by = c(\"model\", \"target_type\")) %>%   summarise_scores(fun = signif, digits = 2) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.  plot_score_table(scores, y = \"model\", by = \"target_type\") +   facet_wrap(~target_type, ncol = 1)   # can also put target description on the y-axis plot_score_table(scores,                  y = c(\"model\", \"target_type\"),                  by = \"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_wis.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot Contributions to the Weighted Interval Score — plot_wis","title":"Plot Contributions to the Weighted Interval Score — plot_wis","text":"Visualise components weighted interval score: penalties -prediction, -prediction high dispersion (lack sharpness)","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_wis.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot Contributions to the Weighted Interval Score — plot_wis","text":"","code":"plot_wis(scores, x = \"model\", relative_contributions = FALSE, flip = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/reference/plot_wis.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot Contributions to the Weighted Interval Score — plot_wis","text":"scores data.frame scores based quantile forecasts produced score() summarised using summarise_scores() x variable scores want show x-Axis. Usually \"model\". relative_contributions show relative contributions instead absolute contributions. Default FALSE functionality available yet. flip boolean (default FALSE), whether flip axes.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_wis.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot Contributions to the Weighted Interval Score — plot_wis","text":"ggplot2 object showing contributions three components weighted interval score","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_wis.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Plot Contributions to the Weighted Interval Score — plot_wis","text":"Bracher J, Ray E, Gneiting T, Reich, N (2020) Evaluating epidemic forecasts interval format. https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/plot_wis.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot Contributions to the Weighted Interval Score — plot_wis","text":"","code":"library(ggplot2) scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\"))  plot_wis(scores,   x = \"model\",   relative_contributions = TRUE ) +   facet_wrap(~target_type)  plot_wis(scores,   x = \"model\",   relative_contributions = FALSE ) +   facet_wrap(~target_type, scales = \"free_x\")"},{"path":"https://epiforecasts.io/scoringutils/reference/prediction_is_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Check if predictions are quantile forecasts — prediction_is_quantile","title":"Check if predictions are quantile forecasts — prediction_is_quantile","text":"Internal helper function check data frame contains quantile forecast predictions. determined checking \"quantile\" column present.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/prediction_is_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check if predictions are quantile forecasts — prediction_is_quantile","text":"","code":"prediction_is_quantile(data)"},{"path":"https://epiforecasts.io/scoringutils/reference/prediction_is_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check if predictions are quantile forecasts — prediction_is_quantile","text":"data Data frame containing forecast predictions","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/prediction_is_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check if predictions are quantile forecasts — prediction_is_quantile","text":"Logical indicating whether predictions quantile forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/print.scoringutils_check.html","id":null,"dir":"Reference","previous_headings":"","what":"Print output from check_forecasts() — print.scoringutils_check","title":"Print output from check_forecasts() — print.scoringutils_check","text":"Helper function prints output generated check_forecasts()","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/print.scoringutils_check.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Print output from check_forecasts() — print.scoringutils_check","text":"","code":"# S3 method for scoringutils_check print(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/reference/print.scoringutils_check.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Print output from check_forecasts() — print.scoringutils_check","text":"x object class 'scoringutils_check' produced check_forecasts() ... additional arguments (used )","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/print.scoringutils_check.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Print output from check_forecasts() — print.scoringutils_check","text":"","code":"check <- check_forecasts(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. print(check) #> Your forecasts seem to be for a target of the following type: #> $target_type #> [1] \"integer\" #>  #> and in the following format: #> $prediction_type #> [1] \"quantile\" #>  #> The unit of a single forecast is defined by: #> $forecast_unit #> [1] \"location\"        \"target_end_date\" \"target_type\"     \"location_name\"   #> [5] \"forecast_date\"   \"model\"           \"horizon\"         #>  #> Cleaned data, rows with NA values in prediction or true_value removed: #> $cleaned_data #>        location target_end_date target_type true_value location_name #>     1:       DE      2021-05-08       Cases     106987       Germany #>     2:       DE      2021-05-08       Cases     106987       Germany #>     3:       DE      2021-05-08       Cases     106987       Germany #>     4:       DE      2021-05-08       Cases     106987       Germany #>     5:       DE      2021-05-08       Cases     106987       Germany #>    ---                                                               #> 20397:       IT      2021-07-24      Deaths         78         Italy #> 20398:       IT      2021-07-24      Deaths         78         Italy #> 20399:       IT      2021-07-24      Deaths         78         Italy #> 20400:       IT      2021-07-24      Deaths         78         Italy #> 20401:       IT      2021-07-24      Deaths         78         Italy #>        forecast_date quantile prediction                 model horizon #>     1:    2021-05-03    0.010      82466 EuroCOVIDhub-ensemble       1 #>     2:    2021-05-03    0.025      86669 EuroCOVIDhub-ensemble       1 #>     3:    2021-05-03    0.050      90285 EuroCOVIDhub-ensemble       1 #>     4:    2021-05-03    0.100      95341 EuroCOVIDhub-ensemble       1 #>     5:    2021-05-03    0.150      99171 EuroCOVIDhub-ensemble       1 #>    ---                                                                 #> 20397:    2021-07-12    0.850        352  epiforecasts-EpiNow2       2 #> 20398:    2021-07-12    0.900        397  epiforecasts-EpiNow2       2 #> 20399:    2021-07-12    0.950        499  epiforecasts-EpiNow2       2 #> 20400:    2021-07-12    0.975        611  epiforecasts-EpiNow2       2 #> 20401:    2021-07-12    0.990        719  epiforecasts-EpiNow2       2 #>  #> Number of unique values per column per model: #> $unique_values #>                    model location target_end_date target_type true_value #> 1: EuroCOVIDhub-ensemble        4              12           2         96 #> 2: EuroCOVIDhub-baseline        4              12           2         96 #> 3:  epiforecasts-EpiNow2        4              12           2         95 #> 4:       UMass-MechBayes        4              12           1         48 #>    location_name forecast_date quantile prediction horizon #> 1:             4            11       23       3969       3 #> 2:             4            11       23       3733       3 #> 3:             4            11       23       3903       3 #> 4:             4            11       23       1058       3 #>  #> $messages #> [1] \"144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.\" #>"},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile Score — quantile_score","title":"Quantile Score — quantile_score","text":"Proper Scoring Rule score quantile predictions. Smaller values better. quantile score closely related Interval score (see interval_score()) quantile equivalent works single quantiles instead central prediction intervals.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile Score — quantile_score","text":"","code":"quantile_score(true_values, predictions, quantiles, weigh = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Quantile Score — quantile_score","text":"true_values vector true observed values size n predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n. quantiles vector size n quantile values corresponding predictions. weigh TRUE, weigh score alpha / 2, can averaged interval score , limit, corresponds CRPS. Alpha value corresponds (alpha/2) (1 - alpha/2) quantiles provided computed quantile. Alpha decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1). Default: TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Quantile Score — quantile_score","text":"vector scoring values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Quantile Score — quantile_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Quantile Score — quantile_score","text":"","code":"true_values <- rnorm(10, mean = 1:10) alpha <- 0.5  lower <- qnorm(alpha / 2, rnorm(10, mean = 1:10)) upper <- qnorm((1 - alpha / 2), rnorm(10, mean = 1:10))  qs_lower <- quantile_score(true_values,   predictions = lower,   quantiles = alpha / 2 ) qs_upper <- quantile_score(true_values,   predictions = upper,   quantiles = 1 - alpha / 2 ) interval_score <- (qs_lower + qs_upper) / 2"},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_to_range_long.html","id":null,"dir":"Reference","previous_headings":"","what":"Change Data from a Plain Quantile Format to a Long Range Format — quantile_to_range_long","title":"Change Data from a Plain Quantile Format to a Long Range Format — quantile_to_range_long","text":"Transform data format uses quantiles one uses interval ranges denote quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_to_range_long.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Change Data from a Plain Quantile Format to a Long Range Format — quantile_to_range_long","text":"","code":"quantile_to_range_long(data, keep_quantile_col = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_to_range_long.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Change Data from a Plain Quantile Format to a Long Range Format — quantile_to_range_long","text":"data data.frame quantile format keep_quantile_col keep quantile column final output transformation (default FALSE)","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/quantile_to_range_long.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Change Data from a Plain Quantile Format to a Long Range Format — quantile_to_range_long","text":"data.frame long interval range format","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/range_long_to_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Change Data from a Range Format to a Quantile Format — range_long_to_quantile","title":"Change Data from a Range Format to a Quantile Format — range_long_to_quantile","text":"Transform data format uses interval ranges denote quantiles format uses quantiles .","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/range_long_to_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Change Data from a Range Format to a Quantile Format — range_long_to_quantile","text":"","code":"range_long_to_quantile(data, keep_range_col = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/reference/range_long_to_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Change Data from a Range Format to a Quantile Format — range_long_to_quantile","text":"data data.frame following specifications score()) quantile forecasts. keep_range_col keep range boundary columns transformation (default FALSE)","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/range_long_to_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Change Data from a Range Format to a Quantile Format — range_long_to_quantile","text":"data.frame plain quantile format","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Change Data from a Sample Based Format to a Quantile Format — sample_to_quantile","title":"Change Data from a Sample Based Format to a Quantile Format — sample_to_quantile","text":"Transform data format based predictive samples format based plain quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Change Data from a Sample Based Format to a Quantile Format — sample_to_quantile","text":"","code":"sample_to_quantile(data, quantiles = c(0.05, 0.25, 0.5, 0.75, 0.95), type = 7)"},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Change Data from a Sample Based Format to a Quantile Format — sample_to_quantile","text":"data data.frame samples quantiles numeric vector quantiles extract type type argument passed quantile function. information, see quantile()","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Change Data from a Sample Based Format to a Quantile Format — sample_to_quantile","text":"data.frame long interval range format","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Change Data from a Sample Based Format to a Quantile Format — sample_to_quantile","text":"","code":"sample_to_quantile(example_integer) #>       location location_name target_end_date target_type forecast_date #>    1:       DE       Germany      2021-01-02       Cases          <NA> #>    2:       DE       Germany      2021-01-02       Cases          <NA> #>    3:       DE       Germany      2021-01-02       Cases          <NA> #>    4:       DE       Germany      2021-01-02       Cases          <NA> #>    5:       DE       Germany      2021-01-02       Cases          <NA> #>   ---                                                                  #> 5151:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 5152:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 5153:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 5154:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 5155:       IT         Italy      2021-07-24      Deaths    2021-07-12 #>                      model horizon true_value quantile prediction #>    1:                 <NA>      NA     127300     0.05         NA #>    2:                 <NA>      NA     127300     0.25         NA #>    3:                 <NA>      NA     127300     0.50         NA #>    4:                 <NA>      NA     127300     0.75         NA #>    5:                 <NA>      NA     127300     0.95         NA #>   ---                                                             #> 5151: epiforecasts-EpiNow2       2         78     0.05      44.90 #> 5152: epiforecasts-EpiNow2       2         78     0.25      89.25 #> 5153: epiforecasts-EpiNow2       2         78     0.50     151.50 #> 5154: epiforecasts-EpiNow2       2         78     0.75     208.00 #> 5155: epiforecasts-EpiNow2       2         78     0.95     469.10"},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_range_long.html","id":null,"dir":"Reference","previous_headings":"","what":"Change Data from a Sample Based Format to a Long Interval Range Format — sample_to_range_long","title":"Change Data from a Sample Based Format to a Long Interval Range Format — sample_to_range_long","text":"Transform data format based predictive samples format based interval ranges","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_range_long.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Change Data from a Sample Based Format to a Long Interval Range Format — sample_to_range_long","text":"","code":"sample_to_range_long(   data,   range = c(0, 50, 90),   type = 7,   keep_quantile_col = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_range_long.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Change Data from a Sample Based Format to a Long Interval Range Format — sample_to_range_long","text":"data data.frame samples range numeric vector interval ranges extract (e.g. c(0, 50, 90)) type type argument passed quantile function. information, see quantile() keep_quantile_col keep quantile column, default TRUE","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/sample_to_range_long.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Change Data from a Sample Based Format to a Long Interval Range Format — sample_to_range_long","text":"data.frame long interval range format","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score.html","id":null,"dir":"Reference","previous_headings":"","what":"Evaluate forecasts — score","title":"Evaluate forecasts — score","text":"function allows automatic scoring forecasts using range metrics. users workhorse scoring forecasts wraps lower level functions package functions. However, functions also available wish make use independently. range forecasts formats supported, including quantile-based, sample-based, binary forecasts. Prior scoring, users may wish make use check_forecasts() ensure input data supported format though also run internally score(). Examples format also provided (see documentation data check_forecasts()). format set required columns (see ). Additional columns may present indicate grouping forecasts. example, forecasts made different models various locations different time points, several weeks future. important, columns present relevant order group forecasts. combination different columns uniquely define unit single forecast, meaning single forecast defined values columns. Adding additional unrelated columns may alter results. obtain quick overview currently supported evaluation metrics, look metrics data included package. column metrics$Name gives overview available metric names can computed. interested unsupported metric please open feature request consider contributing pull request. additional help examples, check Getting Started Vignette well paper Evaluating Forecasts scoringutils R.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Evaluate forecasts — score","text":"","code":"score(data, metrics = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/reference/score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Evaluate forecasts — score","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary). metrics metrics want output. NULL (default), available metrics computed. list available metrics see available_metrics(),  check metrics data set. ... additional parameters passed score_quantile() (internal function used scoring forecasts quantile-based format).","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Evaluate forecasts — score","text":"data.table unsummarised scores. one score per quantile sample, usually desired, almost always run summarise_scores() unsummarised scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Evaluate forecasts — score","text":"Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, Edmunds WJ (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLoS Comput Biol 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Evaluate forecasts — score","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Evaluate forecasts — score","text":"","code":"library(magrittr) # pipe operator data.table::setDTthreads(1) # only needed to avoid issues on CRAN  check_forecasts(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #> Your forecasts seem to be for a target of the following type: #> $target_type #> [1] \"integer\" #>  #> and in the following format: #> $prediction_type #> [1] \"quantile\" #>  #> The unit of a single forecast is defined by: #> $forecast_unit #> [1] \"location\"        \"target_end_date\" \"target_type\"     \"location_name\"   #> [5] \"forecast_date\"   \"model\"           \"horizon\"         #>  #> Cleaned data, rows with NA values in prediction or true_value removed: #> $cleaned_data #>        location target_end_date target_type true_value location_name #>     1:       DE      2021-05-08       Cases     106987       Germany #>     2:       DE      2021-05-08       Cases     106987       Germany #>     3:       DE      2021-05-08       Cases     106987       Germany #>     4:       DE      2021-05-08       Cases     106987       Germany #>     5:       DE      2021-05-08       Cases     106987       Germany #>    ---                                                               #> 20397:       IT      2021-07-24      Deaths         78         Italy #> 20398:       IT      2021-07-24      Deaths         78         Italy #> 20399:       IT      2021-07-24      Deaths         78         Italy #> 20400:       IT      2021-07-24      Deaths         78         Italy #> 20401:       IT      2021-07-24      Deaths         78         Italy #>        forecast_date quantile prediction                 model horizon #>     1:    2021-05-03    0.010      82466 EuroCOVIDhub-ensemble       1 #>     2:    2021-05-03    0.025      86669 EuroCOVIDhub-ensemble       1 #>     3:    2021-05-03    0.050      90285 EuroCOVIDhub-ensemble       1 #>     4:    2021-05-03    0.100      95341 EuroCOVIDhub-ensemble       1 #>     5:    2021-05-03    0.150      99171 EuroCOVIDhub-ensemble       1 #>    ---                                                                 #> 20397:    2021-07-12    0.850        352  epiforecasts-EpiNow2       2 #> 20398:    2021-07-12    0.900        397  epiforecasts-EpiNow2       2 #> 20399:    2021-07-12    0.950        499  epiforecasts-EpiNow2       2 #> 20400:    2021-07-12    0.975        611  epiforecasts-EpiNow2       2 #> 20401:    2021-07-12    0.990        719  epiforecasts-EpiNow2       2 #>  #> Number of unique values per column per model: #> $unique_values #>                    model location target_end_date target_type true_value #> 1: EuroCOVIDhub-ensemble        4              12           2         96 #> 2: EuroCOVIDhub-baseline        4              12           2         96 #> 3:  epiforecasts-EpiNow2        4              12           2         95 #> 4:       UMass-MechBayes        4              12           1         48 #>    location_name forecast_date quantile prediction horizon #> 1:             4            11       23       3969       3 #> 2:             4            11       23       3733       3 #> 3:             4            11       23       3903       3 #> 4:             4            11       23       1058       3 #>  #> $messages #> [1] \"144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected.\" #>  score(example_quantile) %>%   add_coverage(by = c(\"model\", \"target_type\")) %>%   summarise_scores(by = c(\"model\", \"target_type\")) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model target_type interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline       Cases    28483.57465 4102.50094    10284.972826 #> 2: EuroCOVIDhub-baseline      Deaths      159.40387   91.40625        2.098505 #> 3: EuroCOVIDhub-ensemble       Cases    17943.82383 3663.52458     4237.177310 #> 4: EuroCOVIDhub-ensemble      Deaths       41.42249   30.18099        4.103261 #> 5:       UMass-MechBayes      Deaths       52.65195   26.87239       16.800951 #> 6:  epiforecasts-EpiNow2       Cases    20831.55662 5664.37795     3260.355639 #> 7:  epiforecasts-EpiNow2      Deaths       66.64282   31.85692       15.893314 #>    overprediction coverage_deviation        bias   ae_median coverage_50 #> 1:   14096.100883        -0.11211957  0.09796875 38473.60156   0.3281250 #> 2:      65.899117         0.11614130  0.33906250   233.25781   0.6640625 #> 3:   10043.121943        -0.09785326 -0.05640625 24101.07031   0.3906250 #> 4:       7.138247         0.19528533  0.07265625    53.13281   0.8750000 #> 5:       8.978601        -0.02312500 -0.02234375    78.47656   0.4609375 #> 6:   11906.823030        -0.06660326 -0.07890625 27923.81250   0.4687500 #> 7:      18.892583        -0.04287176 -0.00512605   104.74790   0.4201681 #>    coverage_90 #> 1:   0.8203125 #> 2:   1.0000000 #> 3:   0.8046875 #> 4:   1.0000000 #> 5:   0.8750000 #> 6:   0.7890625 #> 7:   0.9075630  # set forecast unit manually (to avoid issues with scoringutils trying to # determine the forecast unit automatically), check forecasts before scoring example_quantile %>%   set_forecast_unit(     c(\"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\")   ) %>%   check_forecasts() %>%   score() #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>        location target_end_date target_type horizon                 model range #>     1:       DE      2021-05-08       Cases       1 EuroCOVIDhub-baseline     0 #>     2:       DE      2021-05-08       Cases       1 EuroCOVIDhub-baseline    10 #>     3:       DE      2021-05-08       Cases       1 EuroCOVIDhub-baseline    10 #>     4:       DE      2021-05-08       Cases       1 EuroCOVIDhub-baseline    20 #>     5:       DE      2021-05-08       Cases       1 EuroCOVIDhub-baseline    20 #>    ---                                                                          #> 20397:       IT      2021-07-24      Deaths       3  epiforecasts-EpiNow2    90 #> 20398:       IT      2021-07-24      Deaths       3  epiforecasts-EpiNow2    95 #> 20399:       IT      2021-07-24      Deaths       3  epiforecasts-EpiNow2    95 #> 20400:       IT      2021-07-24      Deaths       3  epiforecasts-EpiNow2    98 #> 20401:       IT      2021-07-24      Deaths       3  epiforecasts-EpiNow2    98 #>        interval_score dispersion underprediction overprediction coverage #>     1:       25620.00       0.00               0          25620        0 #>     2:       25599.50     184.50               0          25415        0 #>     3:       25599.50     184.50               0          25415        0 #>     4:       25481.00     556.00               0          24925        0 #>     5:       25481.00     556.00               0          24925        0 #>    ---                                                                   #> 20397:          15.85      15.85               0              0        1 #> 20398:          11.75      11.75               0              0        1 #> 20399:          11.75      11.75               0              0        1 #> 20400:           6.52       6.52               0              0        1 #> 20401:           6.52       6.52               0              0        1 #>        coverage_deviation bias quantile ae_median quantile_coverage #>     1:               0.00 0.95    0.500     25620              TRUE #>     2:              -0.10 0.95    0.450     25620              TRUE #>     3:              -0.10 0.95    0.550     25620              TRUE #>     4:              -0.20 0.95    0.400     25620              TRUE #>     5:              -0.20 0.95    0.600     25620              TRUE #>    ---                                                              #> 20397:               0.10 0.50    0.950        26              TRUE #> 20398:               0.05 0.50    0.025        26             FALSE #> 20399:               0.05 0.50    0.975        26              TRUE #> 20400:               0.02 0.50    0.010        26             FALSE #> 20401:               0.02 0.50    0.990        26              TRUE  # forecast formats with different metrics if (FALSE) { score(example_binary) score(example_quantile) score(example_integer) score(example_continuous) }  # score point forecasts (marked by 'NA' in the quantile column) score(example_point) %>%   summarise_scores(by = \"model\", na.rm = TRUE) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #> Median not available, computing as mean of two innermost quantiles in order to compute bias. #>                    model interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline     14092.7647 2192.26967      4996.92898 #> 2: EuroCOVIDhub-ensemble      8852.4196 1930.80064      2005.56357 #> 3:  epiforecasts-EpiNow2     10659.5125 3084.85850      1604.96135 #> 4:       UMass-MechBayes        51.4781   28.09387        15.32315 #>    overprediction coverage_deviation        bias    ae_point     se_point #> 1:     6903.56605        0.002102273  0.21851562 19353.42969 2.883446e+09 #> 2:     4916.05540        0.050752841  0.00812500 12077.10156 1.945118e+09 #> 3:     5969.69268       -0.057861612 -0.04376518 14521.10526 2.680928e+09 #> 4:        8.06108       -0.024886364 -0.02312500    78.47656 1.170976e+04 #>    ae_median #> 1:       NaN #> 2:       NaN #> 3:       NaN #> 4:       NaN"},{"path":"https://epiforecasts.io/scoringutils/reference/score_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Evaluate forecasts in a Binary Format — score_binary","title":"Evaluate forecasts in a Binary Format — score_binary","text":"Evaluate forecasts Binary Format","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Evaluate forecasts in a Binary Format — score_binary","text":"","code":"score_binary(data, forecast_unit, metrics)"},{"path":"https://epiforecasts.io/scoringutils/reference/score_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Evaluate forecasts in a Binary Format — score_binary","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary). forecast_unit character vector column names define unit single forecast, .e. forecast made combination values forecast_unit. metrics metrics want output. NULL (default), available metrics computed. list available metrics see available_metrics(),  check metrics data set.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Evaluate forecasts in a Binary Format — score_binary","text":"data.table appropriate scores. information see score().","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_binary.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Evaluate forecasts in a Binary Format — score_binary","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Evaluate forecasts in a Quantile-Based Format — score_quantile","title":"Evaluate forecasts in a Quantile-Based Format — score_quantile","text":"Evaluate forecasts Quantile-Based Format","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Evaluate forecasts in a Quantile-Based Format — score_quantile","text":"","code":"score_quantile(   data,   forecast_unit,   metrics,   weigh = TRUE,   count_median_twice = FALSE,   separate_results = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/reference/score_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Evaluate forecasts in a Quantile-Based Format — score_quantile","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary). forecast_unit character vector column names define unit single forecast, .e. forecast made combination values forecast_unit metrics metrics want output. NULL (default), available metrics computed. list available metrics see available_metrics(),  check metrics data set. weigh TRUE, weigh score alpha / 2, can averaged interval score , limit, corresponds CRPS. Alpha decimal value  represents much outside central prediction interval (e.g. 90 percent central prediction interval, alpha 0.1) Default: TRUE. count_median_twice logical controls whether count median twice summarising (default FALSE). Counting median twice conceptually treat 0\\ median lower well upper bound. alternative treat median single quantile forecast instead interval. interval score better understood average quantile scores. separate_results TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Evaluate forecasts in a Quantile-Based Format — score_quantile","text":"data.table appropriate scores. information see score()","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_quantile.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Evaluate forecasts in a Quantile-Based Format — score_quantile","text":"Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, Edmunds WJ (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLoS Comput Biol 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_quantile.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Evaluate forecasts in a Quantile-Based Format — score_quantile","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Evaluate forecasts in a Sample-Based Format (Integer or Continuous) — score_sample","title":"Evaluate forecasts in a Sample-Based Format (Integer or Continuous) — score_sample","text":"Evaluate forecasts Sample-Based Format (Integer Continuous)","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Evaluate forecasts in a Sample-Based Format (Integer or Continuous) — score_sample","text":"","code":"score_sample(data, forecast_unit, metrics, prediction_type)"},{"path":"https://epiforecasts.io/scoringutils/reference/score_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Evaluate forecasts in a Sample-Based Format (Integer or Continuous) — score_sample","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary). forecast_unit character vector column names define unit single forecast, .e. forecast made combination values forecast_unit metrics metrics want output. NULL (default), available metrics computed. list available metrics see available_metrics(),  check metrics data set. prediction_type character, either \"continuous\" \"integer\"","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Evaluate forecasts in a Sample-Based Format (Integer or Continuous) — score_sample","text":"data.table appropriate scores. information see score()","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Evaluate forecasts in a Sample-Based Format (Integer or Continuous) — score_sample","text":"Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, Edmunds WJ (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLoS Comput Biol 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/score_sample.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Evaluate forecasts in a Sample-Based Format (Integer or Continuous) — score_sample","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/scoringutils-package.html","id":null,"dir":"Reference","previous_headings":"","what":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","title":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","text":"Provides collection metrics proper scoring rules (Tilmann Gneiting & Adrian E Raftery (2007) doi:10.1198/016214506000001437 , Jordan, ., Krüger, F., & Lerch, S. (2019) doi:10.18637/jss.v090.i12 ) within consistent framework evaluation, comparison visualisation forecasts. addition proper scoring rules, functions provided assess bias, sharpness calibration (Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) doi:10.1371/journal.pcbi.1006785 ) forecasts. Several types predictions (e.g. binary, discrete, continuous) may come different formats (e.g. forecasts represented predictive samples quantiles predictive distribution) can evaluated. Scoring metrics can used either convenient data.frame format, can applied individual functions vector / matrix format. functionality implemented focus performance robustly tested. Find information scoringutils accompanying paper (Bosse et al., 2022) arXiv:2205.07090v1. Provides collection metrics proper scoring rules (Tilmann Gneiting & Adrian E Raftery (2007) doi:10.1198/016214506000001437 , Jordan, ., Krüger, F., & Lerch, S. (2019) doi:10.18637/jss.v090.i12 ) within consistent framework evaluation, comparison visualisation forecasts. addition proper scoring rules, functions provided assess bias, sharpness calibration (Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) doi:10.1371/journal.pcbi.1006785 ) forecasts. Several types predictions (e.g. binary, discrete, continuous) may come different formats (e.g. forecasts represented predictive samples quantiles predictive distribution) can evaluated. Scoring metrics can used either convenient data.frame format, can applied individual functions vector / matrix format. functionality implemented focus performance robustly tested. Find information scoringutils accompanying paper (Bosse et al., 2022) arXiv:2205.07090v1.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/reference/scoringutils-package.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","text":"Maintainer: Nikos Bosse nikosbosse@gmail.com (ORCID) Authors: Sam Abbott contact@samabbott.co.uk (ORCID) Hugo Gruson hugo.gruson@lshtm.ac.uk (ORCID) Sebastian Funk sebastian.funk@lshtm.ac.uk contributors: Johannes Bracher johannes.bracher@kit.edu (ORCID) [contributor]","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/se_mean_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Squared Error of the Mean (Sample-based Version) — se_mean_sample","title":"Squared Error of the Mean (Sample-based Version) — se_mean_sample","text":"Squared error mean calculated $$   \\textrm{mean}(\\textrm{true\\_value} - \\textrm{prediction})^2 $$","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/se_mean_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Squared Error of the Mean (Sample-based Version) — se_mean_sample","text":"","code":"se_mean_sample(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/se_mean_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Squared Error of the Mean (Sample-based Version) — se_mean_sample","text":"true_values vector true observed values size n predictions nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predictions can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/se_mean_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Squared Error of the Mean (Sample-based Version) — se_mean_sample","text":"vector scoring values","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/reference/se_mean_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Squared Error of the Mean (Sample-based Version) — se_mean_sample","text":"","code":"true_values <- rnorm(30, mean = 1:30) predicted_values <- rnorm(30, mean = 1:30) se_mean_sample(true_values, predicted_values) #>  [1] 1.5369388190 0.7926418467 0.0064358942 8.0875579457 0.6223434196 #>  [6] 5.1973894672 0.3780707728 1.8730178893 4.6789601717 0.1668502145 #> [11] 0.0270798542 0.5391113037 0.8762152838 0.3347385895 5.3358123896 #> [16] 2.6739807804 2.2938975472 2.6748938964 7.3483374250 0.2775697450 #> [21] 3.3677055015 2.1256807972 0.5024278826 0.0082441616 0.9248873844 #> [26] 0.5373266162 0.0382242675 0.0239365969 0.0003035308 0.0570072019"},{"path":"https://epiforecasts.io/scoringutils/reference/set_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Set unit of a single forecast manually — set_forecast_unit","title":"Set unit of a single forecast manually — set_forecast_unit","text":"Helper function set unit single forecast (.e. combination columns uniquely define single forecast) manually. simple function keeps columns specified forecast_unit (plus additional protected columns, e.g. true values, predictions quantile levels) removes duplicate rows. done manually, scoringutils attempts determine unit single forecast automatically simply assuming column names relevant determine forecast unit. may lead unexpected behaviour, setting forecast unit explicitly can help make code easier debug easier read. used part workflow, set_forecast_unit() can directly piped check_forecasts() check everything order.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/set_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Set unit of a single forecast manually — set_forecast_unit","text":"","code":"set_forecast_unit(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/reference/set_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Set unit of a single forecast manually — set_forecast_unit","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary). forecast_unit Character vector names columns uniquely identify single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/set_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Set unit of a single forecast manually — set_forecast_unit","text":"data.table columns kept relevant scoring denote unit single forecast specified user.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/set_forecast_unit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Set unit of a single forecast manually — set_forecast_unit","text":"","code":"set_forecast_unit(   example_quantile,   c(\"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\") ) #>        true_value quantile prediction location target_end_date target_type #>     1:     127300       NA         NA       DE      2021-01-02       Cases #>     2:       4534       NA         NA       DE      2021-01-02      Deaths #>     3:     154922       NA         NA       DE      2021-01-09       Cases #>     4:       6117       NA         NA       DE      2021-01-09      Deaths #>     5:     110183       NA         NA       DE      2021-01-16       Cases #>    ---                                                                     #> 20541:         78    0.850        352       IT      2021-07-24      Deaths #> 20542:         78    0.900        397       IT      2021-07-24      Deaths #> 20543:         78    0.950        499       IT      2021-07-24      Deaths #> 20544:         78    0.975        611       IT      2021-07-24      Deaths #> 20545:         78    0.990        719       IT      2021-07-24      Deaths #>        horizon                model #>     1:      NA                 <NA> #>     2:      NA                 <NA> #>     3:      NA                 <NA> #>     4:      NA                 <NA> #>     5:      NA                 <NA> #>    ---                              #> 20541:       2 epiforecasts-EpiNow2 #> 20542:       2 epiforecasts-EpiNow2 #> 20543:       2 epiforecasts-EpiNow2 #> 20544:       2 epiforecasts-EpiNow2 #> 20545:       2 epiforecasts-EpiNow2"},{"path":"https://epiforecasts.io/scoringutils/reference/squared_error.html","id":null,"dir":"Reference","previous_headings":"","what":"Squared Error — squared_error","title":"Squared Error — squared_error","text":"Squared Error SE calculated $$   (\\textrm{true\\_values} - \\textrm{predicted\\_values})^2 $$","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/squared_error.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Squared Error — squared_error","text":"","code":"squared_error(true_values, predictions)"},{"path":"https://epiforecasts.io/scoringutils/reference/squared_error.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Squared Error — squared_error","text":"true_values vector true observed values size n predictions vector predicted values size n","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/squared_error.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Squared Error — squared_error","text":"vector scoring values","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/squared_error.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Squared Error — squared_error","text":"","code":"true_values <- rnorm(30, mean = 1:30) predicted_values <- rnorm(30, mean = 1:30) squared_error(true_values, predicted_values) #>  [1] 1.815208e-02 8.494946e-01 8.870800e-02 2.570784e+00 1.112817e+01 #>  [6] 2.011951e+00 5.608241e-01 4.543362e+00 2.312538e+00 4.374467e-02 #> [11] 8.841450e-02 1.002675e-02 6.565734e+00 3.997209e+00 1.425201e+00 #> [16] 2.282389e+00 1.362692e+01 5.913843e-04 2.354307e-01 1.745345e+00 #> [21] 2.374719e+00 3.503143e+00 1.722154e+00 6.022786e+00 5.777315e-01 #> [26] 3.155219e+00 4.461945e+00 1.634580e+00 1.122590e+01 8.894234e-02"},{"path":"https://epiforecasts.io/scoringutils/reference/summarise_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Summarise scores as produced by score() — summarise_scores","title":"Summarise scores as produced by score() — summarise_scores","text":"Summarise scores produced score()","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/summarise_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Summarise scores as produced by score() — summarise_scores","text":"","code":"summarise_scores(   scores,   by = NULL,   across = NULL,   fun = mean,   relative_skill = FALSE,   relative_skill_metric = \"auto\",   metric = deprecated(),   baseline = NULL,   ... )  summarize_scores(   scores,   by = NULL,   across = NULL,   fun = mean,   relative_skill = FALSE,   relative_skill_metric = \"auto\",   metric = deprecated(),   baseline = NULL,   ... )"},{"path":"https://epiforecasts.io/scoringutils/reference/summarise_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Summarise scores as produced by score() — summarise_scores","text":"scores data.table scores produced score(). character vector column names summarise scores . Default NULL, meaning summary takes place summarising samples quantiles (case quantile-based forecasts), one score per forecast defined unit single forecast (rather one score every sample quantile). unit single forecast determined columns present input data correspond metric produced score(), indicate indicate grouping forecasts (example may one forecast per day, location model). Adding additional, unrelated, columns may alter results unpredictable way. across character vector column names vector variables define unit single forecast (see ) summarise scores across (meaning specified columns dropped). alternative specifying directly. NULL (default), used inferred internally also specified.  one across  may used time. fun function used summarising scores. Default mean. relative_skill logical, whether compute relative performance models based pairwise comparisons. TRUE (default FALSE), column called 'model' must present input data. information computation relative skill, see pairwise_comparison(). Relative skill calculated aggregation level specified . relative_skill_metric character name metric relative skill shall computed. equal 'auto' (default), either interval score, CRPS Brier score (depending available input data) metric Deprecated 1.1.0. Use relative_skill_metric instead. baseline character string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. ... additional parameters can passed summary function provided fun. information see documentation respective function.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/summarise_scores.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Summarise scores as produced by score() — summarise_scores","text":"","code":"data.table::setDTthreads(1) # only needed to avoid issues on CRAN library(magrittr) # pipe operator  scores <- score(example_continuous) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. summarise_scores(scores) #>      location location_name target_end_date target_type forecast_date #>   1:       DE       Germany      2021-05-08       Cases    2021-05-03 #>   2:       DE       Germany      2021-05-08       Cases    2021-05-03 #>   3:       DE       Germany      2021-05-08       Cases    2021-05-03 #>   4:       DE       Germany      2021-05-08      Deaths    2021-05-03 #>   5:       DE       Germany      2021-05-08      Deaths    2021-05-03 #>  ---                                                                  #> 883:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 884:       IT         Italy      2021-07-24      Deaths    2021-07-05 #> 885:       IT         Italy      2021-07-24      Deaths    2021-07-12 #> 886:       IT         Italy      2021-07-24      Deaths    2021-07-05 #> 887:       IT         Italy      2021-07-24      Deaths    2021-07-12 #>                      model horizon         mad  bias       dss         crps #>   1: EuroCOVIDhub-ensemble       1 17641.24334  0.55 20.601386  7482.975177 #>   2: EuroCOVIDhub-baseline       1 19341.68942  0.95 22.443370 20371.250988 #>   3:  epiforecasts-EpiNow2       1 32348.79978  0.80 22.310568 24810.424753 #>   4: EuroCOVIDhub-ensemble       1   267.13585  0.20 11.112388    67.510511 #>   5: EuroCOVIDhub-baseline       1   397.09371 -0.05 11.541040    86.462930 #>  ---                                                                        #> 883: EuroCOVIDhub-baseline       2   168.03093  0.45 12.913561    66.515150 #> 884:       UMass-MechBayes       3    24.76457  0.10  6.353679     6.616381 #> 885:       UMass-MechBayes       2    39.96542  0.80  8.765716    29.446723 #> 886:  epiforecasts-EpiNow2       3   107.93293  0.10 10.283832    28.863742 #> 887:  epiforecasts-EpiNow2       2    93.54245  0.65 10.563729    49.998906 #>      log_score    ae_median      se_mean #>   1: 10.989568  9413.839675 2.303980e+08 #>   2: 11.941135 29378.807513 9.128538e+08 #>   3: 12.007007 36512.810659 1.603609e+09 #>   4:  6.516954    77.077318 8.593954e+03 #>   5:  6.913879    27.429626 2.123067e+03 #>  ---                                     #> 883:  6.067282    58.606302 6.065218e+04 #> 884:  4.242600     5.071742 6.043161e+01 #> 885:  5.321013    42.271396 2.153122e+03 #> 886:  5.709216     7.760566 3.212150e+03 #> 887:  5.736238    73.670518 1.263027e+04   # summarise over samples or quantiles to get one score per forecast scores <- score(example_quantile) #> The following messages were produced when checking inputs: #> 1.  144 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. summarise_scores(scores) #>      location target_end_date target_type location_name forecast_date #>   1:       DE      2021-05-08       Cases       Germany    2021-05-03 #>   2:       DE      2021-05-08       Cases       Germany    2021-05-03 #>   3:       DE      2021-05-08       Cases       Germany    2021-05-03 #>   4:       DE      2021-05-08      Deaths       Germany    2021-05-03 #>   5:       DE      2021-05-08      Deaths       Germany    2021-05-03 #>  ---                                                                  #> 883:       IT      2021-07-24      Deaths         Italy    2021-07-05 #> 884:       IT      2021-07-24      Deaths         Italy    2021-07-12 #> 885:       IT      2021-07-24      Deaths         Italy    2021-07-12 #> 886:       IT      2021-07-24      Deaths         Italy    2021-07-12 #> 887:       IT      2021-07-24      Deaths         Italy    2021-07-12 #>                      model horizon interval_score  dispersion underprediction #>   1: EuroCOVIDhub-baseline       1    16925.04696 1649.220870       0.0000000 #>   2: EuroCOVIDhub-ensemble       1     7990.85478 5440.985217       0.0000000 #>   3:  epiforecasts-EpiNow2       1    25395.96087 8173.700000       0.0000000 #>   4: EuroCOVIDhub-baseline       1       46.79304   44.662609       0.0000000 #>   5: EuroCOVIDhub-ensemble       1       53.88000   53.271304       0.6086957 #>  ---                                                                          #> 883:  epiforecasts-EpiNow2       3       19.76261   14.284348       0.0000000 #> 884: EuroCOVIDhub-baseline       2       80.33696   76.728261       0.0000000 #> 885: EuroCOVIDhub-ensemble       2       18.65870   13.354348       0.0000000 #> 886:       UMass-MechBayes       2       25.58174    7.755652       0.0000000 #> 887:  epiforecasts-EpiNow2       2       66.16174   25.553043       0.0000000 #>      overprediction coverage_deviation  bias ae_median #>   1:   15275.826087        -0.38521739  0.95     25620 #>   2:    2549.869565         0.04956522  0.50     12271 #>   3:   17222.260870        -0.29826087  0.90     44192 #>   4:       2.130435         0.22347826  0.30        15 #>   5:       0.000000         0.39739130 -0.10        14 #>  ---                                                   #> 883:       5.478261         0.04956522  0.50        26 #> 884:       3.608696         0.31043478  0.20        53 #> 885:       5.304348         0.13652174  0.40        30 #> 886:      17.826087        -0.21130435  0.80        46 #> 887:      40.608696        -0.29826087  0.90       108  # get scores by model summarise_scores(scores,by = \"model\") #>                    model interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline    14321.48926 2096.95360      5143.53567 #> 2: EuroCOVIDhub-ensemble     8992.62316 1846.85278      2120.64029 #> 3:  epiforecasts-EpiNow2    10827.40786 2950.73422      1697.23411 #> 4:       UMass-MechBayes       52.65195   26.87239        16.80095 #>    overprediction coverage_deviation        bias   ae_median #> 1:    7081.000000         0.00201087  0.21851562 19353.42969 #> 2:    5025.130095         0.04871603  0.00812500 12077.10156 #> 3:    6179.439535        -0.05516986 -0.04336032 14521.10526 #> 4:       8.978601        -0.02312500 -0.02234375    78.47656  # get scores by model and target type summarise_scores(scores, by = c(\"model\", \"target_type\")) #>                    model target_type interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline       Cases    28483.57465 4102.50094    10284.972826 #> 2: EuroCOVIDhub-ensemble       Cases    17943.82383 3663.52458     4237.177310 #> 3:  epiforecasts-EpiNow2       Cases    20831.55662 5664.37795     3260.355639 #> 4: EuroCOVIDhub-baseline      Deaths      159.40387   91.40625        2.098505 #> 5: EuroCOVIDhub-ensemble      Deaths       41.42249   30.18099        4.103261 #> 6:       UMass-MechBayes      Deaths       52.65195   26.87239       16.800951 #> 7:  epiforecasts-EpiNow2      Deaths       66.64282   31.85692       15.893314 #>    overprediction coverage_deviation        bias   ae_median #> 1:   14096.100883        -0.11211957  0.09796875 38473.60156 #> 2:   10043.121943        -0.09785326 -0.05640625 24101.07031 #> 3:   11906.823030        -0.06660326 -0.07890625 27923.81250 #> 4:      65.899117         0.11614130  0.33906250   233.25781 #> 5:       7.138247         0.19528533  0.07265625    53.13281 #> 6:       8.978601        -0.02312500 -0.02234375    78.47656 #> 7:      18.892583        -0.04287176 -0.00512605   104.74790  # Get scores summarised across horizon, forecast date, and target end date summarise_scores(  scores, across = c(\"horizon\", \"forecast_date\", \"target_end_date\") ) #>     location target_type  location_name                 model interval_score #>  1:       DE       Cases        Germany EuroCOVIDhub-baseline    14506.65500 #>  2:       DE       Cases        Germany EuroCOVIDhub-ensemble     6286.66495 #>  3:       DE       Cases        Germany  epiforecasts-EpiNow2    11684.72865 #>  4:       DE      Deaths        Germany EuroCOVIDhub-baseline      155.91235 #>  5:       DE      Deaths        Germany EuroCOVIDhub-ensemble       44.46077 #>  6:       DE      Deaths        Germany       UMass-MechBayes       68.91583 #>  7:       DE      Deaths        Germany  epiforecasts-EpiNow2       93.33921 #>  8:       FR       Cases         France EuroCOVIDhub-baseline    54147.68308 #>  9:       FR       Cases         France EuroCOVIDhub-ensemble    44537.04769 #> 10:       FR       Cases         France  epiforecasts-EpiNow2    50141.70361 #> 11:       FR      Deaths         France EuroCOVIDhub-baseline      187.63842 #> 12:       FR      Deaths         France EuroCOVIDhub-ensemble       56.70677 #> 13:       FR      Deaths         France       UMass-MechBayes       74.17250 #> 14:       FR      Deaths         France  epiforecasts-EpiNow2       96.69760 #> 15:       GB       Cases United Kingdom EuroCOVIDhub-baseline    36032.52912 #> 16:       GB       Cases United Kingdom EuroCOVIDhub-ensemble    16010.55516 #> 17:       GB       Cases United Kingdom  epiforecasts-EpiNow2    18303.21825 #> 18:       GB      Deaths United Kingdom EuroCOVIDhub-baseline       95.17583 #> 19:       GB      Deaths United Kingdom EuroCOVIDhub-ensemble       20.04649 #> 20:       GB      Deaths United Kingdom       UMass-MechBayes       33.45330 #> 21:       GB      Deaths United Kingdom  epiforecasts-EpiNow2       25.77830 #> 22:       IT       Cases          Italy EuroCOVIDhub-baseline     9247.43141 #> 23:       IT       Cases          Italy EuroCOVIDhub-ensemble     4941.02753 #> 24:       IT       Cases          Italy  epiforecasts-EpiNow2     3196.57595 #> 25:       IT      Deaths          Italy EuroCOVIDhub-baseline      198.88887 #> 26:       IT      Deaths          Italy EuroCOVIDhub-ensemble       44.47594 #> 27:       IT      Deaths          Italy       UMass-MechBayes       34.06615 #> 28:       IT      Deaths          Italy  epiforecasts-EpiNow2       59.20908 #>     location target_type  location_name                 model interval_score #>      dispersion underprediction overprediction coverage_deviation        bias #>  1:  2923.63054    1.147921e+02   1.146823e+04        -0.08630435  0.49062500 #>  2:  2286.83071    4.768886e+02   3.522946e+03        -0.06184783  0.17375000 #>  3:  3382.48817    9.200027e+02   7.382238e+03        -0.14608696  0.13375000 #>  4:    78.26697    0.000000e+00   7.764538e+01         0.02510870  0.52812500 #>  5:    35.27327    4.533967e+00   4.653533e+00         0.23842391 -0.06875000 #>  6:    31.75822    3.715761e+01   0.000000e+00        -0.07271739 -0.63312500 #>  7:    44.37726    3.803397e+01   1.092799e+01        -0.06184783 -0.45937500 #>  8:  6294.16678    8.578344e+03   3.927517e+04        -0.13521739  0.16250000 #>  9:  4834.10747    6.054179e+03   3.364876e+04        -0.13521739  0.02281250 #> 10:  7852.48758    7.281398e+03   3.500782e+04        -0.06456522 -0.05718750 #> 11:   101.19685    0.000000e+00   8.644158e+01         0.07945652  0.46562500 #> 12:    41.62796    7.942935e+00   7.135870e+00         0.20445652  0.06562500 #> 13:    40.59505    2.091984e+01   1.265761e+01         0.05635870  0.08812500 #> 14:    60.31008    1.061815e+01   2.576938e+01         0.02310019 -0.07173913 #> 15:  4300.45982    3.166997e+04   6.210326e+01        -0.18141304 -0.63156250 #> 16:  6016.72500    8.891671e+03   1.102159e+03        -0.09445652 -0.50937500 #> 17: 10391.44923    3.617322e+03   4.294447e+03         0.01695652 -0.28125000 #> 18:    86.38914    8.266304e+00   5.203804e-01         0.29956522 -0.10625000 #> 19:    16.20003    1.241848e+00   2.604620e+00         0.19358696  0.15937500 #> 20:    14.34596    4.891304e-02   1.905842e+01        -0.12978261  0.64468750 #> 21:    13.71580    1.086957e-02   1.205163e+01        -0.01701087  0.54875000 #> 22:  2891.74663    7.767894e+02   5.578895e+03        -0.04554348  0.37031250 #> 23:  1516.43514    1.525970e+03   1.898622e+03        -0.09989130  0.08718750 #> 24:  1031.08682    1.222700e+03   9.427894e+02        -0.07271739 -0.11093750 #> 25:    99.77202    1.277174e-01   9.898913e+01         0.06043478  0.46875000 #> 26:    27.62268    2.694293e+00   1.415897e+01         0.14467391  0.13437500 #> 27:    20.79034    9.077446e+00   4.198370e+00         0.05364130 -0.18906250 #> 28:    17.02701    1.342663e+01   2.875543e+01        -0.09717391 -0.05687500 #>      dispersion underprediction overprediction coverage_deviation        bias #>       ae_median #>  1: 21109.81250 #>  2:  9949.12500 #>  3: 18315.09375 #>  4:   261.03125 #>  5:    51.93750 #>  6:   124.50000 #>  7:   154.62500 #>  8: 67726.87500 #>  9: 52432.93750 #> 10: 60549.84375 #> 11:   293.59375 #> 12:    72.12500 #> 13:    88.53125 #> 14:   143.00000 #> 15: 49934.03125 #> 16: 26422.84375 #> 17: 27901.06250 #> 18:    59.03125 #> 19:    22.96875 #> 20:    55.34375 #> 21:    40.62500 #> 22: 15123.68750 #> 23:  7599.37500 #> 24:  4929.25000 #> 25:   319.37500 #> 26:    65.50000 #> 27:    45.53125 #> 28:    91.50000 #>       ae_median  # get standard deviation summarise_scores(scores, by = \"model\", fun = sd) #>                    model interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline      43157.488  2682.7722      18417.2012 #> 2: EuroCOVIDhub-ensemble      38788.740  3863.3379       7179.5954 #> 3:  epiforecasts-EpiNow2      42478.208  6698.7470       7629.9180 #> 4:       UMass-MechBayes         42.813    25.2045         30.8519 #>    overprediction coverage_deviation      bias   ae_median #> 1:     38723.6131          0.2589550 0.5628605 50186.94967 #> 2:     36978.7730          0.2726646 0.5707267 42500.79359 #> 3:     38832.5598          0.2838865 0.6632842 49800.66917 #> 4:        18.3498          0.2705116 0.6457779    74.79904  # round digits summarise_scores(scores,by = \"model\") %>%   summarise_scores(fun = signif, digits = 2) #>                    model interval_score dispersion underprediction #> 1: EuroCOVIDhub-baseline          14000       2100            5100 #> 2: EuroCOVIDhub-ensemble           9000       1800            2100 #> 3:  epiforecasts-EpiNow2          11000       3000            1700 #> 4:       UMass-MechBayes             53         27              17 #>    overprediction coverage_deviation    bias ae_median #> 1:           7100              0.002  0.2200     19000 #> 2:           5000              0.049  0.0081     12000 #> 3:           6200             -0.055 -0.0430     15000 #> 4:              9             -0.023 -0.0220        78  # get quantiles of scores # make sure to aggregate over ranges first summarise_scores(scores,   by = \"model\", fun = quantile,   probs = c(0.25, 0.5, 0.75) ) #>                     model interval_score dispersion underprediction #>  1: EuroCOVIDhub-baseline      130.23793   87.81804      0.00000000 #>  2: EuroCOVIDhub-baseline      934.41870  163.28739      0.00000000 #>  3: EuroCOVIDhub-baseline    10795.61967 3476.52750      4.42391304 #>  4: EuroCOVIDhub-ensemble       28.33707   23.67087      0.00000000 #>  5: EuroCOVIDhub-ensemble      300.30739  168.95543      0.00000000 #>  6: EuroCOVIDhub-ensemble     5532.28565 2030.41870     44.10869565 #>  7:  epiforecasts-EpiNow2       50.94935   16.90326      0.00000000 #>  8:  epiforecasts-EpiNow2      442.81696  152.12565      0.08695652 #>  9:  epiforecasts-EpiNow2     5902.85957 2033.24370    155.76086957 #> 10:       UMass-MechBayes       18.49435    7.73837      0.00000000 #> 11:       UMass-MechBayes       46.05413   17.15413      0.02173913 #> 12:       UMass-MechBayes       73.27087   38.04620     18.41304348 #>     overprediction coverage_deviation  bias ae_median #>  1:      0.0000000         -0.1243478 -0.20    166.00 #>  2:     54.1304348         -0.0373913  0.40    763.50 #>  3:    496.7065217          0.2234783  0.60  17709.00 #>  4:      0.0000000         -0.1243478 -0.40     27.00 #>  5:      0.2826087          0.1365217  0.10    277.00 #>  6:     43.5217391          0.2452174  0.40   8100.00 #>  7:      0.0000000         -0.2982609 -0.70     68.00 #>  8:      0.0000000         -0.0373913 -0.10    404.00 #>  9:     45.6956522          0.1365217  0.60   7177.00 #> 10:      0.0000000         -0.2113043 -0.60     24.75 #> 11:      0.0000000         -0.0373913 -0.05     52.00 #> 12:     10.2500000          0.2234783  0.60    106.00  # get ranges # summarise_scores(scores, by = \"range\")"},{"path":"https://epiforecasts.io/scoringutils/reference/theme_scoringutils.html","id":null,"dir":"Reference","previous_headings":"","what":"Scoringutils ggplot2 theme — theme_scoringutils","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"theme ggplot2 plots used scoringutils","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/theme_scoringutils.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"","code":"theme_scoringutils()"},{"path":"https://epiforecasts.io/scoringutils/reference/theme_scoringutils.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"ggplot2 theme","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/transform_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Transform forecasts and observed values — transform_forecasts","title":"Transform forecasts and observed values — transform_forecasts","text":"Function transform forecasts true values scoring.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/transform_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Transform forecasts and observed values — transform_forecasts","text":"","code":"transform_forecasts(data, fun = log_shift, append = TRUE, label = \"log\", ...)"},{"path":"https://epiforecasts.io/scoringutils/reference/transform_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Transform forecasts and observed values — transform_forecasts","text":"data data.frame data.table predictions observations. scoring using score(), following columns need present: true_value - true observed values prediction - predictions predictive samples one true value. (need provide prediction column want score quantile forecasts wide range format.) scoring integer continuous forecasts sample column needed: sample - index identify predictive samples prediction column generated one model one true value. necessary continuous integer forecasts, binary predictions. scoring predictions quantile-format forecast provide column called quantile: quantile: quantile prediction corresponds addition model column suggested present flagged added input data forecasts assigned \"unspecified model\"). can check format data using check_forecasts() examples format (example_quantile, example_continuous, example_integer, example_binary). fun function used transform true values predictions. default function log_shift(), custom function essentially log(), additional arguments (offset) allows add offset applying logarithm. often helpful natural log transformation defined zero. common, pragmatic solution, add small offset data applying log transformation. work often used offset 1 precise value depend application. append Logical, defaults TRUE. Whether append transformed version data currently existing data (TRUE). selected, data gets transformed appended existing data frame, making possible use outcome directly score(). additional column, 'scale', gets created denotes rows untransformed ('scale' value \"natural\") transformed ('scale' value passed argument label). label string newly created 'scale' column denote newly transformed values. relevant append = TRUE. ... Additional parameters pass function supplied. default option log_shift() offset argument.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/transform_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Transform forecasts and observed values — transform_forecasts","text":"data.table either transformed version data, one untransformed transformed data. includes original data well transformation original data. one additional column, `scale', present set \"natural\" untransformed forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/transform_forecasts.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Transform forecasts and observed values — transform_forecasts","text":"reasons, depending circumstances, might desirable (check linked reference info). epidemiology, example, may useful log-transform incidence counts evaluating forecasts using scores weighted interval score (WIS) continuous ranked probability score (CRPS). Log-transforming forecasts observations changes interpretation score measure absolute distance forecast observation score evaluates forecast exponential growth rate. Another motivation can apply variance-stabilising transformation standardise incidence counts population. Note want apply transformation, important transform forecasts observations apply score. Applying transformation score risks losing propriety proper scoring rule.","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/transform_forecasts.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Transform forecasts and observed values — transform_forecasts","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722  https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/transform_forecasts.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Transform forecasts and observed values — transform_forecasts","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/reference/transform_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Transform forecasts and observed values — transform_forecasts","text":"","code":"library(magrittr) # pipe operator  # transform forecasts using the natural logarithm # negative values need to be handled (here by replacing them with 0) example_quantile %>%   .[, true_value := ifelse(true_value < 0, 0, true_value)] %>% # Here we use the default function log_shift() which is essentially the same # as log(), but has an additional arguments (offset) that allows you add an # offset before applying the logarithm.   transform_forecasts(append = FALSE) %>%   head() #> Warning: Detected zeros in input values.Try specifying offset = 1 (or any other offset). #> Warning: Detected zeros in input values.Try specifying offset = 1 (or any other offset). #>    location target_end_date target_type true_value location_name forecast_date #> 1:       DE      2021-01-02       Cases  11.754302       Germany          <NA> #> 2:       DE      2021-01-02      Deaths   8.419360       Germany          <NA> #> 3:       DE      2021-01-09       Cases  11.950677       Germany          <NA> #> 4:       DE      2021-01-09      Deaths   8.718827       Germany          <NA> #> 5:       DE      2021-01-16       Cases  11.609898       Germany          <NA> #> 6:       DE      2021-01-16      Deaths   8.677099       Germany          <NA> #>    quantile prediction model horizon #> 1:       NA         NA  <NA>      NA #> 2:       NA         NA  <NA>      NA #> 3:       NA         NA  <NA>      NA #> 4:       NA         NA  <NA>      NA #> 5:       NA         NA  <NA>      NA #> 6:       NA         NA  <NA>      NA  # alternatively, integrating the truncation in the transformation function: example_quantile %>%  transform_forecasts(    fun = function(x) {log_shift(pmax(0, x))}, append = FALSE  ) %>%  head() #> Warning: Detected zeros in input values.Try specifying offset = 1 (or any other offset). #> Warning: Detected zeros in input values.Try specifying offset = 1 (or any other offset). #>    location target_end_date target_type true_value location_name forecast_date #> 1:       DE      2021-01-02       Cases  11.754302       Germany          <NA> #> 2:       DE      2021-01-02      Deaths   8.419360       Germany          <NA> #> 3:       DE      2021-01-09       Cases  11.950677       Germany          <NA> #> 4:       DE      2021-01-09      Deaths   8.718827       Germany          <NA> #> 5:       DE      2021-01-16       Cases  11.609898       Germany          <NA> #> 6:       DE      2021-01-16      Deaths   8.677099       Germany          <NA> #>    quantile prediction model horizon #> 1:       NA         NA  <NA>      NA #> 2:       NA         NA  <NA>      NA #> 3:       NA         NA  <NA>      NA #> 4:       NA         NA  <NA>      NA #> 5:       NA         NA  <NA>      NA #> 6:       NA         NA  <NA>      NA  # specifying an offset for the log transformation removes the # warning caused by zeros in the data example_quantile %>%   .[, true_value := ifelse(true_value < 0, 0, true_value)] %>%   transform_forecasts(offset = 1, append = FALSE) %>%   head() #>    location target_end_date target_type true_value location_name forecast_date #> 1:       DE      2021-01-02       Cases  11.754310       Germany          <NA> #> 2:       DE      2021-01-02      Deaths   8.419580       Germany          <NA> #> 3:       DE      2021-01-09       Cases  11.950683       Germany          <NA> #> 4:       DE      2021-01-09      Deaths   8.718991       Germany          <NA> #> 5:       DE      2021-01-16       Cases  11.609907       Germany          <NA> #> 6:       DE      2021-01-16      Deaths   8.677269       Germany          <NA> #>    quantile prediction model horizon #> 1:       NA         NA  <NA>      NA #> 2:       NA         NA  <NA>      NA #> 3:       NA         NA  <NA>      NA #> 4:       NA         NA  <NA>      NA #> 5:       NA         NA  <NA>      NA #> 6:       NA         NA  <NA>      NA  # adding square root transformed forecasts to the original ones example_quantile %>%   .[, true_value := ifelse(true_value < 0, 0, true_value)] %>%   transform_forecasts(fun = sqrt, label = \"sqrt\") %>%   score() %>%   summarise_scores(by = c(\"model\", \"scale\")) #> The following messages were produced when checking inputs: #> 1.  288 values for `prediction` are NA in the data provided and the corresponding rows were removed. This may indicate a problem if unexpected. #>                    model   scale interval_score   dispersion underprediction #> 1: EuroCOVIDhub-baseline natural   11124.930667 2096.9535954    5143.5356658 #> 2: EuroCOVIDhub-baseline    sqrt      27.742316    7.7296761       9.5936380 #> 3: EuroCOVIDhub-ensemble natural    5796.064569 1846.8527819    2120.6402853 #> 4: EuroCOVIDhub-ensemble    sqrt      14.974344    4.2878323       5.1827454 #> 5:  epiforecasts-EpiNow2 natural    7514.375476 2950.7342158    1697.2341137 #> 6:  epiforecasts-EpiNow2    sqrt      17.704899    5.4112770       5.7235785 #> 7:       UMass-MechBayes natural      52.651946   26.8723947      16.8009511 #> 8:       UMass-MechBayes    sqrt       1.328653    0.5993586       0.4019195 #>    overprediction coverage_deviation        bias    ae_median #> 1:   3884.4414062        0.003369565  0.21816406 16156.871094 #> 2:     10.4190016        0.003369565  0.21816406    39.185406 #> 3:   1828.5715014        0.048716033  0.00812500  8880.542969 #> 4:      5.5037665        0.048716033  0.00812500    22.458900 #> 5:   2866.4071466       -0.055169864 -0.04336032 11208.072874 #> 6:      6.5700431       -0.055169864 -0.04336032    25.585018 #> 7:      8.9786005       -0.023125000 -0.02234375    78.476562 #> 8:      0.3273746       -0.023125000 -0.02234375     2.069103  # adding multiple transformations example_quantile %>%   .[, true_value := ifelse(true_value < 0, 0, true_value)] %>%   transform_forecasts(fun = log_shift, offset = 1) %>%   transform_forecasts(fun = sqrt, label = \"sqrt\") %>%   head() #>    location target_end_date target_type true_value location_name forecast_date #> 1:       DE      2021-01-02       Cases     127300       Germany          <NA> #> 2:       DE      2021-01-02      Deaths       4534       Germany          <NA> #> 3:       DE      2021-01-09       Cases     154922       Germany          <NA> #> 4:       DE      2021-01-09      Deaths       6117       Germany          <NA> #> 5:       DE      2021-01-16       Cases     110183       Germany          <NA> #> 6:       DE      2021-01-16      Deaths       5867       Germany          <NA> #>    quantile prediction model horizon   scale #> 1:       NA         NA  <NA>      NA natural #> 2:       NA         NA  <NA>      NA natural #> 3:       NA         NA  <NA>      NA natural #> 4:       NA         NA  <NA>      NA natural #> 5:       NA         NA  <NA>      NA natural #> 6:       NA         NA  <NA>      NA natural"},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-1-2-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.2","text":"scoringutils now depends R 3.6. change made since packages testthat lifecycle, used scoringutils now require R 3.6. also updated Github action CI check work R 3.6 now.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-1-2-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.1","text":"gets rid preferably package _pkgdown.yml. theme toggle light dark theme didn’t work properly updates gh pages deploy action v4 also cleans files triggered introduces gh action automatically render Readme Readme.Rmd removes links vignettes renamed","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"scoringutils-120","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.0","title":"scoringutils 1.2.0","text":"major release contains range new features bug fixes introduced minor releases since 1.1.0. important changes : Documentation updated reflect changes since version 1.1.0, including new transform workflow functions. New set_forecast_unit() function allows manual setting forecast unit. summarise_scores() gains new across argument summarizing across variables. New transform_forecasts() log_shift() functions allow forecast transformations. See documentation transform_forecasts() details example use case. Input checks test coverage improved bias functions. Bug fix get_prediction_type() integer matrix input. Links scoringutils paper citation updates. Warning added interval_score() small interval ranges. Linting updates improvements. Thanks @nikosbosse, @seabbs, @sbfnk code review contributions. Thanks @bisaloo suggestion use linting GitHub Action triggers changes, @adrian-lison suggestion add warning interval_score() interval range 0 1.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-1-2-0","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.0","text":"documentation updated reflect recent changes since scoringutils 1.1.0. particular, usage functions set_forecast_unit(), check_forecasts() transform_forecasts() now documented Vignettes. introduction functions enhances overall workflow help make code readable. functions designed used together pipe operator. example, one can now use something like following: Documentation transform_forecasts() also extended. functions allows user easily add transformations forecasts, suggested paper “Scoring epidemiological forecasts transformed scales”. epidemiological context, example, may make sense apply natural logarithm first scoring forecasts, order obtain scores reflect well models able predict exponential growth rates, rather absolute values. Users can now something like following score transformed version data addition original one: use log_shift() function apply logarithmic transformation forecasts. function introduced scoringutils 1.1.2 helper function acts just like log(), additional argument offset can add number every prediction observed value applying log transformation.","code":"example_quantile |>    set_forecast_unit(c(\"model\", \"location\", \"forecast_date\", \"horizon\", \"target_type\")) |>    check_forecasts() |>    score() data <- example_quantile[true_value > 0, ] data |>   transform_forecasts(fun = log_shift, offset = 1) |>    score() |>    summarise_scores(by = c(\"model\", \"scale\"))"},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-1-2-0","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.2.0","text":"Made check_forecasts() score() pipeable (see issue #290). means users can now directly use output check_forecasts() input score(). score() otherwise runs check_forecasts() internally anyway simply makes step explicit helps writing clearer code.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"scoringutils-117","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.7","title":"scoringutils 1.1.7","text":"Release @seabbs #305. Reviewed @nikosbosse @sbfnk.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"breaking-changes-1-1-7","dir":"Changelog","previous_headings":"","what":"Breaking changes","title":"scoringutils 1.1.7","text":"prediction_type argument get_forecast_unit() changed dropped. Instead new internal function prediction_is_quantile() used detect quantile variable present. Whilst internal function may impact users accessible via `find_duplicates().","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-1-1-7","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.7","text":"Made imputation median bias_range() bias_quantile() obvious user may cause unexpected behaviour. Simplified bias_range() uses bias_quantile() internally. Added additional input checks bias_range(), bias_quantile(), check_predictions() make sure input valid. Improve coverage unit tests bias_range(), bias_quantile(), bias_sample(). Updated pairwise comparison unit tests use realistic data.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"bug-fixes-1-1-7","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.1.7","text":"Fixed bug get_prediction_type() led unable correctly detect integer (instead categorising continuous) forecasts input matrix. issue impacted bias_sample() also score() used integer forecasts resulting lower bias scores expected.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-1-1-6","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.6","text":"Added new argument, across, summarise_scores(). argument allows user summarise scores across different forecast units alternative specifying . See documentation summarise_scores() details example use case.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-1-1-5","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.5","text":"Added new function, set_forecast_unit() allows user set forecast unit manually. function removes columns relevant uniquely identifying single forecast. done manually, scoringutils attempts determine unit single automatically simply assuming column names relevant determine forecast unit. can lead unexpected behaviour, setting forecast unit explicitly can help make code easier debug easier read (see issue #268). used part workflow, set_forecast_unit() can directly piped check_forecasts() check everything order.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-1-1-4","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.4","text":"Added links scoringutils paper Evaluating Forecasts scoringutils R package. Updated citation formatting comply newer standards.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-1-1-3","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.3","text":"Added warning interval_score() interval range 0 1. Thanks @adrian-lison (see #277) suggestion.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-1-1-3-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.3","text":"Switched linting GitHub Action triggers changes. Inspired @bisaloo recent contribution epinowcast package. Updated package linters extensive. Inspired @bisaloo recent contribution epinowcast package. Resolved flagged linting issues across package.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-1-1-2","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.2","text":"Added new function, transform_forecasts() make easy transform forecasts scoring , suggested Bosse et al. (2023), https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1. Added function, log_shift() implements default transformation function. function allows add offset applying logarithm.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"scoringutils-111","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.1","title":"scoringutils 1.1.1","text":"Added small change interval_score() explicitly converts logical vector numeric one. happen implicitly anyway, now done explicitly order avoid issues may come input vector type doesn’t allow implicit conversion.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"scoringutils-110","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.0","title":"scoringutils 1.1.0","text":"CRAN release: 2023-01-30 minor update package bug fixes minor changes.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-1-1-0","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.0","text":"Removed attach message warned breaking changes 1.0.0. Renamed metric argument summarise_scores() relative_skill_metric. argument now deprecated removed future version package. Please use new argument instead. Updated documentation score() related functions make soft requirement model column input data explicit. Updated documentation score(), pairwise_comparison() summarise_scores() make clearer unit single forecast required computations Simplified function plot_pairwise_comparison() now supports plotting mean score ratios p-values removed hybrid option print time.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"bug-fixes-1-1-0","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.1.0","text":"Missing baseline forecasts pairwise_comparison() now trigger explicit informative error message. requirements table getting started vignette now correct. Added support optional sample column using quantile forecast format. Previously resulted error.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"scoringutils-100","dir":"Changelog","previous_headings":"","what":"scoringutils 1.0.0","title":"scoringutils 1.0.0","text":"CRAN release: 2022-05-13 Major update package package functions lots breaking changes.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-1-0-0","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.0.0","text":"New updated Readme vignette. proposed scoring workflow reworked. Functions changed can easily piped simplified arguments outputs.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"new-functions-and-function-changes-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"New functions and function changes","title":"scoringutils 1.0.0","text":"function eval_forecasts() replaced function score() much reduced set function arguments. Functionality summarise scores add relative skill scores moved function summarise_scores() New function check_forecasts() analyse input data scoring New function correlation() compute correlations different metrics New function add_coverage() add coverage specific central prediction intervals. New function avail_forecasts() allows visualise number available forecasts. New function find_duplicates() find duplicate forecasts cause error. plotting functions renamed begin plot_. Arguments simplified. function pit() now works based data.frames. old pit function renamed pit_sample(). PIT p-values removed entirely. function plot_pit() now works directly input produced pit() Many data-handling functions removed input types score() restricted sample-based, quantile-based binary forecasts. function brier_score() now returns brier scores, rather taking mean returning output. crps(), dss() logs() renamed crps_sample(), dss_sample(), logs_sample()","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"bug-fixes-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"Bug fixes","title":"scoringutils 1.0.0","text":"Testing expanded Minor bugs fixed, example bug sample_to_quantile() function (https://github.com/epiforecasts/scoringutils/pull/223)","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-data-updated-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"Package data updated","title":"scoringutils 1.0.0","text":"Package data now based forecasts submitted European Forecast Hub (https://covid19forecasthub.eu/). example data files renamed begin example_. new data set, summary_metrics included contains summary metrics implemented scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"other-breaking-changes-1-0-0","dir":"Changelog","previous_headings":"","what":"Other breaking changes","title":"scoringutils 1.0.0","text":"‘sharpness’ component weighted interval score renamed dispersion. done make clear component represents maintain consistency used places.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-0-1-8","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.8","text":"Added function check_forecasts() runs basic checks input data provides feedback.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"scoringutils-0172","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.7.2","title":"scoringutils 0.1.7.2","text":"CRAN release: 2021-07-21","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-0-1-7-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.7.2","text":"Minor bug fixes (previously, ‘interval_score’ needed among selected metrics). data.tables now returned table[] rather table, don’t called twice display contents.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"scoringutils-017","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.7","title":"scoringutils 0.1.7","text":"CRAN release: 2021-07-14","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-0-1-7","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.7","text":"Added function, pairwise_comparison() runs pairwise comparisons models output eval_forecasts() Added functionality compute relative skill within eval_forecasts(). Added function visualise pairwise comparisons.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-0-1-7","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.7","text":"WIS definition change introduced version 0.1.5 partly corrected difference weighting introduced summarising scores different interval ranges. “sharpness” renamed ‘mad’ output [score()] sample-based forecasts.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-0-1","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.","text":"eval_forecasts() can now handle separate forecast truth data set input. eval_forecasts() now supports scoring point forecasts along side quantiles quantile-based format. Currently metric used absolute error.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-0-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.","text":"Many functions, especially eval_forecasts() got major rewrite. functionality unchanged, code now easier maintain data-handling functions got renamed, old names supported well now.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-0-1-5","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.5","text":"Changed default definition weighted interval score. Previously, median prediction counted twice, counted . want go back old behaviour, can call interval_score function argument count_median_twice = FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"scoringutils-014","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.4","title":"scoringutils 0.1.4","text":"CRAN release: 2020-11-17","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-0-1-4","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.4","text":"Added basic plotting functionality visualise scores. can now easily obtain diagnostic plots based scores produced score. correlation_plot() shows correlation metrics. plot_ranges() shows contribution different prediction intervals chosen metric. plot_heatmap() visualises scores heatmap. plot_score_table() shows coloured summary table scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-0-1-4","dir":"Changelog","previous_headings":"","what":"package updates","title":"scoringutils 0.1.4","text":"Renamed “calibration” “coverage”. Renamed “true_values” “true_value” data.frames. Renamed “predictions” “prediction” data.frames. Renamed “is_overprediction” “overprediction”. Renamed “is_underprediction” “underprediction”.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"potentially-breaking-changes-0-1-3","dir":"Changelog","previous_headings":"","what":"(Potentially) Breaking changes","title":"scoringutils 0.1.3","text":"argument score now slightly changed meaning. now denotes lowest possible grouping unit, .e. unit one observation needs specified explicitly. default now NULL. reason change metrics need scoring observation level consistent implementation principle. pit function receives grouping now summarise_by. similar spirit, summarise_by specified explicitly e.g. doesn’t assume anymore want ‘range’ included. interval score, weigh = TRUE now default option. Renamed true_values true_value predictions prediction.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-0-1-3","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.3","text":"Updated quantile evaluation metrics score. Bias well calibration now take quantiles account. Included option summarise scores according summarise_by argument score() summary can return mean, standard deviation well arbitrary set quantiles. score() can now return pit histograms. Switched ggplot2 plotting.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"potentially-breaking-changes-0-1-2","dir":"Changelog","previous_headings":"","what":"(Potentially) Breaking changes","title":"scoringutils 0.1.2","text":"scores score consistently renamed lower case. Interval_score now interval_score, CRPS now crps etc.","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-0-1-2","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.2","text":"Included support grouping scores according vector column names score(). Included support passing arguments lower-level functions score() Included support three new metrics score quantiles score(): bias, sharpness calibration","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-0-1-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.2","text":"Example data now horizon column illustrate use grouping. Documentation updated explain listed changes.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"feature-updates-0-1-1","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.1","text":"Included support long well wide input formats quantile forecasts scored score().","code":""},{"path":"https://epiforecasts.io/scoringutils/news/index.html","id":"package-updates-0-1-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.1","text":"Updated documentation score(). Added badges README.","code":""}]