Merge branch 'develop' into examples-range-plots

.Rbuildignore

@@ -12,6 +12,7 @@
 ^Meta$
 ^_pkgdown\.yml$
 ^inst/manuscript/manuscript_cache$
+^inst/manuscript/.trackdown$
 ^\.lintr$
 ^docs$
 ^\.devcontainer$

.github/workflows/R-CMD-check.yaml

@@ -23,7 +23,7 @@ jobs:
           - {os: ubuntu-latest,   r: 'devel', http-user-agent: 'release'}
           - {os: ubuntu-latest,   r: 'release'}
           - {os: ubuntu-latest,   r: 'oldrel-1'}
-          - {os: ubuntu-latest,   r: '3.5'}
+          - {os: ubuntu-latest,   r: '3.6'}
 
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -13,6 +13,7 @@ inst/manuscript/manuscript.blg
 inst/manuscript/manuscript.pdf
 inst/manuscript/manuscript.tex
 inst/manuscript/manuscript_files/
+inst/manuscript/.trackdown
 docs
 ..bfg-report/
 .DS_Store

.lintr

@@ -11,6 +11,6 @@ linters: linters_with_tags(
 exclusions: c(
     list.files("tests", recursive = TRUE, full.names = TRUE),
     list.files("inst", recursive = TRUE, full.names = TRUE),
-    "vignettes/metric-details.Rmd"
+    list.files("vignettes", pattern = ".R$", full.names = TRUE)
   )
 exclude: "# nolint"

DESCRIPTION

@@ -73,5 +73,5 @@ URL: https://doi.org/10.48550/arXiv.2205.07090, https://epiforecasts.io/scoringu
 BugReports: https://github.com/epiforecasts/scoringutils/issues
 VignetteBuilder: knitr
 Depends: 
-    R (>= 3.5)
+    R (>= 3.6)
 Roxygen: list(markdown = TRUE)

NAMESPACE

@@ -1,54 +1,53 @@
 # Generated by roxygen2: do not edit by hand
 
-S3method(plot,scoringutils_available_forecasts)
+S3method(as_forecast,default)
 S3method(print,scoringutils_check)
 S3method(quantile_to_interval,data.frame)
 S3method(quantile_to_interval,numeric)
 S3method(score,default)
-S3method(score,scoringutils_binary)
-S3method(score,scoringutils_point)
-S3method(score,scoringutils_quantile)
-S3method(score,scoringutils_sample)
-S3method(validate,default)
-S3method(validate,scoringutils_binary)
-S3method(validate,scoringutils_point)
-S3method(validate,scoringutils_quantile)
-S3method(validate,scoringutils_sample)
+S3method(score,forecast_binary)
+S3method(score,forecast_point)
+S3method(score,forecast_quantile)
+S3method(score,forecast_sample)
+S3method(validate_forecast,forecast_binary)
+S3method(validate_forecast,forecast_point)
+S3method(validate_forecast,forecast_quantile)
+S3method(validate_forecast,forecast_sample)
 export(abs_error)
 export(add_coverage)
 export(add_pairwise_comparison)
 export(ae_median_quantile)
 export(ae_median_sample)
-export(avail_forecasts)
-export(available_forecasts)
+export(as_forecast)
 export(available_metrics)
 export(bias_quantile)
-export(bias_range)
 export(bias_sample)
 export(brier_score)
 export(correlation)
 export(crps_sample)
 export(dispersion)
 export(dss_sample)
 export(get_duplicate_forecasts)
-export(interval_coverage_deviation_quantile)
+export(get_forecast_counts)
+export(get_forecast_type)
+export(get_forecast_unit)
+export(interval_coverage_dev_quantile)
 export(interval_coverage_quantile)
 export(interval_coverage_sample)
-export(interval_score)
 export(log_shift)
 export(logs_binary)
 export(logs_sample)
 export(mad_sample)
 export(make_NA)
 export(make_na)
 export(merge_pred_and_obs)
-export(new_scoringutils)
+export(new_forecast)
 export(overprediction)
 export(pairwise_comparison)
 export(pit)
 export(pit_sample)
-export(plot_avail_forecasts)
 export(plot_correlation)
+export(plot_forecast_counts)
 export(plot_heatmap)
 export(plot_interval_coverage)
 export(plot_pairwise_comparison)
@@ -59,6 +58,7 @@ export(plot_ranges)
 export(plot_score_table)
 export(plot_wis)
 export(quantile_score)
+export(quantile_to_interval)
 export(run_safely)
 export(sample_to_quantile)
 export(score)
@@ -70,7 +70,7 @@ export(summarize_scores)
 export(theme_scoringutils)
 export(transform_forecasts)
 export(underprediction)
-export(validate)
+export(validate_forecast)
 export(validate_general)
 export(wis)
 importFrom(Metrics,ae)
@@ -85,6 +85,8 @@ importFrom(checkmate,assert_list)
 importFrom(checkmate,assert_logical)
 importFrom(checkmate,assert_number)
 importFrom(checkmate,assert_numeric)
+importFrom(checkmate,assert_string)
+importFrom(checkmate,assert_vector)
 importFrom(checkmate,check_atomic_vector)
 importFrom(checkmate,check_data_frame)
 importFrom(checkmate,check_function)

NEWS.md

@@ -6,6 +6,7 @@ The update introduces breaking changes. If you want to keep using the older vers
 
 ## Package updates
 - In `score()`, required columns "true_value" and "prediction" were renamed and replaced by required columns "observed" and "predicted". Scoring functions now also use the function arguments "observed" and "predicted" everywhere consistently. 
+- The overall scoring workflow was updated. `score()` is now a generic function that dispatches the correct method based on the forecast type. forecast types currently supported are "binary", "point", "sample" and "quantile" with corresponding classes "forecast_binary", "forecast_point", "forecast_sample" and "forecast_quantile". An object of class `forecast_*` can be created using the function `as_forecast()`, which also replaces the previous function `check_forecasts()` (see more information below). 
 - Scoring functions received a consistent interface and input checks:
   - metrics for binary forecasts:
     - `observed`: factor with exactly 2 levels
@@ -20,15 +21,18 @@ The update introduces breaking changes. If you want to keep using the older vers
     - `observed`: numeric, either a scalar or a vector
     - `predicted`: numeric, a vector (if `observed` is a scalar) or a matrix (if `observed` is a vector)
     - `quantile`: numeric, a vector with quantile-levels. Can alternatively be a matrix of the same shape as `predicted`.
-- `check_forecasts()` was replaced by a new function `validate()`. `validate()` validates the input and in that sense fulfills the purpose of `check_forecasts()`. It has different methods: `validate.default()` assigns the input a class based on their forecast type. Other methods validate the input specifically for the various forecast types.
+- `check_forecasts()` was replaced by a different workflow. There now is a function, `as_forecast()`, that determines forecast type of the data, constructs a forecasting object and validates it using the function `validate_forecast()` (a generic that dispatches the correct method based on the forecast type). Objects of class `forecast_binary`, `forecast_point`, `forecast_sample` and `forecast_quantile` have print methods that fulfill the functionality of `check_forecasts()`.
 - The functionality for computing pairwise comparisons was now split from `summarise_scores()`. Instead of doing pairwise comparisons as part of summarising scores, a new function, `add_pairwise_comparison()`, was introduced that takes summarised scores as an input and adds pairwise comparisons to it. 
 - `add_coverage()` was reworked completely. It's new purpose is now to add coverage information to the raw forecast data (essentially fulfilling some of the functionality that was previously covered by `score_quantile()`)
+- Support for the interval format was mostly dropped (see PR #525 by @nikosbosse and reviewed by @seabbs)
+    - The function `bias_range()` was removed (users should now use `bias_quantile()` instead)
+    - The function `interval_score()` was made an internal function rather than being exported to users. We recommend using `wis()` instead. 
 - The function `find_duplicates()` was renamed to `get_duplicate_forecasts()`
 - Changes to `avail_forecasts()` and `plot_avail_forecasts()`:
-  - The function `avail_forecasts()` was renamed to `available_forecasts()` for consistency with `available_metrics()`. The old function, `avail_forecasts()` is still available as an alias, but will be removed in the future.
-  - For clarity, the output column in `avail_forecasts()` was renamed from "Number forecasts" to "count".
-  - `available_forecasts()` now also displays combinations where there are 0 forecasts, instead of silently dropping corresponding rows.
-  - `plot_avail_forecasts()` has been deprecated in favour of an S3 method for `plot()`. An alias is still available, but will be removed in the future.
+  - The function `avail_forecasts()` was renamed to `get_forecast_counts()`. This represents a change in the naming convention where we aim to name functions that provide the user with additional useful information about the data with a prefix "get_". Sees Issue #403 and #521 and PR #511 by @nikosbosse and reviewed by @seabbs for details. 
+  - For clarity, the output column in `get_forecast_counts()` was renamed from "Number forecasts" to "count".
+  - `get_forecast_counts()` now also displays combinations where there are 0 forecasts, instead of silently dropping corresponding rows.
+  - `plot_avail_forecasts()` was renamed `plot_forecast_counts()` in line with the change in the function name. The `x` argument no longer has a default value, as the value will depend on the data provided by the user.
 - The deprecated `..density..` was replaced with `after_stat(density)` in ggplot calls.
 - Files ending in ".Rda" were renamed to ".rds" where appropriate when used together with `saveRDS()` or `readRDS()`.
 - added documentation for the return value of `summarise_scores()`. 
@@ -188,7 +192,7 @@ to a function `summarise_scores()`
 - New function `check_forecasts()` to analyse input data before scoring
 - New function `correlation()` to compute correlations between different metrics
 - New function `add_coverage()` to add coverage for specific central prediction intervals.
-- New function `available_forecasts()` allows to visualise the number of available forecasts.
+- New function `avail_forecasts()` allows to visualise the number of available forecasts.
 - New function `find_duplicates()` to find duplicate forecasts which cause an error.
 - All plotting functions were renamed to begin with `plot_`. Arguments were
 simplified.

R/add_coverage.R

@@ -47,17 +47,13 @@
 #' @export
 add_coverage <- function(data) {
   stored_attributes <- get_scoringutils_attributes(data)
-  data <- validate(data)
+  data <- as_forecast(data)
   forecast_unit <- get_forecast_unit(data)
   data_cols <- colnames(data) # store so we can reset column order later
 
-  # what happens if quantiles are not symmetric around the median?
-  # should things error? Also write tests for that.
   interval_data <- quantile_to_interval(data, format = "wide")
-  interval_data[, interval_coverage := ifelse(
-    observed <= upper & observed >= lower,
-    TRUE,
-    FALSE)
+  interval_data[,
+    interval_coverage := (observed <= upper) & (observed >= lower)
   ][, c("lower", "upper", "observed") := NULL]
 
   data[, range := get_range_from_quantile(quantile)]

R/available_forecasts.R

@@ -31,14 +31,14 @@
 #' @examples
 #' data.table::setDTthreads(1) # only needed to avoid issues on CRAN
 #'
-#' available_forecasts(example_quantile,
+#' get_forecast_counts(example_quantile,
 #'   by = c("model", "target_type")
 #' )
-available_forecasts <- function(data,
+get_forecast_counts <- function(data,
                                 by = NULL,
                                 collapse = c("quantile", "sample_id")) {
 
-  data <- validate(data)
+  data <- as_forecast(data)
   forecast_unit <- attr(data, "forecast_unit")
   data <- remove_na_observed_predicted(data)
 
@@ -58,7 +58,7 @@ available_forecasts <- function(data,
   data <- data[data[, .I[1], by = collapse_by]$V1]
 
   # count number of rows = number of forecasts
-  out <- data[, .(`count` = .N), by = by]
+  out <- data[, .(count = .N), by = by]
 
   # make sure that all combinations in "by" are included in the output (with
   # count = 0). To achieve that, take the unique values in data and expand grid
@@ -70,23 +70,5 @@ available_forecasts <- function(data,
   out <- merge(out, out_empty, by = by, all.y = TRUE)
   out[, count := nafill(count, fill = 0)]
 
-  class(out) <- c("scoringutils_available_forecasts", class(out))
-
   return(out[])
 }
-
-#' @title Count Number of Available Forecasts `r lifecycle::badge("deprecated")`
-#' @details `r lifecycle::badge("deprecated")` Deprecated in 1.2.2. Use
-#' [available_forecasts()] instead.
-#' @inherit available_forecasts
-#' @keywords check-forecasts
-#' @export
-avail_forecasts <- function(data,
-                            by = NULL,
-                            collapse = c("quantile", "sample")) {
-  lifecycle::deprecate_warn(
-    "1.2.2", "avail_forecasts()",
-    "available_forecasts()"
-  )
-  available_forecasts(data, by, collapse)
-}