From 3de253725ce7cbe147a1308fa9c4b16d893ade7e Mon Sep 17 00:00:00 2001
From: Nat DeFries <42820733+nmdefries@users.noreply.github.com>
Date: Thu, 2 Sep 2021 15:47:58 -0400
Subject: [PATCH 1/2] replace evaluate_chu with
evalcast::evaluate_covid_predictions
---
Report/create_reports.R | 6 +++---
Report/score.R | 42 -----------------------------------------
2 files changed, 3 insertions(+), 45 deletions(-)
diff --git a/Report/create_reports.R b/Report/create_reports.R
index fac3761..3d85f08 100644
--- a/Report/create_reports.R
+++ b/Report/create_reports.R
@@ -134,9 +134,9 @@ if ( "confirmed_admissions_covid_1d" %in% unique(state_scores$signal)) {
print("Evaluating national forecasts")
-# COVIDcast does not return national level data, using CovidHubUtils instead
-
-nation_scores = evaluate_chu(nation_predictions, signals, err_measures)
+nation_scores_covidcast = evaluate_covid_predictions(nation_predictions,
+ err_measures,
+ geo_type = "nation")
if ( "confirmed_incidence_num" %in% unique(state_scores$signal)) {
print("Saving nation confirmed incidence...")
diff --git a/Report/score.R b/Report/score.R
index ac61312..888db9b 100644
--- a/Report/score.R
+++ b/Report/score.R
@@ -35,45 +35,3 @@ save_score_cards = function(score_card, geo_type = c("state", "nation"),
file = output_file_name,
compress = "xz")
}
-
-evaluate_chu = function(predictions, signals, err_measures) {
- allowed_signals = c("confirmed_incidence_num",
- "deaths_incidence_num",
- "confirmed_admissions_covid_1d")
- assert_that(all(signals %in% allowed_signals),
- msg = paste("Signal not allowed:",
- setdiff(signals, allowed_signals)))
-
- target_map <- list("confirmed_incidence_num" = "inc case",
- "deaths_incidence_num" = "inc death",
- "confirmed_admissions_covid_1d" = "inc hosp")
- source_map <- list("confirmed_incidence_num" = "JHU",
- "deaths_incidence_num" = "JHU",
- "confirmed_admissions_covid_1d" = "HealthData")
- scores = c()
- for (signal_name in signals) {
- preds_signal = predictions %>%
- filter(signal == signal_name)
- signal <- target_map[[signal_name]]
- source <- source_map[[signal_name]]
- chu_truth = covidHubUtils::load_truth(source, signal)
- chu_truth = chu_truth %>%
- rename(actual = value) %>%
- select(-c(model,
- target_variable,
- location,
- location_name,
- population,
- geo_type,
- abbreviation))
- signal_scores = evaluate_predictions(preds_signal,
- truth_data = chu_truth,
- err_measures,
- grp_vars = c("target_end_date",
- "geo_value",
- "ahead",
- "forecaster"))
- scores = rbind(scores, signal_scores)
- }
- return(scores)
-}
From 47ba974cf663416f7372b1b633aafa0cf3a563b4 Mon Sep 17 00:00:00 2001
From: Nat DeFries <42820733+nmdefries@users.noreply.github.com>
Date: Thu, 2 Sep 2021 16:05:36 -0400
Subject: [PATCH 2/2] fix nation scores names
---
Report/create_reports.R | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/Report/create_reports.R b/Report/create_reports.R
index 3d85f08..818a43f 100644
--- a/Report/create_reports.R
+++ b/Report/create_reports.R
@@ -134,11 +134,11 @@ if ( "confirmed_admissions_covid_1d" %in% unique(state_scores$signal)) {
print("Evaluating national forecasts")
-nation_scores_covidcast = evaluate_covid_predictions(nation_predictions,
+nation_scores = evaluate_covid_predictions(nation_predictions,
err_measures,
geo_type = "nation")
-if ( "confirmed_incidence_num" %in% unique(state_scores$signal)) {
+if ( "confirmed_incidence_num" %in% unique(nation_scores$signal)) {
print("Saving nation confirmed incidence...")
save_score_cards(nation_scores, "nation",
signal_name = "confirmed_incidence_num", output_dir = opt$dir)
@@ -146,7 +146,7 @@ if ( "confirmed_incidence_num" %in% unique(state_scores$signal)) {
warning("Nation confirmed incidence should generally be available. Please
verify that you expect not to have any cases incidence forecasts")
}
-if ( "deaths_incidence_num" %in% unique(state_scores$signal)) {
+if ( "deaths_incidence_num" %in% unique(nation_scores$signal)) {
print("Saving nation deaths incidence...")
save_score_cards(nation_scores, "nation", signal_name = "deaths_incidence_num",
output_dir = opt$dir)
@@ -154,7 +154,7 @@ if ( "deaths_incidence_num" %in% unique(state_scores$signal)) {
warning("Nation deaths incidence should generally be available. Please
verify that you expect not to have any deaths incidence forecasts")
}
-if ( "confirmed_admissions_covid_1d" %in% unique(state_scores$signal)) {
+if ( "confirmed_admissions_covid_1d" %in% unique(nation_scores$signal)) {
print("Saving nation hospitalizations...")
save_score_cards(nation_scores, "nation", signal_name = "confirmed_admissions_covid_1d",
output_dir = opt$dir)