Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace evaluate_chu with evalcast::evaluate_covid_predictions #165

Draft
wants to merge 2 commits into
base: dev
Choose a base branch
from
Draft

Replace evaluate_chu with evalcast::evaluate_covid_predictions #165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3de2537
replace evaluate_chu with evalcast::evaluate_covid_predictions
nmdefries Sep 2, 2021
47ba974
fix nation scores names
nmdefries Sep 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions Report/create_reports.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,27 +134,27 @@ if ( "confirmed_admissions_covid_1d" %in% unique(state_scores$signal)) {


print("Evaluating national forecasts")
# COVIDcast does not return national level data, using CovidHubUtils instead
nation_scores = evaluate_covid_predictions(nation_predictions,
err_measures,
geo_type = "nation")

nation_scores = evaluate_chu(nation_predictions, signals, err_measures)

if ( "confirmed_incidence_num" %in% unique(state_scores$signal)) {
if ( "confirmed_incidence_num" %in% unique(nation_scores$signal)) {
print("Saving nation confirmed incidence...")
save_score_cards(nation_scores, "nation",
signal_name = "confirmed_incidence_num", output_dir = opt$dir)
} else {
warning("Nation confirmed incidence should generally be available. Please
verify that you expect not to have any cases incidence forecasts")
}
if ( "deaths_incidence_num" %in% unique(state_scores$signal)) {
if ( "deaths_incidence_num" %in% unique(nation_scores$signal)) {
print("Saving nation deaths incidence...")
save_score_cards(nation_scores, "nation", signal_name = "deaths_incidence_num",
output_dir = opt$dir)
} else {
warning("Nation deaths incidence should generally be available. Please
verify that you expect not to have any deaths incidence forecasts")
}
if ( "confirmed_admissions_covid_1d" %in% unique(state_scores$signal)) {
if ( "confirmed_admissions_covid_1d" %in% unique(nation_scores$signal)) {
print("Saving nation hospitalizations...")
save_score_cards(nation_scores, "nation", signal_name = "confirmed_admissions_covid_1d",
output_dir = opt$dir)
Expand Down
42 changes: 0 additions & 42 deletions Report/score.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,45 +35,3 @@ save_score_cards = function(score_card, geo_type = c("state", "nation"),
file = output_file_name,
compress = "xz")
}

evaluate_chu = function(predictions, signals, err_measures) {
allowed_signals = c("confirmed_incidence_num",
"deaths_incidence_num",
"confirmed_admissions_covid_1d")
assert_that(all(signals %in% allowed_signals),
msg = paste("Signal not allowed:",
setdiff(signals, allowed_signals)))

target_map <- list("confirmed_incidence_num" = "inc case",
"deaths_incidence_num" = "inc death",
"confirmed_admissions_covid_1d" = "inc hosp")
source_map <- list("confirmed_incidence_num" = "JHU",
"deaths_incidence_num" = "JHU",
"confirmed_admissions_covid_1d" = "HealthData")
scores = c()
for (signal_name in signals) {
preds_signal = predictions %>%
filter(signal == signal_name)
signal <- target_map[[signal_name]]
source <- source_map[[signal_name]]
chu_truth = covidHubUtils::load_truth(source, signal)
chu_truth = chu_truth %>%
rename(actual = value) %>%
select(-c(model,
target_variable,
location,
location_name,
population,
geo_type,
abbreviation))
signal_scores = evaluate_predictions(preds_signal,
truth_data = chu_truth,
err_measures,
grp_vars = c("target_end_date",
"geo_value",
"ahead",
"forecaster"))
scores = rbind(scores, signal_scores)
}
return(scores)
}