diff --git a/DESCRIPTION b/DESCRIPTION index 69d9ce2..32442f9 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: jmpwashdata Type: Package Title: WHO/UNICEF Joint Monitoring Programme Water and Sanitation Data -Version: 0.1.3.9000 +Version: 0.1.4 Author: Nicolas Dickinson Maintainer: Nicolas Dickinson Description: As a convenience, this package reproduces a snapshot of the JMP WASH household (currently as of July 2021), WASH in schools and WASH in health care facilities data that is normally available in Excel sheets on https://washdata.org. diff --git a/data-raw/jmpdatadownload.R b/data-raw/jmpdatadownload.R index de4365c..38d7119 100644 --- a/data-raw/jmpdatadownload.R +++ b/data-raw/jmpdatadownload.R @@ -8,6 +8,8 @@ library(stringr) .refresh_data_files <- function() { + .remove_previous_data() + page <- .download_page() wld_data <- .download_aggregate_files(page, target = "WLD") @@ -28,6 +30,16 @@ library(stringr) compress = "bzip2") } +.remove_previous_data <- function() { + unlink("data-raw/healthcare",recursive=TRUE) + unlink("data-raw/household",recursive=TRUE) + unlink("data-raw/inequalities",recursive=TRUE) + unlink("data-raw/REG",recursive=TRUE) + unlink("data-raw/schools",recursive=TRUE) + unlink("data-raw/WLD",recursive=TRUE) +} + + ## Data is updated no more than annually as of 2021-06-22 # target = "WLD" or "REG" .download_aggregate_files <- function(page, target = "WLD") { @@ -38,10 +50,11 @@ library(stringr) .collect_metadata <- function(links) { lapply(seq_along(links), function(i, x) { .wait_approx(.1) + message("Getting filename for ", x[[i]]) jmp_excel <- HEAD("https://washdata.org/", path=x[[i]]) filename = str_extract( jmp_excel$headers$`content-disposition`, - '(?<=").{1,128}(?=")' + '(?<=attachment; filename\\=).{1,128}(?=)' ) type = str_extract( x[[i]], @@ -64,12 +77,16 @@ library(stringr) files <- .collect_metadata(links) lapply(files, function(x) { print(x[1,"filename"]) + print(paste0("https://washdata.org", + x[1,"path"])) + target_dir <- paste0("data-raw/", if (is.na(folder)) x[1, "type"] else folder) + if (!dir.exists(target_dir)) dir.create(target_dir) .wait_approx(0.5) print(jmp_excel <- RETRY("GET", paste0("https://washdata.org", x[1,"path"]), - write_disk(paste0("data-raw/", if (is.na(folder)) x[1, "type"] else folder,"/",x[1,"filename"]), overwrite = overwrite), + write_disk(path = paste0(target_dir,"/",x[1,"filename"]), overwrite = overwrite), verbose() - )) + ), timeout(1)) }) bind_rows(files) } diff --git a/data-raw/jmpdataextract.R b/data-raw/jmpdataextract.R index 5ad21e0..1f7015e 100644 --- a/data-raw/jmpdataextract.R +++ b/data-raw/jmpdataextract.R @@ -4,6 +4,8 @@ library(rio) library(dplyr) library(tidyr) +library(futile.logger) +library(tryCatchLog) load("data/jmp_files.rda") @@ -130,7 +132,7 @@ var_attr <- function(x, attr_name, unlist = FALSE) { message = c(names(warnings()), error_txt), message_type = c(rep("warning", times = length(names(warnings()))), rep("error", times = length(error_txt))) ) - assign("last.warning", NULL, envir = baseenv()) + tryCatch(assign("last.warning", NULL, envir = baseenv()), warning = function(cond) {invisible()}) }) %>% bind_rows() usethis::use_data( @@ -227,7 +229,7 @@ var_attr <- function(x, attr_name, unlist = FALSE) { jmp_household_watsan_sources <- lapply(countries$geo, function(x) { hh_path <- paste0("data-raw/household/", filter(jmp_files, geo == x, type == "household")$filename) - print(paste0("Watsan summary from: ", hh_path)) + message(paste0("Watsan summary from: ", hh_path)) watsan_summary_data <- readxl::read_excel(hh_path, sheet = "Chart Data", range="A5:CL208", col_names = TRUE, col_types = c(rep("text", 2), rep("numeric", 88))) watsan_summary_data <- watsan_summary_data %>% filter(if_any(everything(), ~ (!is.na(.)&.!=0))) @@ -245,7 +247,7 @@ var_attr <- function(x, attr_name, unlist = FALSE) { jmp_household_hygiene_sources <- lapply(countries$geo, function(x) { hh_path <- paste0("data-raw/household/", filter(jmp_files, geo == x, type == "household")$filename) - print(paste0("Hygiene summary from: ", hh_path)) + message(paste0("Hygiene summary from: ", hh_path)) hyg_summary_data <- readxl::read_excel(hh_path, sheet = "Chart Data", range="CM5:CU208", col_names = TRUE, col_types = c(rep("text", 2), rep("numeric", 7))) hyg_summary_data <- hyg_summary_data %>% filter(if_any(everything(), ~ (!is.na(.)&.!=0))) @@ -271,6 +273,8 @@ var_attr <- function(x, attr_name, unlist = FALSE) { ### Procedure to extract inequality data .extract_inequalities_estimate_data <- function() { + message("--- Starting to extract inequality estimate data ---") + countries <- jmp_files %>% filter(type == "inequalities", !(geo %in% c("WLD", "REG"))) use_data <- usethis::use_data @@ -279,12 +283,12 @@ var_attr <- function(x, attr_name, unlist = FALSE) { lapply(c("water", "sanitation"), function(service_type) { dataset_name <- paste0("jmp_inequality_",service_type,"_estimate") - print(dataset_name) + message(dataset_name) dataset <- lapply(countries$geo, function(x) { ineq_path <- paste0("data-raw/inequalities/", filter(countries, geo == x)$filename) - print(ineq_path) + message(sprintf("Extracting from %s", ineq_path)) .get_watsan_quintile_estimates( ineq_path = ineq_path, @@ -302,6 +306,8 @@ var_attr <- function(x, attr_name, unlist = FALSE) { } .extract_inequalities_region_data <- function(verbose = FALSE) { + message("--- Starting to extract inequality region data ---") + countries <- jmp_files %>% filter(type == "inequalities", !(geo %in% c("WLD", "REG"))) #%>% filter(geo == "NPL") @@ -335,6 +341,8 @@ var_attr <- function(x, attr_name, unlist = FALSE) { } .extract_inequalities_source_data <- function() { + message("--- Starting to extract source data ---") + countries <- jmp_files %>% filter(type == "inequalities", !(geo %in% c("WLD", "REG"))) #%>% slice_head(n = 2) @@ -368,6 +376,8 @@ var_attr <- function(x, attr_name, unlist = FALSE) { .extract_inequalities_data_summary <- function() { + message("--- Starting to extract inequality data summary ---") + countries <- jmp_files %>% filter(type == "inequalities", !(geo %in% c("WLD", "REG"))) #%>% slice_head(n = 2) use_data <- usethis::use_data @@ -557,14 +567,14 @@ var_attr <- function(x, attr_name, unlist = FALSE) { # for later - would be more readable to name the residence ranges lapply(1:3, function(x, ranges) { - message(x) + message(sprintf("ranges$residence[[%d]]", x)) quin_vars <- suppressMessages( readxl::read_excel(ineq_path, sheet = sheet, range=ranges$residence[[x]], col_names = TRUE) ) %>% .estimate_quintile_vars(iso3) lapply(1:5, function(y, quintile_list) { - print(y*1000) + message(sprintf("quintile %d", y)) df_quin <- suppressMessages( readxl::read_excel(ineq_path, sheet = sheet, range=as.character(quintile_list[y]), col_names = TRUE) ) diff --git a/data/jmp_extraction_messages.rda b/data/jmp_extraction_messages.rda index 28cbf89..bd3cda0 100644 Binary files a/data/jmp_extraction_messages.rda and b/data/jmp_extraction_messages.rda differ diff --git a/data/jmp_files.rda b/data/jmp_files.rda index 7edc422..cd9aaeb 100644 Binary files a/data/jmp_files.rda and b/data/jmp_files.rda differ diff --git a/data/jmp_healthcare_reg_env_cleaning.rda b/data/jmp_healthcare_reg_env_cleaning.rda index b53e065..3c9753f 100644 Binary files a/data/jmp_healthcare_reg_env_cleaning.rda and b/data/jmp_healthcare_reg_env_cleaning.rda differ diff --git a/data/jmp_healthcare_reg_hygiene.rda b/data/jmp_healthcare_reg_hygiene.rda index 7dbd91e..02b4eef 100644 Binary files a/data/jmp_healthcare_reg_hygiene.rda and b/data/jmp_healthcare_reg_hygiene.rda differ diff --git a/data/jmp_healthcare_reg_sanitation.rda b/data/jmp_healthcare_reg_sanitation.rda index 7520d65..dcfb102 100644 Binary files a/data/jmp_healthcare_reg_sanitation.rda and b/data/jmp_healthcare_reg_sanitation.rda differ diff --git a/data/jmp_healthcare_reg_waste_man.rda b/data/jmp_healthcare_reg_waste_man.rda index 799a659..a774f8c 100644 Binary files a/data/jmp_healthcare_reg_waste_man.rda and b/data/jmp_healthcare_reg_waste_man.rda differ diff --git a/data/jmp_healthcare_reg_water.rda b/data/jmp_healthcare_reg_water.rda index 9fc744f..c6110de 100644 Binary files a/data/jmp_healthcare_reg_water.rda and b/data/jmp_healthcare_reg_water.rda differ diff --git a/data/jmp_healthcare_wld_env_cleaning.rda b/data/jmp_healthcare_wld_env_cleaning.rda index 4174253..5db60c2 100644 Binary files a/data/jmp_healthcare_wld_env_cleaning.rda and b/data/jmp_healthcare_wld_env_cleaning.rda differ diff --git a/data/jmp_healthcare_wld_hygiene.rda b/data/jmp_healthcare_wld_hygiene.rda index 1dcfa97..4c21566 100644 Binary files a/data/jmp_healthcare_wld_hygiene.rda and b/data/jmp_healthcare_wld_hygiene.rda differ diff --git a/data/jmp_healthcare_wld_sanitation.rda b/data/jmp_healthcare_wld_sanitation.rda index a6c2a82..acf4593 100644 Binary files a/data/jmp_healthcare_wld_sanitation.rda and b/data/jmp_healthcare_wld_sanitation.rda differ diff --git a/data/jmp_healthcare_wld_waste_man.rda b/data/jmp_healthcare_wld_waste_man.rda index f0f1eac..26d7cdf 100644 Binary files a/data/jmp_healthcare_wld_waste_man.rda and b/data/jmp_healthcare_wld_waste_man.rda differ diff --git a/data/jmp_healthcare_wld_water.rda b/data/jmp_healthcare_wld_water.rda index 07fdf36..e2a06f1 100644 Binary files a/data/jmp_healthcare_wld_water.rda and b/data/jmp_healthcare_wld_water.rda differ diff --git a/data/jmp_household_watsan_sources.rda b/data/jmp_household_watsan_sources.rda index 7ea9a30..d851e03 100644 Binary files a/data/jmp_household_watsan_sources.rda and b/data/jmp_household_watsan_sources.rda differ diff --git a/data/jmp_schools_reg_hygiene.rda b/data/jmp_schools_reg_hygiene.rda index 402faff..080d9ec 100644 Binary files a/data/jmp_schools_reg_hygiene.rda and b/data/jmp_schools_reg_hygiene.rda differ diff --git a/data/jmp_schools_reg_sanitation.rda b/data/jmp_schools_reg_sanitation.rda index fd8ccd9..8aac3bc 100644 Binary files a/data/jmp_schools_reg_sanitation.rda and b/data/jmp_schools_reg_sanitation.rda differ diff --git a/data/jmp_schools_reg_water.rda b/data/jmp_schools_reg_water.rda index 9f49c64..2453c7e 100644 Binary files a/data/jmp_schools_reg_water.rda and b/data/jmp_schools_reg_water.rda differ diff --git a/data/jmp_schools_wld_hygiene.rda b/data/jmp_schools_wld_hygiene.rda index 032d0ca..de345a7 100644 Binary files a/data/jmp_schools_wld_hygiene.rda and b/data/jmp_schools_wld_hygiene.rda differ diff --git a/data/jmp_schools_wld_sanitation.rda b/data/jmp_schools_wld_sanitation.rda index a78b3f3..a7fe4e3 100644 Binary files a/data/jmp_schools_wld_sanitation.rda and b/data/jmp_schools_wld_sanitation.rda differ diff --git a/data/jmp_schools_wld_water.rda b/data/jmp_schools_wld_water.rda index 35c2db6..24bf9bd 100644 Binary files a/data/jmp_schools_wld_water.rda and b/data/jmp_schools_wld_water.rda differ