From 4993ce656251df23ed967268b942002eaf8adef6 Mon Sep 17 00:00:00 2001 From: Tony Fujs Date: Fri, 27 Nov 2020 16:25:06 +0100 Subject: [PATCH 1/3] fix get_survey_list() max number of records was hard coded --- DESCRIPTION | 8 +++++--- NAMESPACE | 1 + R/get_survey_list.R | 27 ++++++++++++++++++++++++++- man/connect_mdlib.Rd | 8 ++++++-- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 91868c5..c037d30 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -9,9 +9,11 @@ Description: More about what it does (maybe more than one line) License: MIT + file LICENSE Encoding: UTF-8 LazyData: true -Imports: httr, - jsonlite -RoxygenNote: 6.1.1 +Imports: + httr, + jsonlite, + purrr +RoxygenNote: 7.1.1 Suggests: knitr, rmarkdown VignetteBuilder: knitr diff --git a/NAMESPACE b/NAMESPACE index f670851..847ad90 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -2,6 +2,7 @@ export(connect_mdlib) export(get_access_types) +export(get_data_dictionary) export(get_datafiles_list) export(get_doc_metadata) export(get_public_survey_list) diff --git a/R/get_survey_list.R b/R/get_survey_list.R index 90c636e..6feef53 100644 --- a/R/get_survey_list.R +++ b/R/get_survey_list.R @@ -7,8 +7,33 @@ #' get_survey_list <- function(token){ - out <- connect_mdlib(path = 'index.php/api/catalog/search?ps=10000', token = token) + + n_records <- get_number_of_surveys(token) + path <- paste0('index.php/api/catalog/search?ps=', n_records) + out <- connect_mdlib(path = path, token = token) out <- out$content$result$rows return(out) } + +get_number_of_surveys <- function(token) { + # Should be better integrated with the other functions (URL should not be hard coded) + url <- "http://microdatalib.worldbank.org/index.php/api/catalog/search?ps=1" + + resp <- httr::GET(url, + httr::add_headers(.headers = c('X-API-KEY' = token, + 'charset' = "utf-8")), + httr::accept_json()) + + # Return useful message on error + httr::stop_for_status(resp, task = 'complete request to Microdata library API\n') + # CHECK: datatype is .JSON + if (httr::http_type(resp) != "application/json") { + stop("API did not return json", call. = FALSE) + } + # Parse response + parsed <- jsonlite::fromJSON(httr::content(resp, "text"), simplifyVector = FALSE) + n_records <- parsed$result$total + + return(n_records) +} diff --git a/man/connect_mdlib.Rd b/man/connect_mdlib.Rd index e43e40e..fd869c4 100644 --- a/man/connect_mdlib.Rd +++ b/man/connect_mdlib.Rd @@ -4,8 +4,12 @@ \alias{connect_mdlib} \title{connect_mdlib} \usage{ -connect_mdlib(path, query = NULL, token, - root = "http://microdatalib.worldbank.org") +connect_mdlib( + path, + query = NULL, + token, + root = "http://microdatalib.worldbank.org" +) } \arguments{ \item{path}{character: path to send request} From b6917ca985bce54c4c330c157036a3c1a88f0133 Mon Sep 17 00:00:00 2001 From: Tony Fujs Date: Fri, 27 Nov 2020 16:25:46 +0100 Subject: [PATCH 2/3] add get_data_dictionary() --- R/get_data_dictionary.R | 38 ++++++++++++++++++++++++++++++++++++++ man/get_data_dictionary.Rd | 21 +++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 R/get_data_dictionary.R create mode 100644 man/get_data_dictionary.Rd diff --git a/R/get_data_dictionary.R b/R/get_data_dictionary.R new file mode 100644 index 0000000..4f03d05 --- /dev/null +++ b/R/get_data_dictionary.R @@ -0,0 +1,38 @@ +#' get_data_dictionary +#' +#' @param id numeric: Survey Unique ID +#' @param token character: Microdata API authentication token +#' @param limit character: maximum number of records per page +#' +#' @return list +#' @export +#' + +get_data_dictionary <- function(id, token, limit = NULL){ + if (is.null(limit)) { + path <- paste0('index.php/api/v2/metadata/list_variables/', id) + out <- connect_mdlib(path = path, token = token) + out <- out$content$items + + return(out) + + } else { + out <- vector(mode = 'list', length = 1000) + counter = 1 + + records_number <- limit + while (records_number == limit) { + temp_offset <- (counter - 1) * limit + path <- paste0('index.php/api/v2/metadata/list_variables/', id, '/', limit, '/', temp_offset) + temp_out <- connect_mdlib(path = path, token = token) + temp_out <- temp_out$content$items + out[[counter]] <- temp_out + counter <- counter + 1 + records_number <- length(temp_out) + } + + out <- purrr::flatten(out) + + return(out) + } +} diff --git a/man/get_data_dictionary.Rd b/man/get_data_dictionary.Rd new file mode 100644 index 0000000..7e4cb3a --- /dev/null +++ b/man/get_data_dictionary.Rd @@ -0,0 +1,21 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get_data_dictionary.R +\name{get_data_dictionary} +\alias{get_data_dictionary} +\title{get_data_dictionary} +\usage{ +get_data_dictionary(id, token, limit = NULL) +} +\arguments{ +\item{id}{numeric: Survey Unique ID} + +\item{token}{character: Microdata API authentication token} + +\item{limit}{character: maximum number of records per page} +} +\value{ +list +} +\description{ +get_data_dictionary +} From 9370271cc8be8f828bfe23ea4e1803334de66cb7 Mon Sep 17 00:00:00 2001 From: Tony Fujs Date: Fri, 27 Nov 2020 16:26:14 +0100 Subject: [PATCH 3/3] fix .Rmd check --- ... started with mdlibconnect.Rmd => getting_started.Rmd} | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) rename vignettes/{Getting started with mdlibconnect.Rmd => getting_started.Rmd} (78%) diff --git a/vignettes/Getting started with mdlibconnect.Rmd b/vignettes/getting_started.Rmd similarity index 78% rename from vignettes/Getting started with mdlibconnect.Rmd rename to vignettes/getting_started.Rmd index fa46741..adde35e 100644 --- a/vignettes/Getting started with mdlibconnect.Rmd +++ b/vignettes/getting_started.Rmd @@ -32,10 +32,14 @@ You can get your API KEY by: 1. Loging using your World Bank SecureID token 1. Clicking the __create new key__ button. +API documentation can be accessed [here](http://microdatalibqa.worldbank.org/apidocumentation/#api-Metadata-datafile_variables) + ### List all available surveys ```{r} -library(mdlibconnect) -get_survey_list(token = Sys.getenv('mdlib_token')) +devtools::load_all() +response <- get_survey_list(token = Sys.getenv('mdlib_token')) +str(response, max.level = 2, list.len = 3) + ```