diff --git a/.gitignore b/.gitignore index 91c6237..2853d72 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,9 @@ # Keep tex and png in cloud resources training !cloud-resources/**/*.tex !cloud-resources/**/*.png + + +# Keep tex, png, and pdf in survey review checklists +!survey-review-checklists/**/*.tex +!survey-review-checklists/**/*.png +!survey-review-checklists/**/*.pdf \ No newline at end of file diff --git a/survey-review-checklists/README.md b/survey-review-checklists/README.md new file mode 100644 index 0000000..f466356 --- /dev/null +++ b/survey-review-checklists/README.md @@ -0,0 +1,3 @@ +# DIME Analytics - Survey Review Checklists + +To update details \ No newline at end of file diff --git a/survey-review-checklists/data_assurance.pdf b/survey-review-checklists/data_assurance.pdf new file mode 100644 index 0000000..fde4ac8 Binary files /dev/null and b/survey-review-checklists/data_assurance.pdf differ diff --git a/survey-review-checklists/data_assurance.tex b/survey-review-checklists/data_assurance.tex new file mode 100644 index 0000000..6f30bef --- /dev/null +++ b/survey-review-checklists/data_assurance.tex @@ -0,0 +1,183 @@ + +\documentclass{report} +\usepackage[most]{tcolorbox} + +\usepackage{nopageno} +\usepackage{fancyhdr} +\usepackage{adjustbox} + +\usepackage{booktabs,tabularx} +\usepackage{hyperref} +%\usepackage{showframe} +\usepackage{xcolor} +\usepackage{graphics} +\pagestyle{fancy} +\fancyhf{} +\renewcommand{\headrulewidth}{0pt} +\lfoot{\includegraphics[height=1cm,keepaspectratio]{img/i2i}} +\cfoot{\includegraphics[height=1cm,keepaspectratio]{img/wb}} +\rfoot{\includegraphics[height=1cm,keepaspectratio]{img/analytics}} +\definecolor{fontcolor}{HTML}{7A0569} +\newcommand{\titleBox}[1]{ + \begin{tcolorbox} + [colframe = fontcolor, + colback = fontcolor, + sharp corners, + halign = flush center, + valign = center, + height = 0.2\textwidth, + after skip = 1cm] + #1 + \end{tcolorbox} +} + + + +%\input{checklist-header.tex} + + + + +% ------------------------------ End of preamble --------------------------------------------- +\begin{document} + + \titleBox{ + \textcolor{white}{\Large{DIME Analytics Data Quality Assurance Checklist}} + } + + \section*{Objective} + In the process of acquiring data, especially primary data, many factors could lead to poor data quality. A plan to ensure quality of data is one that identifies the potential issues in the data, and tracks data quality throughout the process of acquiring data. + + \vspace{5mm} %5mm vertical space + \section*{Procurement} + +\newcounter{row} + +\newcommand{\makerow}[1]{% + % #1 = text field + #1 & + \stepcounter{row}% + \mbox{\CheckBox[print,name=YES\therow, width=0.7em, height=0.7em]{}} & + \mbox{\CheckBox[print,name=NO\therow, width=0.7em, height=0.7em]{}} & + \mbox{\CheckBox[print,name=N/A\therow, width=0.7em, height=0.7em]{}} \\ +} + + +\begin{Form} + \noindent + \begin{tabularx}{\textwidth}{Xccc} + \toprule + \textbf{CHECKS} & YES & NO & N/A \\ + \midrule + \makerow{Does the TOR establish \textbf{minimum quality indicators} required for the data to be considered acceptable?} + \midrule + \makerow{Does the TOR establish a \textbf{maximum response time} for the survey firm to address data quality issues identified?} + \bottomrule + \end{tabularx} + + \vspace{5mm} %5mm vertical space + + \section*{Survey Design} + + + + +\begin{tabularx}{\textwidth}{Xccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Does the survey have \textbf{relevance and constraint logic} set up to aid the enumerator?} +\midrule +\makerow{Does the survey have an \textbf{an ID variable} to identify respondents and link them to the respondents database?} +\midrule +\makerow{Does the survey have an \textbf{unique ID generator} for each survey submission (if survey is being done over SurveyCTO this is automatically generated as the ‘key’ variable.} +\midrule +\makerow{Does the survey include calculation of \textbf{duration} of the full survey and each module within the survey?} +\midrule + +\makerow{Has the \textbf{translation} been approved?} +\midrule +\makerow{Does \textbf{enumerator training materials} include explanation of informed consent, Q\&A session, mock interviews and review of best practices?} +\midrule +\makerow{In case of projects with multiple survey instruments, does each survey have it’s own detailed survey manual?} + +\bottomrule +\end{tabularx} + +\vspace{5mm} %5mm vertical space + + + + \section*{High-Frequency Checks (HFC)} + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Is there a plan in place to conduct \textbf{high-frequency checks} on the data being collected?} +\midrule +\makerow{Does the HFC \textbf{monitor survey duration} and start/end times and day of the week of a survey fill-out?} +\midrule +\makerow{Do the HFCs check for key treatment and \textbf{outcome variables?}} +\midrule +\makerow{Do the HFCs check for \textbf{sample completeness?}} +\midrule +\makerow{Do the HFCs report \textbf{duplicates, missing values and errors} in logic that are not automatically corrected by the survey form?} +\midrule +\makerow{Is the process for generating flags from HFCs \textbf{easy and fast to run} on a daily basis for FCs?} +\midrule +\makerow{Does the HFC process account for potential lack of \textbf{access to WiFi} or electricity for team on the field?} +\midrule +\makerow{Do HFCs monitor geographic location of projects based on \textbf{GPS coordinates?}} +\midrule +\makerow{Has the HFC plan been communicated to all \textbf{stakeholders} in the team (incl. survey firm) with enough time before survey round?} +\bottomrule +\end{tabularx} + +\vspace{5mm} %5mm vertical space + +\section*{Backchecks and audits} + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Is there a plan in place to conduct \textbf{back-checks} on the data collected?} +\midrule +\makerow{Does the back-checks survey instrument include \textbf{variables unlikely to change}, as well as variables that stem from difficult or lengthy modules? } +\midrule +\makerow{Is there a plan to audit through \textbf{audio recordings}? If so, is it mentioned in the informed consent?} +\midrule +\makerow{Does the back-checks plan include back-checking \textbf{all enumerators?}} +\midrule +\makerow{Does the back-check plan include back-checking \textbf{20 percent of the sample} in the first 2 weeks and 10 percent of the sample in the following weeks?} +\bottomrule +\end{tabularx} + + \section*{Communication and Field Management} + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} + \toprule + \textbf{CHECKS} & YES & NO & N/A \\ + \midrule + \makerow{Does team have a plan for \textbf{communication and data quality checks feedback} with enumerators?} + \midrule + \makerow{Is there a protocol in place for the field team to \textbf{gather information and respond to flags} raised during data quality checks?} + \midrule + \makerow{Has the team put together \textbf{tracking systems} for enumerators to fill out as part of their daily tasks?} + \midrule + \makerow{Does the team have a plan for \textbf{reconciling data} from the tracking system with the responses on the server} + \midrule + \makerow{If feasible in the context, does the team plan on research staff \textbf{accompanying enumerator}? } + \bottomrule +\end{tabularx} + + +\end{Form} + + + +\end{document} diff --git a/survey-review-checklists/electronic_survey.pdf b/survey-review-checklists/electronic_survey.pdf new file mode 100644 index 0000000..3ce91d7 Binary files /dev/null and b/survey-review-checklists/electronic_survey.pdf differ diff --git a/survey-review-checklists/electronic_survey.tex b/survey-review-checklists/electronic_survey.tex new file mode 100644 index 0000000..eb0ceaf --- /dev/null +++ b/survey-review-checklists/electronic_survey.tex @@ -0,0 +1,180 @@ +\documentclass{report} +\usepackage[left=2cm, right=2cm, top=2cm]{geometry} +\usepackage[most]{tcolorbox} + +\usepackage{nopageno} +\usepackage{fancyhdr} +\usepackage{adjustbox} + +\usepackage{booktabs,tabularx} +\usepackage{hyperref} +%\usepackage{showframe} +\usepackage{xcolor} +\usepackage{graphics} +\pagestyle{fancy} +\fancyhf{} +\renewcommand{\headrulewidth}{0pt} +\lfoot{\includegraphics[height=1cm,keepaspectratio]{img/i2i}} +\cfoot{\includegraphics[height=1cm,keepaspectratio]{img/wb}} +\rfoot{\includegraphics[height=1cm,keepaspectratio]{img/analytics}} +\definecolor{fontcolor}{HTML}{7A0569} +\newcommand{\titleBox}[1]{ + \begin{tcolorbox} + [colframe = fontcolor, + colback = fontcolor, + sharp corners, + halign = flush center, + valign = center, + height = 0.2\textwidth, + after skip = 1cm] + #1 + \end{tcolorbox} +} + + +% ------------------------------ End of preamble --------------------------------------------- +\begin{document} + + +\titleBox{ + \textcolor{white}{\Large{Project: Name}} \\ + \textcolor{white}{\Large{Survey: Round}} \\ + \textcolor{white}{\Large{DIME Analytics Electronic Survey Review Checklist}} +} + +This report details the findings from review of the electronic survey for the \textit{Survey Round} of the \textit{Project name} project. +\section*{Main findings} + +\subsection*{Required practices} +\begin{itemize} + \item \textit{List required practices identified for project} +\end{itemize} +\subsection*{Recommended practices} + +The \href{https://dimewiki.worldbank.org/Ietestform}{ietestform} command (from the iefieldkit package) was run and the results are attached. It is a good practice to run iestestform multiple times during survey development. This is to make sure you’ve incorporated all suggested changes that are relevant and to re-test when new questions are added during the development phase of the survey. +\begin{itemize} + \item \textit{List recommended practices identified for project} +\end{itemize} + +The checklist in the next section details the exact findings. +\newpage + +\section*{Checklist} + + \subsection*{Encryption} + +\newcounter{row} + +\newcommand{\makerow}[1]{% + % #1 = text field + #1 & + \stepcounter{row}% + \mbox{\CheckBox[print,name=YES\therow, width=0.7em, height=0.7em]{}} & + \mbox{\CheckBox[print,name=NO\therow, width=0.7em, height=0.7em]{}} & + \mbox{\CheckBox[print,name=N/A\therow, width=0.7em, height=0.7em]{}} \\ +} + + +\begin{Form} +\noindent +\begin{tabularx}{\textwidth}{Xccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Survey form \textbf{encrypted}} +\midrule +\makerow{Survey form distinguish \textbf{publishable and non-publishable} fields} +\bottomrule +\end{tabularx} + + \subsection*{Ensuring data quality} + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{The survey incorporates \textbf{validation of key variables} (ID number, phone number etc)} +\midrule +\makerow{Answer choices \textbf{systematically coded} (ie, -9 for "Other", -8 for "Do not know", and -7 for "Prefer not to answer”)} +\midrule +\makerow{The survey makes use of \textbf{constraints} on numeric variables to minimize entry errors} +\midrule +\makerow{The survey has appropriate \textbf{skip patterns and relevance} (ie, the question “What level of schooling are you in?” is only relevant if the question “Are you in school” is answered Yes)} +\midrule +\makerow{Potentially required \textbf{calculations} carried out within the survey instrument} +\bottomrule +\end{tabularx} + + +\vspace{5mm} %5mm vertical space +\subsection*{Form usability} + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} + \toprule + \textbf{CHECKS} & YES & NO & N/A \\ + \midrule + \makerow{Survey form is \textbf{translated} to local language} + \midrule + \makerow{If translated, \textbf{supplemental csv’s} is in UTF-8 format (to ensure special characters in translation are shown correctly)} + \midrule + \makerow{\textbf{Combined different modules} into clearly defined groups for easy viewing} + \midrule + \makerow{\textbf{Includes notes} throughout to clearly guide enumerators/respondents} + \bottomrule +\end{tabularx} + +\newpage +\subsection*{ietestform results} +The following checks are identified by the report generated by running the \textbf{ietestform} command. +\vspace{5mm} %5mm vertical space + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Non-note type fields that are required} +\midrule +\makerow{Note type fields that are not required unless required} +\midrule +\makerow{“begin group” and “end group” fields match} +\midrule +\makerow{“begin repeat” and “end repeat” fields match} +\midrule +\makerow{Field names valid for Stata import} +\midrule +\makerow{Field labels valid for Stata import} +\midrule +\makerow{Choice labels valid for Stata import} +\midrule +\makerow{Field labels of appropriate length for Stata import} +\midrule +\makerow{Field names of appropriate length for Stata Import} +\midrule +\makerow{Field names are unique} +\midrule +\makerow{Fields without Leading or Trailing spaces} +\midrule +\makerow{No duplicate Labels in Choice list} +\midrule +\makerow{Only numeric value in choice name or value} +\midrule +\makerow{No unused choice lists} +\midrule +\makerow{No duplicate Values in Choice list} +\midrule +\makerow{No missing label, value or name of choice list} +\midrule +\makerow{Latest version of SurveyCTO syntax is used} +\bottomrule +\end{tabularx} + + + +\end{Form} + + +\end{document} + diff --git a/survey-review-checklists/img/analytics.png b/survey-review-checklists/img/analytics.png new file mode 100644 index 0000000..e989f86 Binary files /dev/null and b/survey-review-checklists/img/analytics.png differ diff --git a/survey-review-checklists/img/i2i.png b/survey-review-checklists/img/i2i.png new file mode 100644 index 0000000..9b2bbe9 Binary files /dev/null and b/survey-review-checklists/img/i2i.png differ diff --git a/survey-review-checklists/img/wb.png b/survey-review-checklists/img/wb.png new file mode 100644 index 0000000..bd31498 Binary files /dev/null and b/survey-review-checklists/img/wb.png differ diff --git a/survey-review-checklists/papersurvey.pdf b/survey-review-checklists/papersurvey.pdf new file mode 100644 index 0000000..ff6471e Binary files /dev/null and b/survey-review-checklists/papersurvey.pdf differ diff --git a/survey-review-checklists/papersurvey.tex b/survey-review-checklists/papersurvey.tex new file mode 100644 index 0000000..99e1f53 --- /dev/null +++ b/survey-review-checklists/papersurvey.tex @@ -0,0 +1,197 @@ +\documentclass{report} +\usepackage[left=2cm, right=2cm, top=2cm]{geometry} +\usepackage[most]{tcolorbox} + +\usepackage{nopageno} +\usepackage{fancyhdr} +\usepackage{adjustbox} + +\usepackage{booktabs,tabularx} +\usepackage{hyperref} +%\usepackage{showframe} +\usepackage{xcolor} +\usepackage{graphics} +\pagestyle{fancy} +\fancyhf{} +\renewcommand{\headrulewidth}{0pt} +\lfoot{\includegraphics[height=1cm,keepaspectratio]{img/i2i}} +\cfoot{\includegraphics[height=1cm,keepaspectratio]{img/wb}} +\rfoot{\includegraphics[height=1cm,keepaspectratio]{img/analytics}} +\definecolor{fontcolor}{HTML}{7A0569} +\newcommand{\titleBox}[1]{ + \begin{tcolorbox} + [colframe = fontcolor, + colback = fontcolor, + sharp corners, + halign = flush center, + valign = center, + height = 0.2\textwidth, + after skip = 1cm] + #1 + \end{tcolorbox} +} + + +% ------------------------------ End of preamble --------------------------------------------- +\begin{document} + + + \titleBox{ + \textcolor{white}{\Large{Project: Name}} \\ + \textcolor{white}{\Large{Survey: Round}} \\ + \textcolor{white}{\Large{DIME Analytics Paper Survey Review Checklist}} + } + + \section{Objective} + Project teams will submit the paper survey along with a pre-analysis plan/i2i table/concept note. The objective of the review is to ensure that the survey designed is collecting all required information in a manner conducive to the respondents. + + \section*{Main findings} + +This report details the findings from review of the paper survey designed for the \textit{Survey Round} of the \textit{Project name} project. + +\subsection*{Required practices} +\begin{itemize} + \item \textit{List required practices identified for project} +\end{itemize} +\subsection*{Recommended practices} + +The \href{https://dimewiki.worldbank.org/Ietestform}{ietestform} command (from the iefieldkit package) was run and the results are attached. It is a good practice to run iestestform multiple times during survey development. This is to make sure you’ve incorporated all suggested changes that are relevant and to re-test when new questions are added during the development phase of the survey. +\begin{itemize} + \item \textit{List recommended practices identified for project} +\end{itemize} + +The checklist in the next section details the exact findings. +\newpage + +\section*{Checklist} + + + \subsection*{General} + +\newcounter{row} + +\newcommand{\makerow}[1]{% + % #1 = text field + #1 & + \stepcounter{row}% + \mbox{\CheckBox[print,name=YES\therow, width=0.7em, height=0.7em]{}} & + \mbox{\CheckBox[print,name=NO\therow, width=0.7em, height=0.7em]{}} & + \mbox{\CheckBox[print,name=N/A\therow, width=0.7em, height=0.7em]{}} \\ +} + + +\begin{Form} +\noindent +\begin{tabularx}{\textwidth}{Xccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Survey begins with informed consent} +\bottomrule +\end{tabularx} + +\vspace{5mm} %5mm vertical space +\textbf{Consent section includes:} +\vspace{5mm} %5mm vertical space + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Project \textbf{Title}, topic and organization leading project} +\midrule +\makerow{\textbf{Duration} of survey and/or project} +\midrule +\makerow{Potential \textbf{Benefits and Risks} to respondents} +\midrule +\makerow{\textbf{Participation} and Withdrawal consequences} +\midrule +\makerow{Privacy and \textbf{Confidentiality}} +\midrule +\makerow{Details of survey firm \textbf{contact point} for respondents' questions} +\midrule +\makerow{\emph{(If in-person survey)} \textbf{COVID-19} risks and survey staff's current health status} +\bottomrule +\end{tabularx} + +\vspace{5mm} %5mm vertical space + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Has the survey been translated to the \textbf{local language(s)}?} +\midrule +\makerow{Are all the \textbf{outcomes of interest} in the PAP/i2i table reflected in the survey?} +\bottomrule +\end{tabularx} + +\vspace{5mm} %5mm vertical space +\subsection*{Survey Flow} +\vspace{5mm} %5mm vertical space + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Is the survey divided into \textbf{multiple modules?}} +\midrule +\makerow{Does each module have an \textbf{introductory script?}} +\midrule +\makerow{Could the \textbf{order} of survey modules be improved for better respondent understanding?} +\midrule +\makerow{Are there \textbf{open-ended} questions which can be phrased as questions with choice lists in the survey?} +\midrule +\makerow{Are there any questions in the survey that could be \textbf{ambiguous} or that the same person could answer in two different ways?} +\midrule +\makerow{If so, does the survey have \textbf{disclaimers} or clarifying statements/examples to guide respondent?} +\bottomrule +\end{tabularx} + +\vspace{5mm} %5mm vertical space + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Are there \textbf{sensitive questions} in the survey that could make the respondent uncomfortable?} +\midrule +\makerow{If so, does the survey incorporate strategies to mitigate this? (ie, \textbf{ensure anonymity}, ask for consent for that module, ensure the enumerator is in private with the respondent, self-administration of module)} +\midrule +\makerow{Are questions asked only when \textbf{relevant}? (ie, the question “What level of schooling are you in?” is only relevant if the question “Are you in school” is answered Yes)} +\midrule +\makerow{Are there modules that repeat for several people or items? (ie, a \textbf{loop})} +\midrule +\makerow{If so, is the loop \textbf{clearly indicated} to enumerators in the survey?} +\bottomrule +\end{tabularx} + +\vspace{5mm} %5mm vertical space +\subsection*{Improving data quality during data collection} +\vspace{5mm} %5mm vertical space + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Does the survey have a \textbf{unique id} indicator for each respondent?} +\midrule +\makerow{Are answer choices \textbf{systematically coded}? (ie, -9 for "Other", -8 for "Do not know", and -7 for "Prefer not to answer”)} +\midrule +\makerow{Are there questions in the survey that could be subject to \textbf{recall bias?}} +\midrule +\makerow{If so, does the survey incorporate strategies to \textbf{mitigate the bias}? (ie, multiple measurements, consistency checks)} +\midrule +\makerow{If there is no way to ask the question while avoiding bias, has the team considered avoiding that question and obtaining that information through \textbf{alternate means}? (ie, administrative data, geospatial data, triangulation)} +\bottomrule +\end{tabularx} +\end{Form} + + +\end{document} + diff --git a/survey-review-checklists/survey_pilot.pdf b/survey-review-checklists/survey_pilot.pdf new file mode 100644 index 0000000..c26d958 Binary files /dev/null and b/survey-review-checklists/survey_pilot.pdf differ diff --git a/survey-review-checklists/survey_pilot.tex b/survey-review-checklists/survey_pilot.tex new file mode 100644 index 0000000..8745950 --- /dev/null +++ b/survey-review-checklists/survey_pilot.tex @@ -0,0 +1,295 @@ +\documentclass{report} +\usepackage[left=2cm, right=2cm, top=2cm]{geometry} +\usepackage[most]{tcolorbox} + +\usepackage{nopageno} +\usepackage{fancyhdr} +\usepackage{adjustbox} + +\usepackage{booktabs,tabularx} +\usepackage{hyperref} +%\usepackage{showframe} +\usepackage{xcolor} +\usepackage{graphics} +\pagestyle{fancy} +\fancyhf{} +\renewcommand{\headrulewidth}{0pt} +\lfoot{\includegraphics[height=1cm,keepaspectratio]{img/i2i}} +\cfoot{\includegraphics[height=1cm,keepaspectratio]{img/wb}} +\rfoot{\includegraphics[height=1cm,keepaspectratio]{img/analytics}} +\definecolor{fontcolor}{HTML}{7A0569} +\newcommand{\titleBox}[1]{ + \begin{tcolorbox} + [colframe = fontcolor, + colback = fontcolor, + sharp corners, + halign = flush center, + valign = center, + height = 0.2\textwidth, + after skip = 1cm] + #1 + \end{tcolorbox} +} + + +% ------------------------------ End of preamble --------------------------------------------- +\begin{document} + + + \titleBox{ + \textcolor{white}{\Large{Project: Name}} \\ + \textcolor{white}{\Large{Survey: Round}} \\ + \textcolor{white}{\Large{DIME Analytics Survey Pilot Review Checklist}} + } + + \section*{Objective} + Project teams will submit a \href{https://www.dropbox.com/scl/fi/px74eyztrn7uv1hsbeoz3/Survey-Pilot-Plan-Template-for-teams.paper?dl=0&rlkey=yaid3m17zzzjpfoconj0zw6fu}{\textcolor{blue}{Survey Pilot Plan Template (for teams)}} and a Pilot timeline GANTT chart. The objective of the review is to ensure that the pilot is set up in such that teams are able to maximize learning from the pilot. \\ + + + + + +\section*{Main findings} + +This report details the findings from review of the survey pilot for the \textit{Survey Round} of the \textit{Project name} project. + +\subsection*{Required practices} +\begin{itemize} + \item \textit{List required practices identified for project} +\end{itemize} +\subsection*{Recommended practices} + +The \href{https://dimewiki.worldbank.org/Ietestform}{ietestform} command (from the iefieldkit package) was run and the results are attached. It is a good practice to run iestestform multiple times during survey development. This is to make sure you’ve incorporated all suggested changes that are relevant and to re-test when new questions are added during the development phase of the survey. +\begin{itemize} + \item \textit{List recommended practices identified for project} +\end{itemize} + +The checklist in the next section details the exact findings. +\newpage + +\section*{Checklist} + \subsection*{Project Identifiers} + +\newcounter{row} + +\newcommand{\makerow}[1]{% + % #1 = text field + #1 & + \stepcounter{row}% + \mbox{\CheckBox[print,name=YES\therow, width=0.7em, height=0.7em]{}} & + \mbox{\CheckBox[print,name=NO\therow, width=0.7em, height=0.7em]{}} & + \mbox{\CheckBox[print,name=N/A\therow, width=0.7em, height=0.7em]{}} \\ +} + + +\begin{Form} +\noindent +\begin{tabularx}{\textwidth}{Xccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Project name} +\midrule +\makerow{TTL} +\midrule +\makerow{Point of contact} +\bottomrule +\end{tabularx} + + \subsection*{Timeline} + +\noindent +\begin{tabularx}{\textwidth}{Xccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{\textbf{Survey Pilot GANTT} timeline specific review \href{https://www.dropbox.com/home/Survey\%20Review/tex\%20checklists?preview=survey_pilot_timeline.pdf}{\textcolor{blue}{checklist}}} +\midrule +\makerow{Does the timeline allow for \textbf{enough time} for each stage based on the pilot sample size and expected survey duration?} +\midrule +\makerow{Does the plan includes \textbf{training of survey team?}} +\midrule +\makerow{Is there a \textbf{scheduled feedback meeting} with the broader team after survey pilot, with (at least) 2 weeks to spare before the survey round starts?} +\bottomrule +\end{tabularx} + +\newpage + \subsection*{Communication plan} + +Has the team put together a communication plan? Does the plan include the following: +\vspace{5mm} %5mm vertical space + +\noindent +\begin{tabularx}{\textwidth}{Xccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Method of \textbf{recording comments} during pilot} +\midrule +\makerow{Expected plan for sharing comments on a \textbf{daily basis}} +\midrule +\makerow{Pilot duration and \textbf{expected sample} size for pilot} +\midrule +\makerow{\textbf{Debrief sessions} with enumerators} +\midrule +\makerow{Who is \textbf{implementing changes} in the survey once discussed} +\midrule +\makerow{Does FC have additional document to \textbf{record observations} that may become useful for enumerator manual and training down the line?} +\bottomrule +\end{tabularx} + + \subsection*{Pre-Pilot specific} + +\noindent +\begin{tabularx}{\textwidth}{Xccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Has the team submitted a \textbf{PAP}?} +\midrule +\makerow{Has the team submitted a \textbf{Literature review}?} +\midrule +\makerow{Has the team submitted a \textbf{List of outcomes of interest}?} +\midrule +\makerow{Did the team use literature and/or material on \textbf{same sector and same country}?} +\midrule +\makerow{If there is a \textbf{consumption module}, is there a strategy to select country-specific items?} +\midrule +\makerow{If there is an \textbf{income module}, is there a strategy to consider country-specific sources of income?} +\midrule +\makerow{If there is a \textbf{nutrition module}, is there a strategy to consider country-specific food items?} +\bottomrule +\end{tabularx} + +\newpage + \subsection*{Content-pilot specific} + +Team provided details about: + +\noindent +\begin{tabularx}{\textwidth}{Xccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{\textbf{Expected start date} of pilot data collection} +\midrule +\makerow{Expected \textbf{survey duration}} +\midrule +\makerow{Number of \textbf{pilot rounds}} +\midrule +\makerow{Number of \textbf{respondents} per pilot} +\midrule +\makerow{Pilot \textbf{respondent selection} mechanism} +\midrule +\makerow{Unit of observation} +\midrule +\makerow{Has the team trained enumerators or provided the paper survey to the \textbf{survey firm}?} +\midrule +\makerow{Has the team chosen \textbf{respondents for the pilo}t that are similar to the IE’s target population but are not part of that sample?} +\bottomrule +\end{tabularx} + +\vspace{2mm} %2mm vertical space +Did the team prepare an informed consent form to be handed to all respondents, containing: +\vspace{2mm} %2mm vertical space + +\noindent +\begin{tabularx}{\textwidth}{Xccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Project Purpose} +\midrule +\makerow{Interview Purpose} +\midrule +\makerow{Overview of survey modules to be piloted} +\midrule +\makerow{Statement of confidentiality} +\midrule +\makerow{Statement of Respondent’s Rights} +\midrule +\makerow{Respondents’ Risks and Benefits of participating} +\midrule +\makerow{Does the survey begin with informed consent?} +\midrule +\makerow{If yes, does the informed consent include all \textbf{required sections} (templates link \href{https://www.dropbox.com/home/Survey\%20Review/tex\%20checklists?preview=survey_pilot_timeline.pdf}{\textcolor{blue}{here}})} +\bottomrule +\end{tabularx} + + +\vspace{5mm} %5mm vertical space + +\noindent +\begin{tabularx}{\textwidth}{Xccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Does the survey have a \textbf{unique id indicator} for each unit of observation / respondent?} +\midrule +\makerow{Does each module have an \textbf{introductory script}?} +\midrule +\makerow{Has the survey been translated to the \textbf{local language(s)}?} +\midrule +\makerow{Are the \textbf{outcomes of interest} in the PAP/i2i table reflected in the survey modules?} +\midrule +\makerow{Are there any questions in the survey that could be \textbf{ambiguous} or that the same person could answer in two different ways?} +\midrule +\makerow{If so, does the survey have \textbf{disclaimers} or clarifying statements/examples to guide respondent?} +\midrule +\makerow{Could the \textbf{order of survey modules} be improved for better respondent understanding?} +\midrule +\makerow{Are there \textbf{open-ended questions} which can be phrased as questions with choice lists in the survey?} +\midrule +\makerow{Are there \textbf{sensitive questions} in the survey that could make the respondent uncomfortable?} +\midrule +\makerow{If so, does the survey incorporate \textbf{strategies to mitigate} this? (ie, ensure anonymity, ask for consent for that module, ensure the enumerator is in private with the respondent, self-administration of module)} +\bottomrule +\end{tabularx} + + \subsection*{Data-pilot specific} + +\noindent +\begin{tabularx}{\textwidth}{Xccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Does the electronic survey \textbf{match} the piloted paper survey?} +\midrule +\makerow{Are the survey answer choices \textbf{coded in a standard way}?} +\midrule +\makerow{Has the team chosen the \textbf{software} to run the survey?} +\midrule +\makerow{Has the team \textbf{set up a server} in that software for data collection?} +\midrule +\makerow{Does the survey \textbf{incorporate validation of key entries} (ID number, phone number etc)?} +\midrule +\makerow{Does the survey have \textbf{consistency checks} (if applicable)? Ie, if the person is unemployed they shouldn’t list work as an activity of their time use} +\midrule +\makerow{Does the survey have appropriate \textbf{skip patterns and relevance}? (ie, the question “What level of schooling are you in?” is only relevant if the question “Are you in school” is answered Yes)} +\midrule +\makerow{Does the survey have \textbf{constraints} on integer answers?} +\midrule +\makerow{Can the survey be submitted as \textbf{incomplete?}} +\midrule +\makerow{Are answer choices \textbf{systematically coded}? (ie, -9 for "Other", -8 for "Do not know", and -7 for "Prefer not to answer”)} +\bottomrule +\end{tabularx} + +\newpage + +\noindent +\begin{tabularx}{\textwidth}{Xccc} +\toprule +\textbf{CHECKS} & YES & NO & N/A \\ +\midrule +\makerow{Are there modules that repeat for several people or items? (ie, a \textbf{loop})} +\midrule +\makerow{If so, is the loop \textbf{clearly indicated} in the survey for enumerators?} +\bottomrule +\end{tabularx} + +\end{Form} + + +\end{document} + diff --git a/survey-review-checklists/survey_pilot_timeline.pdf b/survey-review-checklists/survey_pilot_timeline.pdf new file mode 100644 index 0000000..32e5b38 Binary files /dev/null and b/survey-review-checklists/survey_pilot_timeline.pdf differ diff --git a/survey-review-checklists/survey_pilot_timeline.tex b/survey-review-checklists/survey_pilot_timeline.tex new file mode 100644 index 0000000..c49fcfc --- /dev/null +++ b/survey-review-checklists/survey_pilot_timeline.tex @@ -0,0 +1,165 @@ +\documentclass{report} +\usepackage[left=2cm, right=2cm, top=2cm]{geometry} +\usepackage[most]{tcolorbox} + +\usepackage{nopageno} +\usepackage{fancyhdr} +\usepackage{adjustbox} + +\usepackage{booktabs,tabularx} +\usepackage{hyperref} +%\usepackage{showframe} +\usepackage{xcolor} +\usepackage{graphics} +\pagestyle{fancy} +\fancyhf{} +\renewcommand{\headrulewidth}{0pt} +\lfoot{\includegraphics[height=1cm,keepaspectratio]{img/i2i}} +\cfoot{\includegraphics[height=1cm,keepaspectratio]{img/wb}} +\rfoot{\includegraphics[height=1cm,keepaspectratio]{img/analytics}} +\definecolor{fontcolor}{HTML}{7A0569} +\newcommand{\titleBox}[1]{ + \begin{tcolorbox} + [colframe = fontcolor, + colback = fontcolor, + sharp corners, + halign = flush center, + valign = center, + height = 0.2\textwidth, + after skip = 1cm] + #1 + \end{tcolorbox} +} + + +% ------------------------------ End of preamble --------------------------------------------- +\begin{document} + + + \titleBox{ + \textcolor{white}{\Large{Project: Name}} \\ + \textcolor{white}{\Large{Survey: Round}} \\ + \textcolor{white}{\Large{DIME Analytics Survey Pilot Timeline Review Checklist}} + } + + + \section*{Objective} + We include duration estimates for each activity, based on a 1-hr survey with a pilot sample of 30 households. The steps below are not necessarily in chronological order, and certain teams may find that tasks will overlap (for example, obtaining IRB approval and survey firm hiring are two tasks that might occur simultaneously). + + + +\section*{Main findings} + +This report details the findings from review of the survey pilot timeline for the \textit{Survey Round} of the \textit{Project name} project. + +\subsection*{Required practices} +\begin{itemize} + \item \textit{List required practices identified for project} +\end{itemize} +\subsection*{Recommended practices} + +The \href{https://dimewiki.worldbank.org/Ietestform}{ietestform} command (from the iefieldkit package) was run and the results are attached. It is a good practice to run iestestform multiple times during survey development. This is to make sure you’ve incorporated all suggested changes that are relevant and to re-test when new questions are added during the development phase of the survey. +\begin{itemize} + \item \textit{List recommended practices identified for project} +\end{itemize} + +The checklist in the next section details the exact findings. +\newpage + +\section*{Checklist} + + + \subsection*{Overall} + +\newcounter{row} + +\newcommand{\makerow}[1]{% + % #1 = text field + #1 & + \stepcounter{row}% + \mbox{\CheckBox[print,name=Included\therow, width=0.7em, height=0.7em]{}} & + \mbox{\CheckBox[print,name=Sufficient Time\therow, width=0.7em, height=0.7em]{}} & + \mbox{\CheckBox[print,name=N/A\therow, width=0.7em, height=0.7em]{}} \\ +} + + +\begin{Form} +\noindent +\begin{tabularx}{\textwidth}{Xccc} +\toprule +\textbf{CHECKS} & Included & Sufficient Time & N/A \\ +\midrule +\makerow{\textbf{Enough time} for each stage based on the pilot sample size and expected survey duration} +\bottomrule +\end{tabularx} + + \subsection*{Pre-Pilot} + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & Included & Sufficient Time & N/A \\ +\midrule +\makerow{\textbf{Outline modules} and indicators based on PAP (3-4 weeks)} +\midrule +\makerow{Discuss \textbf{measurement challenges} (3-4 weeks)} +\midrule +\makerow{Review country and sector-specific \textbf{literature} (3-4 weeks)} +\midrule +\makerow{Discuss answer \textbf{choice coding} (2-3 weeks)} +\midrule +\makerow{\textbf{Focus Group} sessions (3-4 weeks)} +\bottomrule +\end{tabularx} + +\vspace{5mm} %5mm vertical space + \subsection*{Content-focused Pilot} + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & Included & Sufficient Time & N/A \\ +\midrule +\makerow{Survey \textbf{translation} (1-2 weeks)} +\midrule +\makerow{\textbf{Pilot Sample} Selection (2-3 weeks)} +\midrule +\makerow{\textbf{Survey Firm} hiring (2-3 weeks)} +\midrule +\makerow{Obtain \textbf{IRB approval} (2-3 weeks)} +\midrule +\makerow{Prepare \textbf{Informed-Consent} forms for respondents (or verbal consent section of the survey) - (1-2 weeks)} +\midrule +\makerow{\textbf{Enumerator Training} (2 weeks)} +\midrule +\makerow{Pilot Survey \textbf{on the field} (2-3 weeks)} +\midrule +\makerow{\textbf{Pilot feedback} and key-takeaway discussions (2-3 weeks)} +\midrule +\makerow{Conduct \textbf{data quality or variation checks} on pilot data collected (1-2 weeks)} +\bottomrule +\end{tabularx} + + \subsection*{Data-Focused Pilot} + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & Included & Sufficient Time & N/A \\ +\midrule +\makerow{Survey \textbf{programming} (3-4 weeks)} +\midrule +\makerow{Submit a \textbf{mock dataset} (or several) - (2-3 weeks)} +\midrule +\makerow{Placing \textbf{consistency checks and constraints} in survey questions (2-3 weeks)} +\midrule +\makerow{Check skip patterns, loops, relevances, special characters in translated supplemental csv’s, and other \textbf{electronic features} of programmed survey (1-2 weeks)} +\bottomrule +\end{tabularx} + + +\end{Form} + + +\end{document} + diff --git a/survey-review-checklists/survey_timeline.pdf b/survey-review-checklists/survey_timeline.pdf new file mode 100644 index 0000000..9790224 Binary files /dev/null and b/survey-review-checklists/survey_timeline.pdf differ diff --git a/survey-review-checklists/survey_timeline.tex b/survey-review-checklists/survey_timeline.tex new file mode 100644 index 0000000..aab291a --- /dev/null +++ b/survey-review-checklists/survey_timeline.tex @@ -0,0 +1,213 @@ +\documentclass{report} +\usepackage[left=2cm, right=2cm, top=2cm]{geometry} +\usepackage[most]{tcolorbox} + +\usepackage{nopageno} +\usepackage{fancyhdr} +\usepackage{adjustbox} + +\usepackage{booktabs,tabularx} +\usepackage{hyperref} +%\usepackage{showframe} +\usepackage{xcolor} +\usepackage{graphics} +\pagestyle{fancy} +\fancyhf{} +\renewcommand{\headrulewidth}{0pt} +\lfoot{\includegraphics[height=1cm,keepaspectratio]{img/i2i}} +\cfoot{\includegraphics[height=1cm,keepaspectratio]{img/wb}} +\rfoot{\includegraphics[height=1cm,keepaspectratio]{img/analytics}} +\definecolor{fontcolor}{HTML}{7A0569} +\newcommand{\titleBox}[1]{ + \begin{tcolorbox} + [colframe = fontcolor, + colback = fontcolor, + sharp corners, + halign = flush center, + valign = center, + height = 0.2\textwidth, + after skip = 1cm] + #1 + \end{tcolorbox} +} + + +% ------------------------------ End of preamble --------------------------------------------- +\begin{document} + + + \titleBox{ + \textcolor{white}{\Large{Project: Name}} \\ + \textcolor{white}{\Large{Survey: Round}} \\ + \textcolor{white}{\Large{DIME Analytics Survey Timeline Review Checklist}} + } + + \section*{Objective} + The project team will provide a filled in GANTT chart (template \href{https://www.dropbox.com/s/h76tf1xstcisfv4/GANTT\%20Survey\%20Timeline.xlsx?dl=0}{\textcolor{blue}{here}}) which details each step of the survey timeline in detail. The main objectives of the Survey Timeline Review are to ensure the team: \\ + + \indent- has accounted for all activities required to ensure good quality of data collection \\ + \indent- is providing sufficient time for each activity \\ + \indent- has clearly defined roles and responsibilities for each activity \\ + + The reviewer should review each item in the GANTT chart template in line with the checklist below. Note that the timelines can vary project to project and are dependent on the complexity of the survey, number of survey instruments, length of each survey instrument and sample size. Please account for each project’s own survey duration and sample size when reviewing the timeline shared. \\ + + + \section*{Main findings} + + This report details the findings from review of the survey timeline for the \textit{Survey Round} of the \textit{Project name} project. + + \subsection*{Required practices} + \begin{itemize} + \item \textit{List required practices identified for project} + \end{itemize} + \subsection*{Recommended practices} + + The \href{https://dimewiki.worldbank.org/Ietestform}{ietestform} command (from the iefieldkit package) was run and the results are attached. It is a good practice to run iestestform multiple times during survey development. This is to make sure you’ve incorporated all suggested changes that are relevant and to re-test when new questions are added during the development phase of the survey. + \begin{itemize} + \item \textit{List recommended practices identified for project} + \end{itemize} + + The checklist in the next section details the exact findings. + \newpage + + \section*{Checklist} + + + + \subsection*{Procurement} + +\newcounter{row} + +\newcommand{\makerow}[1]{% + % #1 = text field + #1 & + \stepcounter{row}% + \mbox{\CheckBox[print,name=Included\therow, width=0.7em, height=0.7em]{}} & + \mbox{\CheckBox[print,name=Sufficient Time\therow, width=0.7em, height=0.7em]{}} & + \mbox{\CheckBox[print,name=N/A\therow, width=0.7em, height=0.7em]{}} \\ +} + + +\begin{Form} +\noindent +\begin{tabularx}{\textwidth}{Xccc} +\toprule +\textbf{CHECKS} & Included & Sufficient Time & N/A \\ +\midrule +\makerow{Devise a \textbf{TOR} for survey firms} +\midrule +\makerow{Hire a \textbf{survey firm}} +\midrule +\makerow{\textbf{Share materials} with confirmed survey firm} +\bottomrule +\end{tabularx} + + \subsection*{Sampling} + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & Included & Sufficient Time & N/A \\ +\midrule +\makerow{Define \textbf{study population}} +\midrule +\makerow{Develop \textbf{sample listing}} +\midrule +\makerow{Define \textbf{study sample} - including preparation of randomization scripts and/or protocols} +\bottomrule +\end{tabularx} + +\vspace{5mm} %5mm vertical space + \subsection*{Approvals} + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & Included & Sufficient Time & N/A \\ +\midrule +\makerow{Seek \textbf{IRB} approval} +\midrule +\makerow{Confirm approvals from \textbf{local government} and other authorities} +\bottomrule +\end{tabularx} + + \subsection*{Survey Development} + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & Included & Sufficient Time & N/A \\ +\midrule +\makerow{\textbf{Pilot Survey} (approx 12-16 weeks, in detail \href{https://www.dropbox.com/scl/fi/p887f03o57115nv6mhvng/Survey-Pilot-Timeline-Review-for-reviewer.paper?dl=0\&rlkey=59dvs8417f3wvwhvv1k0e9du8}{\textcolor{blue}{here}})} +\midrule +\makerow{Finalize \textbf{Paper Survey}} +\midrule +\makerow{\textbf{Translate} Paper Survey} +\midrule +\makerow{\textbf{Code} Electronic Survey} +\midrule +\makerow{\textbf{Test} Electronic Survey} +\bottomrule +\end{tabularx} + +\vspace{5mm} %5mm vertical space + + \subsection*{Survey Preparation} + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & Included & Sufficient Time & N/A \\ +\midrule +\makerow{Define Survey and Team \textbf{Communication Protocols}} +\midrule +\makerow{Create \textbf{Tracking sheets} for enumerators } +\midrule +\makerow{Carry out any necessary \textbf{field visits} for communication with beneficiaries } +\midrule +\makerow{Define \textbf{data quality} check protocols, including HFCs and back-checks} +\midrule +\makerow{\textbf{Test data quality checks} code, progress tracking and outputs} +\bottomrule +\end{tabularx} + +\vspace{5mm} %5mm vertical space + + \subsection*{Enumerator Training} + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & Included & Sufficient Time & N/A \\ +\midrule +\makerow{Hire \textbf{field team}} +\midrule +\makerow{Prepare \textbf{training materials}, including enumerator manual} +\midrule +\makerow{Conduct \textbf{enumerator training}} +\bottomrule +\end{tabularx} + +\vspace{5mm} %5mm vertical space + + \subsection*{Data Collection and Quality Assurance} + +\noindent +\begin{tabularx}{\textwidth}{Xcccc} +\toprule +\textbf{CHECKS} & Included & Sufficient Time & N/A \\ +\midrule +\makerow{\textbf{Collect data} from respondents } +\midrule +\makerow{Conduct \textbf{data quality checks} } +\midrule +\makerow{Conduct \textbf{back-checks} data collection} +\midrule +\makerow{\textbf{Revisits} or data re-collection} +\bottomrule +\end{tabularx} + +\end{Form} + +\end{document} +