data-paper/DataPaperPlotsStatistics_OpenVersion.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4e572911",
   "metadata": {},
   "source": [
    "# Data Paper Figures and Statistics\n",
    "This notebook produces figures and statistics for the data descriptior paper, using the openly available version of the UK COVID-19 Vocal Audio Dataset. Some subfigures use metadata from the protected version of the dataset and cannot be replicated here. All figures are written to .svg format and further processed for appearance and layout. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8105e46a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# python imports \n",
    "import os\n",
    "import pandas as pd\n",
    "#import kaleido # Required for exporting figures\n",
    "import numpy as np\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "from plotly.subplots import make_subplots\n",
    "import datetime as dt\n",
    "import io"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae2e5dae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# reads participant metadata file \n",
    "participant_df =  pd.read_csv('participant_metadata.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f677c7bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# reads audio metadata file \n",
    "audio_df = pd.read_csv('audio_metadata.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c19d707",
   "metadata": {},
   "source": [
    "### Data Paper Statistics "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c90a8a8",
   "metadata": {},
   "source": [
    "Total Participants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf568fc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db96af3b",
   "metadata": {},
   "source": [
    "Positive cases (COVID)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b186822",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df['covid_test_result'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c7ea3b4-f93d-4749-a86e-1e666f699149",
   "metadata": {},
   "outputs": [],
   "source": [
    "100*participant_df['covid_test_result'].value_counts(dropna=False, normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6214ae43",
   "metadata": {},
   "source": [
    "% PCR Test Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c367202-e6ee-4051-8f96-b2372cc49de7",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df[~participant_df['covid_test_result'].isna()].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "faf0b8b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df[(participant_df['covid_test_method'].str.contains('PCR', na=False)) & (~participant_df['covid_test_result'].isna())].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "759cec7a-78d6-49a6-a1c8-05c3bac8a5f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "round((70565/72999)*100, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8fad547",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tos dataset contains 27101 PCR test results\n",
    "70565/27101"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0902ff1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(100*participant_df[(participant_df['covid_test_method'].str.contains('PCR', na=False)) & (~participant_df['covid_test_result'].isna())].shape[0]/participant_df.shape[0],1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8bc5b893",
   "metadata": {},
   "source": [
    "PCR positive participants "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "493843b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df[participant_df['covid_test_method'].str.contains('PCR', na=False)]['covid_test_result'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ff0aecb",
   "metadata": {},
   "source": [
    "% Respiratory Symptoms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4e8b6eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "respiratory_symptoms = ['symptom_new_continuous_cough', 'symptom_cough_any', \n",
    "                        'symptom_runny_or_blocked_nose', 'symptom_shortness_of_breath',\n",
    "       'symptom_sore_throat',]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea2705b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df['symptoms_respiratory']=participant_df.apply(lambda x:x[respiratory_symptoms].sum()>0, axis=1)\n",
    "participant_df['symptoms_respiratory'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4dc3c122",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(100*participant_df['symptoms_respiratory'].sum()/participant_df.shape[0],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c8f3133",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(100*participant_df[participant_df['covid_test_result']=='Positive']['symptoms_respiratory'].value_counts(normalize=True, dropna=False),1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a77a0b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(100*participant_df[participant_df['covid_test_result']=='Negative']['symptoms_respiratory'].value_counts(normalize=True, dropna=False),1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cac96060-3108-4414-91a7-5b4cd157236c",
   "metadata": {},
   "outputs": [],
   "source": [
    "### % participants testing positive for SARS-CoV-2 infection reporting no symptoms\n",
    "\n",
    "participants_covid = participant_df[participant_df['covid_test_result']=='Positive'].shape[0]\n",
    "covid_no_symptoms = participant_df[(participant_df['covid_test_result']=='Positive') & (participant_df['symptom_none']==1)].shape[0]\n",
    "\n",
    "round(100*(covid_no_symptoms/participants_covid), 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be2cfb52-cf95-4f5e-93c8-9dbfe0d280f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "### % participants testing positive for SARS-CoV-2 infection reporting no respiratory symptoms\n",
    "\n",
    "covid_no_respiratory_symptoms = participant_df[(participant_df['covid_test_result']=='Positive') & (participant_df['symptoms_respiratory']==0)].shape[0]\n",
    "\n",
    "round(100*(covid_no_respiratory_symptoms/participants_covid), 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4649da4",
   "metadata": {},
   "source": [
    "% Respiratory conditions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "849d3cbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "respiratory_conditions = ['respiratory_condition_asthma', 'respiratory_condition_other']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a14a70f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df[participant_df[respiratory_conditions].sum(axis=1)>0].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7a07f0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(100*participant_df[participant_df[respiratory_conditions].sum(axis=1)>0].shape[0]/participant_df.shape[0],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7835337e-ac52-4841-bc32-816312c523f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "### % participants testing positive for SARS-CoV-2 infection reporting a respiratory condition\n",
    "\n",
    "participants_covid = participant_df[participant_df['covid_test_result']=='Positive'].shape[0]\n",
    "participants_covid_asthma = participant_df[(participant_df['respiratory_condition_asthma']==1) & (participant_df['covid_test_result']=='Positive')].shape[0]\n",
    "participants_covid_other_condition = participant_df[(participant_df['respiratory_condition_other']==1) & (participant_df['covid_test_result']=='Positive')].shape[0]\n",
    "\n",
    "round(100*(participants_covid_other_condition+participants_covid_asthma)/participants_covid, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b1cd0e7",
   "metadata": {},
   "source": [
    "% Asthma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdcefe6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df[participant_df['respiratory_condition_asthma']==1].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f446c65a",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(100*participant_df[participant_df['respiratory_condition_asthma']==1].shape[0]/participant_df.shape[0],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5852e791-89c8-418d-bf0b-e63245339b8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "### % participants with asthma testing positive for SARS-CoV-2 infection\n",
    "\n",
    "participants_asthma_covid = participant_df[(participant_df['respiratory_condition_asthma']==1) & (participant_df['covid_test_result']=='Positive')].shape[0]\n",
    "participants_asthma = participant_df[participant_df['respiratory_condition_asthma']==1].shape[0]\n",
    "\n",
    "round(100*participants_asthma_covid/participants_asthma, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6efea28-8ab5-497a-8ef8-16094e7ab184",
   "metadata": {},
   "outputs": [],
   "source": [
    "### % participants with asthma reporting another respiratory condition\n",
    "\n",
    "participants_asthma_other_condition = participant_df[(participant_df['respiratory_condition_asthma']==1) & (participant_df['respiratory_condition_other']==1)].shape[0]\n",
    "\n",
    "round(100*participants_asthma_other_condition/participants_asthma, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1dcfe49-dfb1-4ae4-8373-46f5841026f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "### % participants with asthma testing positive for influenza infection\n",
    "\n",
    "participants_asthma_influenza_a = participant_df[(participant_df['respiratory_condition_asthma']==1) & (participant_df['influenza_a_test_result']=='Positive')].shape[0]\n",
    "participants_asthma_influenza_b = participant_df[(participant_df['respiratory_condition_asthma']==1) & (participant_df['influenza_b_test_result']=='Positive')].shape[0]\n",
    "\n",
    "round(100*(participants_asthma_influenza_a + participants_asthma_influenza_b)/participants_asthma, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4d3a0bb",
   "metadata": {},
   "source": [
    "% Influenza A or B test results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84e034f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df['influenza_a_test_result'].value_counts()[['Positive', 'Negative']].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ffcb67e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df['influenza_b_test_result'].value_counts()[['Positive', 'Negative']].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb195314",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(100*participant_df['influenza_a_test_result'].value_counts()[['Positive', 'Negative']].sum()/participant_df.shape[0],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8c5c830",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(100*participant_df['influenza_b_test_result'].value_counts()[['Positive', 'Negative']].sum()/participant_df.shape[0], 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cb3c3b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df['influenza_a_test_result'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e385406",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df['influenza_b_test_result'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a7b3107",
   "metadata": {},
   "source": [
    "% Test method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db72cee8",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df['covid_test_method'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f0ad1ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(100*participant_df['covid_test_method'].value_counts(normalize=True, dropna=False),1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17ca54ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df['covid_test_result'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "928daa2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(100*participant_df['covid_test_result'].value_counts(normalize=True, dropna=False),1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d28e0268",
   "metadata": {},
   "source": [
    "% REACT, Test and Trace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99d11dc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df['recruitment_source'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "813b80a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(100*participant_df['recruitment_source'].value_counts(normalize=True, dropna=False),1) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "376565ea",
   "metadata": {},
   "source": [
    "Recruitment rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e18248b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df[['recruitment_source', 'survey_phase']].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f30873fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# REACT beta phase recruitment rate\n",
    "# REACT beta phase recruitment: 295493 emails\n",
    "round(100*36116/295493,2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "805f75f3",
   "metadata": {},
   "source": [
    "Age"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f37f5314",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df['age'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0a212b0",
   "metadata": {},
   "source": [
    "Gender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02b55eb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df['gender'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e33d67a",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(participant_df['gender'].value_counts(dropna=False, normalize=True)*100, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "433a0d96",
   "metadata": {},
   "source": [
    "% Wearing mask "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14976a43",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df['wearing_mask'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67a3479b",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(100*participant_df['wearing_mask'].value_counts(dropna=False, normalize=True), 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54397918",
   "metadata": {},
   "source": [
    "Audio sample rate stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fb8cc72",
   "metadata": {},
   "outputs": [],
   "source": [
    "audio_df[['cough_sample_rate', 'three_cough_sample_rate', 'exhalation_sample_rate']].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "068a0f81",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(100*audio_df[['cough_sample_rate', 'three_cough_sample_rate', 'exhalation_sample_rate']].value_counts(normalize=True),1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06ac342c",
   "metadata": {},
   "source": [
    "Audio length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2780ead",
   "metadata": {},
   "outputs": [],
   "source": [
    "max([audio_df['cough_length'].max(),\n",
    "      audio_df['three_cough_length'].max(),\n",
    "     audio_df['exhalation_length'].max()])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9d1db95",
   "metadata": {},
   "source": [
    "Survey phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "061c4b4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df['survey_phase'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cbe21eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(100*participant_df['survey_phase'].value_counts(dropna=False, normalize=True),1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c76e0f85",
   "metadata": {},
   "source": [
    "Missing audio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fff311e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "audio_df['missing_audio'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "842e9f70",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(100*audio_df['missing_audio'].value_counts(dropna=False, normalize=True),1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98feca9d",
   "metadata": {},
   "source": [
    "Number of audio files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3be7c97",
   "metadata": {},
   "outputs": [],
   "source": [
    "audio_df[~(audio_df['exhalation_size']<45) & ~(audio_df['exhalation_size'].isna())].shape[0]  + audio_df[~(audio_df['cough_size']<45) & ~(audio_df['cough_size'].isna())].shape[0] + audio_df[~(audio_df['three_cough_size']<45) & ~(audio_df['three_cough_size'].isna())].shape[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49913760",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df.shape[0]*4 - 289696"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59fd3c00",
   "metadata": {},
   "source": [
    "Total size of audio in GB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c94b3146",
   "metadata": {},
   "outputs": [],
   "source": [
    "(audio_df['exhalation_size'].sum()+audio_df['exhalation_size'].sum()+\n",
    " audio_df['cough_size'].sum()+\n",
    " audio_df['three_cough_size'].sum())/1000000000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4b45704",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Metadata size in MB\n",
    "27.8+35.2 #+7.4"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79806954",
   "metadata": {},
   "source": [
    "Viral load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1778ffb",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df[~participant_df['covid_viral_load'].isna()].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb0d08c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(100*participant_df[~participant_df['covid_viral_load'].isna()].shape[0]/participant_df.shape[0],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7584727",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df['covid_viral_load_category'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f14dc48c-8d16-42a3-86cd-0211147b9a93",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b778d314",
   "metadata": {},
   "source": [
    "Variable completeness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c547d4c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_na_df = participant_df.copy()\n",
    "participant_na_df = participant_na_df.replace('Prefer not to say', np.nan)\n",
    "participant_na_df = participant_na_df.replace('Unknown', np.nan)\n",
    "participant_na_df = participant_na_df.replace('Unknown/Void', np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7aee77e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "for column in participant_na_df.columns:\n",
    "    print(column, round(100-(100*participant_na_df[column].isna().sum()/participant_na_df.shape[0]), 1), \"%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09b014e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "for column in audio_df.columns:\n",
    "    print(column, round(100-(100*audio_df[column].isna().sum()/audio_df.shape[0]), 1), \"%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22dc68aa",
   "metadata": {},
   "source": [
    "## Figure 1 - Recruitment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f932c3b3",
   "metadata": {},
   "source": [
    "#### Figure 1.C Survey completion rates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "127212ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data from survey provider, for beta phase only, in order of survey \n",
    "react_data = [43015, 42707, 41500, 41418, 41389, 41386, 41372, 41343, 41326, 41323, 37602, 37150, 36933, 36768]\n",
    "tt_data = [41060, 40531, 28126, 28063, 28045, 28040, 28031, 28018, 28003, 28001, 26260, 26071, 25959, 25853]\n",
    "\n",
    "# List survey questions\n",
    "questions = [\"Privacy confirmation\", \"Participation agreement\", \n",
    "               \"Barcode entry\", \"Symptoms\", \n",
    "               \"Smoker status\", \"Respiratory conditions\", \n",
    "               \"First language\", \"Height\", \n",
    "               \"Weight\", \"Mask\", \n",
    "               \"Sentence audio\", \"Exhalations audio\", \n",
    "               \"Single cough audio\", \"Three coughs audio\",\n",
    "               \"<b>Processed Data Set<b>\"]\n",
    "\n",
    "react_data.append(participant_df[participant_df['recruitment_source'].str.contains('REACT')]['survey_phase' ].value_counts()['beta'])\n",
    "tt_data.append(participant_df[participant_df['recruitment_source']=='Test and Trace']['survey_phase' ].value_counts()['beta'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "235facef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plots figure\n",
    "fig = go.Figure()\n",
    "\n",
    "fig.add_trace(go.Funnel(\n",
    "    name = 'NHS Test and Trace',\n",
    "    orientation = \"h\",\n",
    "    width=0.75, \n",
    "    y = questions,\n",
    "    x = tt_data,\n",
    "    marker_color='#0072B2',\n",
    "    textposition = \"inside\",\n",
    "    textinfo = \"percent initial\"))\n",
    "\n",
    "fig.add_trace(go.Funnel(\n",
    "    name = 'REACT',\n",
    "    orientation = \"h\",\n",
    "    width=0.75,\n",
    "    y = questions,\n",
    "    x = react_data,\n",
    "    marker_color='#D55E00',\n",
    "    textposition = \"inside\",\n",
    "    textinfo = \"percent initial\"))\n",
    "\n",
    "fig.update_layout(template='simple_white', font_family=\"Arial\", font_size=12, width=500)\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e1963d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Writes figure to image\n",
    "fig.write_image('DataPaperFig1C.svg', scale=1)\n",
    "fig.write_image('DataPaperFig1C.png', scale=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a235f7fc",
   "metadata": {},
   "source": [
    "#### Combined subplots for figures 1.D, 1.E.  (combined to ensure matched font scale and width)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4f49fb3",
   "metadata": {},
   "source": [
    "D. Submissions over time "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d49698a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copies participant dataframe and gets number of participants \n",
    "all_df = participant_df.copy()\n",
    "\n",
    "participants = participant_df.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe533fd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adds week and cohort variables \n",
    "all_df['week'] = pd.to_datetime(all_df['submission_date'], errors='coerce').dt.date - pd.to_datetime(all_df['submission_date'], errors='coerce').dt.weekday * np.timedelta64(1, 'D')\n",
    "\n",
    "all_df['cohort'] = all_df['recruitment_source'].apply(lambda x: 'react' if 'REACT' in x else ('tt' if 'Trace' in x else 'None'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b54b36e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Groups by week \n",
    "week_df = all_df.groupby(['cohort', 'covid_test_result', 'week']).size().to_frame('count').reset_index()\n",
    "\n",
    "week_df['percentage']=100*week_df['count']/participants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe3dbd93",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creates plotly traces to plot\n",
    "tt_pos_timeline_trace=go.Scatter(x=week_df[(week_df['cohort']=='tt')&(week_df['covid_test_result']=='Positive')]['week'],\n",
    "                         y=week_df[(week_df['cohort']=='tt')&(week_df['covid_test_result']=='Positive')]['percentage'],\n",
    "                         stackgroup='one', name='NHS Test and Trace<br>COVID Positive', legendgroup=1,\n",
    "                        line=dict(width=0, color='rgba(0, 114, 178, 1)'))\n",
    "\n",
    "tt_neg_timeline_trace=go.Scatter(x=week_df[(week_df['cohort']=='tt')&(week_df['covid_test_result']=='Negative')]['week'],\n",
    "                         y=week_df[(week_df['cohort']=='tt')&(week_df['covid_test_result']=='Negative')]['percentage'],\n",
    "                         stackgroup='one', name='NHS Test and Trace<br>COVID Negative', legendgroup=1,\n",
    "                        line=dict(width=0, color='rgba(86, 180, 233, 1)'),)\n",
    "\n",
    "react_pos_timeline_trace=go.Scatter(x=week_df[(week_df['cohort']=='react')&(week_df['covid_test_result']=='Positive')]['week'],\n",
    "                         y=week_df[(week_df['cohort']=='react')&(week_df['covid_test_result']=='Positive')]['percentage'],\n",
    "                         stackgroup='one', name='REACT<br>COVID Positive', legendgroup=1,\n",
    "                         fillcolor='rgba(213,94,0,1)',\n",
    "                        line=dict(width=0),)\n",
    "\n",
    "react_neg_timeline_trace=go.Scatter(x=week_df[(week_df['cohort']=='react')&(week_df['covid_test_result']=='Negative')]['week'],\n",
    "                         y=week_df[(week_df['cohort']=='react')&(week_df['covid_test_result']=='Negative')]['percentage'],\n",
    "                         stackgroup='one', name='REACT<br>COVID Negative', legendgroup=1,\n",
    "                        line=dict(width=0, color='rgba(230, 159, 0, 1)'),)\n",
    "\n",
    "\n",
    "timeline_traces = [tt_pos_timeline_trace, tt_neg_timeline_trace, react_pos_timeline_trace, react_neg_timeline_trace]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5bb3b4d5",
   "metadata": {},
   "source": [
    "E. Repsonse Gap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c406686b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splits participant dataframe by recuitment cohort \n",
    "tt_df=participant_df[participant_df['recruitment_source']=='Test and Trace']\n",
    "react_df= participant_df[participant_df['recruitment_source'].str.contains('REACT')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95508a55-1ca0-45c9-a4b3-6023b552bb60",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run following cells \n",
    "delay_df=react_df.copy()\n",
    "participants=react_df.shape[0]\n",
    "\n",
    "# format to be in context of submission date\n",
    "delay_df['symptom_onset'] = 0-delay_df['symptom_onset'] \n",
    "# Make symptom onset na when no symptoms \n",
    "delay_df['symptom_onset'] = delay_df.apply(lambda x: np.nan if x['symptom_none']==1 else x['symptom_onset'], axis=1)\n",
    "delay_df['covid_test_date'] = delay_df['covid_test_date'] - delay_df['submission_date']\n",
    "delay_df['covid_test_processed_date'] = delay_df['covid_test_processed_date'] - delay_df['submission_date']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6deefbb-c845-47c6-9e96-290305243d2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print % of participants completing survey within 72 hours \n",
    "delay_df['submission_delay_72hrs'] = delay_df['covid_test_date'].apply(lambda x: 1 if x<-3 else 0)\n",
    "print(\"percentage participants completing survey withing 72hrs of testing: \")\n",
    "print(round(100*delay_df['submission_delay_72hrs'].value_counts(dropna=False, normalize=True), 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77f9deef-910f-47ac-ae3c-0434e080ded6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# format test completion date data\n",
    "test_date_df = delay_df['covid_test_date'].value_counts().reset_index().sort_values(by='covid_test_date', ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcb0a184-709c-49f9-bdf1-d04a8ce02d55",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Formats data for both participant cohorts, prints 72 hours survey delay statistic\n",
    "subplot_traces=[]\n",
    "\n",
    "for cohort_df in [tt_df, react_df]:\n",
    "\n",
    "    # Run following cells \n",
    "    delay_df=cohort_df.copy()\n",
    "    participants=cohort_df.shape[0]\n",
    "\n",
    "    # format to be in context of submission date\n",
    "    delay_df['symptom_onset'] = 0-delay_df['symptom_onset'] \n",
    "    # Make symptom onset na when no symptoms \n",
    "    delay_df['symptom_onset'] = delay_df.apply(lambda x: np.nan if x['symptom_none']==1 else x['symptom_onset'], axis=1)\n",
    "    delay_df['covid_test_date'] = delay_df['covid_test_date'] - delay_df['submission_date']\n",
    "    delay_df['covid_test_processed_date'] = delay_df['covid_test_processed_date'] - delay_df['submission_date']\n",
    "\n",
    "    # Print % of participants completing survey within 72 hours \n",
    "    delay_df['submission_delay_72hrs'] = delay_df['covid_test_date'].apply(lambda x: 1 if x<-3 else 0)\n",
    "    print(\"percentage participants completing survey withing 72hrs of testing: \")\n",
    "    print(round(100*delay_df['submission_delay_72hrs'].value_counts(dropna=False, normalize=True), 1))\n",
    "\n",
    "    # format test completion date data\n",
    "    test_date_df = delay_df['covid_test_date'].value_counts().reset_index().sort_values(by='covid_test_date', ascending=True)\n",
    "    test_date_df.columns=['days','count']\n",
    "    test_date_df['percentage'] = 100*test_date_df['count']/participants\n",
    "    test_date_df['days']= test_date_df['days'].astype(int)\n",
    "    \n",
    "    # get test completion date plotly trace\n",
    "    taken_trace = go.Scatter(\n",
    "    x=test_date_df['days'], \n",
    "    y=test_date_df['percentage'],\n",
    "    name='Test Taken',\n",
    "    fill='tozeroy',\n",
    "    fillcolor='rgba(213, 94, 0, 0.75)',\n",
    "    line=dict(width=0),\n",
    "    legendgroup=2,\n",
    "    showlegend=False,\n",
    "    mode='lines')\n",
    "\n",
    "    # format test processing date data\n",
    "    processed_date_df = delay_df['covid_test_processed_date'].value_counts().reset_index().sort_values(by='covid_test_processed_date', ascending=True)\n",
    "    processed_date_df.columns=['days','count']\n",
    "    processed_date_df['percentage'] = 100*processed_date_df['count']/participants\n",
    "    processed_date_df['days']= processed_date_df['days'].astype(int)\n",
    "    \n",
    "    # get test processing date plotly trace\n",
    "    processed_trace = go.Scatter(\n",
    "    x=processed_date_df['days'], \n",
    "    y=processed_date_df['percentage'],\n",
    "    name='Test Processed',\n",
    "    fill='tozeroy',\n",
    "    fillcolor='rgba(204, 121, 167, 0.75)',\n",
    "    line=dict(width=0),\n",
    "    legendgroup=2,\n",
    "    showlegend=False,\n",
    "    mode='lines')\n",
    "\n",
    "    # format symptom onset data\n",
    "    onset_df = delay_df['symptom_onset'].value_counts().reset_index().sort_values(by='symptom_onset', ascending=True)\n",
    "    onset_df.columns=['days','count']\n",
    "    onset_df['percentage'] = 100*onset_df['count']/participants\n",
    "    \n",
    "    # get symptom onset date plotly trace\n",
    "    onset_trace = go.Scatter(\n",
    "    x=onset_df['days'], \n",
    "    y=onset_df['percentage'],\n",
    "    name='Symptom Onset',\n",
    "    fill='tozeroy',\n",
    "    fillcolor='rgba(0, 114, 178, 0.75)',\n",
    "    line=dict(width=0),\n",
    "    legendgroup=2,\n",
    "    showlegend=False,\n",
    "    mode='lines',\n",
    "    )\n",
    "    \n",
    "    subplot_traces.append([processed_trace, taken_trace, onset_trace])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b723de27",
   "metadata": {},
   "source": [
    "Combined Subplot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a933e50",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Plots above traces \n",
    "fig = make_subplots(rows=2, cols=2,  vertical_spacing=0.25, \n",
    "                    subplot_titles=('<b>Survey participation by week<b>', '<b>REACT<b>', '<b>NHS Test and Trace<b>'),\n",
    "                    specs=[[{\"colspan\": 2}, None],\n",
    "                          [{},{}]]\n",
    "                   )\n",
    "\n",
    "\n",
    "# for trace in timeline_traces:\n",
    "#     fig.add_trace(trace, row=1, col=1)\n",
    "\n",
    "for trace in subplot_traces[1]:\n",
    "    fig.add_trace(trace, row=2, col=1)\n",
    "    \n",
    "for trace in subplot_traces[0]:\n",
    "    fig.add_trace(trace, row=2, col=2)\n",
    "\n",
    "\n",
    "fig.add_vline(x=0, line_width=2, line_dash=\"dash\", opacity=1,\n",
    "              line_color=\"black\", annotation_text = ' Survey<br> submission',\n",
    "              annotation_position='top right', annotation_align='left', row=2)\n",
    "\n",
    "\n",
    "fig.update_yaxes(ticksuffix='%')\n",
    "fig.update_xaxes(title='Survey submission date', row=1)\n",
    "fig.update_yaxes(title='Percentage of particpants', title_standoff=10, range=[0,10], row=1)\n",
    "fig.update_yaxes(title='Percentage of particpants', title_standoff=5, row=2)\n",
    "fig.update_xaxes(range=[-15, 5], title_text='Days relative to survey submission', row=2)\n",
    "fig.update_yaxes(range=[0, 80], row=2)\n",
    "  \n",
    "fig.update_layout(template='simple_white',\n",
    "                  font_family='Arial',\n",
    "                  font_size=12,\n",
    "                  height=600,\n",
    "                 legend=dict(x=0.025,y=1, bgcolor='rgba(255, 255, 255, 0)', borderwidth=0,\n",
    "                             tracegroupgap=130))\n",
    "\n",
    "fig.show() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0400bb50",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Writes plot to image\n",
    "fig.write_image('DataPaperFig1DE.svg', scale=1) \n",
    "fig.write_image('DataPaperFig1DE.png', scale=3) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74c37537",
   "metadata": {},
   "source": [
    "## Figure 2 - Dataset summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea2bd876",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df = participant_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3903d37a",
   "metadata": {},
   "outputs": [],
   "source": [
    "participants = all_df.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "619d6129",
   "metadata": {},
   "source": [
    "#### Audio Plot\n",
    "Audio data shown in the manuscript is recorded by the author and not included in this repository. Similar plots can be recreated using the audio files listed in audio_df and read using scipy.io.wavfile.read"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f9355a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#my_audio_df = pd.read_csv('my_audio_df.csv')\n",
    "#samplerate = 44100 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4aa51f64",
   "metadata": {},
   "outputs": [],
   "source": [
    "# One example audio file for each audio modality \n",
    "#sentence = np.array(my_audio_df['sentence'])\n",
    "#exhalation = np.array(my_audio_df['exhalation'])\n",
    "#one_cough = np.array(my_audio_df['one_cough'])\n",
    "#three_coughs = np.array(my_audio_df['three_coughs'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6002e196",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generates plotly traces and plots subplots\n",
    "\n",
    "# fig = make_subplots(\n",
    "#     rows=4, cols=1,\n",
    "#     shared_xaxes=True,\n",
    "#     shared_yaxes=True,\n",
    "#     subplot_titles=(\"Speech (read sentence)\", \"Three sharp exhalations\", \"One volitional cough\", \"Three volitional coughs\"))\n",
    "\n",
    "# sentence_trace = go.Scatter(\n",
    "#     x=[x/samplerate for x in range(0, len(sentence))],\n",
    "#     y=sentence,\n",
    "# showlegend=False,\n",
    "# line=dict(color=\"#D55E00\"))\n",
    "    \n",
    "# exhalation_trace = go.Scatter(\n",
    "#     x=[x/samplerate for x in range(0, len(exhalation))],\n",
    "#     y=exhalation,\n",
    "# showlegend=False,\n",
    "# line=dict(color=\"#CC79A7\"))\n",
    "    \n",
    "# one_cough_trace = go.Scatter(\n",
    "#     x=[x/samplerate for x in range(0, len(one_cough))],\n",
    "#     y=one_cough,\n",
    "# showlegend=False,\n",
    "# line=dict(color=\"#CC79A7\"))\n",
    "    \n",
    "# three_coughs_trace = go.Scatter(\n",
    "#     x=[x/samplerate for x in range(0, len(three_coughs))],\n",
    "#     y=three_coughs,\n",
    "# showlegend=False,\n",
    "# line=dict(color=\"#CC79A7\"))\n",
    "\n",
    "# fig.add_trace(sentence_trace, col=1, row=1)\n",
    "\n",
    "# fig.add_trace(exhalation_trace, col=1, row=2)\n",
    "\n",
    "# fig.add_trace(one_cough_trace, col=1, row=3)\n",
    "\n",
    "# fig.add_trace(three_coughs_trace, col=1, row=4)\n",
    "\n",
    "# fig.update_xaxes(title='Seconds', row=4)\n",
    "# fig.update_yaxes(showticklabels=False)\n",
    "# fig.update_yaxes(ticks=\"\")\n",
    "# fig.update_layout(template='simple_white', font_family='Arial', font_size=14, width=500)\n",
    "\n",
    "# fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f2993cf",
   "metadata": {},
   "source": [
    "#### COVID Test Results\n",
    "The following subsections generate plotly traces which are plotted later as subplots "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f7241ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Groups all PCR-type tests \n",
    "def test_simplify(test):\n",
    "    try:\n",
    "        if 'PCR' in test:\n",
    "            return 'PCR'\n",
    "        else:\n",
    "            return test     \n",
    "    except:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "438adc0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Format data\n",
    "all_df['covid_test_method'] = all_df['covid_test_method'].apply(lambda x:test_simplify(x))\n",
    "\n",
    "covid_df = all_df.groupby(['covid_test_result', 'covid_test_method']).size().reset_index()\n",
    "\n",
    "covid_df = covid_df[covid_df['covid_test_result']!='Unknown/Void']\n",
    "\n",
    "covid_df = covid_df[covid_df['covid_test_method']!='Unknown']\n",
    "\n",
    "covid_df.columns  =['covid_test_result', 'covid_test_method', 'count']\n",
    "\n",
    "covid_df['percentage'] = 100* covid_df['count']/participants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b9e8c2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generates plotly traces\n",
    "covid_trace_1 = go.Bar(\n",
    "x=covid_df[covid_df['covid_test_method']=='PCR']['covid_test_result'],\n",
    "y=covid_df[covid_df['covid_test_method']=='PCR']['percentage'],\n",
    "name='PCR',\n",
    "offsetgroup='A',\n",
    "marker=dict(line=dict(width=0)),\n",
    "marker_color='#0072B2')\n",
    "\n",
    "covid_trace_2 = go.Bar(\n",
    "x=covid_df[covid_df['covid_test_method']=='LAMP']['covid_test_result'],\n",
    "y=covid_df[covid_df['covid_test_method']=='LAMP']['percentage'],\n",
    "name='LAMP',\n",
    "offsetgroup='A',\n",
    "base = covid_df[covid_df['covid_test_method']=='PCR']['percentage'],\n",
    "marker=dict(line=dict(width=0)),\n",
    "marker_color='#009E73')\n",
    "\n",
    "covid_trace_3 = go.Bar(\n",
    "x=covid_df[covid_df['covid_test_method']=='LFT']['covid_test_result'],\n",
    "y=covid_df[covid_df['covid_test_method']=='LFT']['percentage'],\n",
    "name='LFT',\n",
    "offsetgroup='A',\n",
    "base = covid_df[covid_df['covid_test_method']!='LFT'].groupby('covid_test_result').sum().reset_index()['percentage'],\n",
    "marker=dict(line=dict(width=0)),\n",
    "marker_color='#F0E442')\n",
    "\n",
    "covid_test_traces = [covid_trace_1, covid_trace_2, covid_trace_3]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57d93f23",
   "metadata": {},
   "source": [
    "#### Influenza Test Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86d3d206",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combines influenza A and B test results \n",
    "\n",
    "def flu_label(row):\n",
    "    try:\n",
    "        if (row['influenza_a_test_result']=='Positive') or (row['influenza_b_test_result']=='Positive'):\n",
    "            return 'Positive'\n",
    "        if (row['influenza_a_test_result']=='Negative') and (row['influenza_b_test_result']=='Negative'):\n",
    "            return 'Negative'\n",
    "        else:\n",
    "            return None\n",
    "    except:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63696c48",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Formats data \n",
    "all_df['influenza_test_result'] = all_df.apply(lambda x:flu_label(x), axis=1)\n",
    "\n",
    "flu_df = all_df.groupby('influenza_test_result').size().reset_index()\n",
    "\n",
    "flu_df.columns = ['influenza_test_result', 'count']\n",
    "\n",
    "flu_df['percentage'] = 100* flu_df['count']/participants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "014e20e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generates plotly trace\n",
    "flu_trace = go.Bar(\n",
    "x=flu_df['influenza_test_result'],\n",
    "y=flu_df['percentage'],\n",
    "name='PCR',\n",
    "showlegend=False,\n",
    "marker=dict(line=dict(width=0)),\n",
    "marker_color='#0072B2')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e49adefc",
   "metadata": {},
   "source": [
    "#### Symptoms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2ba4349",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Groups non-respiratory symptoms for plot \n",
    "\n",
    "non_respiratory_symptoms =  [\n",
    "       'symptom_change_to_sense_of_smell_or_taste',\n",
    "       'symptom_abdominal_pain',\n",
    "       'symptom_diarrhoea', 'symptom_fatigue',\n",
    "       'symptom_fever_high_temperature', 'symptom_headache',\n",
    "       'symptom_loss_of_taste',]\n",
    "\n",
    "all_df['symptom_non_resp'] = all_df[non_respiratory_symptoms].any(axis=1).replace({True: 1, False: 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a27abc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# New symptom list with grouped non-respiratory symptoms \n",
    "symptoms = ['symptom_cough_any', 'symptom_new_continuous_cough', 'symptom_runny_or_blocked_nose',\n",
    "            'symptom_sore_throat', 'symptom_shortness_of_breath',\n",
    "       'symptom_non_resp', 'symptom_other', 'symptom_none', 'symptom_prefer_not_to_say',]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d6c0de9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Formats symptom names for plot\n",
    "symptom_dict = {'symptom_change_to_sense_of_smell_or_taste':'Change to sense<br>of smell or taste',\n",
    "       'symptom_new_continuous_cough':'New continuous<br>cough',\n",
    "        'symptom_abdominal_pain':'Abdominal pain',\n",
    "       'symptom_cough_any': 'Cough (any)',\n",
    "        'symptom_diarrhoea':'Diarrhoea',\n",
    "        'symptom_fatigue':'Fatigue',\n",
    "       'symptom_fever_high_temperature':'Fever',\n",
    "        'symptom_headache':'Headache',\n",
    "       'symptom_loss_of_taste':'Loss of taste',\n",
    "        'symptom_runny_or_blocked_nose':'Runny or<br>blocked nose',\n",
    "        'symptom_shortness_of_breath':'Shortness of<br>breath',\n",
    "       'symptom_sore_throat':'Sore throat', \n",
    "        'symptom_other':'Other',\n",
    "        'symptom_none':'None',\n",
    "        'symptom_non_resp':'Non-respiratory<br>symptoms*',\n",
    "        'symptom_prefer_not_to_say':'Prefer not to say',}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9813801d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split by positive an negative COVID test results for plot \n",
    "pos_df = all_df[all_df['covid_test_result']=='Positive']\n",
    "neg_df = all_df[all_df['covid_test_result']=='Negative']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39e23e3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Format data for bar plot \n",
    "symptom_counts = []\n",
    "for symptom in symptoms:\n",
    "    symptom_counts.append([symptom, pos_df[symptom].sum(), neg_df[symptom].sum()])\n",
    "    \n",
    "symptom_df = pd.DataFrame(symptom_counts)\n",
    "symptom_df.columns = ['symptom', 'pos_count', 'neg_count']\n",
    "\n",
    "symptom_df['symptom'] = symptom_df['symptom'].map(symptom_dict)\n",
    "symptom_df = symptom_df[:8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee5215a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate plotly traces \n",
    "pos_symptom_trace = go.Bar(\n",
    "x=symptom_df['symptom'],\n",
    "y=100*symptom_df['pos_count']/participants, \n",
    "name='COVID<br>Positive', \n",
    "marker_color='rgba(230,159,0,1)',\n",
    ")\n",
    "\n",
    "neg_symptom_trace = go.Bar(\n",
    "x=symptom_df['symptom'],\n",
    "y=100*symptom_df['neg_count']/participants, \n",
    "name='COVID<br>Negative', \n",
    "marker_color='rgba(86,180,233,1)', \n",
    ")\n",
    "\n",
    "symptom_traces = [pos_symptom_trace, neg_symptom_trace]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1fa0785",
   "metadata": {},
   "source": [
    "#### Respiratory Conditions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cee20d09-c568-4f56-a829-e293e94d7ac1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create None columns and nan columns \n",
    "all_df['respiratory_condition_prefer_not_to_say'] = all_df.apply(lambda x: 1 if pd.isna(x['respiratory_condition_other']) else 0 , axis=1)\n",
    "all_df['respiratory_condition_prefer_not_to_say'].sum()\n",
    "all_df['respiratory_condition_none'] = all_df.apply(lambda x: 1 if (x['respiratory_condition_other'] + x['respiratory_condition_asthma'] == 0) else 0 , axis=1)\n",
    "all_df['respiratory_condition_none'] = all_df.apply(lambda x: np.nan if x['respiratory_condition_prefer_not_to_say']==1 else x['respiratory_condition_none'], axis=1)\n",
    "all_df['respiratory_condition_none'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9a629f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of respiratory condition variables\n",
    "conditions=['respiratory_condition_asthma',\n",
    "       'respiratory_condition_other',\n",
    "       'respiratory_condition_none',\n",
    "       'respiratory_condition_prefer_not_to_say']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b303790a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename variables for plot \n",
    "condition_dict = {'respiratory_condition_asthma':'Asthma',\n",
    "       'respiratory_condition_other':'Other',\n",
    "       'respiratory_condition_none':'None',\n",
    "       'respiratory_condition_prefer_not_to_say':'Prefer not to say'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "844ce4a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split by positive an negative COVID test results for plot \n",
    "pos_df = all_df[all_df['covid_test_result']=='Positive']\n",
    "neg_df = all_df[all_df['covid_test_result']=='Negative']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb49d2de",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get negative and positive counts \n",
    "condition_counts = []\n",
    "for condition in conditions:\n",
    "    condition_counts.append([condition, pos_df[condition].sum(), neg_df[condition].sum()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35cc9e91",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Format data for bar plot \n",
    "condition_df = pd.DataFrame(condition_counts)\n",
    "\n",
    "condition_df.columns = ['condition', 'pos_count', 'neg_count']\n",
    "\n",
    "condition_df['condition'] = condition_df['condition'].map(condition_dict)\n",
    "\n",
    "condition_df = condition_df[:3] # Removes 'prefer not to say' option"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03bfffd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate plotly traces\n",
    "pos_condition_trace = go.Bar(\n",
    "x=condition_df['condition'],\n",
    "y=100*condition_df['pos_count']/participants, \n",
    "name='COVID Positive', \n",
    "showlegend=False,\n",
    "marker_color='rgba(230,159,0,1)',\n",
    ")\n",
    "\n",
    "neg_condition_trace = go.Bar(\n",
    "x=condition_df['condition'],\n",
    "y=100*condition_df['neg_count']/participants, \n",
    "name='COVID Negative', \n",
    "showlegend=False,\n",
    "marker_color='rgba(86,180,233,1)',\n",
    ")\n",
    "\n",
    "\n",
    "condition_traces = [pos_condition_trace, neg_condition_trace]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f69a53e",
   "metadata": {},
   "source": [
    "#### Smoker Status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb203a0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Group all variable options for current smokers\n",
    "all_df['smoker_status']=all_df['smoker_status'].apply(lambda x:'Current smoker' if 'Current smoker' in x else x)\n",
    "all_df['smoker_status']=all_df['smoker_status'].apply(lambda x: None if 'Prefer not to say' in x else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a5f1e09",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Format data for bar plot \n",
    "smoker_df = all_df.groupby(['smoker_status' ,'covid_test_result']).size().to_frame().reset_index().rename(columns={0:'counts'})\n",
    "smoker_df = smoker_df[0:8] # Removes prefer not to say' option "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0c9e70e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate plotly traces \n",
    "pos_smoker_trace = go.Bar(\n",
    "    x=smoker_df[smoker_df['covid_test_result']=='Positive']['smoker_status'],\n",
    "    y=100*smoker_df[smoker_df['covid_test_result']=='Positive']['counts']/participants,\n",
    "    name='COVID Positive',\n",
    "    showlegend=False,\n",
    "    marker_color='rgba(230,159,0,1)', \n",
    "    offsetgroup='Pos',\n",
    ")\n",
    "\n",
    "neg_smoker_trace = go.Bar(\n",
    "    x=smoker_df[smoker_df['covid_test_result']=='Negative']['smoker_status'],\n",
    "    y=100*smoker_df[smoker_df['covid_test_result']=='Negative']['counts']/participants,\n",
    "    name='COVID Negative',\n",
    "    showlegend=False,\n",
    "    marker_color='rgba(86,180,233,1)',\n",
    "    offsetgroup='Neg',\n",
    ")\n",
    "\n",
    "smoker_traces = [pos_smoker_trace, neg_smoker_trace]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e1adf30",
   "metadata": {},
   "source": [
    "#### Age"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d670f6d-d15c-4361-b897-0be1f8586da7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Format data for bar plot \n",
    "age_df = all_df.groupby(['age' ,'covid_test_result']).size().to_frame().reset_index().rename(columns={0:'counts'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f77996d1-6752-4699-9ece-b3dc3db246b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate plotly traces\n",
    "pos_age_trace = go.Bar(\n",
    "    x=age_df[age_df['covid_test_result']=='Positive']['age'],\n",
    "    y=100*age_df[age_df['covid_test_result']=='Positive']['counts']/participants,\n",
    "    name='COVID Positive',\n",
    "    showlegend=False,\n",
    "    marker_color='rgba(230,159,0,1)',\n",
    ")\n",
    "\n",
    "neg_age_trace = go.Bar(\n",
    "    x=age_df[age_df['covid_test_result']=='Negative']['age'],\n",
    "    y=100*age_df[age_df['covid_test_result']=='Negative']['counts']/participants,\n",
    "    name='COVID Negative',\n",
    "    showlegend=False,\n",
    "    marker_color='rgba(86,180,233,1)'\n",
    ")\n",
    "\n",
    "age_traces = [pos_age_trace, neg_age_trace]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b80b582",
   "metadata": {},
   "source": [
    "#### Gender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ef04eaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Renames variables, formats data for bar plot, removes no repsonse\n",
    "\n",
    "all_df['gender']=all_df['gender'].apply(lambda x:'Women' if x=='Female' else x)\n",
    "all_df['gender']=all_df['gender'].apply(lambda x:'Men' if x=='Male' else x)\n",
    "all_df['gender']=all_df['gender'].apply(lambda x:'Not Recorded' if pd.isna(x) else x)\n",
    "\n",
    "gender_df = all_df.groupby(['gender' ,'covid_test_result']).size().to_frame().reset_index().rename(columns={0:'counts'})\n",
    "gender_df=gender_df[gender_df['gender']!='Not Recorded'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01e7e577",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Geneates plotly traces\n",
    "gender_pos_trace = go.Bar(\n",
    "    x=gender_df[gender_df['covid_test_result']=='Positive']['gender'],\n",
    "    y=100*gender_df[gender_df['covid_test_result']=='Positive']['counts']/participants,\n",
    "    name='COVID Positive',\n",
    "    showlegend=False,\n",
    "    marker_color='rgba(230,159,0,1)',\n",
    ")\n",
    "\n",
    "gender_neg_trace = go.Bar(\n",
    "    x=gender_df[gender_df['covid_test_result']=='Negative']['gender'],\n",
    "    y=100*gender_df[gender_df['covid_test_result']=='Negative']['counts']/participants,\n",
    "    name='COVID Negative',\n",
    "    showlegend=False,\n",
    "    marker_color='rgba(86,180,233,1)'\n",
    ")\n",
    "\n",
    "gender_traces = [gender_pos_trace, gender_neg_trace]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f18094ee",
   "metadata": {},
   "source": [
    "#### Combined subplot for all Figure 2 traces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f03ee135",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = make_subplots(rows=4, cols=4, vertical_spacing=0.115,\n",
    "                     subplot_titles=('<b>Exhalation<b>', '<b>COVID Test Results<b>', '<b>Influenza Test Results<b>',\n",
    "                                   '<b>One Cough<b>', '<b>Age<b>', '<b>Gender<b>',\n",
    "                                   '<b>Three Coughs<b>', '<b>Respiratory Conditions<b>', '<b>Smoker Status<b>',\n",
    "                                   '<b>\"I love nothing more than an afternoon cream tea\"<b>', '<b>Symptoms<b>' ),\n",
    "                    specs = [[{\"colspan\":2}, None, {}, {}],\n",
    "                    [{\"colspan\":2}, None, {}, {}],\n",
    "                    [{\"colspan\":2}, None, {}, {}],\n",
    "                    [{\"colspan\":2}, None, {\"colspan\":2}, None]] \n",
    "                        \n",
    "                        )\n",
    "\n",
    "# Audio traces can be uncommented to include if they are generated above \n",
    "#fig.add_trace(sentence_trace, row=4, col=1) \n",
    "#fig.add_trace(exhalation_trace, row=1, col=1) \n",
    "#fig.add_trace(one_cough_trace, row=2, col=1) \n",
    "#fig.add_trace(three_coughs_trace, row=3, col=1) \n",
    "\n",
    "for trace in covid_test_traces: \n",
    "    fig.add_trace(trace, row=1, col=3)\n",
    "\n",
    "fig.add_trace(flu_trace, row=1, col=4) \n",
    "\n",
    "for trace in age_traces: \n",
    "    fig.add_trace(trace, row=2, col=3)\n",
    "\n",
    "for trace in gender_traces:\n",
    "    fig.add_trace(trace, row=2, col=4)\n",
    "\n",
    "for trace in condition_traces: \n",
    "    fig.add_trace(trace, row=3, col=3)\n",
    "\n",
    "for trace in smoker_traces: \n",
    "    fig.add_trace(trace, row=3, col=4)\n",
    "\n",
    "for trace in symptom_traces: \n",
    "    fig.add_trace(trace, row=4, col=3)\n",
    "    \n",
    "    \n",
    "fig.for_each_trace(\n",
    "    lambda trace: trace.update(colorbar=dict(x=0.925, y=0.16, len=0.15, borderwidth=0)) if trace.type == \"choropleth\" else ()\n",
    ")\n",
    "\n",
    "fig.for_each_trace(\n",
    "    lambda trace: trace.update(width=0.3, marker=dict(line=dict(width=0))) if trace.type == \"bar\" else ()\n",
    ")\n",
    "\n",
    "fig.for_each_trace(\n",
    "    lambda trace: trace.update(offset=0) if trace.type == \"bar\" and 'Negative' in trace.name else ()\n",
    ")\n",
    "\n",
    "fig.for_each_trace(\n",
    "    lambda trace: trace.update(offset=-0.3) if trace.type == \"bar\" and 'Positive' in trace.name else ()\n",
    ")\n",
    "\n",
    "fig.for_each_trace(\n",
    "    lambda trace: trace.update(legendgroup='A') if trace.type == \"bar\" and trace.offsetgroup == 'A' else trace.update(legendgroup='B')\n",
    ")\n",
    "\n",
    "\n",
    "fig.update_yaxes(showticklabels=False, ticks=\"\", col=1, row=1)\n",
    "fig.update_yaxes(showticklabels=False, ticks=\"\", col=1, row=2)\n",
    "fig.update_yaxes(showticklabels=False, ticks=\"\", col=1, row=3)\n",
    "fig.update_yaxes(showticklabels=False, ticks=\"\", col=1, row=4)\n",
    "fig.update_xaxes(title='Seconds', col=1, row=1)\n",
    "fig.update_xaxes(title='Seconds', col=1, row=2)\n",
    "fig.update_xaxes(title='Seconds', col=1, row=3)\n",
    "fig.update_xaxes(title='Seconds', col=1, row=4)\n",
    "fig.update_yaxes(ticksuffix=\"%\", col=3)\n",
    "fig.update_yaxes(ticksuffix=\"%\", col=4)\n",
    "fig.update_yaxes(ticksuffix=\"%\", row=5, col=2)\n",
    "fig.update_xaxes(tickangle=35, col=3)\n",
    "fig.update_xaxes(tickangle=35, col=4)\n",
    "fig.update_xaxes(tickangle=35, row=4, col=3)\n",
    "fig.update_layout(template='simple_white',\n",
    "                  font_family='Arial', font_size=12, \n",
    "                  width=1140, height=1040,\n",
    "                  legend_tracegroupgap=100,\n",
    "                 boxmode='group')\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c6634f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write plot to image \n",
    "fig.write_image('DataPaperFig2.svg', scale=1)\n",
    "fig.write_image('DataPaperFig2.png', scale=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec4fb21c",
   "metadata": {},
   "source": [
    "# Figure 3a.-c. - Technical Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "189857c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get statistic for number of audio channels in all audio files\n",
    "audio_df['cough_channels'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af5e6865",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot data\n",
    "\n",
    "fig = make_subplots(rows=1, cols=3, vertical_spacing=0.25, horizontal_spacing=0.15)\n",
    "\n",
    "\n",
    "# Length subplot\n",
    "\n",
    "# fig.add_trace(go.Violin(\n",
    "# y=audio_df['sentence_length'],\n",
    "# name='Sentence',\n",
    "# marker_color = '#D55E00',\n",
    "# points=False, \n",
    "# showlegend=False,\n",
    "# fillcolor='rgba(204, 121, 167, 0.25)', \n",
    "# scalegroup='A', \n",
    "# scalemode='count', \n",
    "# spanmode='manual', \n",
    "# span=[audio_df['sentence_length'].quantile(0.005), audio_df['sentence_length'].quantile(0.995)], \n",
    "# box=dict(visible=True)\n",
    "# ), row=1, col=1)\n",
    "\n",
    "fig.add_trace(go.Violin(\n",
    "y=audio_df['exhalation_length'],\n",
    "name='Exhalation',\n",
    "marker_color = '#CC79A7',\n",
    "points=False, \n",
    "showlegend=False,\n",
    "fillcolor='rgba(204, 121, 167, 0.25)', \n",
    "scalegroup='A', \n",
    "scalemode='count', \n",
    "spanmode='manual', \n",
    "span=[audio_df['exhalation_length'].quantile(0.005), audio_df['exhalation_length'].quantile(0.995)], \n",
    "box=dict(visible=True)\n",
    "), row=1, col=1)\n",
    "\n",
    "fig.add_trace(go.Violin(\n",
    "y=audio_df['cough_length'],\n",
    "name='One Cough',\n",
    "marker_color = '#CC79A7',\n",
    "points=False, \n",
    "showlegend=False,\n",
    "fillcolor='rgba(204, 121, 167, 0.25)',  \n",
    "scalemode='count',\n",
    "scalegroup='A', \n",
    "spanmode='manual', \n",
    "span=[audio_df['cough_length'].quantile(0.005), audio_df['cough_length'].quantile(0.995)],\n",
    "box=dict(visible=True)\n",
    "), row=1, col=1)\n",
    "\n",
    "fig.add_trace(go.Violin(\n",
    "y=audio_df['three_cough_length'],\n",
    "name='Three Coughs',\n",
    "marker_color = '#CC79A7',\n",
    "scalegroup='A', \n",
    "scalemode='count', \n",
    "spanmode='manual', \n",
    "span=[audio_df['three_cough_length'].quantile(0.005), audio_df['three_cough_length'].quantile(0.995)], \n",
    "points=False, \n",
    "showlegend=False,\n",
    "fillcolor='rgba(204, 121, 167, 0.25)', \n",
    "box=dict(visible=True)\n",
    "), row=1, col=1)\n",
    "\n",
    "\n",
    "# Amplitude subplot\n",
    "\n",
    "# fig.add_trace(go.Violin(\n",
    "# y=audio_df['sentence_amplitude'],\n",
    "# name='Sentence',\n",
    "# marker_color = '#D55E00',\n",
    "# scalegroup='B', \n",
    "# scalemode='count', \n",
    "# spanmode='manual', \n",
    "# span=[audio_df['sentence_amplitude'].quantile(0.005), audio_df['sentence_amplitude'].quantile(0.995)], \n",
    "# points=False, \n",
    "# showlegend=False,\n",
    "# fillcolor='rgba(204, 121, 167, 0.25)', \n",
    "# box=dict(visible=True)\n",
    "# ), row=1, col=2)\n",
    "\n",
    "fig.add_trace(go.Violin(\n",
    "y=audio_df['exhalation_amplitude'],\n",
    "name='Exhalation',\n",
    "marker_color = '#CC79A7',\n",
    "scalegroup='B', \n",
    "scalemode='count', \n",
    "spanmode='manual', \n",
    "span=[audio_df['exhalation_amplitude'].quantile(0.005), audio_df['exhalation_amplitude'].quantile(0.995)],\n",
    "points=False, \n",
    "showlegend=False,\n",
    "fillcolor='rgba(204, 121, 167, 0.25)', \n",
    "box=dict(visible=True)\n",
    "), row=1, col=2)\n",
    "\n",
    "fig.add_trace(go.Violin(\n",
    "y=audio_df['cough_amplitude'],\n",
    "name='One Cough',\n",
    "marker_color = '#CC79A7',\n",
    "scalegroup='B', \n",
    "scalemode='count', \n",
    "spanmode='manual', \n",
    "span=[audio_df['cough_amplitude'].quantile(0.005), audio_df['cough_amplitude'].quantile(0.995)],\n",
    "points=False, \n",
    "showlegend=False,\n",
    "fillcolor='rgba(204, 121, 167, 0.25)', \n",
    "box=dict(visible=True)\n",
    "), row=1, col=2)\n",
    "\n",
    "fig.add_trace(go.Violin(\n",
    "y=audio_df['three_cough_amplitude'],\n",
    "name='Three Coughs',\n",
    "marker_color = '#CC79A7',\n",
    "scalegroup='B', \n",
    "scalemode='count', \n",
    "spanmode='manual', \n",
    "span=[audio_df['three_cough_amplitude'].quantile(0.005), audio_df['three_cough_amplitude'].quantile(0.995)],\n",
    "points=False, \n",
    "showlegend=False,\n",
    "fillcolor='rgba(204, 121, 167, 0.25)',  \n",
    "box=dict(visible=True)\n",
    "), row=1, col=2)\n",
    "\n",
    "\n",
    "# Variance subplot\n",
    "\n",
    "# fig.add_trace(go.Violin(\n",
    "# y=audio_df['sentence_signal_mean_std_ratio'],\n",
    "# name='Sentence',\n",
    "# marker_color = '#D55E00',\n",
    "# scalegroup='C', \n",
    "# scalemode='count', \n",
    "# spanmode='manual',\n",
    "# span=[audio_df['sentence_signal_mean_std_ratio'].quantile(0.005), audio_df['sentence_signal_mean_std_ratio'].quantile(0.995)],\n",
    "# points=False, \n",
    "# showlegend=False,\n",
    "# fillcolor='rgba(204, 121, 167, 0.25)', \n",
    "# box=dict(visible=True)\n",
    "# ), row=1, col=3)\n",
    "\n",
    "fig.add_trace(go.Violin(\n",
    "y=audio_df['exhalation_signal_mean_std_ratio'],\n",
    "name='Exhalation',\n",
    "marker_color = '#CC79A7',\n",
    "scalegroup='C',\n",
    "scalemode='count', \n",
    "spanmode='manual', \n",
    "span=[audio_df['exhalation_signal_mean_std_ratio'].quantile(0.005), audio_df['exhalation_signal_mean_std_ratio'].quantile(0.995)],\n",
    "points=False, \n",
    "showlegend=False,\n",
    "fillcolor='rgba(204, 121, 167, 0.25)',  \n",
    "box=dict(visible=True)\n",
    "), row=1, col=3)\n",
    "\n",
    "fig.add_trace(go.Violin(\n",
    "y=audio_df['cough_signal_mean_std_ratio'],\n",
    "name='One Cough',\n",
    "marker_color = '#CC79A7',\n",
    "scalegroup='C',\n",
    "scalemode='count', \n",
    "spanmode='manual',\n",
    "span=[audio_df['cough_signal_mean_std_ratio'].quantile(0.005), audio_df['cough_signal_mean_std_ratio'].quantile(0.995)],\n",
    "points=False, \n",
    "showlegend=False,\n",
    "fillcolor='rgba(204, 121, 167, 0.25)', \n",
    "box=dict(visible=True)\n",
    "), row=1, col=3)\n",
    "\n",
    "fig.add_trace(go.Violin(\n",
    "y=audio_df['three_cough_signal_mean_std_ratio'],\n",
    "name='Three Coughs',\n",
    "marker_color = '#CC79A7',\n",
    "scalegroup='C',\n",
    "scalemode='count', \n",
    "spanmode='manual', \n",
    "span=[audio_df['three_cough_signal_mean_std_ratio'].quantile(0.005), audio_df['three_cough_signal_mean_std_ratio'].quantile(0.995)],\n",
    "points=False, \n",
    "showlegend=False,\n",
    "fillcolor='rgba(204, 121, 167, 0.25)', \n",
    "box=dict(visible=True)\n",
    "), row=1, col=3)\n",
    "\n",
    "fig.update_annotations(dict(ay=-15))\n",
    "fig.update_xaxes(tickangle=45, row=1)\n",
    "fig.update_yaxes(range=[0,20], row=1, col=1)\n",
    "fig.update_yaxes(type='log', row=1, range=[-1.25, 0], col=3)\n",
    "fig.update_xaxes(title='Normalised Outlier Score', row=3, col=1)\n",
    "fig.update_yaxes(title='Length (seconds)', title_standoff=0.05, row=1, col=1)\n",
    "fig.update_yaxes(title='Amplitude (AU)', title_standoff=0.05, row=1, col=2)\n",
    "fig.update_yaxes(title='Mean/SD (ratio)', title_standoff=0.05, nticks=2, row=1, col=3)\n",
    "fig.update_layout(template='simple_white', violingap=0, violingroupgap=0.1, legend=dict(y=0.45, x=0.05),\n",
    "                  font_family='Arial', font_size=13, width=750, height=300)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f7b5982",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write plot to image \n",
    "fig.write_image(\"DataPaperFig3.svg\", scale=1)\n",
    "fig.write_image(\"DataPaperFig3.png\", scale=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01591cf3",
   "metadata": {},
   "source": [
    "## Figure 4 - Bias\n",
    "The gender bias subfigures can be recreated from the open dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5aff446-0ae4-4374-90f1-130557377453",
   "metadata": {},
   "outputs": [],
   "source": [
    "## ONS Census 2021 data\n",
    "# from https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationestimates/datasets/populationandhouseholdestimatesenglandandwalescensus2021\n",
    "\n",
    "data = ',Gender,Total,% Total\\n0,Women,28833500,51.04195801719956\\n1,Men,27656300,48.95804198280044\\n'\n",
    "census_gender_df = pd.read_csv(io.StringIO(data), header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05cc144b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TT Data \n",
    "# from Weekly statistics for NHS Test and Trace (England) 2 to 15 June 2022. 22 https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/1085136/NHS-test-and-trace-23-june-2022.pdf (2022)\n",
    "\n",
    "data = ',Gender,Positive,Total,% Total,% Positive\\n0,Women,5913615.0,68650051.0,59.40748549683927,5.1174469971827286\\n1,Men,5239446.0,46907863.0,40.59251450316073,4.534043423456052\\n'\n",
    "tt_gender_df = pd.read_csv(io.StringIO(data), header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1d0440a",
   "metadata": {},
   "outputs": [],
   "source": [
    "## REACT data \n",
    "# from various REACT reports listed here https://www.imperial.ac.uk/medicine/research-and-impact/groups/react-study/real-time-assessment-of-community-transmission-findings/\n",
    "\n",
    "data = ',Gender,Positive,Total,% Total,% Positive\\n1,Women,5798,330449,55.716971429053174,0.9776001753542916\\n0,Men,4888,262613,44.27915054334539,0.8241651702538422\\n2,Unknown,0,23,0.0038780276014399287,0.0\\n'\n",
    "react_gender_df = pd.read_csv(io.StringIO(data), header=0)\n",
    "react_gender_df = react_gender_df.drop(index=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b36a5cc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "bam_total_df = participant_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1fb10634",
   "metadata": {},
   "outputs": [],
   "source": [
    "gender_dict = {\n",
    "    'Female':'Women',\n",
    "    'Male':'Men',\n",
    "    'Unknown': 'Unknown'\n",
    "}\n",
    "\n",
    "bam_total_df['gender'] = bam_total_df['gender'].map(gender_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97be21d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "bam_pos_total_df = bam_total_df[bam_total_df['covid_test_result']=='Positive']\n",
    "\n",
    "# TT data only PCR & LAMP (filtered by testing types containing 'P')\n",
    "bam_tt_df = bam_total_df[(bam_total_df['recruitment_source']=='Test and Trace') & (bam_total_df['covid_test_method'].str.contains('P', na=False))]\n",
    "bam_pos_tt_df = bam_tt_df[bam_tt_df['covid_test_result']=='Positive']\n",
    "\n",
    "bam_react_df = bam_total_df[bam_total_df['recruitment_source'].str.contains('REACT')]\n",
    "bam_pos_react_df = bam_react_df[bam_react_df['covid_test_result']=='Positive']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61e08242",
   "metadata": {},
   "outputs": [],
   "source": [
    "bam_tt = bam_tt_df.shape[0]\n",
    "bam_react = bam_react_df.shape[0]\n",
    "bam_total = bam_total_df.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2603e23d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get gender data for each dataset subset \n",
    "\n",
    "# TT\n",
    "bam_tt_gender_df = bam_tt_df['gender'].value_counts(dropna=False).reset_index()\n",
    "bam_tt_gender_df.columns = ['Gender', 'Total']\n",
    "bam_tt_gender_df['% Total'] = 100*bam_tt_gender_df['Total']/bam_tt\n",
    "bam_tt_gender_df = bam_tt_gender_df[0:2]\n",
    "\n",
    "bam_pos_tt_gender_df = bam_pos_tt_df['gender'].value_counts(dropna=False).reset_index()\n",
    "bam_pos_tt_gender_df.columns = ['Gender', 'Positive']\n",
    "bam_pos_tt_gender_df['% Positive'] = 100*bam_pos_tt_gender_df['Positive']/bam_tt\n",
    "bam_pos_tt_gender_df = bam_pos_tt_gender_df[0:2]\n",
    "\n",
    "# REACT\n",
    "bam_react_gender_df = bam_react_df['gender'].value_counts(dropna=False).reset_index()\n",
    "bam_react_gender_df.columns = ['Gender', 'Total']\n",
    "bam_react_gender_df['% Total'] = 100*bam_react_gender_df['Total']/bam_react\n",
    "bam_react_gender_df = bam_react_gender_df[0:2]\n",
    "\n",
    "bam_pos_react_gender_df = bam_pos_react_df['gender'].value_counts(dropna=False).reset_index()\n",
    "bam_pos_react_gender_df.columns = ['Gender', 'Positive']\n",
    "bam_pos_react_gender_df['% Positive'] = 100*bam_pos_react_gender_df['Positive']/bam_react\n",
    "bam_pos_react_gender_df = bam_pos_react_gender_df[0:2]\n",
    "\n",
    "# Total \n",
    "bam_total_gender_df = bam_total_df['gender'].value_counts(dropna=False).reset_index()\n",
    "bam_total_gender_df.columns = ['Gender', 'Total']\n",
    "bam_total_gender_df['% Total'] = 100*bam_total_gender_df['Total']/bam_total\n",
    "bam_total_gender_df = bam_total_gender_df[0:2]\n",
    "\n",
    "bam_pos_total_gender_df = bam_pos_total_df['gender'].value_counts(dropna=False).reset_index()\n",
    "bam_pos_total_gender_df.columns = ['Gender', 'Positive']\n",
    "bam_pos_total_gender_df['% Positive'] = 100*bam_pos_total_gender_df['Positive']/bam_total\n",
    "bam_pos_total_gender_df = bam_pos_total_gender_df[0:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d576575c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot\n",
    "\n",
    "fig=make_subplots(rows=4, cols=3,\n",
    "                 vertical_spacing=0.175, horizontal_spacing=0.15,\n",
    "                 subplot_titles=(\"<b>Census 2021<b>\", \"<b>NHS Test and Trace<b>\", \"<b>REACT<b>\"))\n",
    "\n",
    "\n",
    "# Census gender\n",
    "fig.add_trace(go.Bar(\n",
    "x=census_gender_df['Gender'],\n",
    "y=census_gender_df['% Total'],\n",
    "width=0.4,\n",
    "offset=0,\n",
    "showlegend=False,\n",
    "marker_color=\"#56B4E9\",\n",
    "marker=dict(line=dict(width=0)),\n",
    "name=\"Census Total\"), \n",
    "             row=2, col=1)\n",
    "\n",
    "\n",
    "fig.add_trace(go.Bar(\n",
    "x=bam_total_gender_df['Gender'],\n",
    "y=bam_total_gender_df['% Total'],\n",
    "width=0.4,\n",
    "offset=-0.4,\n",
    "showlegend=False,\n",
    "marker_color=\"#E69F00\",\n",
    "marker=dict(line=dict(width=0)),\n",
    "name=\"This Study\"),\n",
    "row=2, col=1)\n",
    "\n",
    "fig.add_trace(go.Bar(\n",
    "x=bam_pos_total_gender_df['Gender'],\n",
    "y=bam_pos_total_gender_df['% Positive'],\n",
    "width=0.4,\n",
    "offset=-0.4,\n",
    "showlegend=False,\n",
    "marker_color=\"#D55E00\",\n",
    "marker=dict(line=dict(width=0)),\n",
    "name=\"This Study\"),\n",
    "row=2, col=1)\n",
    "\n",
    "\n",
    "# TT gender\n",
    "fig.add_trace(go.Bar(\n",
    "x=tt_gender_df['Gender'],\n",
    "y=tt_gender_df['% Total'],\n",
    "width=0.4,\n",
    "offset=0,\n",
    "showlegend=False,\n",
    "marker_color=\"#56B4E9\",\n",
    "marker=dict(line=dict(width=0)),\n",
    "name=\"NHS Test and Trace Total\"), \n",
    "             row=2, col=2)\n",
    "\n",
    "fig.add_trace(go.Bar(\n",
    "x=tt_gender_df['Gender'],\n",
    "y=tt_gender_df['% Positive'],\n",
    "width=0.4,\n",
    "offset=0,\n",
    "showlegend=False,\n",
    "marker_color=\"#0072B2\",\n",
    "marker=dict(line=dict(width=0)),\n",
    "name=\"NHS Test and Trace Positive\"),\n",
    "             row=2, col=2)\n",
    "\n",
    "\n",
    "fig.add_trace(go.Bar(\n",
    "x=bam_tt_gender_df['Gender'],\n",
    "y=bam_tt_gender_df['% Total'],\n",
    "width=0.4,\n",
    "offset=-0.4,\n",
    "showlegend=False,\n",
    "marker_color=\"#E69F00\",\n",
    "marker=dict(line=dict(width=0)),\n",
    "name=\"This Study Total (Test and Trace recruited)\"),\n",
    "row=2, col=2)\n",
    "\n",
    "fig.add_trace(go.Bar(\n",
    "x=bam_pos_tt_gender_df['Gender'],\n",
    "y=bam_pos_tt_gender_df['% Positive'],\n",
    "width=0.4,\n",
    "offset=-0.4,\n",
    "showlegend=False,\n",
    "marker_color=\"#D55E00\",\n",
    "marker=dict(line=dict(width=0)),\n",
    "name=\"This Study Positive (Test and Trace recruited)\"),\n",
    "row=2, col=2)\n",
    "\n",
    "\n",
    "# REACT gender\n",
    "fig.add_trace(go.Bar(\n",
    "x=react_gender_df['Gender'],\n",
    "y=react_gender_df['% Total'],\n",
    "width=0.4,\n",
    "offset=0,\n",
    "showlegend=False,\n",
    "marker_color=\"#56B4E9\",\n",
    "marker=dict(line=dict(width=0)),\n",
    "name=\"REACT Study Total\"),\n",
    "             row=2, col=3)\n",
    "\n",
    "fig.add_trace(go.Bar(\n",
    "x=react_gender_df['Gender'],\n",
    "y=react_gender_df['% Positive'],\n",
    "width=0.4,\n",
    "offset=0,\n",
    "showlegend=False,\n",
    "marker_color=\"#0072B2\",\n",
    "marker=dict(line=dict(width=0)),\n",
    "name=\"REACT Study Positive\"),\n",
    "             row=2, col=3)\n",
    "\n",
    "fig.add_trace(go.Bar(\n",
    "x=bam_react_gender_df['Gender'],\n",
    "y=bam_react_gender_df['% Total'],\n",
    "width=0.4,\n",
    "offset=-0.4,\n",
    "showlegend=False,\n",
    "marker_color=\"#E69F00\",\n",
    "marker=dict(line=dict(width=0)),\n",
    "name=\"This Study Total (REACT recruited)\"),\n",
    "row=2, col=3)\n",
    "\n",
    "fig.add_trace(go.Bar(\n",
    "x=bam_pos_react_gender_df['Gender'],\n",
    "y=bam_pos_react_gender_df['% Positive'],\n",
    "width=0.4,\n",
    "offset=-0.4,\n",
    "showlegend=False,\n",
    "marker_color=\"#D55E00\",\n",
    "marker=dict(line=dict(width=0)),\n",
    "name=\"This Study Positive (REACT recruited)\"),\n",
    "row=2, col=3)\n",
    "\n",
    "\n",
    "\n",
    "fig.update_xaxes(title='Age Group', row=1)\n",
    "fig.update_yaxes(range=[0,26], row=1)\n",
    "fig.update_xaxes(title='Gender', row=2)\n",
    "fig.update_yaxes(range=[0, 60], row=2)\n",
    "fig.update_xaxes(title='Ethnicity', row=3)\n",
    "fig.update_xaxes(title='Region', row=4)\n",
    "fig.update_yaxes(range=[0, 20], row=4)\n",
    "fig.update_xaxes(tickangle = 45, title_standoff = 5)\n",
    "fig.update_yaxes(title='% of Total')\n",
    "\n",
    "fig.layout.annotations[0].update(y=1.07)\n",
    "fig.layout.annotations[1].update(y=1.07)\n",
    "fig.layout.annotations[2].update(y=1.07)\n",
    "fig.update_layout(template='simple_white', font_family='Arial', font_size=14, height=850, width=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "774ceaa9",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig.write_image(\"DataPaperFig4.svg\", scale=1)\n",
    "fig.write_image(\"DataPaperFig4.png\", scale=3)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}