temporal-network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.lines import Line2D\n",
    "from pycountry_convert import country_alpha2_to_continent_code, country_name_to_country_alpha2, country_name_to_country_alpha3\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of dataframe: (1274, 45)\n",
      "Columns:  ['ID', 'COUNTRY_NAME', 'ISO3', 'ISO2', 'POLICY_TYPE', 'POLICY_SUBTYPE', 'START_DATE', 'END_DATE', 'AIR', 'AIR_TYPE', 'TARGETS_AIR', 'LAND', 'LAND_TYPE', 'TARGETS_LAND', 'SEA', 'SEA_TYPE', 'TARGETS_SEA', 'CITIZEN', 'CITIZEN_LIST', 'HISTORY_BAN', 'HISTORY_BAN_LIST', 'REFUGEE', 'REFUGEE_LIST', 'VISA_BAN', 'VISA_BAN_TYPE', 'VISA_BAN_LIST', 'CITIZEN_EXCEP', 'CITIZEN_EXCEP_LIST', 'COUNTRY_EXCEP', 'COUNTRY_EXCEP_LIST', 'WORK_EXCEP', 'SOURCE_QUALITY', 'SOURCE_TYPE', 'INTERNAL_GOVT_SOURCE', 'AIRLINE_SOURCE', 'INSURANCE_SOURCE', 'GOVT_SOCIAL_MED_SOURCE', 'EXT_GOVT_SOURCE', 'INTERNAL_MEDIA_SOURCE', 'EXT_MEDIA_SOURCE', 'OTHER_SOURCE', 'END_SOURCE', 'COMMENTS', 'OLD_ID', 'DURATION']\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('data/policy_list.csv')\n",
    "df.START_DATE = pd.to_datetime(df.START_DATE, format = \"%m_%d_%y\", errors = 'coerce')\n",
    "df.END_DATE = pd.to_datetime(df.END_DATE, format = \"%m_%d_%y\", errors = 'coerce')\n",
    "#fixing wrong dates\n",
    "df.loc[878, ['END_DATE']] = pd.to_datetime('2021-01-10')\n",
    "df.loc[69, ['END_DATE']] = pd.to_datetime('2020-04-28')\n",
    "df.loc[421, ['START_DATE']] = pd.to_datetime('2020-08-17')\n",
    "df.loc[595, ['END_DATE']] = pd.to_datetime('2021-01-04')\n",
    "df.loc[918, ['END_DATE']] = pd.to_datetime('2021-02-01')\n",
    "df.loc[1196, ['END_DATE']] = pd.to_datetime('2021-09-21')\n",
    "\n",
    "df = df.sort_values('START_DATE')\n",
    "\n",
    "#remove data with no duration\n",
    "df['DURATION'] = df.END_DATE - df.START_DATE\n",
    "df = df[df.DURATION > pd.Timedelta(seconds = 0)]\n",
    "\n",
    "#fixing policys with country equal to 'European Union'\n",
    "european_union_countries = ['Austria',  'Belgium', 'Bulgaria', 'Croatia', 'Cyprus',\n",
    "                            'Czech Republic', 'Denmark', 'Estonia', 'Finland', 'France',\n",
    "                            'Germany', 'Greece', 'Hungary', 'Ireland', 'Italy', 'Latvia',\n",
    "                            'Lithuania', 'Luxembourg', 'Malta', 'Netherlands', 'Poland',\n",
    "                            'Portugal','Romania', 'Slovakia', 'Slovenia', 'Spain', 'Sweden', 'United Kingdom']\n",
    "df_wrong_country = df[df.COUNTRY_NAME == 'European Union']\n",
    "new_rows = dict([(a, []) for a in df.columns])\n",
    "\n",
    "for row in df_wrong_country.iterrows():\n",
    "    for i, country in enumerate(european_union_countries):\n",
    "        for col in df.columns:\n",
    "            if col == 'COUNTRY_NAME':\n",
    "                new_rows[col].append(country)\n",
    "            elif col == 'ISO2':\n",
    "                new_rows[col].append(country_name_to_country_alpha2(country))\n",
    "            elif col == 'ISO3':\n",
    "                new_rows[col].append(country_name_to_country_alpha3(country))\n",
    "            else:\n",
    "                new_rows[col].append(row[1][col])\n",
    "df = df[df.COUNTRY_NAME != 'European Union']\n",
    "df = pd.concat([df, pd.DataFrame(new_rows)])\n",
    "\n",
    "print(f\"Size of dataframe: {df.shape}\")\n",
    "print(\"Columns: \", list(df.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique countries: 221\n",
      "Initial start date: 2020-01-24 00:00:00\n",
      "Last end date: 2021-10-01 00:00:00\n",
      "Policy with smallest duration: 1 days 00:00:00\n",
      "Policy duration mean: 59 days 06:14:07.723704867\n",
      "\n",
      "Type of policys:\n",
      "PARTIAL     824\n",
      "COMPLETE    450\n",
      "Name: POLICY_TYPE, dtype: int64\n",
      "\n",
      "Closures:\n",
      "Total of air closures: 505/1274\n",
      "Total of land closures: 214/1274\n",
      "Total of sea closures: 178/1274\n",
      "\n",
      "Partial closures params:\n",
      "citizen: 71/1274\n",
      "history_ban: 140/1274\n",
      "refugee: 1/1274\n",
      "visa_ban: 41/1274\n",
      "\n",
      "Complete closure exceptions:\n",
      "citizen_excep: 408/1274\n",
      "country_excep: 186/1274\n",
      "work_excep: 208/1274\n"
     ]
    }
   ],
   "source": [
    "print(f\"Unique countries: {len(df.COUNTRY_NAME.unique())}\")\n",
    "print(f\"Initial start date: {df.START_DATE.min()}\")\n",
    "print(f\"Last end date: {df.END_DATE.max()}\")\n",
    "print(f\"Policy with smallest duration: {df.DURATION.min()}\")\n",
    "print(f\"Policy duration mean: {df.DURATION.mean()}\")\n",
    "print(\"\\nType of policys:\")\n",
    "print(df.POLICY_TYPE.value_counts())\n",
    "print(\"\\nClosures:\")\n",
    "print(f\"Total of air closures: {df[df.AIR == 1].shape[0]}/{df.shape[0]}\")\n",
    "print(f\"Total of land closures: {df[df.LAND == 1].shape[0]}/{df.shape[0]}\")\n",
    "print(f\"Total of sea closures: {df[df.SEA == 1].shape[0]}/{df.shape[0]}\")\n",
    "print(\"\\nPartial closures params:\")\n",
    "for col in ['CITIZEN', 'HISTORY_BAN', 'REFUGEE', 'VISA_BAN']:\n",
    "    print(f\"{col.lower()}: {df[df[col] == 1].shape[0]}/{df.shape[0]}\")\n",
    "print(\"\\nComplete closure exceptions:\")    \n",
    "for col in ['CITIZEN_EXCEP', 'COUNTRY_EXCEP', 'WORK_EXCEP']:\n",
    "    print(f\"{col.lower()}: {df[df[col] == 1].shape[0]}/{df.shape[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[(df.AIR == 1) | (df.POLICY_TYPE == 'COMPLETE')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Source</th>\n",
       "      <th>Target</th>\n",
       "      <th>time_start</th>\n",
       "      <th>time_end</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Israel</td>\n",
       "      <td>China</td>\n",
       "      <td>2020-01-30</td>\n",
       "      <td>2020-03-18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Italy</td>\n",
       "      <td>China</td>\n",
       "      <td>2020-01-30</td>\n",
       "      <td>2020-02-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Pakistan</td>\n",
       "      <td>China</td>\n",
       "      <td>2020-01-31</td>\n",
       "      <td>2020-03-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Palau</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>2020-02-29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Palau</td>\n",
       "      <td>Macau</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>2020-02-29</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Source     Target time_start   time_end\n",
       "0    Israel      China 2020-01-30 2020-03-18\n",
       "1     Italy      China 2020-01-30 2020-02-09\n",
       "2  Pakistan      China 2020-01-31 2020-03-21\n",
       "3     Palau  Hong Kong 2020-02-01 2020-02-29\n",
       "4     Palau      Macau 2020-02-01 2020-02-29"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add values in a list to a dictionary\n",
    "def add_dict(row,dic):\n",
    "    for i, key in enumerate(dic):\n",
    "        dic[key].append(row[i])\n",
    "\n",
    "# parse the string to get the countries involved\n",
    "def parse_country_list(string):\n",
    "    string = str(string)\n",
    "    possible_seps = ['or','and']\n",
    "    for sep in possible_seps:\n",
    "        string.replace(sep, ',')\n",
    "    list_string = string.split(sep=',')\n",
    "    for i in range(len(list_string)):\n",
    "        list_string[i] = list_string[i].split(sep='(')[0].strip()\n",
    "    list_string = [string for string in list_string if string in countries]\n",
    "    return list_string\n",
    "\n",
    "# only considering these countries\n",
    "with open('data/countries.json', 'r') as f:\n",
    "    countries = json.load(f)\n",
    "countries_map = dict([(countries[i], i) for i in range(len(countries))])\n",
    "\n",
    "# our main data object\n",
    "restrictions = {'Source':[],'Target':[], 'time_start':[],'time_end':[]}\n",
    "# iterate over each row, finding out which type is TRUE and what countries are in the restriction\n",
    "for index, row in df.iterrows():\n",
    "    country1 = row['COUNTRY_NAME']\n",
    "    if country1 not in countries:\n",
    "        continue\n",
    "    if row['AIR'] == 1:\n",
    "        if row['AIR_TYPE'] == 'All':\n",
    "            for country2 in countries:\n",
    "                add_dict([country1,country2, row['START_DATE'],row['END_DATE']], restrictions)\n",
    "        else:\n",
    "            countries2 = parse_country_list(row['TARGETS_AIR'])\n",
    "            for country2 in countries2:\n",
    "                if country2 in countries:\n",
    "                    add_dict([country1,country2, row['START_DATE'],row['END_DATE']], restrictions)           \n",
    "    else:\n",
    "        if pd.isna(row['COUNTRY_EXCEP_LIST']):\n",
    "            for country2 in countries:\n",
    "                add_dict([country1,country2, row['START_DATE'],row['END_DATE']], restrictions)\n",
    "        else:\n",
    "            countries2 = parse_country_list(row['COUNTRY_EXCEP_LIST'])\n",
    "            for country2 in countries:\n",
    "                if not country2 in countries2:\n",
    "                    add_dict([country1,country2, row['START_DATE'],row['END_DATE']], restrictions)\n",
    "restrictions = pd.DataFrame(restrictions)\n",
    "\n",
    "restrictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of 3D adjacency matrix: (549, 225, 225)\n"
     ]
    }
   ],
   "source": [
    "number_of_nodes = len(countries)\n",
    "\n",
    "basedate = restrictions.time_start.min()\n",
    "restrictions['time_start_num'] = (restrictions['time_start'] - basedate).dt.days\n",
    "restrictions['time_end_num'] = (restrictions['time_end'] - basedate).dt.days\n",
    "\n",
    "number_of_days = max(restrictions.time_start_num.max(), restrictions.time_end_num.max())\n",
    "\n",
    "with open('data/initial_network.np', 'rb') as f:\n",
    "    initial_network = np.load(f)\n",
    "temporal_adj = np.tile(initial_network, (number_of_days, 1, 1))\n",
    "\n",
    "print(f\"Shape of 3D adjacency matrix: {temporal_adj.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index, row in restrictions.iterrows():\n",
    "    source = row['Source']\n",
    "    source_i = countries_map[source]\n",
    "    target = row['Target']\n",
    "    t1 = row['time_start_num']\n",
    "    t2 = row['time_end_num']\n",
    "    target_i = countries_map[target]    \n",
    "    temporal_adj[t1:t2, source_i, target_i] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/temporal_network.np', 'wb') as f:\n",
    "    np.save(f, temporal_adj)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}