Toronto Neighborhoods.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### In the next cell, the libraries needed for the project are imported ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Libraries imported.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd \n",
    "import requests\n",
    "import lxml.html as lh\n",
    "print('Libraries imported.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Url is saved, as well as the table content ### "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "url='https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M'\n",
    "page = requests.get(url)\n",
    "doc = lh.fromstring(page.content)\n",
    "tr_elements = doc.xpath('//tr')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Separate the first line of the table and save it in a list to be the table header ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1:\"Postcode\"\n",
      "2:\"Borough\"\n",
      "3:\"Neighbourhood\n",
      "\"\n"
     ]
    }
   ],
   "source": [
    "tr_elements = doc.xpath('//tr')\n",
    "col=[]\n",
    "i=0\n",
    "for t in tr_elements[0]:\n",
    "    i+=1\n",
    "    name=t.text_content()\n",
    "    print('%d:\"%s\"'%(i,name))\n",
    "    col.append((name,[]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add the following data to the list associating it with the data previously added ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "for j in range(1,len(tr_elements)):\n",
    "    T=tr_elements[j]\n",
    "\n",
    "    if len(T)!=3:\n",
    "        break\n",
    "    \n",
    "    i=0\n",
    "    \n",
    "    for t in T.iterchildren():\n",
    "        data=t.text_content() \n",
    "\n",
    "        col[i][1].append(data)\n",
    "\n",
    "        i+=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Turn list into a dictionary and then into a dataframe ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "Dict={title:column for (title,column) in col}\n",
    "df=pd.DataFrame(Dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove rows in de dataframe where Borough is Not assigned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df.Borough != 'Not assigned']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rename df column 'Neighbourhood\\n' to 'Neighbourhood'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Postcode', 'Borough', 'Neighborhood'], dtype='object')"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.rename(columns={'Neighbourhood\\n': 'Neighborhood'})\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Remove /n from the Neighbourhood column values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Neighborhood'] = df['Neighborhood'].str[:-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Replace Neighbourhood column values that are equal to 'Not assigned' to the Borough column value of the same row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df.Neighborhood == 'Not assigned', 'Neighborhood'] = df['Borough']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataframe shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(210, 3)"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}