step6.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ODI Queensland workshop - Web Scraping \n",
    "\n",
    "## QUT DMRC - 2015\n",
    "\n",
    "### Restructure the code for clarity\n",
    "\n",
    "This notebook is identical to the [previous step](step5.ipynb), but the code is structured in a different way that will be easier to extend to multiple pages and an extra column is added to the table to store the type of sign."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import bs4\n",
    "import requests\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialise global variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# this is the base_url\n",
    "base_url = \"http://www.qld.gov.au/transport/safety/signs/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# columns labels\n",
    "colnames = [\"sign_name\", \"sign_type\", \"description\", \"images\"]\n",
    "\n",
    "# not using these for now - but needed to clean up display of images column\n",
    "images_cols = [\"image_name\", \"url\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "### Function definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# processes a beautiful_soup data structure and returns new signs in a dataframe\n",
    "def get_itemlist(thesoup):\n",
    "    \n",
    "    # find all the tables on the page\n",
    "    tables = thesoup.findAll('table')\n",
    "    thelist = []\n",
    "\n",
    "    for table in tables:\n",
    "        # find all the table rows\n",
    "        lotsofitems = table.findAll('tr')\n",
    "\n",
    "        # check if the first row contains a 'th' elements (table header)\n",
    "        if lotsofitems[0].find('th'): \n",
    "\n",
    "            # get all header elements\n",
    "            temp = lotsofitems[0].findAll('th')\n",
    "\n",
    "            # check that the table header has the text we expect for the signs table\n",
    "            if temp[0].get_text() == 'Sign' and temp[1].get_text() == 'Meaning':\n",
    "\n",
    "                # print('Traffic sign table found')\n",
    "\n",
    "                # process the table of traffic signs **** THIS IS THE UPDATED SECTION *****\n",
    "                for an_item in lotsofitems[1:]: \n",
    "                    theitem = []\n",
    "\n",
    "                    # sign description & title\n",
    "                    sign_text = an_item.findAll(\"p\")\n",
    "                    description = ''\n",
    "                    for para in sign_text:\n",
    "                        if para.find(\"strong\"):\n",
    "                            # extract sign name (this assumes only the sign name is in bold)\n",
    "                            temptemp = para.find(\"strong\").get_text()\n",
    "                            temptemp = temptemp.split()\n",
    "                            sign_name = \" \".join(temptemp)\n",
    "                        else:\n",
    "                            # extract sign description (may be multiple paragraphs)\n",
    "                            temptemp = para.get_text()\n",
    "                            temptemp = temptemp.split()\n",
    "                            description += \" \".join(temptemp) + '\\n'\n",
    "\n",
    "                    theitem += [sign_name]\n",
    "                    theitem += [sign_type]\n",
    "                    theitem += [description]\n",
    "\n",
    "                    # sign images (may be more than one image per sign name) - save image name & image url\n",
    "                    images = []\n",
    "                    for image in  an_item.findAll(\"img\"):\n",
    "                        # get the image name & image url\n",
    "                        images += [[image.attrs['alt'], image.attrs['src']]]\n",
    "                    theitem += [images]\n",
    "\n",
    "                    thelist += [theitem]\n",
    "\n",
    "#            else:\n",
    "#                print('Different table - with header row:', temp)\n",
    "#        else:\n",
    "#            print('Different table - no header row:', lotsofitems[0])\n",
    "\n",
    "    return pd.DataFrame(thelist,columns=colnames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# reset the dataframe\n",
    "signs = pd.DataFrame(columns=colnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# select which page to scrape based on the type of road sign\n",
    "sign_type = \"regulatory\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# build the url\n",
    "thepage = base_url + sign_type + '/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# call the url\n",
    "stuff = requests.get(thepage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# transform to soup using lxml parser\n",
    "soup = bs4.BeautifulSoup(stuff.text, \"lxml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# extract the new signs from this page\n",
    "new_signs = get_itemlist(soup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# add the new signs to the dataframe\n",
    "signs = signs.append(new_signs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# print something to show how the process develops\n",
    "print(\"URL:\",thepage,flush=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check the result and save to disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# how many signs are there in the dataframe?\n",
    "len(signs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# have a look at the first five items\n",
    "signs[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# save the data as a csv file\n",
    "signs.to_csv(\"signs.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we are ready to move onto the seventh notebook - [Plotting, tiny stat analysis and improved I/O](step7.ipynb)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}