6.4 web scraping demonstration.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 6 -  Data Sourcing via Web\n",
    "## Segment 4 - Web scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import urllib.request\n",
    "from IPython.display import HTML\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bs4.BeautifulSoup"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = urllib.request.urlopen('https://analytics.usa.gov/').read()\n",
    "soup = BeautifulSoup(r, \"lxml\")\n",
    "type(soup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html lang=\"en\">\n",
      " <!-- Initalize title and data source variables -->\n",
      " <head>\n",
      "  <!--\n",
      "\n",
      "    Hi! Welcome to our source code.\n",
      "\n",
      "    This dashboard uses data from the Digital Analytics Program, a US\n",
      "    government team inside the General Services Administration.\n",
      "\n",
      "\n",
      "    For a detailed tech breakdown of how 18F and friends built this site:\n",
      "\n",
      "    https://18f.gsa.gov/2015/03/19/how-we-built-analytics-usa-gov/\n",
      "\n",
      "\n",
      "    This is a fully open source project, and your contributions are welcome.\n",
      "\n",
      "    Frontend static site: https://github.com/18F/analytics.usa.gov\n",
      "    Backend data reporting: https://github.com/18F/analytics-reporter\n",
      "\n",
      "    -->\n",
      "  <meta charset=\"utf-8\"/>\n",
      "  <meta content=\"IE=Edge\" http-equiv=\"X-UA-Compatible\"/>\n",
      "  <meta content=\"NjbZn6hQe7OwV-nTsa6nLmtrOUcSGPRyFjxm5zkmCcg\" name=\"google-site-verification\"/>\n",
      "  <link href=\"/css/vendor/css/uswds.v0.9.6.css\" rel=\"stylesheet\"/>\n",
      "  <link href=\"/css/public_analytics.css\" rel=\"stylesheet\"/>\n",
      "  <link href=\"/images/analytics-favicon.ico\" rel=\"ic\n"
     ]
    }
   ],
   "source": [
    "print(soup.prettify()[:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/\n",
      "#explanation\n",
      "https://analytics.usa.gov/data/\n",
      "https://open.gsa.gov/api/dap/\n",
      "data/\n",
      "#top-pages-realtime\n",
      "#top-pages-7-days\n",
      "#top-pages-30-days\n",
      "https://analytics.usa.gov/data/live/all-pages-realtime.csv\n",
      "https://analytics.usa.gov/data/live/all-domains-30-days.csv\n",
      "https://www.digitalgov.gov/services/dap/\n",
      "https://www.digitalgov.gov/services/dap/common-questions-about-dap-faq/#part-4\n",
      "https://support.google.com/analytics/answer/2763052?hl=en\n",
      "https://analytics.usa.gov/data/live/second-level-domains.csv\n",
      "https://analytics.usa.gov/data/live/sites.csv\n",
      "mailto:DAP@support.digitalgov.gov\n",
      "https://analytics.usa.gov/data/\n",
      "https://open.gsa.gov/api/dap/\n",
      "mailto:DAP@support.digitalgov.gov\n",
      "https://github.com/GSA/analytics.usa.gov/issues\n",
      "https://github.com/GSA/analytics.usa.gov\n",
      "https://github.com/18F/analytics-reporter\n",
      "http://www.gsa.gov/\n",
      "https://www.digitalgov.gov/services/dap/\n",
      "https://cloud.gov/\n"
     ]
    }
   ],
   "source": [
    "for link in soup.find_all('a'):\n",
    "    print(link.get('href'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<a href=\"https://analytics.usa.gov/data/\">Data</a>\n",
      "<a href=\"https://open.gsa.gov/api/dap/\" target=\"_blank\">API</a>\n",
      "<a href=\"https://analytics.usa.gov/data/live/all-pages-realtime.csv\">Download the full dataset.</a>\n",
      "<a href=\"https://analytics.usa.gov/data/live/all-domains-30-days.csv\">Download the full dataset.</a>\n",
      "<a class=\"external-link\" href=\"https://www.digitalgov.gov/services/dap/\">Digital Analytics Program</a>\n",
      "<a class=\"external-link\" href=\"https://www.digitalgov.gov/services/dap/common-questions-about-dap-faq/#part-4\">does not track individuals</a>\n",
      "<a class=\"external-link\" href=\"https://support.google.com/analytics/answer/2763052?hl=en\">anonymizes the IP addresses</a>\n",
      "<a class=\"external-link\" href=\"https://analytics.usa.gov/data/live/second-level-domains.csv\">400 executive branch government domains</a>\n",
      "<a class=\"external-link\" href=\"https://analytics.usa.gov/data/live/sites.csv\">about 5,700 total websites</a>\n",
      "<a href=\"https://analytics.usa.gov/data/\">download the data here.</a>\n",
      "<a href=\"https://open.gsa.gov/api/dap/\" target=\"_blank\"> API project</a>\n",
      "<a class=\"usa-button usa-button-secondary-inverse\" href=\"https://github.com/GSA/analytics.usa.gov/issues\">\n",
      "<img alt=\"Github Icon\" class=\"github-icon\" src=\"/images/github-logo-white.svg\"/>\n",
      "                  Suggest a feature or report an issue\n",
      "            </a>\n",
      "<a href=\"https://github.com/GSA/analytics.usa.gov\">\n",
      "<img alt=\"Github Icon\" class=\"github-icon\" src=\"/images/github-logo.svg\"/>\n",
      "              View our code on GitHub</a>\n",
      "<a href=\"https://github.com/18F/analytics-reporter\">\n",
      "<img alt=\"Github Icon\" class=\"github-icon\" src=\"/images/github-logo.svg\"/>\n",
      "              View our code for the data on GitHub</a>\n",
      "<a href=\"http://www.gsa.gov/\">\n",
      "<img alt=\"GSA\" src=\"/images/gsa-logo.svg\"/>\n",
      "</a>\n",
      "<a href=\"https://www.digitalgov.gov/services/dap/\">Digital Analytics Program</a>\n",
      "<a href=\"https://cloud.gov/\">cloud.gov</a>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "bs4.element.Tag"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for link in soup.findAll('a', attrs={'href': re.compile(\"^http\")}):\n",
    "    print(link)\n",
    "type(link)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "analytics.usa.gov | The US government's web traffic.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':\n",
      "    new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],\n",
      "    j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=\n",
      "    'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);\n",
      "    })(window,document,'script','dataLayer','GTM-MQSGZS');\n",
      "\n",
      "\n",
      "    (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){\n",
      "  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),\n",
      "  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)\n",
      "  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');\n",
      "\n",
      "    ga('create', 'UA-48605964-36', 'auto');\n",
      "    ga('set', 'anonymizeIp', true);\n",
      "    ga('set', 'forceSSL', true);\n",
      "    ga('send', 'pageview');\n",
      "\n",
      "  \n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "analytics.usa.gov\n",
      "              \n",
      "\n",
      "\n",
      "About this site\n",
      "Data | API\n",
      "\n",
      "\n",
      "\n",
      "Select an agency\n",
      "\n",
      "All Participating Websites\n",
      "Agency for International Development\n",
      "Department of Agriculture\n",
      "Department of Commerce\n",
      "Department of Defense\n",
      "Department of Education\n",
      "Department of Energy\n",
      "Department of Health and Human Services\n",
      "Department of Homeland Security\n",
      "Department of Housing and Urban Development\n",
      "Department of Justice\n",
      "Department of Labor\n",
      "Department of State\n",
      "Department of Transportation\n",
      "Department of Veterans Affairs\n",
      "Department of the Interior\n",
      "Department of the Treasury\n",
      "Environmental Protection Agency\n",
      "Executive Office of the President\n",
      "General Services Administration\n",
      "National Aeronautics and Space Administration\n",
      "National Archives and Records Administration\n",
      "National Science Foundation\n",
      "Nuclear Regulatory Commission\n",
      "Office of Personnel Management\n",
      "Postal Service\n",
      "Small Business Administration\n",
      "Social Security Administration\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "...\n",
      "people on government websites now\n",
      "\n",
      "\n",
      "Visits Today\n",
      "Eastern Time\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Visits in the Past 90 Days\n",
      "\n",
      "\n",
      "          There were ... visits over the past 90 days.\n",
      "        \n",
      "\n",
      "Devices\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "            Based on rough network segmentation data, we estimate that less than 5% of all traffic across all agencies comes from US federal government networks.\n",
      "          \n",
      "\n",
      "            Much more detailed data is available in downloadable CSV and JSON. This includes data on combined browser and OS usage.\n",
      "          \n",
      "\n",
      "\n",
      "Browsers\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Internet Explorer\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Operating Systems\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Windows\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Visitor Locations Right Now\n",
      "\n",
      "\n",
      "Cities\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Countries\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "International & Territories\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Top Pages\n",
      "\n",
      "Now\n",
      "7 Days\n",
      "30 Days\n",
      "\n",
      "\n",
      "\n",
      "              People on a single, specific page now. We only count pages with at least 10 people on the page.\n",
      "              Download the full dataset.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Visits over the last week to domains, including traffic to all pages within that domain.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "              Visits over the last month to domains, including traffic to all pages within that domain. We only count pages with at least 1,000 visits in the last month.\n",
      "              Download the full dataset.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Top Downloads\n",
      "Total file downloads yesterday on government domains.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "About this Site\n",
      "\n",
      "            This data provides a window into how people are interacting with the government online.\n",
      "             The data comes from a unified Google Analytics account for U.S. federal government agencies known as the Digital Analytics Program.\n",
      "              This program helps government agencies understand how people find, access, and use government services online. The program does not track individuals,\n",
      "               and anonymizes the IP addresses of visitors.\n",
      "          \n",
      "\n",
      "            Not every government website is represented in this data. \n",
      "            Currently, the Digital Analytics Program collects web traffic from around 400 executive branch government domains,\n",
      "             across about 5,700 total websites,\n",
      "              including every cabinet department.\n",
      "               We continue to pursue and add more sites frequently; to add your site, email the Digital Analytics Program.\n",
      "          \n",
      "\n",
      "\n",
      "Download the data\n",
      "You can download the data here. Available in JSON and CSV format.\n",
      " Additionally, you can access data via our  API project (currently in Beta).\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Have a question or problem? \n",
      "              \n",
      "              Get in touch.\n",
      "              \n",
      "\n",
      "\n",
      "                  Suggest a feature or report an issue\n",
      "            \n",
      "\n",
      "\n",
      "\n",
      "\n",
      "              View our code on GitHub\n",
      "\n",
      "\n",
      "              View our code for the data on GitHub\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Analytics.usa.gov is a project of GSA’s Digital Analytics Program.\n",
      "This website is hosted on cloud.gov.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "window._ie9 = false;\n",
      "\n",
      "\n",
      "\n",
      "    window._ie10 = (\"onpropertychange\" in document && !!window.matchMedia);\n",
      "    window._ie11 = !!navigator.userAgent.match(/Trident\\/7.0; rv 11/);\n",
      "    window._ie = window._ie9 || window._ie10 || window._ie11;\n",
      "  \n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(soup.get_text())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html lang=\"en\">\n",
      " <!-- Initalize title and data source variables -->\n",
      " <head>\n",
      "  <!--\n",
      "\n",
      "    Hi! Welcome to our source code.\n",
      "\n",
      "    This dashboard uses data from the Digital Analytics Program, a US\n",
      "    government team inside the General Services Administration.\n",
      "\n",
      "\n",
      "    For a detailed tech breakdown of how 18F and friends built this site:\n",
      "\n",
      "    https://18f.gsa.gov/2015/03/19/how-we-built-analytics-usa-gov/\n",
      "\n",
      "\n",
      "    This is a fully open source project, and your contributions are welcome.\n",
      "\n",
      "    Frontend static site: https://github.com/18F/analytics.usa.gov\n",
      "    Backend data reporting: https://github.com/18F/analytics-reporter\n",
      "\n",
      "    -->\n",
      "  <meta charset=\"utf-8\"/>\n",
      "  <meta content=\"IE=Edge\" http-equiv=\"X-UA-Compatible\"/>\n",
      "  <meta content=\"NjbZn6hQe7OwV-nTsa6nLmtrOUcSGPRyFjxm5zkmCcg\" name=\"google-site-verification\"/>\n",
      "  <link href=\"/css/vendor/css/uswds.v0.9.6.css\" rel=\"stylesheet\"/>\n",
      "  <link href=\"/css/public_analytics.css\" rel=\"stylesheet\"/>\n",
      "  <link href=\"/images/analytics-favicon.ico\" rel=\"ic\n"
     ]
    }
   ],
   "source": [
    "print(soup.prettify()[0:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<a href=\"https://analytics.usa.gov/data/\">Data</a>\n",
      "<a href=\"https://open.gsa.gov/api/dap/\" target=\"_blank\">API</a>\n",
      "<a href=\"https://analytics.usa.gov/data/live/all-pages-realtime.csv\">Download the full dataset.</a>\n",
      "<a href=\"https://analytics.usa.gov/data/live/all-domains-30-days.csv\">Download the full dataset.</a>\n",
      "<a class=\"external-link\" href=\"https://www.digitalgov.gov/services/dap/\">Digital Analytics Program</a>\n",
      "<a class=\"external-link\" href=\"https://www.digitalgov.gov/services/dap/common-questions-about-dap-faq/#part-4\">does not track individuals</a>\n",
      "<a class=\"external-link\" href=\"https://support.google.com/analytics/answer/2763052?hl=en\">anonymizes the IP addresses</a>\n",
      "<a class=\"external-link\" href=\"https://analytics.usa.gov/data/live/second-level-domains.csv\">400 executive branch government domains</a>\n",
      "<a class=\"external-link\" href=\"https://analytics.usa.gov/data/live/sites.csv\">about 5,700 total websites</a>\n",
      "<a href=\"https://analytics.usa.gov/data/\">download the data here.</a>\n",
      "<a href=\"https://open.gsa.gov/api/dap/\" target=\"_blank\"> API project</a>\n",
      "<a class=\"usa-button usa-button-secondary-inverse\" href=\"https://github.com/GSA/analytics.usa.gov/issues\">\n",
      "<img alt=\"Github Icon\" class=\"github-icon\" src=\"/images/github-logo-white.svg\"/>\n",
      "                  Suggest a feature or report an issue\n",
      "            </a>\n",
      "<a href=\"https://github.com/GSA/analytics.usa.gov\">\n",
      "<img alt=\"Github Icon\" class=\"github-icon\" src=\"/images/github-logo.svg\"/>\n",
      "              View our code on GitHub</a>\n",
      "<a href=\"https://github.com/18F/analytics-reporter\">\n",
      "<img alt=\"Github Icon\" class=\"github-icon\" src=\"/images/github-logo.svg\"/>\n",
      "              View our code for the data on GitHub</a>\n",
      "<a href=\"http://www.gsa.gov/\">\n",
      "<img alt=\"GSA\" src=\"/images/gsa-logo.svg\"/>\n",
      "</a>\n",
      "<a href=\"https://www.digitalgov.gov/services/dap/\">Digital Analytics Program</a>\n",
      "<a href=\"https://cloud.gov/\">cloud.gov</a>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "bs4.element.Tag"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for link in soup.findAll('a', attrs={'href': re.compile(\"^http\")}):\n",
    "    print(link)\n",
    "type(link)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<a href=\"https://analytics.usa.gov/data/\">Data</a>\n",
      "<a href=\"https://open.gsa.gov/api/dap/\" target=\"_blank\">API</a>\n",
      "<a href=\"https://analytics.usa.gov/data/live/all-pages-realtime.csv\">Download the full dataset.</a>\n",
      "<a href=\"https://analytics.usa.gov/data/live/all-domains-30-days.csv\">Download the full dataset.</a>\n",
      "<a class=\"external-link\" href=\"https://www.digitalgov.gov/services/dap/\">Digital Analytics Program</a>\n",
      "<a class=\"external-link\" href=\"https://www.digitalgov.gov/services/dap/common-questions-about-dap-faq/#part-4\">does not track individuals</a>\n",
      "<a class=\"external-link\" href=\"https://support.google.com/analytics/answer/2763052?hl=en\">anonymizes the IP addresses</a>\n",
      "<a class=\"external-link\" href=\"https://analytics.usa.gov/data/live/second-level-domains.csv\">400 executive branch government domains</a>\n",
      "<a class=\"external-link\" href=\"https://analytics.usa.gov/data/live/sites.csv\">about 5,700 total websites</a>\n",
      "<a href=\"https://analytics.usa.gov/data/\">download the data here.</a>\n",
      "<a href=\"https://open.gsa.gov/api/dap/\" target=\"_blank\"> API project</a>\n",
      "<a class=\"usa-button usa-button-secondary-inverse\" href=\"https://github.com/GSA/analytics.usa.gov/issues\">\n",
      "<img alt=\"Github Icon\" class=\"github-icon\" src=\"/images/github-logo-white.svg\"/>\n",
      "                  Suggest a feature or report an issue\n",
      "            </a>\n",
      "<a href=\"https://github.com/GSA/analytics.usa.gov\">\n",
      "<img alt=\"Github Icon\" class=\"github-icon\" src=\"/images/github-logo.svg\"/>\n",
      "              View our code on GitHub</a>\n",
      "<a href=\"https://github.com/18F/analytics-reporter\">\n",
      "<img alt=\"Github Icon\" class=\"github-icon\" src=\"/images/github-logo.svg\"/>\n",
      "              View our code for the data on GitHub</a>\n",
      "<a href=\"http://www.gsa.gov/\">\n",
      "<img alt=\"GSA\" src=\"/images/gsa-logo.svg\"/>\n",
      "</a>\n",
      "<a href=\"https://www.digitalgov.gov/services/dap/\">Digital Analytics Program</a>\n",
      "<a href=\"https://cloud.gov/\">cloud.gov</a>\n"
     ]
    }
   ],
   "source": [
    "file = open(\"parsed_data.txt\", \"w\")\n",
    "for link in soup.findAll('a', attrs={'href': re.compile(\"^http\")}):\n",
    "    soup_link = str(link)\n",
    "    print(soup_link)\n",
    "    file.write(soup_link)\n",
    "file.flush()\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\Andre\\\\Desktop\\\\Projects\\\\ecourses\\\\PythonDataScienceEssentials'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}