01-up-scrape-pdf-links.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}\n",
    "\n",
    "s = requests.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = s.get('https://ceouttarpradesh.nic.in/rollpdf/rollpdf.aspx', headers=headers)\n",
    "r.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(r.content)\n",
    "\n",
    "select_elem = soup.find(\"select\", {\"name\": \"ctl00$ContentPlaceHolder1$DDLDistrict\"})\n",
    "\n",
    "options = []\n",
    "for option_elem in select_elem.find_all(\"option\"):\n",
    "    option_value = option_elem[\"value\"]\n",
    "    option_text = option_elem.text\n",
    "    options.append((option_value, option_text))\n",
    "\n",
    "districts = options[1:]\n",
    "districts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for d in districts:\n",
    "    print(d)\n",
    "    r = s.get('https://ceouttarpradesh.nic.in/rollpdf/rollpdf.aspx', headers=headers)\n",
    "    print(r.status_code)\n",
    "    soup = BeautifulSoup(r.content)            \n",
    "    viewstategenerator = soup.find('input', {'id': '__VIEWSTATEGENERATOR'})['value']\n",
    "    eventvalidation = soup.find('input', {'id': '__EVENTVALIDATION'})['value']\n",
    "    viewstate = soup.select_one('#aspnetForm input[name*=\"__VIEWSTATE\"]')['value']\n",
    "    data = {'__EVENTTARGET': 'ctl00$ContentPlaceHolder1$DDLDistrict',\n",
    "            '__EVENTARGUMENT': '',\n",
    "            '__LASTFOCUS': '',\n",
    "            '__VIEWSTATE': viewstate,\n",
    "            '__VIEWSTATEGENERATOR': viewstategenerator,\n",
    "            '__EVENTVALIDATION': eventvalidation,\n",
    "            'ctl00$ContentPlaceHolder1$DDLDistrict': d[0],\n",
    "           }\n",
    "    r = s.post('https://ceouttarpradesh.nic.in/rollpdf/rollpdf.aspx', headers=headers, data=data)\n",
    "    print(r.status_code)\n",
    "    soup = BeautifulSoup(r.content)\n",
    "    select_elem = soup.find(\"select\", {\"name\": \"ctl00$ContentPlaceHolder1$DDL_AC\"})\n",
    "    options = []\n",
    "    for option_elem in select_elem.find_all(\"option\"):\n",
    "        option_value = option_elem[\"value\"]\n",
    "        option_text = option_elem.text\n",
    "        options.append((option_value, option_text))\n",
    "\n",
    "    acs = options[1:]\n",
    "    print(acs)\n",
    "    first = True\n",
    "    for ac in acs:\n",
    "        print(ac)\n",
    "        if not first:\n",
    "            r = s.get('https://ceouttarpradesh.nic.in/rollpdf/rollpdf.aspx', headers=headers)\n",
    "            print(r.status_code)\n",
    "            soup = BeautifulSoup(r.content)\n",
    "            viewstategenerator = soup.find('input', {'id': '__VIEWSTATEGENERATOR'})['value']\n",
    "            eventvalidation = soup.find('input', {'id': '__EVENTVALIDATION'})['value']\n",
    "            viewstate = soup.select_one('#aspnetForm input[name*=\"__VIEWSTATE\"]')['value']\n",
    "            data = {'__EVENTTARGET': 'ctl00$ContentPlaceHolder1$DDLDistrict',\n",
    "                    '__EVENTARGUMENT': '',\n",
    "                    '__LASTFOCUS': '',\n",
    "                    '__VIEWSTATE': viewstate,\n",
    "                    '__VIEWSTATEGENERATOR': viewstategenerator,\n",
    "                    '__EVENTVALIDATION': eventvalidation,\n",
    "                    'ctl00$ContentPlaceHolder1$DDLDistrict': d[0],\n",
    "                   }\n",
    "            r = s.post('https://ceouttarpradesh.nic.in/rollpdf/rollpdf.aspx', headers=headers, data=data)\n",
    "            print(r.status_code)\n",
    "            soup = BeautifulSoup(r.content)\n",
    "        first = False\n",
    "        viewstategenerator = soup.find('input', {'id': '__VIEWSTATEGENERATOR'})['value']\n",
    "        eventvalidation = soup.find('input', {'id': '__EVENTVALIDATION'})['value']\n",
    "        viewstate = soup.select_one('#aspnetForm input[name*=\"__VIEWSTATE\"]')['value']\n",
    "        data = {'__EVENTTARGET': '',\n",
    "                '__EVENTARGUMENT': '',\n",
    "                '__LASTFOCUS': '',\n",
    "                '__VIEWSTATE': viewstate,\n",
    "                '__VIEWSTATEGENERATOR': viewstategenerator,\n",
    "                '__EVENTVALIDATION': eventvalidation,\n",
    "                'ctl00$ContentPlaceHolder1$DDLDistrict': d[0],\n",
    "                'ctl00$ContentPlaceHolder1$DDL_AC': ac[0],\n",
    "                'ctl00$ContentPlaceHolder1$Button1': 'Show'\n",
    "               }\n",
    "        r = s.post('https://ceouttarpradesh.nic.in/rollpdf/rollpdf.aspx', headers=headers, data=data)\n",
    "        print(r.status_code)\n",
    "        soup = BeautifulSoup(r.content)\n",
    "        table = soup.find('table', {'id': 'ctl00_ContentPlaceHolder1_ElecRollGrd'})\n",
    "        a = table.find_all('a')[-1]\n",
    "        m = re.findall('Page\\$(\\d+)', a.attrs['href'])\n",
    "        if m:\n",
    "            trs = table.find_all('tr')[1:-2]\n",
    "        else:\n",
    "            trs = table.find_all('tr')[1:]\n",
    "        for tr in trs:\n",
    "            cols = tr.find_all('td')\n",
    "            row = {'District': d[1],\n",
    "                   'AC_Name': ac[1],\n",
    "                   'AC_Number': cols[0].text,\n",
    "                   'Part_Number': cols[1].text,\n",
    "                   'Polling_Station_Name': cols[2].text,\n",
    "                   'ElectorRollPdf': 'https://ceouttarpradesh.nic.in/rollpdf/' + cols[4].find('a').attrs['href']\n",
    "                  }\n",
    "            rows.append(row)\n",
    "        if len(m) == 0:\n",
    "            continue\n",
    "        p = 2\n",
    "        while True:\n",
    "            viewstategenerator = soup.find('input', {'id': '__VIEWSTATEGENERATOR'})['value']\n",
    "            eventvalidation = soup.find('input', {'id': '__EVENTVALIDATION'})['value']\n",
    "            viewstate = soup.select_one('#aspnetForm input[name*=\"__VIEWSTATE\"]')['value']\n",
    "            data = {'__EVENTTARGET': 'ctl00$ContentPlaceHolder1$ElecRollGrd',\n",
    "                    '__EVENTARGUMENT': 'Page$%d' % p,\n",
    "                    '__LASTFOCUS': '',\n",
    "                    '__VIEWSTATE': viewstate,\n",
    "                    '__VIEWSTATEGENERATOR': viewstategenerator,\n",
    "                    '__EVENTVALIDATION': eventvalidation,\n",
    "                    'ctl00$ContentPlaceHolder1$DDLDistrict': d[0],\n",
    "                    'ctl00$ContentPlaceHolder1$DDL_AC': ac[0],\n",
    "                   }\n",
    "            r = s.post('https://ceouttarpradesh.nic.in/rollpdf/rollpdf.aspx', headers=headers, data=data)\n",
    "            #print(p, r.status_code)\n",
    "            soup = BeautifulSoup(r.content)\n",
    "            table = soup.find('table', {'id': 'ctl00_ContentPlaceHolder1_ElecRollGrd'})\n",
    "            for tr in table.find_all('tr')[1:-2]:\n",
    "                cols = tr.find_all('td')\n",
    "                row = {'District': d[1],\n",
    "                       'AC_Name': ac[1],\n",
    "                       'AC_Number': cols[0].text,\n",
    "                       'Part_Number': cols[1].text,\n",
    "                       'Polling_Station_Name': cols[2].text,\n",
    "                       'ElectorRollPdf': 'https://ceouttarpradesh.nic.in/rollpdf/' + cols[4].find('a').attrs['href']\n",
    "                      }\n",
    "                rows.append(row)\n",
    "            a = table.find_all('a')[-1]\n",
    "            m = re.findall('Page\\$(\\d+)', a.attrs['href'])\n",
    "            last_page = int(m[0])\n",
    "            p += 1\n",
    "            if p > last_page:\n",
    "                break\n",
    "        #break\n",
    "    #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(rows)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop_duplicates(inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data/up-2023-electoral-rolls.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}