data_scrape_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2b7fc435",
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium.webdriver.common.by import By"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0561af40",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a080418a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.support.ui import Select\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "17f112de",
   "metadata": {},
   "outputs": [
    {
     "ename": "WebDriverException",
     "evalue": "Message: Process unexpectedly closed with status 0\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mWebDriverException\u001b[0m                        Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-0ddf81537b20>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdriver\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mwebdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mFirefox\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0murl\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34mr\"https://www.nba.com/stats/players/advanced/?sort=GP&dir=-1&Season=2021-22&SeasonType=Regular%20Season\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\firefox\\webdriver.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, firefox_profile, firefox_binary, capabilities, proxy, executable_path, options, service_log_path, service_args, service, desired_capabilities, log_path, keep_alive)\u001b[0m\n\u001b[0;32m    175\u001b[0m             \u001b[0mremote_server_addr\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mservice\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mservice_url\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    176\u001b[0m             ignore_proxy=options._ignore_local_proxy)\n\u001b[1;32m--> 177\u001b[1;33m         RemoteWebDriver.__init__(\n\u001b[0m\u001b[0;32m    178\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    179\u001b[0m             \u001b[0mcommand_executor\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mexecutor\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, command_executor, desired_capabilities, browser_profile, proxy, keep_alive, file_detector, options)\u001b[0m\n\u001b[0;32m    273\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_authenticator_id\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    274\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstart_client\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 275\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstart_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcapabilities\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbrowser_profile\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    276\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    277\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__repr__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mstart_session\u001b[1;34m(self, capabilities, browser_profile)\u001b[0m\n\u001b[0;32m    363\u001b[0m         \u001b[0mw3c_caps\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_make_w3c_caps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcapabilities\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    364\u001b[0m         \u001b[0mparameters\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;34m\"capabilities\"\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mw3c_caps\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 365\u001b[1;33m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCommand\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mNEW_SESSION\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    366\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;34m'sessionId'\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    367\u001b[0m             \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'value'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    428\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    429\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 430\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    431\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[0;32m    432\u001b[0m                 response.get('value', None))\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    245\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'alert'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    246\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# type: ignore[call-arg]  # mypy is not smart enough here\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 247\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    248\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    249\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_value_or_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mMapping\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0m_KT\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_VT\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0m_KT\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0m_VT\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[0m_VT\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mWebDriverException\u001b[0m: Message: Process unexpectedly closed with status 0\n"
     ]
    }
   ],
   "source": [
    "driver = webdriver.Firefox()\n",
    "url = r\"https://www.nba.com/stats/players/advanced/?sort=GP&dir=-1&Season=2021-22&SeasonType=Regular%20Season\"\n",
    "driver.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad1796a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "e96366ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "select = Select(driver.find_element(By.XPATH,\"/html/body/main/div/div/div[2]/div/div/div[1]/div[1]/div/div/label/select\"))\n",
    "select.select_by_index(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "2568ada3",
   "metadata": {},
   "outputs": [],
   "source": [
    "select = Select(driver.find_element(By.XPATH,\"/html/body/main/div/div/div[2]/div/div/nba-stat-table/div[1]/div/div/select\"))\n",
    "select.select_by_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "6951f799",
   "metadata": {},
   "outputs": [],
   "source": [
    "src = driver.page_source\n",
    "parser = BeautifulSoup(src,\"lxml\")\n",
    "table = parser.find(\"div\",attrs = {\"class\" : \"nba-stat-table__overflow\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "a3b90683",
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = table.find_all('th')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "655c8017",
   "metadata": {},
   "outputs": [],
   "source": [
    "headerlist = [h.text.strip() for h in headers[1:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "4b270d15",
   "metadata": {},
   "outputs": [],
   "source": [
    "headerlist1 = [a for a in headerlist if not \"RANK\" in a]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "11f80577",
   "metadata": {},
   "outputs": [],
   "source": [
    "rows = table.findAll('tr')[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "cf678770",
   "metadata": {},
   "outputs": [],
   "source": [
    "player_stats = [[td.getText().strip() for td in rows[i].findAll('td')[1:]] for i in range(len(rows))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "dd32bab2",
   "metadata": {},
   "outputs": [],
   "source": [
    "headerlist1 = headerlist1[:-5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "4952207e",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = pd.DataFrame(player_stats,columns=headerlist1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "bf5362e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame.to_excel(stats,\"./nba2022.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "f88df66d",
   "metadata": {},
   "outputs": [
    {
     "ename": "InvalidSessionIdException",
     "evalue": "Message: Tried to run command without establishing a connection\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mInvalidSessionIdException\u001b[0m                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-62-ad0c9a213c39>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mclose\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    941\u001b[0m                 \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    942\u001b[0m         \"\"\"\n\u001b[1;32m--> 943\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCommand\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCLOSE\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    944\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    945\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mquit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    428\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    429\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 430\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    431\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[0;32m    432\u001b[0m                 response.get('value', None))\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    245\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'alert'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    246\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# type: ignore[call-arg]  # mypy is not smart enough here\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 247\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    248\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    249\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_value_or_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mMapping\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0m_KT\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_VT\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0m_KT\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0m_VT\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[0m_VT\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mInvalidSessionIdException\u001b[0m: Message: Tried to run command without establishing a connection\n"
     ]
    }
   ],
   "source": [
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "c50508c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2'"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c951321b",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Firefox()\n",
    "url = r\"https://www.nba.com/stats/players/traditional/?SeasonType=Regular%20Season&sort=PTS&dir=-1&Season=2021-22\"\n",
    "driver.get(url)\n",
    "        #select a year\n",
    "select = Select(driver.find_element(By.XPATH,\"/html/body/main/div/div/div[2]/div/div/div[1]/div[1]/div/div/label/select\"))\n",
    "select.select_by_index(0)\n",
    "        \n",
    "time.sleep(15)\n",
    "# select the page number to all\n",
    "select = Select(driver.find_element(By.XPATH,\"/html/body/main/div/div/div[2]/div/div/nba-stat-table/div[1]/div/div/select\"))\n",
    "select.select_by_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "401f16f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "src = driver.page_source\n",
    "parser = BeautifulSoup(src,\"lxml\")\n",
    "table = parser.find(\"div\",attrs = {\"class\" : \"nba-stat-table__overflow\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fcaa89bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = table.find_all('th')\n",
    "headerlist = [h.text.strip() for h in headers[1:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a6ecf5a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "headerlist1 = [a for a in headerlist if not \"RANK\" in a]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a7fd520c",
   "metadata": {},
   "outputs": [],
   "source": [
    "rows = table.findAll('tr')[1:]\n",
    "player_stats = [[td.getText().strip() for td in rows[i].findAll('td')[1:]] for i in range(len(rows))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "511f2ec6",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = pd.DataFrame(player_stats,columns=headerlist1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2d74adb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame.to_excel(stats,\"./regular_example.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b898d4f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f02193a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "        url = r\"https://www.nba.com/stats/players/traditional/?SeasonType=Regular%20Season&sort=PTS&dir=-1&Season=2021-22&PerMode=Per100Possessions\"\n",
    "        driver.get(url)\n",
    "        \n",
    "        #select a year\n",
    "        time.sleep(5)\n",
    "        select = Select(driver.find_element(By.XPATH,\"/html/body/main/div/div/div[2]/div/div/div[1]/div[1]/div/div/label/select\"))\n",
    "        select.select_by_index(i+year_to_2022)\n",
    "        \n",
    "        time.sleep(10)\n",
    "        # select the page number to all\n",
    "        select = Select(driver.find_element(By.XPATH,\"/html/body/main/div/div/div[2]/div/div/nba-stat-table/div[1]/div/div/select\"))\n",
    "        select.select_by_index(0)\n",
    "        \n",
    "        #find the table \n",
    "        src = driver.page_source\n",
    "        parser = BeautifulSoup(src,\"lxml\")\n",
    "        table = parser.find(\"div\",attrs = {\"class\" : \"nba-stat-table__overflow\"})\n",
    "        \n",
    "        #find the header of the table and remove the attributes that are not shown\n",
    "        headers = table.find_all('th')\n",
    "        headerlist = [h.text.strip() for h in headers[1:]]\n",
    "        headerlist1 = [a for a in headerlist if not \"RANK\" in a]\n",
    "        \n",
    "        #get the data of from all the data cells\n",
    "        rows = table.findAll('tr')[1:]\n",
    "        player_stats = [[td.getText().strip() for td in rows[i].findAll('td')[1:]] for i in range(len(rows))]\n",
    "        \n",
    "        #drop the data into the pdDataFrame\n",
    "        stats = pd.DataFrame(player_stats,columns=headerlist1)\n",
    "        \n",
    "        #add the year infront of the player's name\n",
    "        def add_year(x):\n",
    "            return \"_\"+str(year_b - i)+\"_\"+x\n",
    "        stats[\"PLAYER\"] = stats[\"PLAYER\"].map(add_year)\n",
    "        #save the data as an excel table\n",
    "        pd.DataFrame.to_excel(stats,\"./newdata/per_100_regular\"+str(year_b-i)+\".xlsx\")\n",
    "        driver.close()\n",
    "        print(\"data of \"+str(year_b-i)+\" has been saved\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "3a28eeec",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Firefox()\n",
    "url = r\"https://www.nba.com/stats/players/traditional/?SeasonType=Regular%20Season&sort=PTS&dir=-1&Season=2021-22&PerMode=Per100Possessions\"\n",
    "driver.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "39af0cc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Firefox()\n",
    "url = r\"https://www.nba.com/stats/players/defense/?sort=DEF_WS&dir=-1&Season=2021-22&SeasonType=Regular%20Season\"\n",
    "driver.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ed94cc53",
   "metadata": {},
   "outputs": [],
   "source": [
    "select = Select(driver.find_element(By.XPATH,\"/html/body/main/div/div/div[2]/div/div/div[1]/div[1]/div/div/label/select\"))\n",
    "select.select_by_index(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f92e61c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "select = Select(driver.find_element(By.XPATH,\"/html/body/main/div/div/div[2]/div/div/nba-stat-table/div[1]/div/div/select\"))\n",
    "select.select_by_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "31387ef8",
   "metadata": {},
   "outputs": [],
   "source": [
    "src = driver.page_source\n",
    "parser = BeautifulSoup(src,\"lxml\")\n",
    "table = parser.find(\"div\",attrs = {\"class\" : \"nba-stat-table__overflow\"})\n",
    "\n",
    "#find the header of the table and remove the attributes that are not shown\n",
    "headers = table.find_all('th')\n",
    "headerlist = [h.text.strip() for h in headers[1:]]\n",
    "headerlist1 = [a for a in headerlist if not \"RANK\" in a]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b3b1f139",
   "metadata": {},
   "outputs": [],
   "source": [
    "#get the data of from all the data cells\n",
    "rows = table.findAll('tr')[1:]\n",
    "player_stats = [[td.getText().strip() for td in rows[i].findAll('td')[1:]] for i in range(len(rows))]\n",
    "stats = pd.DataFrame(player_stats,columns=headerlist1)\n",
    "pd.DataFrame.to_excel(stats,\"./newdata/defense2022.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "bc58c32a",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Firefox()\n",
    "url = r\"https://www.nba.com/stats/players/usage/?sort=PCT_PTS&dir=-1&Season=2021-22&SeasonType=Regular%20Season\"\n",
    "driver.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "43177a5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "select = Select(driver.find_element(By.XPATH,\"/html/body/main/div/div/div[2]/div/div/div[1]/div[1]/div/div/label/select\"))\n",
    "select.select_by_index(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f4d928ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "select = Select(driver.find_element(By.XPATH,\"/html/body/main/div/div/div[2]/div/div/nba-stat-table/div[1]/div/div/select\"))\n",
    "select.select_by_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "587d4b1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "src = driver.page_source\n",
    "parser = BeautifulSoup(src,\"lxml\")\n",
    "table = parser.find(\"div\",attrs = {\"class\" : \"nba-stat-table__overflow\"})\n",
    "\n",
    "#find the header of the table and remove the attributes that are not shown\n",
    "headers = table.find_all('th')\n",
    "headerlist = [h.text.strip() for h in headers[1:]]\n",
    "headerlist1 = [a for a in headerlist if not \"RANK\" in a]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "b056d9d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Player',\n",
       " 'TEAM',\n",
       " 'AGE',\n",
       " 'GP',\n",
       " 'W',\n",
       " 'L',\n",
       " 'MIN',\n",
       " 'USG%',\n",
       " '%FGM',\n",
       " '%FGA',\n",
       " '%3PM',\n",
       " '%3PA',\n",
       " '%FTM',\n",
       " '%FTA',\n",
       " '%OREB',\n",
       " '%DREB',\n",
       " '%REB',\n",
       " '%AST',\n",
       " '%TOV',\n",
       " '%STL',\n",
       " '%BLK',\n",
       " '%BLKA',\n",
       " '%PF',\n",
       " '%PFD',\n",
       " '%PTS']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "headerlist1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "e0e59b4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#get the data of from all the data cells\n",
    "rows = table.findAll('tr')[1:]\n",
    "player_stats = [[td.getText().strip() for td in rows[i].findAll('td')[1:]] for i in range(len(rows))]\n",
    "stats = pd.DataFrame(player_stats,columns=headerlist1)\n",
    "pd.DataFrame.to_excel(stats,\"./newdata/opponent2021.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "a1b74f7a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0          Bacon, Dwayne\n",
       "1      Bogdanovic, Bojan\n",
       "2         Bridges, Mikal\n",
       "3       Edwards, Anthony\n",
       "4          Freedom, Enes\n",
       "             ...        \n",
       "621       Haslem, Udonis\n",
       "622      Jackson, Justin\n",
       "623         Magnay, Will\n",
       "624    Pasecniks, Anzejs\n",
       "625        Pelle, Norvel\n",
       "Name: Vs Player, Length: 626, dtype: object"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats[\"Vs Player\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1413f636",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "39644f1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Firefox()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "662553f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://www.nba.com/stats/teams/boxscores?Season=2021-22\"\n",
    "driver.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dfbdc715",
   "metadata": {},
   "outputs": [],
   "source": [
    "select = Select(driver.find_element(By.XPATH,\"/html/body/div[1]/div[2]/div[2]/div[3]/section[2]/div/div[2]/div[2]/div[1]/div[3]/div/label/div/select\"))\n",
    "select.select_by_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "730274f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "src = driver.page_source\n",
    "parser = BeautifulSoup(src,\"lxml\")\n",
    "table = parser.find(\"div\",attrs = {\"class\" : \"Crom_container__C45Ti\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "830c6387",
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = table.find_all('th')\n",
    "headerlist = [h.text.strip() for h in headers]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "34150301",
   "metadata": {},
   "outputs": [],
   "source": [
    "rows = table.findAll('tr')[1:]\n",
    "player_stats = [[td.getText().strip() for td in rows[i].findAll('td')] for i in range(len(rows))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "06627063",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = pd.DataFrame(player_stats,columns=headerlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "353c16b8",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Team</th>\n",
       "      <th>Match Up</th>\n",
       "      <th>Game Date</th>\n",
       "      <th>W/L</th>\n",
       "      <th>MIN</th>\n",
       "      <th>PTS</th>\n",
       "      <th>FGM</th>\n",
       "      <th>FGA</th>\n",
       "      <th>FG%</th>\n",
       "      <th>3PM</th>\n",
       "      <th>...</th>\n",
       "      <th>FT%</th>\n",
       "      <th>OREB</th>\n",
       "      <th>DREB</th>\n",
       "      <th>REB</th>\n",
       "      <th>AST</th>\n",
       "      <th>STL</th>\n",
       "      <th>BLK</th>\n",
       "      <th>TOV</th>\n",
       "      <th>PF</th>\n",
       "      <th>+/-</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CHI</td>\n",
       "      <td>CHI @ MIN</td>\n",
       "      <td>04/10/2022</td>\n",
       "      <td>W</td>\n",
       "      <td>240</td>\n",
       "      <td>124</td>\n",
       "      <td>44</td>\n",
       "      <td>83</td>\n",
       "      <td>53.0</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>78.8</td>\n",
       "      <td>16</td>\n",
       "      <td>32</td>\n",
       "      <td>48</td>\n",
       "      <td>22</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>23</td>\n",
       "      <td>22</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MIN</td>\n",
       "      <td>MIN vs. CHI</td>\n",
       "      <td>04/10/2022</td>\n",
       "      <td>L</td>\n",
       "      <td>240</td>\n",
       "      <td>120</td>\n",
       "      <td>46</td>\n",
       "      <td>91</td>\n",
       "      <td>50.5</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>77.3</td>\n",
       "      <td>9</td>\n",
       "      <td>23</td>\n",
       "      <td>32</td>\n",
       "      <td>30</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>13</td>\n",
       "      <td>23</td>\n",
       "      <td>-4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PHI</td>\n",
       "      <td>PHI vs. DET</td>\n",
       "      <td>04/10/2022</td>\n",
       "      <td>W</td>\n",
       "      <td>240</td>\n",
       "      <td>118</td>\n",
       "      <td>46</td>\n",
       "      <td>88</td>\n",
       "      <td>52.3</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>91.3</td>\n",
       "      <td>10</td>\n",
       "      <td>32</td>\n",
       "      <td>42</td>\n",
       "      <td>25</td>\n",
       "      <td>13</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>23</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DET</td>\n",
       "      <td>DET @ PHI</td>\n",
       "      <td>04/10/2022</td>\n",
       "      <td>L</td>\n",
       "      <td>240</td>\n",
       "      <td>106</td>\n",
       "      <td>38</td>\n",
       "      <td>83</td>\n",
       "      <td>45.8</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>65.5</td>\n",
       "      <td>15</td>\n",
       "      <td>27</td>\n",
       "      <td>42</td>\n",
       "      <td>26</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>16</td>\n",
       "      <td>-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>BOS</td>\n",
       "      <td>BOS @ MEM</td>\n",
       "      <td>04/10/2022</td>\n",
       "      <td>W</td>\n",
       "      <td>240</td>\n",
       "      <td>139</td>\n",
       "      <td>54</td>\n",
       "      <td>99</td>\n",
       "      <td>54.5</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>100</td>\n",
       "      <td>14</td>\n",
       "      <td>42</td>\n",
       "      <td>56</td>\n",
       "      <td>34</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>20</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2455</th>\n",
       "      <td>IND</td>\n",
       "      <td>IND @ CHA</td>\n",
       "      <td>10/20/2021</td>\n",
       "      <td>L</td>\n",
       "      <td>240</td>\n",
       "      <td>122</td>\n",
       "      <td>42</td>\n",
       "      <td>90</td>\n",
       "      <td>46.7</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>87.5</td>\n",
       "      <td>8</td>\n",
       "      <td>43</td>\n",
       "      <td>51</td>\n",
       "      <td>29</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>17</td>\n",
       "      <td>24</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2456</th>\n",
       "      <td>BKN</td>\n",
       "      <td>BKN @ MIL</td>\n",
       "      <td>10/19/2021</td>\n",
       "      <td>L</td>\n",
       "      <td>240</td>\n",
       "      <td>104</td>\n",
       "      <td>37</td>\n",
       "      <td>84</td>\n",
       "      <td>44.0</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>56.5</td>\n",
       "      <td>5</td>\n",
       "      <td>39</td>\n",
       "      <td>44</td>\n",
       "      <td>19</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "      <td>-23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2457</th>\n",
       "      <td>MIL</td>\n",
       "      <td>MIL vs. BKN</td>\n",
       "      <td>10/19/2021</td>\n",
       "      <td>W</td>\n",
       "      <td>240</td>\n",
       "      <td>127</td>\n",
       "      <td>48</td>\n",
       "      <td>105</td>\n",
       "      <td>45.7</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>77.8</td>\n",
       "      <td>13</td>\n",
       "      <td>41</td>\n",
       "      <td>54</td>\n",
       "      <td>25</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>19</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2458</th>\n",
       "      <td>GSW</td>\n",
       "      <td>GSW @ LAL</td>\n",
       "      <td>10/19/2021</td>\n",
       "      <td>W</td>\n",
       "      <td>240</td>\n",
       "      <td>121</td>\n",
       "      <td>41</td>\n",
       "      <td>93</td>\n",
       "      <td>44.1</td>\n",
       "      <td>14</td>\n",
       "      <td>...</td>\n",
       "      <td>83.3</td>\n",
       "      <td>9</td>\n",
       "      <td>41</td>\n",
       "      <td>50</td>\n",
       "      <td>30</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2459</th>\n",
       "      <td>LAL</td>\n",
       "      <td>LAL vs. GSW</td>\n",
       "      <td>10/19/2021</td>\n",
       "      <td>L</td>\n",
       "      <td>240</td>\n",
       "      <td>114</td>\n",
       "      <td>45</td>\n",
       "      <td>95</td>\n",
       "      <td>47.4</td>\n",
       "      <td>15</td>\n",
       "      <td>...</td>\n",
       "      <td>47.4</td>\n",
       "      <td>5</td>\n",
       "      <td>40</td>\n",
       "      <td>45</td>\n",
       "      <td>21</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>25</td>\n",
       "      <td>-7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2460 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Team     Match Up   Game Date W/L  MIN  PTS FGM  FGA   FG% 3PM  ...  \\\n",
       "0     CHI    CHI @ MIN  04/10/2022   W  240  124  44   83  53.0  10  ...   \n",
       "1     MIN  MIN vs. CHI  04/10/2022   L  240  120  46   91  50.5  11  ...   \n",
       "2     PHI  PHI vs. DET  04/10/2022   W  240  118  46   88  52.3   5  ...   \n",
       "3     DET    DET @ PHI  04/10/2022   L  240  106  38   83  45.8  11  ...   \n",
       "4     BOS    BOS @ MEM  04/10/2022   W  240  139  54   99  54.5  18  ...   \n",
       "...   ...          ...         ...  ..  ...  ...  ..  ...   ...  ..  ...   \n",
       "2455  IND    IND @ CHA  10/20/2021   L  240  122  42   90  46.7  17  ...   \n",
       "2456  BKN    BKN @ MIL  10/19/2021   L  240  104  37   84  44.0  17  ...   \n",
       "2457  MIL  MIL vs. BKN  10/19/2021   W  240  127  48  105  45.7  17  ...   \n",
       "2458  GSW    GSW @ LAL  10/19/2021   W  240  121  41   93  44.1  14  ...   \n",
       "2459  LAL  LAL vs. GSW  10/19/2021   L  240  114  45   95  47.4  15  ...   \n",
       "\n",
       "       FT% OREB DREB REB AST STL BLK TOV  PF  +/-  \n",
       "0     78.8   16   32  48  22   9   3  23  22    4  \n",
       "1     77.3    9   23  32  30   7   9  13  23   -4  \n",
       "2     91.3   10   32  42  25  13   6  11  23   12  \n",
       "3     65.5   15   27  42  26   4   4  20  16  -12  \n",
       "4      100   14   42  56  34   5   2  15  20   29  \n",
       "...    ...  ...  ...  ..  ..  ..  ..  ..  ..  ...  \n",
       "2455  87.5    8   43  51  29   2  10  17  24   -1  \n",
       "2456  56.5    5   39  44  19   3   9  13  17  -23  \n",
       "2457  77.8   13   41  54  25   8   9   8  19   23  \n",
       "2458  83.3    9   41  50  30   9   2  17  18    7  \n",
       "2459  47.4    5   40  45  21   7   4  18  25   -7  \n",
       "\n",
       "[2460 rows x 24 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1f5a29c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Firefox()\n",
    "url = \"https://www.nba.com/stats/players/boxscores\"\n",
    "driver.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "cbfefccc",
   "metadata": {},
   "outputs": [],
   "source": [
    "select = Select(driver.find_element(By.XPATH,\"/html/body/div[1]/div[2]/div[2]/div[3]/section[2]/div/div[2]/div[2]/div[1]/div[3]/div/label/div/select\"))\n",
    "select.select_by_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b7ce9ca0",
   "metadata": {},
   "outputs": [],
   "source": [
    "src = driver.page_source\n",
    "parser = BeautifulSoup(src,\"lxml\")\n",
    "table = parser.find(\"div\",attrs = {\"class\" : \"Crom_container__C45Ti\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "13b1656d",
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = table.find_all('th')\n",
    "headerlist = [h.text.strip() for h in headers]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "228015d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "rows = table.findAll('tr')[1:]\n",
    "player_stats = [[td.getText().strip() for td in rows[i].findAll('td')] for i in range(len(rows))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "da55242c",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats_player = pd.DataFrame(player_stats,columns=headerlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "221eaf15",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PLAYER</th>\n",
       "      <th>TEAM</th>\n",
       "      <th>MATCH UP</th>\n",
       "      <th>GAME DATE</th>\n",
       "      <th>W/L</th>\n",
       "      <th>MIN</th>\n",
       "      <th>PTS</th>\n",
       "      <th>FGM</th>\n",
       "      <th>FGA</th>\n",
       "      <th>FG%</th>\n",
       "      <th>...</th>\n",
       "      <th>OREB</th>\n",
       "      <th>DREB</th>\n",
       "      <th>REB</th>\n",
       "      <th>AST</th>\n",
       "      <th>STL</th>\n",
       "      <th>BLK</th>\n",
       "      <th>TOV</th>\n",
       "      <th>PF</th>\n",
       "      <th>+/-</th>\n",
       "      <th>FP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Shaedon Sharpe</td>\n",
       "      <td>POR</td>\n",
       "      <td>POR @ PHX</td>\n",
       "      <td>11/05/2022</td>\n",
       "      <td>L</td>\n",
       "      <td>27</td>\n",
       "      <td>13</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>50.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>-4</td>\n",
       "      <td>18.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>John Butler Jr.</td>\n",
       "      <td>POR</td>\n",
       "      <td>POR @ PHX</td>\n",
       "      <td>11/05/2022</td>\n",
       "      <td>L</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>50.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>6.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dario Saric</td>\n",
       "      <td>PHX</td>\n",
       "      <td>PHX vs. POR</td>\n",
       "      <td>11/05/2022</td>\n",
       "      <td>W</td>\n",
       "      <td>21</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>44.4</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>22.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Damion Lee</td>\n",
       "      <td>PHX</td>\n",
       "      <td>PHX vs. POR</td>\n",
       "      <td>11/05/2022</td>\n",
       "      <td>W</td>\n",
       "      <td>28</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>60.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>15.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Josh Okogie</td>\n",
       "      <td>PHX</td>\n",
       "      <td>PHX vs. POR</td>\n",
       "      <td>11/05/2022</td>\n",
       "      <td>W</td>\n",
       "      <td>16</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>25.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>10.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3049</th>\n",
       "      <td>Moses Moody</td>\n",
       "      <td>GSW</td>\n",
       "      <td>GSW vs. LAL</td>\n",
       "      <td>10/18/2022</td>\n",
       "      <td>W</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>50.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3050</th>\n",
       "      <td>Scotty Pippen Jr.</td>\n",
       "      <td>LAL</td>\n",
       "      <td>LAL @ GSW</td>\n",
       "      <td>10/18/2022</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3051</th>\n",
       "      <td>Max Christie</td>\n",
       "      <td>LAL</td>\n",
       "      <td>LAL @ GSW</td>\n",
       "      <td>10/18/2022</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>4.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3052</th>\n",
       "      <td>Ryan Rollins</td>\n",
       "      <td>GSW</td>\n",
       "      <td>GSW vs. LAL</td>\n",
       "      <td>10/18/2022</td>\n",
       "      <td>W</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>-3</td>\n",
       "      <td>2.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3053</th>\n",
       "      <td>Cole Swider</td>\n",
       "      <td>LAL</td>\n",
       "      <td>LAL @ GSW</td>\n",
       "      <td>10/18/2022</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2.7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3054 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 PLAYER TEAM     MATCH UP   GAME DATE W/L MIN PTS FGM FGA  \\\n",
       "0        Shaedon Sharpe  POR    POR @ PHX  11/05/2022   L  27  13   6  12   \n",
       "1       John Butler Jr.  POR    POR @ PHX  11/05/2022   L   9   2   1   2   \n",
       "2           Dario Saric  PHX  PHX vs. POR  11/05/2022   W  21   9   4   9   \n",
       "3            Damion Lee  PHX  PHX vs. POR  11/05/2022   W  28   9   3   5   \n",
       "4           Josh Okogie  PHX  PHX vs. POR  11/05/2022   W  16   4   1   4   \n",
       "...                 ...  ...          ...         ...  ..  ..  ..  ..  ..   \n",
       "3049        Moses Moody  GSW  GSW vs. LAL  10/18/2022   W   8   5   2   4   \n",
       "3050  Scotty Pippen Jr.  LAL    LAL @ GSW  10/18/2022   L   2   2   0   0   \n",
       "3051       Max Christie  LAL    LAL @ GSW  10/18/2022   L   2   3   1   1   \n",
       "3052       Ryan Rollins  GSW  GSW vs. LAL  10/18/2022   W   2   0   0   1   \n",
       "3053        Cole Swider  LAL    LAL @ GSW  10/18/2022   L   2   0   0   1   \n",
       "\n",
       "       FG%  ... OREB DREB REB AST STL BLK TOV PF +/-    FP  \n",
       "0     50.0  ...    0    4   4   0   0   1   2  2  -4  18.8  \n",
       "1     50.0  ...    0    1   1   0   0   1   0  0   8   6.2  \n",
       "2     44.4  ...    4    5   9   1   1   0   2  3   2  22.3  \n",
       "3     60.0  ...    1    3   4   1   0   0   0  1  -1  15.3  \n",
       "4     25.0  ...    0    3   3   0   1   1   3  3   2  10.6  \n",
       "...    ...  ...  ...  ...  ..  ..  ..  ..  .. ..  ..   ...  \n",
       "3049  50.0  ...    0    1   1   0   0   0   0  0   1   6.2  \n",
       "3050     -  ...    0    0   0   0   0   0   0  0   3   2.0  \n",
       "3051   100  ...    0    1   1   0   0   0   0  0   3   4.2  \n",
       "3052   0.0  ...    0    1   1   1   0   0   0  2  -3   2.7  \n",
       "3053   0.0  ...    0    1   1   1   0   0   0  0   3   2.7  \n",
       "\n",
       "[3054 rows x 26 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats_player"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b22058a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}