Web scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Beautiful Soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "html_doc = \"\"\"<html><head><title>The Dormouse's story</title></head>\n",
    "<body>\n",
    "<p class=\"title\"><b>The Dormouse's story</b></p>\n",
    "\n",
    "<p class=\"story\">Once upon a time there were three little sisters; and their names were\n",
    "<a href=\"http://example.com/elsie\" class=\"sister\" id=\"link1\">Elsie</a>,\n",
    "<a href=\"http://example.com/lacie\" class=\"sister\" id=\"link2\">Lacie</a> and\n",
    "<a href=\"http://example.com/tillie\" class=\"sister\" id=\"link3\">Tillie</a>;\n",
    "and they lived at the bottom of a well.</p>\n",
    "\n",
    "<p class=\"story\">...</p>\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<html>\n",
      " <head>\n",
      "  <title>\n",
      "   The Dormouse's story\n",
      "  </title>\n",
      " </head>\n",
      " <body>\n",
      "  <p class=\"title\">\n",
      "   <b>\n",
      "    The Dormouse's story\n",
      "   </b>\n",
      "  </p>\n",
      "  <p class=\"story\">\n",
      "   Once upon a time there were three little sisters; and their names were\n",
      "   <a class=\"sister\" href=\"http://example.com/elsie\" id=\"link1\">\n",
      "    Elsie\n",
      "   </a>\n",
      "   ,\n",
      "   <a class=\"sister\" href=\"http://example.com/lacie\" id=\"link2\">\n",
      "    Lacie\n",
      "   </a>\n",
      "   and\n",
      "   <a class=\"sister\" href=\"http://example.com/tillie\" id=\"link3\">\n",
      "    Tillie\n",
      "   </a>\n",
      "   ;\n",
      "and they lived at the bottom of a well.\n",
      "  </p>\n",
      "  <p class=\"story\">\n",
      "   ...\n",
      "  </p>\n",
      " </body>\n",
      "</html>\n"
     ]
    }
   ],
   "source": [
    "soup = BeautifulSoup(html_doc, 'html.parser')\n",
    "\n",
    "print(soup.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<title>The Dormouse's story</title>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'title'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.title.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"The Dormouse's story\""
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.title.string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'head'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.title.parent.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<p class=\"title\"><b>The Dormouse's story</b></p>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['title']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.p['class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<a class=\"sister\" href=\"http://example.com/elsie\" id=\"link1\">Elsie</a>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<a class=\"sister\" href=\"http://example.com/elsie\" id=\"link1\">Elsie</a>,\n",
       " <a class=\"sister\" href=\"http://example.com/lacie\" id=\"link2\">Lacie</a>,\n",
       " <a class=\"sister\" href=\"http://example.com/tillie\" id=\"link3\">Tillie</a>]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all('a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<a class=\"sister\" href=\"http://example.com/tillie\" id=\"link3\">Tillie</a>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find(id=\"link3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<p class=\"title\"><b>The Dormouse's story</b></p>\n",
      "<p class=\"story\">Once upon a time there were three little sisters; and their names were\n",
      "<a class=\"sister\" href=\"http://example.com/elsie\" id=\"link1\">Elsie</a>,\n",
      "<a class=\"sister\" href=\"http://example.com/lacie\" id=\"link2\">Lacie</a> and\n",
      "<a class=\"sister\" href=\"http://example.com/tillie\" id=\"link3\">Tillie</a>;\n",
      "and they lived at the bottom of a well.</p>\n",
      "<p class=\"story\">...</p>\n",
      "----------------------------------\n",
      "['title']\n",
      "['story']\n",
      "['story']\n"
     ]
    }
   ],
   "source": [
    "p = soup.find_all('p')\n",
    "for j in p:\n",
    "    print(j)\n",
    "print(\"----------------------------------\")\n",
    "for i in p:\n",
    "    print(i['class'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['title']"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.p['class']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# User-Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n",
      "{'User-Agent': 'python-requests/2.21.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}\n"
     ]
    }
   ],
   "source": [
    "r = requests.get(\"http://google.com\")\n",
    "print(r.status_code)\n",
    "print(r.request.headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "503\n",
      "{'User-Agent': 'python-requests/2.21.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}\n"
     ]
    }
   ],
   "source": [
    "r1 = requests.get(\"https://www.amazon.in/Nike-Runallday-P-Platinum-C-Grey-Running-898464-018/dp/B07HBCRFL1/ref=sr_1_10?dchild=1&keywords=jogging+shoes+for+men+nike&qid=1607181938&sr=8-10\")\n",
    "print(r1.status_code)\n",
    "print(r1.request.headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n",
      "{'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}\n"
     ]
    }
   ],
   "source": [
    "Agents = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'\n",
    "r2 = requests.get(\"https://www.amazon.in/Nike-Runallday-P-Platinum-C-Grey-Running-898464-018/dp/B07HBCRFL1/ref=sr_1_10?dchild=1&keywords=jogging+shoes+for+men+nike&qid=1607181938&sr=8-10\",headers={'User-Agent': Agents})\n",
    "print(r2.status_code)\n",
    "print(r2.request.headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<div class=\"dir-ltr\" dir=\"ltr\">\n",
      "          🚀 Exploring the universe and our home planet.\n",
      "        </div>, <div class=\"dir-ltr\">\n",
      "<a class=\"twitter-timeline-link activeLink dir-ltr tco-link\" data-url=\"nasa.gov\" dir=\"ltr\" href=\"https://t.co/HMJJbimQpV\" rel=\"nofollow\" target=\"_blank\">nasa.gov</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  Launch Update 🚀 <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"11348282\" data-screenname=\"NASA\" dir=\"ltr\" href=\"/NASA\">@NASA</a> and <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"34743251\" data-screenname=\"SpaceX\" dir=\"ltr\" href=\"/SpaceX\">@SpaceX</a> are now targeting Sunday, Dec. 6 at 11:17 a.m. ET for the 21st Commercial Resupply Services mission to the <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"1451773004\" data-screenname=\"Space_Station\" dir=\"ltr\" href=\"/Space_Station\">@Space_Station</a>. \n",
      "\n",
      "Live coverage will begin tomorrow at 10:45 a.m. ET on NASA TV: <a class=\"twitter_external_link dir-ltr tco-link\" data-expanded-url=\"https://www.nasa.gov/nasalive\" data-url=\"https://www.nasa.gov/nasalive\" dir=\"ltr\" href=\"https://t.co/1XQVlkaW6V\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https://www.nasa.gov/nasalive\">nasa.gov/nasalive</a> <a class=\"twitter_external_link dir-ltr tco-link\" data-expanded-url=\"https://twitter.com/spacex/status/1335198069759995905\" data-url=\"https://twitter.com/spacex/status/1335198069759995905\" dir=\"ltr\" href=\"https://t.co/sBAU4NzrSN\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https://twitter.com/spacex/status/1335198069759995905\">twitter.com/spacex/status/…</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  🚀 The <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"34743251\" data-screenname=\"SpaceX\" dir=\"ltr\" href=\"/SpaceX\">@SpaceX</a> resupply mission launching Sat., Dec. 5 will make history for many firsts, including the first time 2 Dragon spacecraft will be docked to the <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"1451773004\" data-screenname=\"Space_Station\" dir=\"ltr\" href=\"/Space_Station\">@Space_Station</a>, &amp; the first cargo Dragon to splash down in the Atlantic: <a class=\"twitter_external_link dir-ltr tco-link\" data-expanded-url=\"http://youtu.be/ZdobdzBO-3k\" data-url=\"http://youtu.be/ZdobdzBO-3k\" dir=\"ltr\" href=\"https://t.co/PJGMhAsM58\" rel=\"nofollow noopener\" target=\"_blank\" title=\"http://youtu.be/ZdobdzBO-3k\">youtu.be/ZdobdzBO-3k</a>\n",
      "\n",
      " Get to know more ⤵️⤵️⤵️ <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1335033147805868033\" data-expanded-path=\"/NASA/status/1335033202843463680/video/1\" data-pre-embedded=\"true\" data-tco-id=\"1IemIuX1Rq\" data-url=\"https://twitter.com/NASA/status/1335033202843463680/video/1\" dir=\"ltr\" href=\"https://t.co/1IemIuX1Rq\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/1IemIuX1Rq</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  Project Scientist Bob Pappalardo of <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"19802879\" data-screenname=\"NASAJPL\" dir=\"ltr\" href=\"/NASAJPL\">@NASAJPL</a> discusses our upcoming <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"2997418453\" data-screenname=\"EuropaClipper\" dir=\"ltr\" href=\"/EuropaClipper\">@EuropaClipper</a> mission as well as the possibility of life on Jupiter's moon Europa on this episode of our <a class=\"twitter-hashtag dir-ltr\" data-query-source=\"hashtag_click\" dir=\"ltr\" href=\"/hashtag/GravityAssist?src=hash\">#GravityAssist</a> podcast. \n",
      "\n",
      "🎧 Listen and subscribe: <a class=\"twitter_external_link dir-ltr tco-link\" data-expanded-url=\"https://go.nasa.gov/2JCz2PR\" data-url=\"https://go.nasa.gov/2JCz2PR\" dir=\"ltr\" href=\"https://t.co/wIOVSBh8qX\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https://go.nasa.gov/2JCz2PR\">go.nasa.gov/2JCz2PR</a> <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1335009769422204928\" data-expanded-path=\"/NASA/status/1335010227310223361/video/1\" data-pre-embedded=\"true\" data-tco-id=\"sbfnUXk0fc\" data-url=\"https://twitter.com/NASA/status/1335010227310223361/video/1\" dir=\"ltr\" href=\"https://t.co/sbfnUXk0fc\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/sbfnUXk0fc</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  What investigations &amp; science activities will be part of <a class=\"twitter-hashtag dir-ltr\" data-query-source=\"hashtag_click\" dir=\"ltr\" href=\"/hashtag/Artemis?src=hash\">#Artemis</a> III, which will land <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"43166813\" data-screenname=\"NASA_Astronauts\" dir=\"ltr\" href=\"/NASA_Astronauts\">@NASA_Astronauts</a> on the lunar surface?\n",
      "\n",
      "On Mon., Dec. 7 at 1pm ET, <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"260380078\" data-screenname=\"Dr_ThomasZ\" dir=\"ltr\" href=\"/Dr_ThomasZ\">@Dr_ThomasZ</a> &amp; agency leaders in science and human spaceflight discuss priorities. How to tune in: <a class=\"twitter_external_link dir-ltr tco-link\" data-expanded-url=\"https://go.nasa.gov/37uAATO\" data-url=\"https://go.nasa.gov/37uAATO\" dir=\"ltr\" href=\"https://t.co/ylYT4mgxX4\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https://go.nasa.gov/37uAATO\">go.nasa.gov/37uAATO</a> <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1334993110426849284\" data-expanded-path=\"/NASA/status/1334993136171552773/photo/1\" data-pre-embedded=\"true\" data-tco-id=\"OJKTc62if7\" data-url=\"https://twitter.com/NASA/status/1334993136171552773/photo/1\" dir=\"ltr\" href=\"https://t.co/OJKTc62if7\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/OJKTc62if7</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  ☁️ Weather is 50% GO for launch tomorrow. \n",
      "\n",
      "Melody Lovin, launch weather officer for the <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"242828495\" data-screenname=\"45thSpaceWing\" dir=\"ltr\" href=\"/45thSpaceWing\">@45thSpaceWing</a> describes it in two words: “It's complicated.” \n",
      "\n",
      "Listen for a breakdown: <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1334980564625969153\" data-expanded-path=\"/NASA/status/1334981174209335296/video/1\" data-pre-embedded=\"true\" data-tco-id=\"D6NjOU3Lgd\" data-url=\"https://twitter.com/NASA/status/1334981174209335296/video/1\" dir=\"ltr\" href=\"https://t.co/D6NjOU3Lgd\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/D6NjOU3Lgd</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  🛰️ Stays on <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"1451773004\" data-screenname=\"Space_Station\" dir=\"ltr\" href=\"/Space_Station\">@Space_Station</a> for 75 days \n",
      "🧪Double the amount of powered cargo (good for life sciences) \n",
      "✔️Certified for reuse up to 5Xs\n",
      "\n",
      "Here's what makes Cargo Dragon 2 unique from its previous version as described by Sarah Walker, director, <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"34743251\" data-screenname=\"SpaceX\" dir=\"ltr\" href=\"/SpaceX\">@SpaceX</a> Dragon Mission Management: <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1334974312629952512\" data-expanded-path=\"/NASA/status/1334976996279865344/video/1\" data-pre-embedded=\"true\" data-tco-id=\"N6CG8doLUX\" data-url=\"https://twitter.com/NASA/status/1334976996279865344/video/1\" dir=\"ltr\" href=\"https://t.co/N6CG8doLUX\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/N6CG8doLUX</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  ❓ Fun fact: If the weather holds on Saturday, this will be the third year in a row that <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"34743251\" data-screenname=\"SpaceX\" dir=\"ltr\" href=\"/SpaceX\">@SpaceX</a> launches a commercial NASA resupply mission on 12/5! \n",
      "\n",
      "Learn about this &amp; more launch opportunities from Sarah Walker, director, Dragon Mission Management, SpaceX: <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1334972760645226497\" data-expanded-path=\"/NASA/status/1334974946783608832/video/1\" data-pre-embedded=\"true\" data-tco-id=\"87U8kXO3aw\" data-url=\"https://twitter.com/NASA/status/1334974946783608832/video/1\" dir=\"ltr\" href=\"https://t.co/87U8kXO3aw\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/87U8kXO3aw</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  🤔 Wondering what’s flying on board the upgraded Cargo Dragon?  ",
      " ",
      "Take a behind-the-scenes look at how some of our researchers have been working across the nation to get their science ready for tomorrow’s mission: <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1334971004624367616\" data-expanded-path=\"/NASA/status/1334972891864055813/video/1\" data-pre-embedded=\"true\" data-tco-id=\"UP6Hrckbmb\" data-url=\"https://twitter.com/NASA/status/1334972891864055813/video/1\" dir=\"ltr\" href=\"https://t.co/UP6Hrckbmb\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/UP6Hrckbmb</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  “The <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"1451773004\" data-screenname=\"Space_Station\" dir=\"ltr\" href=\"/Space_Station\">@Space_Station</a> is receiving its first  <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"34743251\" data-screenname=\"SpaceX\" dir=\"ltr\" href=\"/SpaceX\">@SpaceX</a> upgraded Cargo 2 vehicle.”  ",
      " ",
      "\n",
      "\n",
      "Learn about out all the new capabilities this spacecraft comes with from Kirt Costello, chief scientist, for the International Space Station: <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1334969673532624898\" data-expanded-path=\"/NASA/status/1334970021311705088/video/1\" data-pre-embedded=\"true\" data-tco-id=\"X1yWSdqV2y\" data-url=\"https://twitter.com/NASA/status/1334970021311705088/video/1\" dir=\"ltr\" href=\"https://t.co/X1yWSdqV2y\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/X1yWSdqV2y</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  “It’s the first time we will have a couple of Dragons on board – a Crew Dragon and a Cargo Dragon.”  ",
      " ",
      "\n",
      "\n",
      "Hear about some of the science heading to space, including a new commercial airlock, from Kenny Todd, deputy program manager, <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"1451773004\" data-screenname=\"Space_Station\" dir=\"ltr\" href=\"/Space_Station\">@Space_Station</a> Program Office: <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1334968568782278656\" data-expanded-path=\"/NASA/status/1334969260464979968/video/1\" data-pre-embedded=\"true\" data-tco-id=\"oYlIiaqMIK\" data-url=\"https://twitter.com/NASA/status/1334969260464979968/video/1\" dir=\"ltr\" href=\"https://t.co/oYlIiaqMIK\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/oYlIiaqMIK</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  🔴LIVE FROM <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"16580226\" data-screenname=\"NASAKennedy\" dir=\"ltr\" href=\"/NASAKennedy\">@NASAKennedy</a>: Experts discuss tomorrow’s <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"34743251\" data-screenname=\"SpaceX\" dir=\"ltr\" href=\"/SpaceX\">@SpaceX</a> resupply mission to the <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"1451773004\" data-screenname=\"Space_Station\" dir=\"ltr\" href=\"/Space_Station\">@Space_Station</a> – the first launch featuring a newly upgraded Dragon cargo spacecraft. \n",
      "\n",
      "Have questions? Use <a class=\"twitter-hashtag dir-ltr\" data-query-source=\"hashtag_click\" dir=\"ltr\" href=\"/hashtag/askNASA?src=hash\">#askNASA</a> for a chance to be answered on air!   <a class=\"twitter_external_link dir-ltr tco-link\" data-expanded-url=\"https://twitter.com/i/broadcasts/1MYGNmaOVgzKw\" data-url=\"https://twitter.com/i/broadcasts/1MYGNmaOVgzKw\" dir=\"ltr\" href=\"https://t.co/aTNTZcIsTA\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https://twitter.com/i/broadcasts/1MYGNmaOVgzKw\">twitter.com/i/broadcasts/1…</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  ⏳ In 30 minutes, experts will preview tomorrow’s SpaceX resupply mission set to send ~6,400 pounds of cargo to the International Space Station. Participants include:\n",
      "\n",
      "<a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"1451773004\" data-screenname=\"Space_Station\" dir=\"ltr\" href=\"/Space_Station\">@Space_Station</a>\n",
      "<a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"34743251\" data-screenname=\"SpaceX\" dir=\"ltr\" href=\"/SpaceX\">@SpaceX</a>\n",
      "<a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"242828495\" data-screenname=\"45thSpaceWing\" dir=\"ltr\" href=\"/45thSpaceWing\">@45thSpaceWing</a>\n",
      "\n",
      "Tune in and comment your ?s below using <a class=\"twitter-hashtag dir-ltr\" data-query-source=\"hashtag_click\" dir=\"ltr\" href=\"/hashtag/askNASA?src=hash\">#askNASA</a>: <a class=\"twitter_external_link dir-ltr tco-link\" data-expanded-url=\"http://www.nasa.gov/live\" data-url=\"http://www.nasa.gov/live\" dir=\"ltr\" href=\"https://t.co/mzKW5uV4hS\" rel=\"nofollow noopener\" target=\"_blank\" title=\"http://www.nasa.gov/live\">nasa.gov/live</a> <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1334958153000169472\" data-expanded-path=\"/NASA/status/1334958154870894593/photo/1\" data-pre-embedded=\"true\" data-tco-id=\"tHVspRYXWg\" data-url=\"https://twitter.com/NASA/status/1334958154870894593/photo/1\" dir=\"ltr\" href=\"https://t.co/tHVspRYXWg\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/tHVspRYXWg</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  🔥 Engine tests continue for our <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"467739426\" data-screenname=\"NASA_SLS\" dir=\"ltr\" href=\"/NASA_SLS\">@NASA_SLS</a> rocket, with systems powering up in a rehearsal of the final stage: a hot fire of all four engines to simulate an <a class=\"twitter-hashtag dir-ltr\" data-query-source=\"hashtag_click\" dir=\"ltr\" href=\"/hashtag/Artemis?src=hash\">#Artemis</a> launch to the Moon.\n",
      "\n",
      "Tune in at 12pm ET on Thurs., Dec. 10 for a Green Run series update: <a class=\"twitter_external_link dir-ltr tco-link\" data-expanded-url=\"https://go.nasa.gov/3lCvl9O\" data-url=\"https://go.nasa.gov/3lCvl9O\" dir=\"ltr\" href=\"https://t.co/hbidjrjnip\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https://go.nasa.gov/3lCvl9O\">go.nasa.gov/3lCvl9O</a> <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1334953119571210240\" data-expanded-path=\"/NASA/status/1334953121240543232/photo/1\" data-pre-embedded=\"true\" data-tco-id=\"F8vugTtmoz\" data-url=\"https://twitter.com/NASA/status/1334953121240543232/photo/1\" dir=\"ltr\" href=\"https://t.co/F8vugTtmoz\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/F8vugTtmoz</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  NEWS: <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"11348282\" data-screenname=\"NASA\" dir=\"ltr\" href=\"/NASA\">@NASA</a> teams at <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"44134466\" data-screenname=\"NASAStennis\" dir=\"ltr\" href=\"/NASAStennis\">@NASAStennis</a> have given the “go” to start the seventh test of the Green Run test series. During wet dress rehearsal, the <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"467739426\" data-screenname=\"NASA_SLS\" dir=\"ltr\" href=\"/NASA_SLS\">@NASA_SLS</a> core stage will be fully loaded with liquid propellants for the first time. More: <a class=\"twitter_external_link dir-ltr tco-link\" data-expanded-url=\"https://blogs.nasa.gov/artemis/2020/12/04/nasa-go-for-green-run-wet-dress-rehearsal/\" data-url=\"https://blogs.nasa.gov/artemis/2020/12/04/nasa-go-for-green-run-wet-dress-rehearsal/\" dir=\"ltr\" href=\"https://t.co/r7zwjXf6BD\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https://blogs.nasa.gov/artemis/2020/12/04/nasa-go-for-green-run-wet-dress-rehearsal/\">blogs.nasa.gov/artemis/2020/1…</a> <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1334948440732733444\" data-expanded-path=\"/JimBridenstine/status/1334948445182943233/photo/1\" data-pre-embedded=\"true\" data-tco-id=\"nblUBV5Q0z\" data-url=\"https://twitter.com/JimBridenstine/status/1334948445182943233/photo/1\" dir=\"ltr\" href=\"https://t.co/nblUBV5Q0z\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/nblUBV5Q0z</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  ✔️ The launch readiness review for the 21st Commercial Resupply Services mission to the <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"1451773004\" data-screenname=\"Space_Station\" dir=\"ltr\" href=\"/Space_Station\">@Space_Station</a> has concluded!\n",
      "\n",
      "<a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"11348282\" data-screenname=\"NASA\" dir=\"ltr\" href=\"/NASA\">@NASA</a> and <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"34743251\" data-screenname=\"SpaceX\" dir=\"ltr\" href=\"/SpaceX\">@SpaceX</a> are proceeding toward launch at 11:39 a.m. ET on Dec. 5. Tune in today at 4 p.m. ET for the prelaunch news conference: <a class=\"twitter_external_link dir-ltr tco-link\" data-expanded-url=\"https://www.nasa.gov/nasalive\" data-url=\"https://www.nasa.gov/nasalive\" dir=\"ltr\" href=\"https://t.co/1XQVljTkIl\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https://www.nasa.gov/nasalive\">nasa.gov/nasalive</a> <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1334944880636747778\" data-expanded-path=\"/NASAKennedy/status/1334946158561566720/photo/1\" data-pre-embedded=\"true\" data-tco-id=\"p4bZmaQz6y\" data-url=\"https://twitter.com/NASAKennedy/status/1334946158561566720/photo/1\" dir=\"ltr\" href=\"https://t.co/p4bZmaQz6y\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/p4bZmaQz6y</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  Launching to the <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"1451773004\" data-screenname=\"Space_Station\" dir=\"ltr\" href=\"/Space_Station\">@Space_Station</a> in the trunk of a Dragon capsule tomorrow is a new commercial airlock. The <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"107589725\" data-screenname=\"Nanoracks\" dir=\"ltr\" href=\"/Nanoracks\">@Nanoracks</a> Bishop Airlock will allow for new CubeSat deployments and science experiments aboard the space station. <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1328779725716684801\" data-expanded-path=\"/ISS_Research/status/1334909668028608515/video/1\" data-pre-embedded=\"true\" data-tco-id=\"kzxIudK0mH\" data-url=\"https://twitter.com/ISS_Research/status/1334909668028608515/video/1\" dir=\"ltr\" href=\"https://t.co/kzxIudK0mH\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/kzxIudK0mH</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  🔴 WE’RE LIVE: Our virtual <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"52758395\" data-screenname=\"NASASocial\" dir=\"ltr\" href=\"/NASASocial\">@NASASocial</a> Science &amp; Station Q&amp;A starts now! 🚀\n",
      "\n",
      "Ahead of tomorrow's launch, we are chatting with the experts behind the science &amp; technology aboard the upgraded <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"34743251\" data-screenname=\"SpaceX\" dir=\"ltr\" href=\"/SpaceX\">@SpaceX</a> Dragon spacecraft. Have a question? Submit using <a class=\"twitter-hashtag dir-ltr\" data-query-source=\"hashtag_click\" dir=\"ltr\" href=\"/hashtag/AskNASA?src=hash\">#AskNASA</a>: <a class=\"twitter_external_link dir-ltr tco-link\" data-expanded-url=\"http://www.nasa.gov/live\" data-url=\"http://www.nasa.gov/live\" dir=\"ltr\" href=\"https://t.co/mzKW5uV4hS\" rel=\"nofollow noopener\" target=\"_blank\" title=\"http://www.nasa.gov/live\">nasa.gov/live</a> <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1334920411126095872\" data-expanded-path=\"/NASA/status/1334920412707385347/photo/1\" data-pre-embedded=\"true\" data-tco-id=\"tFslw61vmV\" data-url=\"https://twitter.com/NASA/status/1334920412707385347/photo/1\" dir=\"ltr\" href=\"https://t.co/tFslw61vmV\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/tFslw61vmV</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  We are one day away from <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"34743251\" data-screenname=\"SpaceX\" dir=\"ltr\" href=\"/SpaceX\">@SpaceX</a>'s 21st Commercial Resupply Services (CRS-21) mission! 🚀 Join us today for our prelaunch events:\n",
      "\n",
      "1⃣ 1pm ET: Virtual <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"52758395\" data-screenname=\"NASASocial\" dir=\"ltr\" href=\"/NASASocial\">@NASASocial</a> Science &amp; Station Q&amp;A\n",
      "2⃣ TBD: Prelaunch News Conference\n",
      "\n",
      "Both will air live on <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"11348282\" data-screenname=\"NASA\" dir=\"ltr\" href=\"/NASA\">@NASA</a> TV: <a class=\"twitter_external_link dir-ltr tco-link\" data-expanded-url=\"https://www.nasa.gov/nasalive\" data-url=\"https://www.nasa.gov/nasalive\" dir=\"ltr\" href=\"https://t.co/1XQVljTkIl\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https://www.nasa.gov/nasalive\">nasa.gov/nasalive</a> <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1334886649088524292\" data-expanded-path=\"/NASAKennedy/status/1334889811065274368/photo/1\" data-pre-embedded=\"true\" data-tco-id=\"IyFxgd41g8\" data-url=\"https://twitter.com/NASAKennedy/status/1334889811065274368/photo/1\" dir=\"ltr\" href=\"https://t.co/IyFxgd41g8\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/IyFxgd41g8</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  This week, preparations are underway for <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"34743251\" data-screenname=\"SpaceX\" dir=\"ltr\" href=\"/SpaceX\">@SpaceX</a>'s next resupply launch to the <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"1451773004\" data-screenname=\"Space_Station\" dir=\"ltr\" href=\"/Space_Station\">@Space_Station</a>. This first flight of the upgraded Dragon will deliver ~6,400 pounds of cargo, and a new door to space: the <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"107589725\" data-screenname=\"Nanoracks\" dir=\"ltr\" href=\"/Nanoracks\">@Nanoracks</a> Bishop airlock. More on <a class=\"twitter-hashtag dir-ltr\" data-query-source=\"hashtag_click\" dir=\"ltr\" href=\"/hashtag/SpacetoGround?src=hash\">#SpacetoGround</a>: <a class=\"twitter_external_link dir-ltr tco-link\" data-expanded-url=\"https://youtu.be/sH0hz7Z0JPc\" data-url=\"https://youtu.be/sH0hz7Z0JPc\" dir=\"ltr\" href=\"https://t.co/sUTAeFqetv\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https://youtu.be/sH0hz7Z0JPc\">youtu.be/sH0hz7Z0JPc</a> <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1334892745685676032\" data-expanded-path=\"/NASA/status/1334893035776315398/video/1\" data-pre-embedded=\"true\" data-tco-id=\"wceCMrCD2A\" data-url=\"https://twitter.com/NASA/status/1334893035776315398/video/1\" dir=\"ltr\" href=\"https://t.co/wceCMrCD2A\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/wceCMrCD2A</a>\n",
      "</div>, <div class=\"dir-ltr\" dir=\"ltr\">  Excited to reveal our Crew-2 mission patch!  The determined expression of the Dragon in the patch reflects the strength of the team and their contribution to the exploration of space  <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"437520768\" data-screenname=\"Thom_astro\" dir=\"ltr\" href=\"/Thom_astro\">@Thom_astro</a> <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"1286753931834646529\" data-screenname=\"Astro_Megan\" dir=\"ltr\" href=\"/Astro_Megan\">@Astro_Megan</a> <a class=\"twitter-atreply dir-ltr\" data-mentioned-user-id=\"390455433\" data-screenname=\"Aki_Hoshide\" dir=\"ltr\" href=\"/Aki_Hoshide\">@Aki_Hoshide</a> <a class=\"twitter-hashtag dir-ltr\" data-query-source=\"hashtag_click\" dir=\"ltr\" href=\"/hashtag/nasa?src=hash\">#nasa</a> <a class=\"twitter-hashtag dir-ltr\" data-query-source=\"hashtag_click\" dir=\"ltr\" href=\"/hashtag/spacex?src=hash\">#spacex</a> <a class=\"twitter-hashtag dir-ltr\" data-query-source=\"hashtag_click\" dir=\"ltr\" href=\"/hashtag/iss?src=hash\">#iss</a> <a class=\"twitter-hashtag dir-ltr\" data-query-source=\"hashtag_click\" dir=\"ltr\" href=\"/hashtag/space?src=hash\">#space</a> <a class=\"twitter_external_link dir-ltr tco-link has-expanded-path\" data-entity-id=\"1334657470606798850\" data-expanded-path=\"/astro_kimbrough/status/1334657473513480193/photo/1\" data-pre-embedded=\"true\" data-tco-id=\"0W3OwlQXKA\" data-url=\"https://twitter.com/astro_kimbrough/status/1334657473513480193/photo/1\" dir=\"ltr\" href=\"https://t.co/0W3OwlQXKA\" rel=\"nofollow\" target=\"_top\">pic.twitter.com/0W3OwlQXKA</a>\n",
      "</div>]\n"
     ]
    }
   ],
   "source": [
    "CONTENT_CLASS_NAME = \"dir-ltr\"\n",
    "CONTENT_CONTAINER_TAGS = [\"div\"]\n",
    "\n",
    "TWITTER_URL = \"https://twitter.com/\"\n",
    "twitter_handle = \"nasa\"\n",
    "AGENTS= 'Nokia5310XpressMusic_CMCC/2.0 (10.10) Profile/MIDP-2.1 '\\\n",
    "'Configuration/CLDC-1.1 UCWEB/2.0 (Java; U; MIDP-2.0; en-US; '\\\n",
    "'Nokia5310XpressMusic) U2/1.0.0 UCBrowser/9.5.0.449 U2/1.0.0 Mobile'\n",
    "\n",
    "url = TWITTER_URL + twitter_handle\n",
    "response = requests.get(url, headers={'User-Agent': AGENTS})\n",
    "html1 = response.content\n",
    "\n",
    "soup1 = BeautifulSoup(html1, \"html.parser\")\n",
    "\n",
    "elements = soup1.find_all(CONTENT_CONTAINER_TAGS, {\"class\": CONTENT_CLASS_NAME})\n",
    "print(elements)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\n          🚀 Exploring the universe and our home planet.\\n        ', 'nasa.gov', '  Launch Update 🚀 ', '@NASA', ' and ', '@SpaceX', ' are now targeting Sunday, Dec. 6 at 11:17 a.m. ET for the 21st Commercial Resupply Services mission to the ', '@Space_Station', '. \\n\\nLive coverage will begin tomorrow at 10:45 a.m. ET on NASA TV: ', 'nasa.gov/nasalive', ' ', 'twitter.com/spacex/status/…', '  🚀 The ', '@SpaceX', ' resupply mission launching Sat., Dec. 5 will make history for many firsts, including the first time 2 Dragon spacecraft will be docked to the ', '@Space_Station', ', & the first cargo Dragon to splash down in the Atlantic: ', 'youtu.be/ZdobdzBO-3k', '\\n\\n Get to know more ⤵️⤵️⤵️ ', 'pic.twitter.com/1IemIuX1Rq', '  Project Scientist Bob Pappalardo of ', '@NASAJPL', ' discusses our upcoming ', '@EuropaClipper', \" mission as well as the possibility of life on Jupiter's moon Europa on this episode of our \", '#GravityAssist', ' podcast. \\n\\n🎧 Listen and subscribe: ', 'go.nasa.gov/2JCz2PR', ' ', 'pic.twitter.com/sbfnUXk0fc', '  What investigations & science activities will be part of ', '#Artemis', ' III, which will land ', '@NASA_Astronauts', ' on the lunar surface?\\n\\nOn Mon., Dec. 7 at 1pm ET, ', '@Dr_ThomasZ', ' & agency leaders in science and human spaceflight discuss priorities. How to tune in: ', 'go.nasa.gov/37uAATO', ' ', 'pic.twitter.com/OJKTc62if7', '  ☁️ Weather is 50% GO for launch tomorrow. \\n\\nMelody Lovin, launch weather officer for the ', '@45thSpaceWing', \" describes it in two words: “It's complicated.” \\n\\nListen for a breakdown: \", 'pic.twitter.com/D6NjOU3Lgd', '  🛰️ Stays on ', '@Space_Station', \" for 75 days \\n🧪Double the amount of powered cargo (good for life sciences) \\n✔️Certified for reuse up to 5Xs\\n\\nHere's what makes Cargo Dragon 2 unique from its previous version as described by Sarah Walker, director, \", '@SpaceX', ' Dragon Mission Management: ', 'pic.twitter.com/N6CG8doLUX', '  ❓ Fun fact: If the weather holds on Saturday, this will be the third year in a row that ', '@SpaceX', ' launches a commercial NASA resupply mission on 12/5! \\n\\nLearn about this & more launch opportunities from Sarah Walker, director, Dragon Mission Management, SpaceX: ', 'pic.twitter.com/87U8kXO3aw', '  🤔 Wondering what’s flying on board the upgraded Cargo Dragon? \\u2028\\u2028Take a behind-the-scenes look at how some of our researchers have been working across the nation to get their science ready for tomorrow’s mission: ', 'pic.twitter.com/UP6Hrckbmb', '  “The ', '@Space_Station', ' is receiving its first  ', '@SpaceX', ' upgraded Cargo 2 vehicle.” \\u2028\\u2028\\n\\nLearn about out all the new capabilities this spacecraft comes with from Kirt Costello, chief scientist, for the International Space Station: ', 'pic.twitter.com/X1yWSdqV2y', '  “It’s the first time we will have a couple of Dragons on board – a Crew Dragon and a Cargo Dragon.” \\u2028\\u2028\\n\\nHear about some of the science heading to space, including a new commercial airlock, from Kenny Todd, deputy program manager, ', '@Space_Station', ' Program Office: ', 'pic.twitter.com/oYlIiaqMIK', '  🔴LIVE FROM ', '@NASAKennedy', ': Experts discuss tomorrow’s ', '@SpaceX', ' resupply mission to the ', '@Space_Station', ' – the first launch featuring a newly upgraded Dragon cargo spacecraft. \\n\\nHave questions? Use ', '#askNASA', ' for a chance to be answered on air!   ', 'twitter.com/i/broadcasts/1…', '  ⏳ In 30 minutes, experts will preview tomorrow’s SpaceX resupply mission set to send ~6,400 pounds of cargo to the International Space Station. Participants include:\\n\\n', '@Space_Station', '@SpaceX', '@45thSpaceWing', '\\n\\nTune in and comment your ?s below using ', '#askNASA', ': ', 'nasa.gov/live', ' ', 'pic.twitter.com/tHVspRYXWg', '  🔥 Engine tests continue for our ', '@NASA_SLS', ' rocket, with systems powering up in a rehearsal of the final stage: a hot fire of all four engines to simulate an ', '#Artemis', ' launch to the Moon.\\n\\nTune in at 12pm ET on Thurs., Dec. 10 for a Green Run series update: ', 'go.nasa.gov/3lCvl9O', ' ', 'pic.twitter.com/F8vugTtmoz', '  NEWS: ', '@NASA', ' teams at ', '@NASAStennis', ' have given the “go” to start the seventh test of the Green Run test series. During wet dress rehearsal, the ', '@NASA_SLS', ' core stage will be fully loaded with liquid propellants for the first time. More: ', 'blogs.nasa.gov/artemis/2020/1…', ' ', 'pic.twitter.com/nblUBV5Q0z', '  ✔️ The launch readiness review for the 21st Commercial Resupply Services mission to the ', '@Space_Station', ' has concluded!\\n\\n', '@NASA', ' and ', '@SpaceX', ' are proceeding toward launch at 11:39 a.m. ET on Dec. 5. Tune in today at 4 p.m. ET for the prelaunch news conference: ', 'nasa.gov/nasalive', ' ', 'pic.twitter.com/p4bZmaQz6y', '  Launching to the ', '@Space_Station', ' in the trunk of a Dragon capsule tomorrow is a new commercial airlock. The ', '@Nanoracks', ' Bishop Airlock will allow for new CubeSat deployments and science experiments aboard the space station. ', 'pic.twitter.com/kzxIudK0mH', '  🔴 WE’RE LIVE: Our virtual ', '@NASASocial', \" Science & Station Q&A starts now! 🚀\\n\\nAhead of tomorrow's launch, we are chatting with the experts behind the science & technology aboard the upgraded \", '@SpaceX', ' Dragon spacecraft. Have a question? Submit using ', '#AskNASA', ': ', 'nasa.gov/live', ' ', 'pic.twitter.com/tFslw61vmV', '  We are one day away from ', '@SpaceX', \"'s 21st Commercial Resupply Services (CRS-21) mission! 🚀 Join us today for our prelaunch events:\\n\\n1⃣ 1pm ET: Virtual \", '@NASASocial', ' Science & Station Q&A\\n2⃣ TBD: Prelaunch News Conference\\n\\nBoth will air live on ', '@NASA', ' TV: ', 'nasa.gov/nasalive', ' ', 'pic.twitter.com/IyFxgd41g8', '  This week, preparations are underway for ', '@SpaceX', \"'s next resupply launch to the \", '@Space_Station', '. This first flight of the upgraded Dragon will deliver ~6,400 pounds of cargo, and a new door to space: the ', '@Nanoracks', ' Bishop airlock. More on ', '#SpacetoGround', ': ', 'youtu.be/sH0hz7Z0JPc', ' ', 'pic.twitter.com/wceCMrCD2A', '  Excited to reveal our Crew-2 mission patch!  The determined expression of the Dragon in the patch reflects the strength of the team and their contribution to the exploration of space  ', '@Thom_astro', ' ', '@Astro_Megan', ' ', '@Aki_Hoshide', ' ', '#nasa', ' ', '#spacex', ' ', '#iss', ' ', '#space', ' ', 'pic.twitter.com/0W3OwlQXKA']\n"
     ]
    }
   ],
   "source": [
    "tweets = []\n",
    "EMPTY_ITEMS = [None, \"None\", '\\n']\n",
    "for post in elements:\n",
    "    for text in post.contents:\n",
    "        if text.string not in EMPTY_ITEMS:\n",
    "            tweets.append(text.string)\n",
    "print(tweets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Regular Expression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Match\n",
      "<re.Match object; span=(0, 10), match='01-12-2020'>\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(re.match(\"([0-9]{2}\\-[0-9]{2}\\-[0-9]{4})\", \"01-12-2020.txt\"))\n",
    "print(re.match(\"([0-9]{2}\\-[0-9]{2}\\-[0-9]{4})\", \"file - 01-12-2020.txt\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(0, 10), match='01-12-2020'>\n",
      "<re.Match object; span=(7, 17), match='01-12-2020'>\n",
      "<re.Match object; span=(0, 10), match='01-12-2020'>\n"
     ]
    }
   ],
   "source": [
    "print(re.search(\"([0-9]{2}\\-[0-9]{2}\\-[0-9]{4})\", \"01-12-2020.txt\"))\n",
    "print(re.search(\"([0-9]{2}\\-[0-9]{2}\\-[0-9]{4})\", \"file - 01-12-2020.txt\"))\n",
    "print(re.search(\"([0-9]{2}\\-[0-9]{2}\\-[0-9]{4})\", \"01-12-2020.txt 02-12-2020.py\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Findall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['01-12-2020', '02-12-2020']\n"
     ]
    }
   ],
   "source": [
    "print(re.findall(\"([0-9]{2}\\-[0-9]{2}\\-[0-9]{4})\", \"01-12-2020.txt 02-12-2020.py\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['01-12-2020', '02-12-2020']\n"
     ]
    }
   ],
   "source": [
    "# Compile\n",
    "date = re.compile('[0-9]{2}\\-[0-9]{2}\\-[0-9]{4}')\n",
    "print(date.findall(\"01-12-2020.txt 02-12-2020.py\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "01-01-2021.txt\n"
     ]
    }
   ],
   "source": [
    "sub_date = \"01-01-2021\"\n",
    "print(date.sub(sub_date,\"01-12-2020.txt\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}