scrapper_shwcase.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7a987f9d-e8c6-4ea0-aeb6-89e0cefb816f",
   "metadata": {},
   "source": [
    "### Go to this given URL and solve the following questions\n",
    ">URL: https://www.youtube.com/@PW-Foundation/videos"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67dd3965-4ef7-41f3-ad2f-5a65702ed654",
   "metadata": {},
   "source": [
    "### Q1. Write a python program to extract the video URL of the first five videos."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de20eb90-342f-4a98-b952-a5ced4712d19",
   "metadata": {},
   "source": [
    "### Q2. Write a python program to extract the URL of the video thumbnails of the first five videos.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a672c42-c8a1-487b-88af-25929ab4bba3",
   "metadata": {},
   "source": [
    "#### Q3. Write a python program to extract the title of the first five videos.\n",
    "#### Q4. Write a python program to extract the number of views of the first five videos.\n",
    "### Q5. Write a python program to extract the time of posting of video for the first five videos."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae85b550-104a-4bfd-be21-3a27e17890d9",
   "metadata": {
    "tags": []
   },
   "source": [
    ">> Get the Channel name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e435f47-c1ca-4cb1-972e-2052609e4d46",
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.service import Service as ChromeService\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from selenium.webdriver.common.by import By\n",
    "# from selenium.webdriver.common.keys import Keys\n",
    "import time\n",
    "\n",
    "# from webelement_highlighter import WebElementHighlighter\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "driver = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()))\n",
    "\n",
    "URL = \"https://www.youtube.com/@PW-Foundation/videos\"\n",
    "\n",
    "driver.get(URL)\n",
    "driver.maximize_window()\n",
    "\n",
    "driver.find_element(By.XPATH, '//*[@id=\"tabsContent\"]/tp-yt-paper-tab[7]/div').click()\n",
    "time.sleep(0.5)\n",
    "driver.find_element(By.XPATH, '//*[@id=\"tabsContent\"]/tp-yt-paper-tab[7]/div').click()\n",
    "time.sleep(0.5)\n",
    "next_url = driver.current_url\n",
    "driver.get(next_url)\n",
    "# print(next_url)\n",
    "time.sleep(3)\n",
    "driver.find_element(By.ID, 'channel-info').click()\n",
    "time.sleep(3)\n",
    "channel_url = str(driver.current_url).split('/')[-1]\n",
    "\n",
    "print(channel_url)\n",
    "\n",
    "\n",
    "time.sleep(3)\n",
    "driver.delete_all_cookies()\n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "daa5e94b-ceed-499b-9b42-72cc38d8d21d",
   "metadata": {},
   "source": [
    "# Get details from all data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96345f37-aefe-4cd2-95b1-f582f3a9a065",
   "metadata": {},
   "outputs": [],
   "source": [
    "from googleapiclient.discovery import build\n",
    "scopes = [\"https://www.googleapis.com/auth/youtube.readonly\"]\n",
    "import pandas as pd\n",
    "api_key = secretAPI\n",
    "api_service_name = \"youtube\"\n",
    "api_version = \"v3\"\n",
    "client_secrets_file = client.json\n",
    "\n",
    "\n",
    "youtube = build('youtube','v3', developerKey=api_key)\n",
    "\n",
    "# Accessing the playlist id\n",
    "playlist_id= channel_url\n",
    "\n",
    "# we need create function to get video id to access the video detail \n",
    "\n",
    "def get_video_ids(youtube,playlist_id):\n",
    "    \n",
    "    request = youtube.playlistItems().list(\n",
    "                part='contentDetails',\n",
    "                playlistId=playlist_id,\n",
    "                maxResults = 50)\n",
    "    response=request.execute()\n",
    "    \n",
    "    video_ids=[]\n",
    "    \n",
    "    for i in range(len(response['items'])):\n",
    "        video_ids.append(response['items'][i]['contentDetails']['videoId'])\n",
    "        \n",
    "## while loop to check if we have more pages than 50 \n",
    "        \n",
    "    next_page_token = response.get('nextPageToken')\n",
    "    more_pages = True\n",
    "    \n",
    "    while more_pages:\n",
    "        if next_page_token is None:\n",
    "            more_pages = False\n",
    "        else:\n",
    "            request = youtube.playlistItems().list(\n",
    "                        part='contentDetails',\n",
    "                        playlistId=playlist_id,\n",
    "                        maxResults = 50,\n",
    "                        pageToken = next_page_token)\n",
    "            response=request.execute()\n",
    "            \n",
    "            for i in range(len(response['items'])):\n",
    "                video_ids.append(response['items'][i]['contentDetails']['videoId'])\n",
    "                \n",
    "            next_page_token = response.get('nextPageToken')\n",
    "            \n",
    "    return (video_ids)\n",
    "    \n",
    "video_ids=get_video_ids(youtube,playlist_id)\n",
    "# fot play url\n",
    "video_url = []\n",
    "for name in video_ids:\n",
    "    urlname = f\"https://www.youtube.com/watch?v={name}\"\n",
    "    video_url.append(urlname)\n",
    "\n",
    "def get_video_details(youtube, video_ids):\n",
    "    all_video_stats = []\n",
    "    \n",
    "    for i in range(0, len(video_ids), 50): ## processing 50 videos at a time as # limitation of youtube-we can fetch maxm 50 video ids at a time\n",
    "        request = youtube.videos().list(\n",
    "                part='snippet,statistics',\n",
    "                id=','.join(video_ids[i:i+50]))\n",
    "        response = request.execute()\n",
    "        a = 1\n",
    "        try:\n",
    "            for video in response['items']:\n",
    "                # print(video['statistics'])\n",
    "                # if a == 1:\n",
    "                #     break\n",
    "                video_stats = dict(Title = video['snippet']['title'],\n",
    "                                Published_date = video['snippet']['publishedAt'],\n",
    "                                Views = video['statistics']['viewCount'],\n",
    "                                Likes = video['statistics']['likeCount'],\n",
    "                                Comments = video['statistics']['commentCount']\n",
    "                                )\n",
    "                all_video_stats.append(video_stats)\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "    \n",
    "    return all_video_stats\n",
    "\n",
    "video_details=get_video_details(youtube, video_ids)\n",
    "\n",
    "video_data = pd.DataFrame(video_details)\n",
    "print(video_data)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}