extract_links_from_webpage.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "934dc54b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting requests\n",
      "  Downloading requests-2.28.1-py3-none-any.whl (62 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.8/62.8 kB\u001b[0m \u001b[31m266.1 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m1m232.8 kB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting certifi>=2017.4.17\n",
      "  Downloading certifi-2022.12.7-py3-none-any.whl (155 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m155.3/155.3 kB\u001b[0m \u001b[31m590.5 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m kB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25hCollecting urllib3<1.27,>=1.21.1\n",
      "  Downloading urllib3-1.26.13-py2.py3-none-any.whl (140 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m140.6/140.6 kB\u001b[0m \u001b[31m356.0 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m kB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25hCollecting charset-normalizer<3,>=2\n",
      "  Downloading charset_normalizer-2.1.1-py3-none-any.whl (39 kB)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/sina/notebook/jupyterenv/lib/python3.10/site-packages (from requests) (3.4)\n",
      "Installing collected packages: urllib3, charset-normalizer, certifi, requests\n",
      "Successfully installed certifi-2022.12.7 charset-normalizer-2.1.1 requests-2.28.1 urllib3-1.26.13\n"
     ]
    }
   ],
   "source": [
    "!pip install requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ef410c1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting bs4\n",
      "  Using cached bs4-0.0.1-py3-none-any.whl\n",
      "Requirement already satisfied: beautifulsoup4 in /home/sina/notebook/jupyterenv/lib/python3.10/site-packages (from bs4) (4.11.1)\n",
      "Requirement already satisfied: soupsieve>1.2 in /home/sina/notebook/jupyterenv/lib/python3.10/site-packages (from beautifulsoup4->bs4) (2.3.2.post1)\n",
      "Installing collected packages: bs4\n",
      "Successfully installed bs4-0.0.1\n"
     ]
    }
   ],
   "source": [
    "!pip install bs4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "09cc433f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests as rq\n",
    "#from bs4 import BeautifulSoap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2c921c5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import bs4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d952794a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter Link: https://distrowatch.com/\n"
     ]
    }
   ],
   "source": [
    "url = input(\"Enter Link: \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "08713497",
   "metadata": {},
   "outputs": [],
   "source": [
    "if(\"https\" or \"http\") in url:\n",
    "    data = rq.get(url)\n",
    "else:\n",
    "    data = rq.get(\"https://\" + url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9390ae38",
   "metadata": {},
   "outputs": [],
   "source": [
    "soap = bs4.BeautifulSoup(data.text, \"html.parser\")\n",
    "links = []\n",
    "for link in soap.find_all(\"a\"):\n",
    "    links.append(link.get(\"href\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e8a68a80",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"mylinks.txt\", \"a\") as saved:\n",
    "    print(links[:10], file=saved)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7093f8a2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}