Undergrad_Calendar.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Undergraduate Calendar for Mathematics\n",
        "This is the Jupyter notebook where the idea was first developed. \n",
        "All relevant contents can be found in this [Python script](scraper.py), which can be run with [main.py](main.py)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "54uD6L38mhxm"
      },
      "outputs": [],
      "source": [
        "from bs4 import BeautifulSoup\n",
        "import requests\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "rgTZ27BW5yOZ",
        "outputId": "581f72d8-0596-430b-b558-d4d799460673"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "https://academic-calendar-archive.uwaterloo.ca/undergraduate-studies/2019-2020/page/MATH-List-of-Academic-Programs-or-Plans.html\n",
            "Actuarial Science \n",
            "Joint Actuarial Science\n",
            "Actuarial Science/Finance Option\n",
            "\n",
            "Actuarial Science/Predictive Analytics Option\n",
            "Applied Mathematics\n",
            "Joint Applied Mathematics\n",
            "Applied Mathematics/Scientific Computing\n",
            "Mathematical Physics\n",
            "Biology Specialization\n",
            "Economics Specialization\n",
            "Engineering Specialization: Fluids and Heat\n",
            "Engineering Specialization: Communication and Control\n",
            "Engineering Specialization: Heat and Mass Transfer\n",
            "Physics Specialization\n",
            "Combinatorics and Optimization\n",
            "Joint Combinatorics and Optimization\n",
            "Mathematical Optimization\n",
            "Computer Science\n",
            "Joint Computer Science\n",
            "Artificial Intelligence Specialization\n",
            "Bioinformatics Specialization\n",
            "Business Specialization\n",
            "Computational Fine Art Specialization\n",
            "Digital Hardware Specialization\n",
            "Human-Computer Interaction Specialization\n",
            "Software Engineering Specialization\n",
            "Pure Mathematics \n",
            "Joint Pure Mathematics\n",
            "Pure Mathematics/Teaching\n",
            "Statistics\n",
            "Joint Statistics\n",
            "Biostatistics\n",
            "Data Science\n",
            "Mathematical Finance\n",
            "Information Technology Management\n",
            "Mathematical Economics\n",
            "Mathematics/Business Administration\n",
            "Mathematics/Chartered Professional Accountancy\n",
            "Mathematics/Chartered Professional Accountancy â Finance Option\n",
            "Mathematics/Financial Analysis and Risk Management\n",
            "Mathematics/Teaching\n",
            "Computational Mathematics\n",
            "Mathematical Studies\n",
            "Mathematical Studies - Business Specialization\n",
            "Computer Science\n",
            "Joint Computer Science\n",
            "Artificial Intelligence Specialization\n",
            "Bioinformatics Specialization\n",
            "Business Specialization\n",
            "Computational Fine Art Specialization\n",
            "Digital Hardware Specialization\n",
            "Human-Computer Interaction Specialization\n",
            "Software Engineering Specialization\n",
            "Data Science\n",
            "Business Administration and Mathematics\n",
            "Business Administration and Computer Science\n",
            "Computing and Financial Management\n",
            "Software Engineering\n"
          ]
        }
      ],
      "source": [
        "# Get the academic list for a specific year\n",
        "\n",
        "# Set up the URL\n",
        "year = \"2019-2020\"\n",
        "url_for_list = \"https://academic-calendar-archive.uwaterloo.ca/undergraduate-studies/\" + year + \"/page/MATH-List-of-Academic-Programs-or-Plans.html\"\n",
        "print(url_for_list)\n",
        "\n",
        "# Webscrape for BeautifulSoup\n",
        "listpage = requests.get(url_for_list)\n",
        "listsoup = BeautifulSoup(listpage.text, 'html')\n",
        "\n",
        "# Get all the names\n",
        "major_names = listsoup.find_all('a')\n",
        "\n",
        "# Remove everything before \"Amendments\"\n",
        "i = 0\n",
        "for name in major_names:\n",
        "  if \"Amendments\" in name.get_text():\n",
        "    break\n",
        "  i += 1\n",
        "major_names = major_names[i + 1:]\n",
        "\n",
        "# Print out all the names\n",
        "for name in major_names:\n",
        "  print(name.get_text());"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "Mq9lt41hX5nZ"
      },
      "outputs": [],
      "source": [
        "# Get the choices and setup the URLS\n",
        "\n",
        "# Link header to the academic calendar\n",
        "link_header = \"https://academic-calendar-archive.uwaterloo.ca\"\n",
        "\n",
        "# Random choices\n",
        "choice1 = major_names[5]\n",
        "choice2 = major_names[7]\n",
        "\n",
        "# Make URL\n",
        "choice1_url = link_header + choice1.get('href')\n",
        "choice2_url = link_header + choice2.get('href')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "YITnQb-fc9Of"
      },
      "outputs": [],
      "source": [
        "# Function that filters out invalid course codes\n",
        "import re\n",
        "\n",
        "def is_valid_course(course_code):\n",
        "    pattern = r'^[A-Za-z]{2,}\\s\\d{3}[A-Za-z]?$'\n",
        "    return re.match(pattern, course_code, re.IGNORECASE) is not None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "UVWV4l5Pmvrg",
        "outputId": "b888d00c-606e-4c4c-b83a-90680f9d64fa"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "AMATH 250\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH250\n",
            "MATH 237\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-MATH.html#MATH237\n",
            "MATH 247\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-MATH.html#MATH247\n",
            "AMATH 231\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH231\n",
            "AMATH 242\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH242\n",
            "CS 371\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-CS.html#CS371\n",
            "AMATH 251\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH251\n",
            "AMATH 332\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH332\n",
            "PMATH 332\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-PMATH.html#PMATH332\n",
            "AMATH 351\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH351\n",
            "AMATH 353\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH353\n",
            "PHYS 121\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-PHYS.html#PHYS121\n",
            "AMATH 331\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH331\n",
            "PMATH 331\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-PMATH.html#PMATH331\n"
          ]
        }
      ],
      "source": [
        "# Get all the course names from choice1\n",
        "\n",
        "# Setup BeautifulSoup\n",
        "choice1_page = requests.get(choice1_url)\n",
        "choice1_soup = BeautifulSoup(choice1_page.text, 'html')\n",
        "choice1_courses = choice1_soup.find_all('a')\n",
        "\n",
        "# Filter out invalid courses\n",
        "tmp = []\n",
        "for course in choice1_courses:\n",
        "  if is_valid_course(course.get_text()):\n",
        "    tmp.append(course)\n",
        "choice1_courses = tmp\n",
        "\n",
        "for name in choice1_courses:\n",
        "  print(name.get_text())\n",
        "  print(name.get('href'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "L7rYTU89cQNx",
        "outputId": "18aebd47-663c-43e3-93db-0b677dff7850"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "AMATH 250\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH250\n",
            "MATH 237\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-MATH.html#MATH237\n",
            "MATH 247\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-MATH.html#MATH247\n",
            "AMATH 231\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH231\n",
            "AMATH 242\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH242\n",
            "CS 371\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-CS.html#CS371\n",
            "AMATH 251\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH251\n",
            "AMATH 271\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH271\n",
            "AMATH 331\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH331\n",
            "PMATH 331\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-PMATH.html#PMATH331\n",
            "AMATH 332\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH332\n",
            "PMATH 332\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-PMATH.html#PMATH332\n",
            "AMATH 351\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH351\n",
            "AMATH 353\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH353\n",
            "AMATH 361\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH361\n",
            "AMATH 373\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH373\n",
            "AMATH 473\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH473\n",
            "PHYS 454\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-PHYS.html#PHYS454\n",
            "AMATH 475\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH475\n",
            "PHYS 476\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-PHYS.html#PHYS476\n",
            "PHYS 121\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-PHYS.html#PHYS121\n",
            "PHYS 122\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-PHYS.html#PHYS122\n",
            "PHYS 234\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-PHYS.html#PHYS234\n",
            "PHYS 242\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-PHYS.html#PHYS242\n",
            "PHYS 342\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-PHYS.html#PHYS342\n",
            "PHYS 358\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-PHYS.html#PHYS358\n",
            "ECE 403\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-ECE.html#ECE403\n",
            "PHYS 359\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-PHYS.html#PHYS359\n",
            "PHYS 363\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-PHYS.html#PHYS363\n"
          ]
        }
      ],
      "source": [
        "# Get all the course names from choice2\n",
        "\n",
        "# Setup BeautifulSoup\n",
        "choice2_page = requests.get(choice2_url)\n",
        "choice2_soup = BeautifulSoup(choice2_page.text, 'html')\n",
        "choice2_courses = choice2_soup.find_all('a')\n",
        "\n",
        "# Filter out invalid courses\n",
        "tmp = []\n",
        "for course in choice2_courses:\n",
        "  if is_valid_course(course.get_text()):\n",
        "    tmp.append(course)\n",
        "choice2_courses = tmp\n",
        "\n",
        "for name in choice2_courses:\n",
        "  print(name.get_text())\n",
        "  print(name.get('href'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "Scd5cRkd81rI",
        "outputId": "2b416fb5-87fb-4199-e31d-4ba3a509fe6c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Ergonomics and Injury Prevention Minor\n",
            "Gerontology Minor\n",
            "Human Nutrition Minor\n",
            "Medical Physiology Minor\n",
            "Rehabilitation Sciences Minor\n",
            "Anthropology Minor\n",
            "Applied Language Studies Minor\n",
            "Canadian Studies Minor\n",
            "Catholic Studies Minor\n",
            "Church Music and Worship Minor\n",
            "Classical Studies Minor\n",
            "Cognitive Science Minor\n",
            "Cultural Identities Minor\n",
            "Digital Arts Communication Minor\n",
            "East Asian Studies Minor\n",
            "Economic Theory Minor\n",
            "Economics Minor\n",
            "English Minor\n",
            "Fine Arts Studio Minor\n",
            "French Minor\n",
            "Gender and Social Justice Minor\n",
            "German Minor\n",
            "Greek Minor\n",
            "History Minor\n",
            "Human Resources Management Minor\n",
            "Human Rights Minor\n",
            "Human Sciences Minor\n",
            "Indigenous Studies Minor\n",
            "International Studies Minor\n",
            "International Trade Minor\n",
            "Italian Studies Minor\n",
            "Jewish Studies Minor\n",
            "Latin Minor\n",
            "Legal Studies Minor\n",
            "Management Studies Minor\n",
            "Medieval Studies Minor\n",
            "Mennonite Studies Minor\n",
            "Music Minor\n",
            "Peace and Conflict Studies Minor\n",
            "Philosophy Minor\n",
            "Political Science Minor\n",
            "Psychology Minor\n",
            "Public Policy and Administration Minor\n",
            "Religious Studies Minor\n",
            "Russian and East European Studies Minor\n",
            "Sexuality, Marriage, and Family Studies Minor\n",
            "Social Development Studies Minor\n",
            "Sociology Minor\n",
            "Spanish Minor\n",
            "Speech Communication Minor\n",
            "Studies in Islam Minor\n",
            "Technical Writing Minor\n",
            "Theatre and Performance Minor\n",
            "Visual Culture in a Global Context Minor\n",
            "Entrepreneurship Minor\n",
            "Environment, Resources and Sustainability Minor\n",
            "Geography and Environmental Management Minor\n",
            "International Development Minor\n",
            "Knowledge Integration Minor\n",
            "Urban Studies Minor\n",
            "Actuarial Science Minor\n",
            "Applied Mathematics Minor\n",
            "Combinatorics and Optimization Minor\n",
            "Computational Mathematics Minor\n",
            "Computer Science Minor\n",
            "Mathematics Minor\n",
            "Pure Mathematics Minor\n",
            "Statistics Minor\n",
            "Astrophysics Minor\n",
            "Biochemistry Minor\n",
            "Biology Minor\n",
            "Biophysics Minor\n",
            "Chemistry Minor\n",
            "Earth Sciences Minor\n",
            "Medical Physiology Minor\n",
            "Physics Minor\n",
            "\n",
            "Aging Studies Option\n",
            "Health Informatics Option\n",
            "Tourism Option\n",
            "Artificial Intelligence Option\n",
            "Biomechanics Option\n",
            "Computer Engineering Option\n",
            "Entrepreneurship Option\n",
            "Environmental Engineering Option\n",
            "International Studies in Engineering Option\n",
            "Life Sciences Option\n",
            "Management Sciences Option\n",
            "Mathematics Option\n",
            "Mechatronics Option\n",
            "Physical Sciences Option\n",
            "Society, Technology and Values Option\n",
            "Software Engineering Option\n",
            "Statistics Option\n",
            "Water Resources Option\n",
            "Business Option\n",
            "Parks Option\n",
            "Tourism Option\n",
            "Computing Option\n",
            "\n",
            "Diploma in Gerontology\n",
            "Diploma in Applied Language Studies\n",
            "Diploma in Arabic Language\n",
            "Diploma in Chinese Language\n",
            "Diploma in Church Music and Worship\n",
            "Diploma in East Asian Studies\n",
            "Diplomas in French Language\n",
            "Diploma in Gender and Social Justice\n",
            "Diploma in Human Resources Management\n",
            "Diploma in Japanese Language\n",
            "Diploma in Korean Language\n",
            "Diploma in Latin American Studies\n",
            "Diploma in Peace and Conflict Studies\n",
            "Diploma in Sexuality, Marriage, and Family Studies\n",
            "Diplomas in Spanish Language\n",
            "Diploma in Studies in Islam\n",
            "Diploma in Ecological Restoration and Rehabilitation\n",
            "Diploma in Environmental Assessment\n",
            "Diploma in Excellence in Geographic Information Systems\n",
            "Diploma in Sustainability\n",
            "\n",
            "Certificates in Arabic Language\n",
            "Certificates in Chinese Language\n",
            "Certificate in Croatian Language\n",
            "Certificate in Dutch Language\n",
            "Certificate in German Language\n",
            "Certificates in Japanese Language\n",
            "Certificates in Korean Language\n",
            "Certificate in Russian Language\n",
            "Certificate in Co-op Research\n",
            "Certificate in Experiential Education (EDGE)\n",
            "\n",
            "Certificate in Global Experience (GEC)\n",
            "Certificate in English for Multilingual Speakers\n"
          ]
        }
      ],
      "source": [
        "# Get Minors/Certifications/Options/Diplomas\n",
        "# Set up URL\n",
        "minor_url = \"https://academic-calendar-archive.uwaterloo.ca/undergraduate-studies/\" + year + \"/page/Minors-Options-Diplomas-Certificates.html\"\n",
        "\n",
        "# Set up BeautifulSoup\n",
        "minor_page = requests.get(minor_url)\n",
        "minor_soup = BeautifulSoup(minor_page.text, 'html')\n",
        "minor_names = minor_soup.find_all('a')\n",
        "\n",
        "# Get rid of everything before admendments and a few others\n",
        "i = 0\n",
        "for name in minor_names:\n",
        "  if \"Amendments\" in name.get_text():\n",
        "    break\n",
        "  i += 1\n",
        "minor_names = minor_names[i + 5:]\n",
        "\n",
        "# Print out all the names\n",
        "for name in minor_names:\n",
        "  print(name.get_text())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JYtKwyjstt_L",
        "outputId": "1c34e64b-9748-440e-d650-20a4e24b1413"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "https://academic-calendar-archive.uwaterloo.ca/undergraduate-studies/2019-2020/page/MATH-Applied-Mathematics-Minor1.html\n",
            "AMATH 250\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH250\n",
            "MATH 237\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-MATH.html#MATH237\n",
            "MATH 247\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-MATH.html#MATH247\n",
            "AMATH 231\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH231\n",
            "AMATH 251\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH251\n",
            "AMATH 351\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH351\n",
            "AMATH 353\n",
            "https://ucalendar.uwaterloo.ca/1920/COURSE/course-AMATH.html#AMATH353\n"
          ]
        }
      ],
      "source": [
        "# Random choice\n",
        "minor_choice = minor_names[62]\n",
        "minor_choice_url = link_header + minor_choice.get('href')\n",
        "\n",
        "# Setup BeautifulSoup\n",
        "minor_choice_page = requests.get(minor_choice_url)\n",
        "minor_choice_soup = BeautifulSoup(minor_choice_page.text, 'html')\n",
        "minor_choice_courses = minor_choice_soup.find_all('a')\n",
        "\n",
        "# Filter out invalid courses\n",
        "tmp = []\n",
        "for course in minor_choice_courses:\n",
        "  if is_valid_course(course.get_text()):\n",
        "    tmp.append(course)\n",
        "minor_choice_courses = tmp\n",
        "\n",
        "# Print\n",
        "print(minor_choice_url)\n",
        "for name in minor_choice_courses:\n",
        "  print(name.get_text())\n",
        "  print(name.get('href'))"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "authorship_tag": "ABX9TyMhK/4gYboFVRLlLLDTq/AL",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.12.3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}