Internship_DDB_Telkom_Reports_Crawler.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOVPA1Qm4HJcd48BlpBl0C3",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/MirzaQusyairi/internship-ddb-telkom-reports-crawler/blob/main/Internship_DDB_Telkom_Reports_Crawler.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 68,
      "metadata": {
        "id": "BOaQrKQGWrJU"
      },
      "outputs": [],
      "source": [
        "from datetime import timedelta, datetime\n",
        "\n",
        "import pandas as pd\n",
        "import requests"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "url = 'https://api-internship.ddbtelkom.id/api/user/v1/login'\n",
        "headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36',\n",
        "           'Authorization': 'Basic dGVsa29tOmRhMWMyNWQ4LTM3YzgtNDFiMS1hZmUyLTQyZGQ0ODI1YmZlYQ=='}\n",
        "\n",
        "obj = {\n",
        "    'email': 'YOUR_EMAIL',\n",
        "    'password': 'YOUR_PASSWORD'\n",
        "}\n",
        "\n",
        "response = requests.post(url, json=obj, headers=headers)\n",
        "token = response.json()['data'].get('token')\n",
        "# replace the 0 with 1 if get extend and want to get the first batch reports\n",
        "internshipDataID = response.json()['data'].get('internshipData')[0].get('_id')"
      ],
      "metadata": {
        "id": "XGRiiTnwXIU6"
      },
      "execution_count": 165,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "headers = dict()\n",
        "headers[\"Accept\"] = \"application/json\"\n",
        "headers[\"Authorization\"] = f'Bearer {token}'\n",
        "headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36'"
      ],
      "metadata": {
        "id": "C0yscN3EZonM"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "weekly_reports = []\n",
        "daily_reports = []\n",
        "j = 1\n",
        "\n",
        "for i in range(25):\n",
        "  url = f'https://api-internship.ddbtelkom.id/api/user/v1/internship-data/{internshipDataID}/idt/group-by-week/{i+1}'\n",
        "  response = requests.get(url, headers=headers)\n",
        "  reports = response.json().get('data')\n",
        "\n",
        "  if (reports != ''):\n",
        "    if (reports.get('weeklyIdt') is not None):\n",
        "      weekly_activity = reports.get('weeklyIdt').get('activity')\n",
        "      date = reports.get('weeklyIdt').get('fromDate')\n",
        "      date = datetime.strptime(date, '%Y-%m-%dT%H:%M:%S%z')\n",
        "\n",
        "      date_format = date.strftime(\"%B %Y\")\n",
        "      weekly_reports.append([j, date_format, weekly_activity])\n",
        "      j = j+1\n",
        "\n",
        "    for report in reports.get('dailyIdt'):\n",
        "      date = report.get('date')\n",
        "      date = datetime.strptime(date, '%Y-%m-%dT%H:%M:%S%z')\n",
        "\n",
        "      date_format = date.strftime(\"%A, %d %B %Y\")\n",
        "      daily_reports.append([date_format, report.get('activity')])\n",
        "  else:\n",
        "    break"
      ],
      "metadata": {
        "id": "QrHpSwbnaw8z"
      },
      "execution_count": 161,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_daily = pd.DataFrame(daily_reports, columns=['Hari', 'Aktifitas'])\n",
        "df_daily"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 582
        },
        "id": "bb_9WMMXr1W7",
        "outputId": "145457df-a05f-4f0d-c95d-7d33dbe16cdb"
      },
      "execution_count": 162,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                          Hari  \\\n",
              "0      Monday, 02 January 2023   \n",
              "1     Tuesday, 03 January 2023   \n",
              "2   Wednesday, 04 January 2023   \n",
              "3    Thursday, 05 January 2023   \n",
              "4      Friday, 06 January 2023   \n",
              "5      Monday, 09 January 2023   \n",
              "6     Tuesday, 10 January 2023   \n",
              "7   Wednesday, 11 January 2023   \n",
              "8    Thursday, 12 January 2023   \n",
              "9      Friday, 13 January 2023   \n",
              "10     Monday, 16 January 2023   \n",
              "11    Tuesday, 17 January 2023   \n",
              "12  Wednesday, 18 January 2023   \n",
              "13   Thursday, 19 January 2023   \n",
              "14     Friday, 20 January 2023   \n",
              "15     Monday, 23 January 2023   \n",
              "16    Tuesday, 24 January 2023   \n",
              "\n",
              "                                            Aktifitas  \n",
              "0   melanjutkan pengerjaan BDA 6061 implementasi t...  \n",
              "1   melanjutkan pengerjaan BDA 6061 implementasi t...  \n",
              "2   melanjutkan pengerjaan BDA 6061 implementasi t...  \n",
              "3   melanjutkan pengerjaan BDA 6061 implementasi t...  \n",
              "4   melanjutkan pengerjaan BDA 6061 implementasi t...  \n",
              "5   mengerjakan BDA 6279 implementasi delete colle...  \n",
              "6   melanjutkan pengerjaan BDA 6279 implementasi d...  \n",
              "7   melanjutkan pengerjaan BDA 6279 implementasi d...  \n",
              "8   melanjutkan pengerjaan BDA 6279 implementasi d...  \n",
              "9   sprint review dan sprint retro dipagi hari mem...  \n",
              "10  sprint planning merencanakan task untuk 2 ming...  \n",
              "11  mengerjakan task BDA 5956 implementasi popup v...  \n",
              "12  melanjutkan pengerjaan task BDA 5956 implement...  \n",
              "13  melanjutkan pengerjaan task BDA 5956 implement...  \n",
              "14  mendapatkan task dadakan BDA 6346 change secti...  \n",
              "15                                               None  \n",
              "16  mengerjakan BDA 6322 hide competition page, la...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-973f0a3e-2c24-44a5-8007-6df1e0a69fc8\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Hari</th>\n",
              "      <th>Aktifitas</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Monday, 02 January 2023</td>\n",
              "      <td>melanjutkan pengerjaan BDA 6061 implementasi t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Tuesday, 03 January 2023</td>\n",
              "      <td>melanjutkan pengerjaan BDA 6061 implementasi t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Wednesday, 04 January 2023</td>\n",
              "      <td>melanjutkan pengerjaan BDA 6061 implementasi t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Thursday, 05 January 2023</td>\n",
              "      <td>melanjutkan pengerjaan BDA 6061 implementasi t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Friday, 06 January 2023</td>\n",
              "      <td>melanjutkan pengerjaan BDA 6061 implementasi t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Monday, 09 January 2023</td>\n",
              "      <td>mengerjakan BDA 6279 implementasi delete colle...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Tuesday, 10 January 2023</td>\n",
              "      <td>melanjutkan pengerjaan BDA 6279 implementasi d...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Wednesday, 11 January 2023</td>\n",
              "      <td>melanjutkan pengerjaan BDA 6279 implementasi d...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Thursday, 12 January 2023</td>\n",
              "      <td>melanjutkan pengerjaan BDA 6279 implementasi d...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Friday, 13 January 2023</td>\n",
              "      <td>sprint review dan sprint retro dipagi hari mem...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>Monday, 16 January 2023</td>\n",
              "      <td>sprint planning merencanakan task untuk 2 ming...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>Tuesday, 17 January 2023</td>\n",
              "      <td>mengerjakan task BDA 5956 implementasi popup v...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>Wednesday, 18 January 2023</td>\n",
              "      <td>melanjutkan pengerjaan task BDA 5956 implement...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>Thursday, 19 January 2023</td>\n",
              "      <td>melanjutkan pengerjaan task BDA 5956 implement...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>Friday, 20 January 2023</td>\n",
              "      <td>mendapatkan task dadakan BDA 6346 change secti...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>Monday, 23 January 2023</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>Tuesday, 24 January 2023</td>\n",
              "      <td>mengerjakan BDA 6322 hide competition page, la...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-973f0a3e-2c24-44a5-8007-6df1e0a69fc8')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-973f0a3e-2c24-44a5-8007-6df1e0a69fc8 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-973f0a3e-2c24-44a5-8007-6df1e0a69fc8');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 162
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_weekly = pd.DataFrame(weekly_reports, columns=['Minggu ke','Bulan', 'Aktifitas'])\n",
        "df_weekly"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "Nk17gZjNr5gU",
        "outputId": "8067ff9f-27a4-4dfd-c1e1-8390a82b96b4"
      },
      "execution_count": 163,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Minggu ke         Bulan                                          Aktifitas\n",
              "0          1  January 2023  belajar membuat tampilan baru dari sebuah hala...\n",
              "1          2  January 2023                   belajar menerapkan endpoint baru\n",
              "2          3  January 2023           pada minggu ini belajar memperbaiki bug "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2475caaa-ced6-4545-87c9-f0c9083d32bd\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Minggu ke</th>\n",
              "      <th>Bulan</th>\n",
              "      <th>Aktifitas</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>January 2023</td>\n",
              "      <td>belajar membuat tampilan baru dari sebuah hala...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>January 2023</td>\n",
              "      <td>belajar menerapkan endpoint baru</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>January 2023</td>\n",
              "      <td>pada minggu ini belajar memperbaiki bug</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2475caaa-ced6-4545-87c9-f0c9083d32bd')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2475caaa-ced6-4545-87c9-f0c9083d32bd button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2475caaa-ced6-4545-87c9-f0c9083d32bd');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 163
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_weekly.to_csv('Weekly Report.csv', index=False)\n",
        "df_daily.to_csv('Daily Report.csv', index=False)"
      ],
      "metadata": {
        "id": "JsK4enWybdVJ"
      },
      "execution_count": 164,
      "outputs": []
    }
  ]
}