Fix data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 1;\n",
       "                var nbb_unformatted_code = \"import os\\nimport datetime\\nimport pandas as pd\\nfrom tqdm import tqdm\\nfrom utils import (\\n    get_usage_last_14_days,\\n    create_connection,\\n    create_table_from_df,\\n    delete_table,\\n)\\n\\n%load_ext nb_black\";\n",
       "                var nbb_formatted_code = \"import os\\nimport datetime\\nimport pandas as pd\\nfrom tqdm import tqdm\\nfrom utils import (\\n    get_usage_last_14_days,\\n    create_connection,\\n    create_table_from_df,\\n    delete_table,\\n)\\n\\n%load_ext nb_black\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "import datetime\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from utils import (\n",
    "    get_usage_last_14_days,\n",
    "    create_connection,\n",
    "    create_table_from_df,\n",
    "    delete_table,\n",
    ")\n",
    "\n",
    "%load_ext nb_black"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get all historical data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 3;\n",
       "                var nbb_unformatted_code = \"today_str = \\\"05-03-2022\\\"\\ncumulative_df = pd.read_pickle(\\n    f\\\"data/prod_archive/{today_str}/cumulative_tracking_{today_str}.pkl\\\"\\n)\";\n",
       "                var nbb_formatted_code = \"today_str = \\\"05-03-2022\\\"\\ncumulative_df = pd.read_pickle(\\n    f\\\"data/prod_archive/{today_str}/cumulative_tracking_{today_str}.pkl\\\"\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "today_str = \"05-03-2022\"\n",
    "cumulative_df = pd.read_pickle(\n",
    "    f\"data/prod_archive/{today_str}/cumulative_tracking_{today_str}.pkl\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clones_unique</th>\n",
       "      <th>clones_total</th>\n",
       "      <th>views_unique</th>\n",
       "      <th>views_total</th>\n",
       "      <th>repo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2021-02-10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Churn_Prediction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2021-02-10</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Image_Analysis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2021-02-10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Anomaly_Detection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2021-02-10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NeuralQA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2021-02-10</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>41</td>\n",
       "      <td>Structural_Time_Series</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Train_Gensim_W2V</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Tensorboard_on_CML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Continuous_Model_Monitoring</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Video_Classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ML-Prototypes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6131 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            clones_unique  clones_total  views_unique  views_total  \\\n",
       "2021-02-10              1             1             0            0   \n",
       "2021-02-10              2             2             0            0   \n",
       "2021-02-10              0             0             0            0   \n",
       "2021-02-10              0             0             0            0   \n",
       "2021-02-10              7            10             8           41   \n",
       "...                   ...           ...           ...          ...   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              1             1             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "\n",
       "                                   repo  \n",
       "2021-02-10             Churn_Prediction  \n",
       "2021-02-10               Image_Analysis  \n",
       "2021-02-10            Anomaly_Detection  \n",
       "2021-02-10                     NeuralQA  \n",
       "2021-02-10       Structural_Time_Series  \n",
       "...                                 ...  \n",
       "2022-05-02             Train_Gensim_W2V  \n",
       "2022-05-02           Tensorboard_on_CML  \n",
       "2022-05-02  Continuous_Model_Monitoring  \n",
       "2022-05-02         Video_Classification  \n",
       "2022-05-02                ML-Prototypes  \n",
       "\n",
       "[6131 rows x 5 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 4;\n",
       "                var nbb_unformatted_code = \"cumulative_df\";\n",
       "                var nbb_formatted_code = \"cumulative_df\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cumulative_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clones_unique</th>\n",
       "      <th>clones_total</th>\n",
       "      <th>views_unique</th>\n",
       "      <th>views_total</th>\n",
       "      <th>repo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>Churn_Prediction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Image_Analysis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>Anomaly_Detection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>NeuralQA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>Structural_Time_Series</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SpaCy_Entity_Extraction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>Explainability_LIME_SHAP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>Question_Answering</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Active_Learning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>MLFlow_Tracking</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Few-Shot_Text_Classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Object_Detection_Inference</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Canceled_Flight_Prediction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>Streamlit_on_CML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>APIv2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>AutoML_with_TPOT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Summarize</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Train_Gensim_W2V</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Tensorboard_on_CML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Continuous_Model_Monitoring</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Video_Classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ML-Prototypes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            clones_unique  clones_total  views_unique  views_total  \\\n",
       "2022-05-02              4             5             4           12   \n",
       "2022-05-02              1             2             1            4   \n",
       "2022-05-02              1             1             2            2   \n",
       "2022-05-02              0             0             2            2   \n",
       "2022-05-02              1             1             3            3   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              0             0             2            3   \n",
       "2022-05-02              0             0             3            7   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              1             1             1            1   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              0             0             2            3   \n",
       "2022-05-02              1             1             0            0   \n",
       "2022-05-02              1             1             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              1             1             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "\n",
       "                                    repo  \n",
       "2022-05-02              Churn_Prediction  \n",
       "2022-05-02                Image_Analysis  \n",
       "2022-05-02             Anomaly_Detection  \n",
       "2022-05-02                      NeuralQA  \n",
       "2022-05-02        Structural_Time_Series  \n",
       "2022-05-02       SpaCy_Entity_Extraction  \n",
       "2022-05-02      Explainability_LIME_SHAP  \n",
       "2022-05-02            Question_Answering  \n",
       "2022-05-02               Active_Learning  \n",
       "2022-05-02               MLFlow_Tracking  \n",
       "2022-05-02  Few-Shot_Text_Classification  \n",
       "2022-05-02    Object_Detection_Inference  \n",
       "2022-05-02    Canceled_Flight_Prediction  \n",
       "2022-05-02              Streamlit_on_CML  \n",
       "2022-05-02                         APIv2  \n",
       "2022-05-02              AutoML_with_TPOT  \n",
       "2022-05-02                     Summarize  \n",
       "2022-05-02              Train_Gensim_W2V  \n",
       "2022-05-02            Tensorboard_on_CML  \n",
       "2022-05-02   Continuous_Model_Monitoring  \n",
       "2022-05-02          Video_Classification  \n",
       "2022-05-02                 ML-Prototypes  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 5;\n",
       "                var nbb_unformatted_code = \"cumulative_df.loc[\\\"05-02-2022\\\"]\";\n",
       "                var nbb_formatted_code = \"cumulative_df.loc[\\\"05-02-2022\\\"]\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cumulative_df.loc[\"05-02-2022\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect data from daily archives for the missing dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 6;\n",
       "                var nbb_unformatted_code = \"missing_range = pd.date_range(start=\\\"03-13-2022\\\", end=\\\"05-03-2022\\\")\";\n",
       "                var nbb_formatted_code = \"missing_range = pd.date_range(start=\\\"03-13-2022\\\", end=\\\"05-03-2022\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "missing_range = pd.date_range(start=\"03-13-2022\", end=\"05-03-2022\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:00<00:00, 397.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No archive for 03-13-2022\n",
      "No archive for 04-15-2022\n",
      "No archive for 04-23-2022\n",
      "No archive for 04-24-2022\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 7;\n",
       "                var nbb_unformatted_code = \"daily_archives = []\\nfor date in tqdm(missing_range):\\n    try:\\n        temp_daily_df = pd.read_pickle(\\n            f'data/daily_archive/{date.strftime(\\\"%m-%d-%Y\\\")}/daily_tracking_{date.strftime(\\\"%m-%d-%Y\\\")}.pkl'\\n        )\\n        daily_archives.append(temp_daily_df)\\n    except FileNotFoundError:\\n        print(f'No archive for {date.strftime(\\\"%m-%d-%Y\\\")}')\";\n",
       "                var nbb_formatted_code = \"daily_archives = []\\nfor date in tqdm(missing_range):\\n    try:\\n        temp_daily_df = pd.read_pickle(\\n            f'data/daily_archive/{date.strftime(\\\"%m-%d-%Y\\\")}/daily_tracking_{date.strftime(\\\"%m-%d-%Y\\\")}.pkl'\\n        )\\n        daily_archives.append(temp_daily_df)\\n    except FileNotFoundError:\\n        print(f'No archive for {date.strftime(\\\"%m-%d-%Y\\\")}')\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "daily_archives = []\n",
    "for date in tqdm(missing_range):\n",
    "    try:\n",
    "        temp_daily_df = pd.read_pickle(\n",
    "            f'data/daily_archive/{date.strftime(\"%m-%d-%Y\")}/daily_tracking_{date.strftime(\"%m-%d-%Y\")}.pkl'\n",
    "        )\n",
    "        daily_archives.append(temp_daily_df)\n",
    "    except FileNotFoundError:\n",
    "        print(f'No archive for {date.strftime(\"%m-%d-%Y\")}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clones_unique</th>\n",
       "      <th>clones_total</th>\n",
       "      <th>views_unique</th>\n",
       "      <th>views_total</th>\n",
       "      <th>repo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>Churn_Prediction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Image_Analysis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>Anomaly_Detection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>NeuralQA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>Structural_Time_Series</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SpaCy_Entity_Extraction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>Explainability_LIME_SHAP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>Question_Answering</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Active_Learning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>MLFlow_Tracking</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Few-Shot_Text_Classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Object_Detection_Inference</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Canceled_Flight_Prediction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>Streamlit_on_CML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>APIv2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>AutoML_with_TPOT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Summarize</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Train_Gensim_W2V</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Tensorboard_on_CML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Continuous_Model_Monitoring</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Video_Classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2022-05-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ML-Prototypes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            clones_unique  clones_total  views_unique  views_total  \\\n",
       "2022-05-02              4             5             4           12   \n",
       "2022-05-02              1             2             1            4   \n",
       "2022-05-02              1             1             2            2   \n",
       "2022-05-02              0             0             2            2   \n",
       "2022-05-02              1             1             3            3   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              0             0             2            3   \n",
       "2022-05-02              0             0             3            7   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              1             1             1            1   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              0             0             2            3   \n",
       "2022-05-02              1             1             0            0   \n",
       "2022-05-02              1             1             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              1             1             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "2022-05-02              0             0             0            0   \n",
       "\n",
       "                                    repo  \n",
       "2022-05-02              Churn_Prediction  \n",
       "2022-05-02                Image_Analysis  \n",
       "2022-05-02             Anomaly_Detection  \n",
       "2022-05-02                      NeuralQA  \n",
       "2022-05-02        Structural_Time_Series  \n",
       "2022-05-02       SpaCy_Entity_Extraction  \n",
       "2022-05-02      Explainability_LIME_SHAP  \n",
       "2022-05-02            Question_Answering  \n",
       "2022-05-02               Active_Learning  \n",
       "2022-05-02               MLFlow_Tracking  \n",
       "2022-05-02  Few-Shot_Text_Classification  \n",
       "2022-05-02    Object_Detection_Inference  \n",
       "2022-05-02    Canceled_Flight_Prediction  \n",
       "2022-05-02              Streamlit_on_CML  \n",
       "2022-05-02                         APIv2  \n",
       "2022-05-02              AutoML_with_TPOT  \n",
       "2022-05-02                     Summarize  \n",
       "2022-05-02              Train_Gensim_W2V  \n",
       "2022-05-02            Tensorboard_on_CML  \n",
       "2022-05-02   Continuous_Model_Monitoring  \n",
       "2022-05-02          Video_Classification  \n",
       "2022-05-02                 ML-Prototypes  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 8;\n",
       "                var nbb_unformatted_code = \"daily_archives[-1]\";\n",
       "                var nbb_formatted_code = \"daily_archives[-1]\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "daily_archives[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 9;\n",
       "                var nbb_unformatted_code = \"daily_archives_df = pd.concat(daily_archives)\";\n",
       "                var nbb_formatted_code = \"daily_archives_df = pd.concat(daily_archives)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "daily_archives_df = pd.concat(daily_archives)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine historical with missing records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 10;\n",
       "                var nbb_unformatted_code = \"new_cumulative_df = pd.concat(\\n    [\\n        cumulative_df[~cumulative_df.index.isin([\\\"05-01-2022\\\", \\\"05-02-2022\\\"])], # filter out latest two days, they're in daily_archives_df\\n        daily_archives_df,\\n    ]\\n)\";\n",
       "                var nbb_formatted_code = \"new_cumulative_df = pd.concat(\\n    [\\n        cumulative_df[\\n            ~cumulative_df.index.isin([\\\"05-01-2022\\\", \\\"05-02-2022\\\"])\\n        ],  # filter out latest two days, they're in daily_archives_df\\n        daily_archives_df,\\n    ]\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "new_cumulative_df = pd.concat(\n",
    "    [\n",
    "        cumulative_df[\n",
    "            ~cumulative_df.index.isin([\"05-01-2022\", \"05-02-2022\"])\n",
    "        ],  # filter out latest two days, they're in daily_archives_df\n",
    "        daily_archives_df,\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7143, 5)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 20;\n",
       "                var nbb_unformatted_code = \"new_cumulative_df.shape\";\n",
       "                var nbb_formatted_code = \"new_cumulative_df.shape\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "new_cumulative_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clones_unique</th>\n",
       "      <th>clones_total</th>\n",
       "      <th>views_unique</th>\n",
       "      <th>views_total</th>\n",
       "      <th>repo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2021-02-10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Churn_Prediction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2021-02-10</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Image_Analysis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2021-02-10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Anomaly_Detection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2021-02-10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NeuralQA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2021-02-10</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>41</td>\n",
       "      <td>Structural_Time_Series</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            clones_unique  clones_total  views_unique  views_total  \\\n",
       "2021-02-10              1             1             0            0   \n",
       "2021-02-10              2             2             0            0   \n",
       "2021-02-10              0             0             0            0   \n",
       "2021-02-10              0             0             0            0   \n",
       "2021-02-10              7            10             8           41   \n",
       "\n",
       "                              repo  \n",
       "2021-02-10        Churn_Prediction  \n",
       "2021-02-10          Image_Analysis  \n",
       "2021-02-10       Anomaly_Detection  \n",
       "2021-02-10                NeuralQA  \n",
       "2021-02-10  Structural_Time_Series  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 21;\n",
       "                var nbb_unformatted_code = \"new_cumulative_df.head()\";\n",
       "                var nbb_formatted_code = \"new_cumulative_df.head()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "new_cumulative_df_cumulative_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save new cumulative_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 16;\n",
       "                var nbb_unformatted_code = \"today_str = \\\"05-03-2022\\\"\\nnew_cumulative_df.to_pickle(\\n    f\\\"data/prod_archive/{today_str}/cumulative_tracking_{today_str}.pkl\\\"\\n)\";\n",
       "                var nbb_formatted_code = \"today_str = \\\"05-03-2022\\\"\\nnew_cumulative_df.to_pickle(\\n    f\\\"data/prod_archive/{today_str}/cumulative_tracking_{today_str}.pkl\\\"\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "today_str = \"05-03-2022\"\n",
    "new_cumulative_df.to_pickle(\n",
    "    f\"data/prod_archive/{today_str}/cumulative_tracking_{today_str}.pkl\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Update SQL db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 17;\n",
       "                var nbb_unformatted_code = \"# 4. Delete existing SQLite tables\\nconn = create_connection(f\\\"{os.getcwd()}/db/pythonsqlite.db\\\")\";\n",
       "                var nbb_formatted_code = \"# 4. Delete existing SQLite tables\\nconn = create_connection(f\\\"{os.getcwd()}/db/pythonsqlite.db\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 4. Delete existing SQLite tables\n",
    "conn = create_connection(f\"{os.getcwd()}/db/pythonsqlite.db\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 18;\n",
       "                var nbb_unformatted_code = \"delete_table(\\\"amp_tracking\\\", conn)\\n# delete_table(\\\"amp_referring\\\", conn)\";\n",
       "                var nbb_formatted_code = \"delete_table(\\\"amp_tracking\\\", conn)\\n# delete_table(\\\"amp_referring\\\", conn)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "delete_table(\"amp_tracking\", conn)\n",
    "# delete_table(\"amp_referring\", conn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 19;\n",
       "                var nbb_unformatted_code = \"# 5. Create new tables to refresh data\\n\\ncreate_table_from_df(\\\"amp_tracking\\\", conn, new_cumulative_df)\\n# create_table_from_df(\\\"amp_referring\\\", conn, amp_referring_df)\";\n",
       "                var nbb_formatted_code = \"# 5. Create new tables to refresh data\\n\\ncreate_table_from_df(\\\"amp_tracking\\\", conn, new_cumulative_df)\\n# create_table_from_df(\\\"amp_referring\\\", conn, amp_referring_df)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 5. Create new tables to refresh data\n",
    "\n",
    "create_table_from_df(\"amp_tracking\", conn, new_cumulative_df)\n",
    "# create_table_from_df(\"amp_referring\", conn, amp_referring_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}