Intro- DataFrames and StudentLife Basics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib as plt\n",
    "import seaborn as sns\n",
    "from numpy import *\n",
    "import glob\n",
    "import scipy \n",
    "from datetime import datetime\n",
    "import sklearn\n",
    "sns.style = 'darkgrid'\n",
    "path = '' # make sure this is the file directory to your dataset. We use this to tell the computer where to read the data. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pandas Basics with Piazza and Grades Data\n",
    "\n",
    "## Loading Dataframes\n",
    "To begin, we will try reading the data, which is stored as a dataframe object. Dataframes and series are the main objects of pandas, and you will see many examples throughout this exercies. A series can be thought of as an array with built in, mutable indices, and it has nice functions you can use to avoid having to use tons of loops in your code. A dataframe is basically a matrix, or a list of series. \n",
    "\n",
    "We read our dataframe with pandas builtin pd.read_csv. You can view a small part of the dataframe with .head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>gpa all</th>\n",
       "      <th>gpa 13s</th>\n",
       "      <th>cs 65</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>u01</td>\n",
       "      <td>2.863</td>\n",
       "      <td>1.777778</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>u02</td>\n",
       "      <td>3.505</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>u04</td>\n",
       "      <td>3.029</td>\n",
       "      <td>3.500000</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>u05</td>\n",
       "      <td>3.679</td>\n",
       "      <td>3.777778</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>u07</td>\n",
       "      <td>3.474</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid   gpa all   gpa 13s   cs 65\n",
       "0  u01     2.863  1.777778     3.0\n",
       "1  u02     3.505  4.000000     4.0\n",
       "2  u04     3.029  3.500000     4.0\n",
       "3  u05     3.679  3.777778     4.0\n",
       "4  u07     3.474  4.000000     4.0"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grades = pd.read_csv(path + \"dataset/education/grades.csv\") \n",
    "# if this doesn't work, try changing your path to where files are stored\n",
    "grades.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Viewing rows and columns\n",
    "to just look at one column, use dataframe_name.column, dataframe_name[\"column\"], or dataframe_name[[\"column\"]].\n",
    "to look at multiple columns, use dataframe_name[[\"column_1\", \"column_2\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     u01\n",
       "1     u02\n",
       "2     u04\n",
       "3     u05\n",
       "4     u07\n",
       "5     u08\n",
       "6     u09\n",
       "7     u10\n",
       "8     u12\n",
       "9     u14\n",
       "10    u15\n",
       "11    u16\n",
       "12    u17\n",
       "13    u18\n",
       "14    u19\n",
       "15    u22\n",
       "16    u24\n",
       "17    u25\n",
       "18    u27\n",
       "19    u30\n",
       "20    u32\n",
       "21    u33\n",
       "22    u41\n",
       "23    u43\n",
       "24    u46\n",
       "25    u49\n",
       "26    u52\n",
       "27    u54\n",
       "28    u57\n",
       "29    u59\n",
       "Name: uid, dtype: object"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's try three ways to look at the uid column for grades. \n",
    "grades[\"uid\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     u01\n",
       "1     u02\n",
       "2     u04\n",
       "3     u05\n",
       "4     u07\n",
       "5     u08\n",
       "6     u09\n",
       "7     u10\n",
       "8     u12\n",
       "9     u14\n",
       "10    u15\n",
       "11    u16\n",
       "12    u17\n",
       "13    u18\n",
       "14    u19\n",
       "15    u22\n",
       "16    u24\n",
       "17    u25\n",
       "18    u27\n",
       "19    u30\n",
       "20    u32\n",
       "21    u33\n",
       "22    u41\n",
       "23    u43\n",
       "24    u46\n",
       "25    u49\n",
       "26    u52\n",
       "27    u54\n",
       "28    u57\n",
       "29    u59\n",
       "Name: uid, dtype: object"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grades.uid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>u01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>u02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>u04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>u05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>u07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>u08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>u09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>u10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>u12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>u14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>u15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>u16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>u17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>u18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>u19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>u22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>u24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>u25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>u27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>u30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>u32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>u33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>u41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>u43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>u46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>u49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>u52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>u54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>u57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>u59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    uid\n",
       "0   u01\n",
       "1   u02\n",
       "2   u04\n",
       "3   u05\n",
       "4   u07\n",
       "5   u08\n",
       "6   u09\n",
       "7   u10\n",
       "8   u12\n",
       "9   u14\n",
       "10  u15\n",
       "11  u16\n",
       "12  u17\n",
       "13  u18\n",
       "14  u19\n",
       "15  u22\n",
       "16  u24\n",
       "17  u25\n",
       "18  u27\n",
       "19  u30\n",
       "20  u32\n",
       "21  u33\n",
       "22  u41\n",
       "23  u43\n",
       "24  u46\n",
       "25  u49\n",
       "26  u52\n",
       "27  u54\n",
       "28  u57\n",
       "29  u59"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grades[[\"uid\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "note that the third way returns a dataframe object, while the first two return a series object (I prefer the third way). \n",
    "\n",
    "Now, let's try looking at rows. To look at a specific index, use dataframe_name.iloc[row_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "uid             u01\n",
       " gpa all      2.863\n",
       " gpa 13s    1.77778\n",
       " cs 65            3\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grades.iloc[0] # take the zero-indexed first row"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filtering dataframes\n",
    "\n",
    "To filter data based on specific properties and return a new dataframe, use dataframe_name[dataframe_name[\"column\"] boolean]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'gpa all'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001b[0m in \u001b[0;36mget_loc\u001b[1;34m(self, key, method, tolerance)\u001b[0m\n\u001b[0;32m   2645\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2646\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2647\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'gpa all'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-31-20b669aa68fa>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#take only gpas above 3.5\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mgrades\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mgrades\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'gpa all'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m3.5\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   2798\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnlevels\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2799\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2800\u001b[1;33m             \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2801\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2802\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001b[0m in \u001b[0;36mget_loc\u001b[1;34m(self, key, method, tolerance)\u001b[0m\n\u001b[0;32m   2646\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2647\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2648\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2649\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2650\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mindexer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m1\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mindexer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'gpa all'"
     ]
    }
   ],
   "source": [
    "#take only gpas above 3.5\n",
    "grades[grades['gpa all'] > 3.5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There's an error! It looks like gpa all is in our column names, let's check the columns to see!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['uid', ' gpa all', ' gpa 13s', ' cs 65'], dtype='object')"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grades.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "looks like there's an empty space before 'gpa all' (this sometimes happens in the studentLife dataset). looking at the columns can be helpful if you're stuck."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>gpa all</th>\n",
       "      <th>gpa 13s</th>\n",
       "      <th>cs 65</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>u02</td>\n",
       "      <td>3.505</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>u05</td>\n",
       "      <td>3.679</td>\n",
       "      <td>3.777778</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>u08</td>\n",
       "      <td>3.705</td>\n",
       "      <td>3.333333</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>u09</td>\n",
       "      <td>3.806</td>\n",
       "      <td>3.777778</td>\n",
       "      <td>3.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>u10</td>\n",
       "      <td>3.667</td>\n",
       "      <td>3.777778</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>u19</td>\n",
       "      <td>3.947</td>\n",
       "      <td>3.777778</td>\n",
       "      <td>3.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>u22</td>\n",
       "      <td>3.889</td>\n",
       "      <td>3.916667</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>u27</td>\n",
       "      <td>3.719</td>\n",
       "      <td>3.333333</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>u30</td>\n",
       "      <td>3.930</td>\n",
       "      <td>3.916667</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>u32</td>\n",
       "      <td>3.826</td>\n",
       "      <td>3.888889</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>u41</td>\n",
       "      <td>3.652</td>\n",
       "      <td>3.777778</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>u43</td>\n",
       "      <td>3.790</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>u46</td>\n",
       "      <td>3.646</td>\n",
       "      <td>1.111111</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>u49</td>\n",
       "      <td>3.625</td>\n",
       "      <td>3.777778</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>u59</td>\n",
       "      <td>3.519</td>\n",
       "      <td>3.555556</td>\n",
       "      <td>3.666667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    uid   gpa all   gpa 13s     cs 65\n",
       "1   u02     3.505  4.000000  4.000000\n",
       "3   u05     3.679  3.777778  4.000000\n",
       "5   u08     3.705  3.333333  4.000000\n",
       "6   u09     3.806  3.777778  3.666667\n",
       "7   u10     3.667  3.777778  4.000000\n",
       "14  u19     3.947  3.777778  3.333333\n",
       "15  u22     3.889  3.916667  4.000000\n",
       "18  u27     3.719  3.333333  3.000000\n",
       "19  u30     3.930  3.916667  4.000000\n",
       "20  u32     3.826  3.888889  4.000000\n",
       "22  u41     3.652  3.777778  4.000000\n",
       "23  u43     3.790  4.000000  4.000000\n",
       "24  u46     3.646  1.111111  4.000000\n",
       "25  u49     3.625  3.777778  4.000000\n",
       "29  u59     3.519  3.555556  3.666667"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# now it works!\n",
    "grades[grades[' gpa all'] > 3.5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>gpa all</th>\n",
       "      <th>gpa 13s</th>\n",
       "      <th>cs 65</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>u02</td>\n",
       "      <td>3.505</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>u05</td>\n",
       "      <td>3.679</td>\n",
       "      <td>3.777778</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid   gpa all   gpa 13s   cs 65\n",
       "1  u02     3.505  4.000000     4.0\n",
       "3  u05     3.679  3.777778     4.0"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# You can filter with the isin command too. This can help extract the dataset for only users that you want.  \n",
    "grades[grades['uid'].isin(['u02', 'u05'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>days online</th>\n",
       "      <th>views</th>\n",
       "      <th>contributions</th>\n",
       "      <th>questions</th>\n",
       "      <th>notes</th>\n",
       "      <th>answers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>u00</td>\n",
       "      <td>49</td>\n",
       "      <td>162</td>\n",
       "      <td>144</td>\n",
       "      <td>0</td>\n",
       "      <td>67</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>u01</td>\n",
       "      <td>29</td>\n",
       "      <td>299</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>u02</td>\n",
       "      <td>57</td>\n",
       "      <td>299</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>u03</td>\n",
       "      <td>47</td>\n",
       "      <td>262</td>\n",
       "      <td>30</td>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>u04</td>\n",
       "      <td>27</td>\n",
       "      <td>101</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid  days online  views  contributions  questions  notes  answers\n",
       "0  u00           49    162            144          0     67       22\n",
       "1  u01           29    299              5          1      1        0\n",
       "2  u02           57    299              0          0      0        0\n",
       "3  u03           47    262             30         15      2        6\n",
       "4  u04           27    101              1          1      0        0"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's read another dataframe, that may be useful for our gpa data!\n",
    "piazza = pd.read_csv(path + \"dataset/education/piazza.csv\")\n",
    "piazza.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merging\n",
    "\n",
    "In order to view our dataframes together, we can merge them, since some users aren't in grades, that are in piazza. \n",
    "Use .merge with param how='inner' to find the intersection the variable listed in param on. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>days online</th>\n",
       "      <th>views</th>\n",
       "      <th>contributions</th>\n",
       "      <th>questions</th>\n",
       "      <th>notes</th>\n",
       "      <th>answers</th>\n",
       "      <th>gpa all</th>\n",
       "      <th>gpa 13s</th>\n",
       "      <th>cs 65</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>u01</td>\n",
       "      <td>29</td>\n",
       "      <td>299</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2.863</td>\n",
       "      <td>1.777778</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>u02</td>\n",
       "      <td>57</td>\n",
       "      <td>299</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.505</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>u04</td>\n",
       "      <td>27</td>\n",
       "      <td>101</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.029</td>\n",
       "      <td>3.500000</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>u05</td>\n",
       "      <td>67</td>\n",
       "      <td>301</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.679</td>\n",
       "      <td>3.777778</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>u07</td>\n",
       "      <td>43</td>\n",
       "      <td>201</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.474</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid  days online  views  contributions  questions  notes  answers  \\\n",
       "0  u01           29    299              5          1      1        0   \n",
       "1  u02           57    299              0          0      0        0   \n",
       "2  u04           27    101              1          1      0        0   \n",
       "3  u05           67    301              1          0      0        0   \n",
       "4  u07           43    201             12          1      0        0   \n",
       "\n",
       "    gpa all   gpa 13s   cs 65  \n",
       "0     2.863  1.777778     3.0  \n",
       "1     3.505  4.000000     4.0  \n",
       "2     3.029  3.500000     4.0  \n",
       "3     3.679  3.777778     4.0  \n",
       "4     3.474  4.000000     4.0  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "piazza_and_grades = piazza.merge(grades, on = 'uid', how = 'inner')\n",
    "piazza_and_grades.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sensor Data\n",
    "\n",
    "Now, we'll shift focus to the sensor data. The first difference you'll notice from the grades data is that sensor data is collected and stored for each user indiviudally. Here's an example. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>activity inference</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1364356801</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1364356804</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1364356807</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1364356809</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1364356992</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    timestamp   activity inference\n",
       "0  1364356801                    0\n",
       "1  1364356804                    0\n",
       "2  1364356807                    0\n",
       "3  1364356809                    0\n",
       "4  1364356992                    0"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# activity data for u00\n",
    "activity_00 = pd.read_csv(path + \"dataset/sensing/activity/activity_u00.csv\")\n",
    "activity_00.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The timestamp in unix time is ugly, use use pd.to_datetime to convert timestamps to datetimes, and we'll access the timestamp column as described above. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>activity inference</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2013-03-27 04:00:01</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2013-03-27 04:00:04</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2013-03-27 04:00:07</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2013-03-27 04:00:09</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2013-03-27 04:03:12</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            timestamp   activity inference\n",
       "0 2013-03-27 04:00:01                    0\n",
       "1 2013-03-27 04:00:04                    0\n",
       "2 2013-03-27 04:00:07                    0\n",
       "3 2013-03-27 04:00:09                    0\n",
       "4 2013-03-27 04:03:12                    0"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "activity_00['timestamp'] = pd.to_datetime(activity_00['timestamp'], unit = 's')\n",
    "activity_00.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Getting the activity data for one user is great, but what if we want to do it for all users? We can use glob to help us find all the files for activity data in our computer and concatenate these into one large dataframe. In the next notebook, we will use this type of code a lot for processing sensor data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>activity inference</th>\n",
       "      <th>uid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1364356801</td>\n",
       "      <td>0</td>\n",
       "      <td>u00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1364356804</td>\n",
       "      <td>0</td>\n",
       "      <td>u00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1364356807</td>\n",
       "      <td>0</td>\n",
       "      <td>u00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1364356809</td>\n",
       "      <td>0</td>\n",
       "      <td>u00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1364356992</td>\n",
       "      <td>0</td>\n",
       "      <td>u00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22842186</th>\n",
       "      <td>1370059191</td>\n",
       "      <td>0</td>\n",
       "      <td>u59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22842187</th>\n",
       "      <td>1370059193</td>\n",
       "      <td>0</td>\n",
       "      <td>u59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22842188</th>\n",
       "      <td>1370059195</td>\n",
       "      <td>0</td>\n",
       "      <td>u59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22842189</th>\n",
       "      <td>1370059197</td>\n",
       "      <td>0</td>\n",
       "      <td>u59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22842190</th>\n",
       "      <td>1370059199</td>\n",
       "      <td>0</td>\n",
       "      <td>u59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>22842191 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           timestamp   activity inference  uid\n",
       "0         1364356801                    0  u00\n",
       "1         1364356804                    0  u00\n",
       "2         1364356807                    0  u00\n",
       "3         1364356809                    0  u00\n",
       "4         1364356992                    0  u00\n",
       "...              ...                  ...  ...\n",
       "22842186  1370059191                    0  u59\n",
       "22842187  1370059193                    0  u59\n",
       "22842188  1370059195                    0  u59\n",
       "22842189  1370059197                    0  u59\n",
       "22842190  1370059199                    0  u59\n",
       "\n",
       "[22842191 rows x 3 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get iterable for all activity and mood files\n",
    "activity_files = glob.glob('dataset/sensing/activity/activity_' + '*.csv')\n",
    "# the start of our uid is after this string, so the start of our slice for the uid is this integer. \n",
    "uid_start = len('dataset/sensing/activity/activity_')\n",
    "# initialize a dataframe to store all our activities\n",
    "activities = pd.DataFrame()\n",
    "\n",
    "for file in activity_files:\n",
    "    uid = file[uid_start:uid_start+3]\n",
    "    \n",
    "    # for other uid's, add to the conversations dataframe\n",
    "    single_act_df = pd.read_csv(file)\n",
    "    # set the uid for this round of data equal to the current uid\n",
    "    single_act_df['uid'] = uid\n",
    "    activities = pd.concat([activities, single_act_df], ignore_index = True)\n",
    "\n",
    "# this is the dataframe containing the start and end time of conversation for every user. \n",
    "activities"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Surveys \n",
    "\n",
    "Another key part of the StudentLife dataset is surveys from the beginning and end of the semester. Here we can take a quick look at how to process the PHQ-9 survey."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>type</th>\n",
       "      <th>Little interest or pleasure in doing things</th>\n",
       "      <th>Feeling down, depressed, hopeless.</th>\n",
       "      <th>Trouble falling or staying asleep, or sleeping too much.</th>\n",
       "      <th>Feeling tired or having little energy</th>\n",
       "      <th>Poor appetite or overeating</th>\n",
       "      <th>Feeling bad about yourself or that you are a failure or have let yourself or your family down</th>\n",
       "      <th>Trouble concentrating on things, such as reading the newspaper or watching television</th>\n",
       "      <th>Moving or speaking so slowly that other people could have noticed. Or the opposite being so figety or restless that you have been moving around a lot more than usual</th>\n",
       "      <th>Thoughts that you would be better off dead, or of hurting yourself</th>\n",
       "      <th>Response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>u00</td>\n",
       "      <td>pre</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Several days</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Several days</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Not difficult at all</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>u01</td>\n",
       "      <td>pre</td>\n",
       "      <td>Several days</td>\n",
       "      <td>Several days</td>\n",
       "      <td>Several days</td>\n",
       "      <td>Several days</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Several days</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Very difficult</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>u02</td>\n",
       "      <td>pre</td>\n",
       "      <td>More than half the days</td>\n",
       "      <td>Several days</td>\n",
       "      <td>More than half the days</td>\n",
       "      <td>More than half the days</td>\n",
       "      <td>More than half the days</td>\n",
       "      <td>Several days</td>\n",
       "      <td>Several days</td>\n",
       "      <td>More than half the days</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Somewhat difficult</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>u03</td>\n",
       "      <td>pre</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Several days</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Several days</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Somewhat difficult</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>u04</td>\n",
       "      <td>pre</td>\n",
       "      <td>Several days</td>\n",
       "      <td>Several days</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Several days</td>\n",
       "      <td>Several days</td>\n",
       "      <td>Several days</td>\n",
       "      <td>Several days</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Somewhat difficult</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid type Little interest or pleasure in doing things  \\\n",
       "0  u00  pre                                  Not at all   \n",
       "1  u01  pre                                Several days   \n",
       "2  u02  pre                     More than half the days   \n",
       "3  u03  pre                                  Not at all   \n",
       "4  u04  pre                                Several days   \n",
       "\n",
       "  Feeling down, depressed, hopeless.  \\\n",
       "0                       Several days   \n",
       "1                       Several days   \n",
       "2                       Several days   \n",
       "3                       Several days   \n",
       "4                       Several days   \n",
       "\n",
       "  Trouble falling or staying asleep, or sleeping too much.  \\\n",
       "0                                         Not at all         \n",
       "1                                       Several days         \n",
       "2                            More than half the days         \n",
       "3                                         Not at all         \n",
       "4                                         Not at all         \n",
       "\n",
       "  Feeling tired or having little energy Poor appetite or overeating  \\\n",
       "0                          Several days                  Not at all   \n",
       "1                          Several days                  Not at all   \n",
       "2               More than half the days     More than half the days   \n",
       "3                            Not at all                  Not at all   \n",
       "4                          Several days                Several days   \n",
       "\n",
       "  Feeling bad about yourself or that you are a failure or have let yourself or your family down  \\\n",
       "0                                         Not at all                                              \n",
       "1                                       Several days                                              \n",
       "2                                       Several days                                              \n",
       "3                                         Not at all                                              \n",
       "4                                       Several days                                              \n",
       "\n",
       "  Trouble concentrating on things, such as reading the newspaper or watching television  \\\n",
       "0                                         Not at all                                      \n",
       "1                                         Not at all                                      \n",
       "2                                       Several days                                      \n",
       "3                                         Not at all                                      \n",
       "4                                       Several days                                      \n",
       "\n",
       "  Moving or speaking so slowly that other people could have noticed. Or the opposite being so figety or restless that you have been moving around a lot more than usual  \\\n",
       "0                                         Not at all                                                                                                                      \n",
       "1                                         Not at all                                                                                                                      \n",
       "2                            More than half the days                                                                                                                      \n",
       "3                                       Several days                                                                                                                      \n",
       "4                                         Not at all                                                                                                                      \n",
       "\n",
       "  Thoughts that you would be better off dead, or of hurting yourself  \\\n",
       "0                                         Not at all                   \n",
       "1                                         Not at all                   \n",
       "2                                         Not at all                   \n",
       "3                                         Not at all                   \n",
       "4                                         Not at all                   \n",
       "\n",
       "               Response  \n",
       "0  Not difficult at all  \n",
       "1        Very difficult  \n",
       "2    Somewhat difficult  \n",
       "3    Somewhat difficult  \n",
       "4    Somewhat difficult  "
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# again, let's read and look at some of the data\n",
    "phq_df = pd.read_csv(path + \"dataset/survey/PHQ-9.csv\")\n",
    "phq_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These non-numerical responses are problematic, so we should create a function to get rid of them. Here, we'll use the pandas 'applymap' function, where we can apply a function to each cell of a dataframe. If you want to apply a function to each column in the dataframe (series object), use 'apply'.\n",
    "\n",
    "First, let's define our function to applymap. After use the sum function to sum along each row in order to find out the a user's total score. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "the data is in strings for each response, so I'll transform the data frames into numerical results \n",
    "then sum along each row to get the total result. \n",
    "\"\"\"\n",
    "\n",
    "def phq_num_from_str(response):\n",
    "    \"\"\"\n",
    "    input: response string from PHQ-9 survey\n",
    "    output: the numerical value of this response in the PHQ-9 survey\n",
    "    \"\"\"\n",
    "    \n",
    "    if response == \"Not at all\": \n",
    "        return 0\n",
    "    elif response == \"Several days\": \n",
    "        return 1\n",
    "    elif response == \"More than half the days\": \n",
    "        return 2\n",
    "    elif response == \"Nearly every day\": \n",
    "        return 3\n",
    "    else:\n",
    "        return response # this is so the response difficulties and uids don't affect the results. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>type</th>\n",
       "      <th>Little interest or pleasure in doing things</th>\n",
       "      <th>Feeling down, depressed, hopeless.</th>\n",
       "      <th>Trouble falling or staying asleep, or sleeping too much.</th>\n",
       "      <th>Feeling tired or having little energy</th>\n",
       "      <th>Poor appetite or overeating</th>\n",
       "      <th>Feeling bad about yourself or that you are a failure or have let yourself or your family down</th>\n",
       "      <th>Trouble concentrating on things, such as reading the newspaper or watching television</th>\n",
       "      <th>Moving or speaking so slowly that other people could have noticed. Or the opposite being so figety or restless that you have been moving around a lot more than usual</th>\n",
       "      <th>Thoughts that you would be better off dead, or of hurting yourself</th>\n",
       "      <th>Response</th>\n",
       "      <th>total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>u00</td>\n",
       "      <td>pre</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Not difficult at all</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>u01</td>\n",
       "      <td>pre</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Very difficult</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>u02</td>\n",
       "      <td>pre</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>Somewhat difficult</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>u03</td>\n",
       "      <td>pre</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Somewhat difficult</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>u04</td>\n",
       "      <td>pre</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Somewhat difficult</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid type  Little interest or pleasure in doing things  \\\n",
       "0  u00  pre                                            0   \n",
       "1  u01  pre                                            1   \n",
       "2  u02  pre                                            2   \n",
       "3  u03  pre                                            0   \n",
       "4  u04  pre                                            1   \n",
       "\n",
       "   Feeling down, depressed, hopeless.  \\\n",
       "0                                   1   \n",
       "1                                   1   \n",
       "2                                   1   \n",
       "3                                   1   \n",
       "4                                   1   \n",
       "\n",
       "   Trouble falling or staying asleep, or sleeping too much.  \\\n",
       "0                                                  0          \n",
       "1                                                  1          \n",
       "2                                                  2          \n",
       "3                                                  0          \n",
       "4                                                  0          \n",
       "\n",
       "   Feeling tired or having little energy  Poor appetite or overeating  \\\n",
       "0                                      1                            0   \n",
       "1                                      1                            0   \n",
       "2                                      2                            2   \n",
       "3                                      0                            0   \n",
       "4                                      1                            1   \n",
       "\n",
       "   Feeling bad about yourself or that you are a failure or have let yourself or your family down  \\\n",
       "0                                                  0                                               \n",
       "1                                                  1                                               \n",
       "2                                                  1                                               \n",
       "3                                                  0                                               \n",
       "4                                                  1                                               \n",
       "\n",
       "   Trouble concentrating on things, such as reading the newspaper or watching television  \\\n",
       "0                                                  0                                       \n",
       "1                                                  0                                       \n",
       "2                                                  1                                       \n",
       "3                                                  0                                       \n",
       "4                                                  1                                       \n",
       "\n",
       "   Moving or speaking so slowly that other people could have noticed. Or the opposite being so figety or restless that you have been moving around a lot more than usual  \\\n",
       "0                                                  0                                                                                                                       \n",
       "1                                                  0                                                                                                                       \n",
       "2                                                  2                                                                                                                       \n",
       "3                                                  1                                                                                                                       \n",
       "4                                                  0                                                                                                                       \n",
       "\n",
       "   Thoughts that you would be better off dead, or of hurting yourself  \\\n",
       "0                                                  0                    \n",
       "1                                                  0                    \n",
       "2                                                  0                    \n",
       "3                                                  0                    \n",
       "4                                                  0                    \n",
       "\n",
       "               Response  total  \n",
       "0  Not difficult at all      2  \n",
       "1        Very difficult      5  \n",
       "2    Somewhat difficult     13  \n",
       "3    Somewhat difficult      2  \n",
       "4    Somewhat difficult      6  "
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# now use applymap to transform the phq_df matrix applying the above function. \n",
    "phq_df = phq_df.applymap(phq_num_from_str)\n",
    "\n",
    "# now create a new column for the total score of each row\n",
    "phq_df['total'] = phq_df.sum(axis = 1, skipna = True)\n",
    "phq_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's practice using the merge and selection functions to see the differences between pre and post surveys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare_pre_and_post(df, desired_column): \n",
    "    \"\"\"\n",
    "    given an input dataframe and desired column, returns the dataframe of the desired column \n",
    "    separated into pre and post results and their respective uids. \n",
    "    \"\"\"\n",
    "    pre_df = df[[desired_column, 'uid']][df['type'] == 'pre']\n",
    "    post_df = df[[desired_column, 'uid']][df['type'] == 'post']\n",
    "    \n",
    "    # merge the two dataframes removing uids that aren't in both lists. rename the columns to make sense. \n",
    "    merged = pre_df.merge(post_df, on = 'uid', how = 'inner')\n",
    "    # you can comment out this line to see what the dataframe looks like without this renaming\n",
    "    merged.rename(columns={desired_column + '_x': desired_column + \" pre-term\", \\\n",
    "                           desired_column + '_y': desired_column + \" post-term\"}, inplace=True)\n",
    "    \n",
    "    return merged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>total pre-term</th>\n",
       "      <th>uid</th>\n",
       "      <th>total post-term</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>u00</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>u01</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13</td>\n",
       "      <td>u02</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>u03</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>u04</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   total pre-term  uid  total post-term\n",
       "0               2  u00                3\n",
       "1               5  u01                4\n",
       "2              13  u02                5\n",
       "3               2  u03                4\n",
       "4               6  u04                8"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pre_vs_post = compare_pre_and_post(phq_df, 'total')\n",
    "pre_vs_post.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we have a dataframe that will be much easier to analyze!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EMA's \n",
    "\n",
    "EMA's are different than the rest of the data we've looked at because they're stored in json files. This means we have to change our function to read ema files, but we still return them as dataframes that can use all the methods already described. We'll do more with EMA's in future exercises. For now, here's an example with the mood EMA. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>happy</th>\n",
       "      <th>happyornot</th>\n",
       "      <th>location</th>\n",
       "      <th>resp_time</th>\n",
       "      <th>sad</th>\n",
       "      <th>sadornot</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>43.75921839,-72.32919741</td>\n",
       "      <td>2013-04-25 06:09:55</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>2013-04-25 02:40:33</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>43.71332016,-72.30876457</td>\n",
       "      <td>2013-05-16 15:43:07</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>43.75880824,-72.32925862</td>\n",
       "      <td>2013-05-15 15:24:22</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>43.75929012,-72.32904032</td>\n",
       "      <td>2013-05-17 14:59:10</td>\n",
       "      <td>1</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   happy happyornot                  location           resp_time  sad  \\\n",
       "0      1          2  43.75921839,-72.32919741 2013-04-25 06:09:55    3   \n",
       "1      2          1                   Unknown 2013-04-25 02:40:33    4   \n",
       "2      2          1  43.71332016,-72.30876457 2013-05-16 15:43:07    1   \n",
       "3      3          1  43.75880824,-72.32925862 2013-05-15 15:24:22    1   \n",
       "4      2          1  43.75929012,-72.32904032 2013-05-17 14:59:10    1   \n",
       "\n",
       "  sadornot  \n",
       "0        1  \n",
       "1        1  \n",
       "2        1  \n",
       "3        2  \n",
       "4     null  "
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading the mood data for user 00\n",
    "mood_00 = pd.read_json(path + \"dataset/EMA/response/Mood/Mood_u00.json\")\n",
    "mood_00.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}