average_classroom_time_solutions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "% matplotlib inline\n",
    "\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>id</th>\n",
       "      <th>group</th>\n",
       "      <th>total_days</th>\n",
       "      <th>completed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2015-08-10 17:06:01.032740</td>\n",
       "      <td>610019</td>\n",
       "      <td>experiment</td>\n",
       "      <td>97</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2015-08-10 17:15:28.950975</td>\n",
       "      <td>690224</td>\n",
       "      <td>control</td>\n",
       "      <td>75</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2015-08-10 17:34:40.920384</td>\n",
       "      <td>564994</td>\n",
       "      <td>experiment</td>\n",
       "      <td>128</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2015-08-10 17:50:39.847374</td>\n",
       "      <td>849588</td>\n",
       "      <td>experiment</td>\n",
       "      <td>66</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2015-08-10 19:10:40.650599</td>\n",
       "      <td>849826</td>\n",
       "      <td>experiment</td>\n",
       "      <td>34</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    timestamp      id       group  total_days  completed\n",
       "0  2015-08-10 17:06:01.032740  610019  experiment          97       True\n",
       "1  2015-08-10 17:15:28.950975  690224     control          75      False\n",
       "2  2015-08-10 17:34:40.920384  564994  experiment         128       True\n",
       "3  2015-08-10 17:50:39.847374  849588  experiment          66      False\n",
       "4  2015-08-10 19:10:40.650599  849826  experiment          34      False"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('classroom_actions.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(73.368990384615387, 74.671593533487297)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get the average classroom time for control group\n",
    "control_mean = df.query('group == \"control\"').total_days.mean()\n",
    "\n",
    "# get the average classroom time for experiment group\n",
    "experiment_mean = df.query('group == \"experiment\"').total_days.mean()\n",
    "\n",
    "# display average classroom time for each group\n",
    "control_mean, experiment_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.3026031488719099"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# compute observed difference in classroom time\n",
    "obs_diff = experiment_mean - control_mean\n",
    "\n",
    "# display observed difference\n",
    "obs_diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create sampling distribution of difference in average classroom times\n",
    "# with boostrapping\n",
    "diffs = []\n",
    "size = df.shape[0]\n",
    "for _ in range(10000):\n",
    "    b_samp = df.sample(size, replace=True)\n",
    "    control_mean = b_samp.query('group == \"control\"').total_days.mean()\n",
    "    experiment_mean = b_samp.query('group == \"experiment\"').total_days.mean()\n",
    "    diffs.append(experiment_mean - control_mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# convert to numpy array\n",
    "diffs = np.array(diffs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEQFJREFUeJzt3X+s3XV9x/HnywJqphs4Lgzbuktc\nt4hmFtNgF/5hoFDAWFxGAsm0cST1j5JgYrIV/QN/jASzCYuZstTRiBvaNVNCI92wIoaYCLRorZTK\nuEMm1za0roAaMrbie3/cb+cBbnvPvffce3rv5/lITs73+/5+vuf7+QR6Xuf786aqkCS151XD7oAk\naTgMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjThp2B47n9NNPr9HR0WF3Q5IW\nlIcffvhnVTUyVbsTOgBGR0fZtWvXsLshSQtKkv/sp52HgCSpUQaAJDXKAJCkRk0ZAElek+ShJD9I\nsjfJJ7r62UkeTPJ4kn9OckpXf3U3P9YtH+35rOu7+mNJLpmrQUmSptbPHsALwIVV9XZgJbAmyWrg\n08AtVbUCeAa4pmt/DfBMVf0ecEvXjiTnAFcBbwXWAJ9PsmSQg5Ek9W/KAKgJv+xmT+5eBVwI/EtX\nvx24opte283TLb8oSbr6lqp6oap+DIwB5w1kFJKkaevrHECSJUl2AweBHcB/AM9W1ZGuyTiwtJte\nCjwF0C1/Dvjt3vok6/Rua32SXUl2HTp0aPojkiT1pa8AqKoXq2olsIyJX+1vmaxZ955jLDtW/eXb\n2lRVq6pq1cjIlPcxSJJmaFpXAVXVs8C3gdXAqUmO3ki2DNjfTY8DywG65b8FHO6tT7KOJGmeTXkn\ncJIR4H+r6tkkrwXexcSJ3fuAPwW2AOuAu7pVtnXz3+2Wf6uqKsk24MtJbgbeCKwAHhrweKR5M7rx\n7qFs98mbLh/KdrX49PMoiLOA27srdl4FbK2qryd5FNiS5K+A7wO3de1vA/4xyRgTv/yvAqiqvUm2\nAo8CR4ANVfXiYIcjSerXlAFQVXuAcyepP8EkV/FU1X8DVx7js24Ebpx+NyVJg+adwJLUKANAkhpl\nAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaA\nJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ16qRhd0CajdGNdw+7C9KC5R6A\nJDXKAJCkRk0ZAEmWJ7kvyb4ke5Nc19U/nuSnSXZ3r8t61rk+yViSx5Jc0lNf09XGkmycmyFJkvrR\nzzmAI8BHqup7SV4PPJxkR7fslqr6m97GSc4BrgLeCrwR+GaS3+8Wfw54NzAO7EyyraoeHcRAJEnT\nM2UAVNUB4EA3/Ysk+4Clx1llLbClql4AfpxkDDivWzZWVU8AJNnStTUAJGkIpnUOIMkocC7wYFe6\nNsmeJJuTnNbVlgJP9aw23tWOVX/5NtYn2ZVk16FDh6bTPUnSNPQdAEleB3wV+HBV/Ry4FXgzsJKJ\nPYTPHG06yep1nPpLC1WbqmpVVa0aGRnpt3uSpGnq6z6AJCcz8eV/R1V9DaCqnu5Z/gXg693sOLC8\nZ/VlwP5u+lh1SdI86+cqoAC3Afuq6uae+lk9zd4HPNJNbwOuSvLqJGcDK4CHgJ3AiiRnJzmFiRPF\n2wYzDEnSdPWzB3A+8H7gh0l2d7WPAlcnWcnEYZwngQ8BVNXeJFuZOLl7BNhQVS8CJLkWuAdYAmyu\nqr0DHIskaRr6uQroO0x+/H77cda5Ebhxkvr2460nSZo/3gksSY0yACSpUQaAJDXKAJCkRhkAktQo\nA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIA\nJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUVMGQJLlSe5Lsi/J3iTXdfU3\nJNmR5PHu/bSuniSfTTKWZE+Sd/R81rqu/eNJ1s3dsCRJU+lnD+AI8JGqeguwGtiQ5BxgI3BvVa0A\n7u3mAS4FVnSv9cCtMBEYwA3AO4HzgBuOhoYkaf5NGQBVdaCqvtdN/wLYBywF1gK3d81uB67optcC\nX6oJDwCnJjkLuATYUVWHq+oZYAewZqCjkST1bVrnAJKMAucCDwJnVtUBmAgJ4Iyu2VLgqZ7Vxrva\nseov38b6JLuS7Dp06NB0uidJmoa+AyDJ64CvAh+uqp8fr+kktTpO/aWFqk1VtaqqVo2MjPTbPUnS\nNPUVAElOZuLL/46q+lpXfro7tEP3frCrjwPLe1ZfBuw/Tl2SNAT9XAUU4DZgX1Xd3LNoG3D0Sp51\nwF099Q90VwOtBp7rDhHdA1yc5LTu5O/FXU2SNAQn9dHmfOD9wA+T7O5qHwVuArYmuQb4CXBlt2w7\ncBkwBjwPfBCgqg4n+RSws2v3yao6PJBRSJKmbcoAqKrvMPnxe4CLJmlfwIZjfNZmYPN0OihJmhve\nCSxJjTIAJKlR/ZwDkHQCGd1499C2/eRNlw9t2xo89wAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhS\nowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXK\nAJCkRhkAktQoA0CSGmUASFKjpgyAJJuTHEzySE/t40l+mmR397qsZ9n1ScaSPJbkkp76mq42lmTj\n4IciSZqOfvYAvgismaR+S1Wt7F7bAZKcA1wFvLVb5/NJliRZAnwOuBQ4B7i6aytJGpKTpmpQVfcn\nGe3z89YCW6rqBeDHScaA87plY1X1BECSLV3bR6fdY0nSQMzmHMC1SfZ0h4hO62pLgad62ox3tWPV\nJUlDMtMAuBV4M7ASOAB8pqtnkrZ1nPorJFmfZFeSXYcOHZph9yRJU5lRAFTV01X1YlX9CvgCvz7M\nMw4s72m6DNh/nPpkn72pqlZV1aqRkZGZdE+S1IcZBUCSs3pm3wccvUJoG3BVklcnORtYATwE7ARW\nJDk7ySlMnCjeNvNuS5Jma8qTwEm+AlwAnJ5kHLgBuCDJSiYO4zwJfAigqvYm2crEyd0jwIaqerH7\nnGuBe4AlwOaq2jvw0UiS+tbPVUBXT1K+7TjtbwRunKS+Hdg+rd5JkuaMdwJLUqMMAElqlAEgSY0y\nACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1asqngUr9GN1497C7\nIGma3AOQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBI\nUqOmDIAkm5McTPJIT+0NSXYkebx7P62rJ8lnk4wl2ZPkHT3rrOvaP55k3dwMR5LUr372AL4IrHlZ\nbSNwb1WtAO7t5gEuBVZ0r/XArTARGMANwDuB84AbjoaGJGk4pgyAqrofOPyy8lrg9m76duCKnvqX\nasIDwKlJzgIuAXZU1eGqegbYwStDRZI0j2Z6DuDMqjoA0L2f0dWXAk/1tBvvaseqS5KGZNAngTNJ\nrY5Tf+UHJOuT7Eqy69ChQwPtnCTp12YaAE93h3bo3g929XFgeU+7ZcD+49Rfoao2VdWqqlo1MjIy\nw+5JkqYy0wDYBhy9kmcdcFdP/QPd1UCrgee6Q0T3ABcnOa07+XtxV5MkDcmUfxM4yVeAC4DTk4wz\ncTXPTcDWJNcAPwGu7JpvBy4DxoDngQ8CVNXhJJ8CdnbtPllVLz+xLEmaR1MGQFVdfYxFF03StoAN\nx/iczcDmafVOkjRnvBNYkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBI\nUqMMAElqlAEgSY2a8mFwknTU6Ma7h7LdJ2+6fCjbXezcA5CkRhkAktQoA0CSGmUASFKjDABJapQB\nIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWrUrAIgyZNJfphkd5JdXe0NSXYkebx7\nP62rJ8lnk4wl2ZPkHYMYgCRpZgaxB/DHVbWyqlZ18xuBe6tqBXBvNw9wKbCie60Hbh3AtiVJMzQX\nh4DWArd307cDV/TUv1QTHgBOTXLWHGxfktSH2QZAAd9I8nCS9V3tzKo6ANC9n9HVlwJP9aw73tUk\nSUMw278Idn5V7U9yBrAjyY+O0zaT1OoVjSaCZD3Am970pll2T5J0LLPaA6iq/d37QeBO4Dzg6aOH\ndrr3g13zcWB5z+rLgP2TfOamqlpVVatGRkZm0z1J0nHMOACS/EaS1x+dBi4GHgG2Aeu6ZuuAu7rp\nbcAHuquBVgPPHT1UJEmaf7M5BHQmcGeSo5/z5ar6tyQ7ga1JrgF+AlzZtd8OXAaMAc8DH5zFtiVJ\nszTjAKiqJ4C3T1L/L+CiSeoFbJjp9iRJg+WdwJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAk\nNWq2zwLSCWZ0493D7oKkBcI9AElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJ\napQBIEmNMgAkqVE+C0jSCW+Yz7h68qbLh7btueYegCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWrU\nvAdAkjVJHksylmTjfG9fkjRhXu8DSLIE+BzwbmAc2JlkW1U9Op/9mGv+XV5JC8F83wh2HjBWVU8A\nJNkCrAUWVQBIWjyG9YNuPm5Am+8AWAo81TM/DrxzrjbmL3FJOrb5DoBMUquXNEjWA+u72ReSPDLn\nvZp/pwM/G3YnBmwxjgkW57gW45hgkY0rnwZmPqbf7afRfAfAOLC8Z34ZsL+3QVVtAjYBJNlVVavm\nr3vzYzGOazGOCRbnuBbjmGBxjmuuxzTfVwHtBFYkOTvJKcBVwLZ57oMkiXneA6iqI0muBe4BlgCb\nq2rvfPZBkjRh3h8HXVXbge19Nt80l30ZosU4rsU4Jlic41qMY4LFOa45HVOqaupWkqRFx0dBSFKj\nTvgASPLXSX6UZE+SO5OcOuw+zVaSK5PsTfKrJAv+qoXF+HiPJJuTHFxMlyEnWZ7kviT7uv//rht2\nn2YryWuSPJTkB92YPjHsPg1KkiVJvp/k63O1jRM+AIAdwNuq6g+BfweuH3J/BuER4E+A+4fdkdnq\nebzHpcA5wNVJzhlurwbii8CaYXdiwI4AH6mqtwCrgQ2L4L/VC8CFVfV2YCWwJsnqIfdpUK4D9s3l\nBk74AKiqb1TVkW72ASbuHVjQqmpfVT027H4MyP8/3qOq/gc4+niPBa2q7gcOD7sfg1RVB6rqe930\nL5j4clk63F7NTk34ZTd7cvda8Cc2kywDLgf+YS63c8IHwMv8OfCvw+6EXmKyx3ss6C+VFiQZBc4F\nHhxuT2avO1SyGzgI7KiqBT8m4G+BvwB+NZcbmffLQCeT5JvA70yy6GNVdVfX5mNM7MLeMZ99m6l+\nxrRITPl4D51YkrwO+Crw4ar6+bD7M1tV9SKwsjs/eGeSt1XVgj13k+Q9wMGqejjJBXO5rRMiAKrq\nXcdbnmQd8B7golog161ONaZFZMrHe+jEkeRkJr7876iqrw27P4NUVc8m+TYT524WbAAA5wPvTXIZ\n8BrgN5P8U1X92aA3dMIfAkqyBvhL4L1V9fyw+6NX8PEeC0SSALcB+6rq5mH3ZxCSjBy9MjDJa4F3\nAT8abq9mp6qur6plVTXKxL+nb83Flz8sgAAA/g54PbAjye4kfz/sDs1WkvclGQf+CLg7yT3D7tNM\ndSfojz7eYx+wdTE83iPJV4DvAn+QZDzJNcPu0wCcD7wfuLD7t7S7+5W5kJ0F3JdkDxM/RnZU1Zxd\nNrnYeCewJDVqIewBSJLmgAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKj/g+U44onNgt/\ngQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7febaa203320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot sampling distribution\n",
    "plt.hist(diffs);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# simulate distribution under the null hypothesis\n",
    "null_vals = np.random.normal(0, diffs.std(), diffs.size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.lines.Line2D at 0x7febaa0aa048>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEL5JREFUeJzt3W+snnV9x/H3xwJqphk4DsjaukNm\nZ0Qzi2mwC08YKBQwKy5jKQ+0U5L6oCSa+GBFl+A/EsymRDNkqaGxLMzaDAmNdMPKIMRkQIurSKmM\nM+zk2IZWi6ghYyl+9+C+Om/gtOc+59w9d9vf+5Xcua/re/2u6/r9Ujifc/09qSokSe15zag7IEka\nDQNAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1KhTRt2BoznzzDNrfHx81N2QTj5P\nPtn7ftvbRtsPHROPPvroz6pqbLp2x3UAjI+Ps2PHjlF3Qzr5XHRR7/uBB0bZCx0jSf57kHaeApKk\nRhkAktQoA0CSGmUASFKjDABJapQBIEmNmjYAkrwuySNJfpBkV5LPdPVzkzyc5Kkk30xyWld/bTc/\n0S0f79vW9V39ySSXHatBSZKmN8gRwIvAxVX1LmApsCLJcuALwM1VtQR4Dri2a38t8FxVvRW4uWtH\nkvOAVcA7gBXAV5MsGOZgJEmDmzYAqufX3eyp3aeAi4F/7uobgau66ZXdPN3yS5Kkq2+qqher6sfA\nBHDBUEYhSZqxgZ4E7n5TfxR4K3AL8F/AL6rqUNdkEljYTS8EngGoqkNJngd+r6s/1LfZ/nWkWRlf\nd8/I9r3npitHtm9pGAa6CFxVL1XVUmARvd/a3z5Vs+47R1h2pPrLJFmTZEeSHQcOHBike5KkWZjR\nXUBV9QvgAWA5cHqSw0cQi4C93fQksBigW/67wMH++hTr9O9jfVUtq6plY2PTvstIkjRLg9wFNJbk\n9G769cB7gd3A/cBfdM1WA3d301u6ebrl/1ZV1dVXdXcJnQssAR4Z1kAkSTMzyDWAc4CN3XWA1wCb\nq+rbSZ4ANiX5PPAfwG1d+9uAf0wyQe83/1UAVbUryWbgCeAQsLaqXhrucCRJg5o2AKrqMeD8KepP\nM8VdPFX1P8DVR9jWjcCNM++mJGnYfBJYkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS\n1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmN\nMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSo6YNgCSLk9yfZHeSXUk+1tU/neSnSXZ2nyv61rk+\nyUSSJ5Nc1ldf0dUmkqw7NkOSJA3ilAHaHAI+UVXfT/JG4NEk27plN1fV3/U3TnIesAp4B/D7wHeT\n/FG3+BbgfcAksD3Jlqp6YhgDkSTNzLQBUFX7gH3d9K+S7AYWHmWVlcCmqnoR+HGSCeCCbtlEVT0N\nkGRT19YAkKQRmNE1gCTjwPnAw13puiSPJdmQ5IyuthB4pm+1ya52pPor97EmyY4kOw4cODCT7kmS\nZmCQU0AAJHkDcCfw8ar6ZZJbgc8B1X1/EfgIkClWL6YOm3pVoWo9sB5g2bJlr1qu49P4untG3QVJ\nMzRQACQ5ld4P/zuq6lsAVfVs3/KvAd/uZieBxX2rLwL2dtNHqkuS5tkgdwEFuA3YXVVf6quf09fs\nA8Dj3fQWYFWS1yY5F1gCPAJsB5YkOTfJafQuFG8ZzjAkSTM1yBHAhcAHgR8m2dnVPglck2QpvdM4\ne4CPAlTVriSb6V3cPQSsraqXAJJcB9wLLAA2VNWuIY5FkjQDg9wF9D2mPq+/9Sjr3AjcOEV969HW\nkyTNH58ElqRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUA\nSFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAk\nNcoAkKRGnTJdgySLgduBNwO/AdZX1ZeTvAn4JjAO7AH+sqqeSxLgy8AVwAvAX1XV97ttrQb+ptv0\n56tq43CHI82f8XX3jGS/e266ciT71clnkCOAQ8AnqurtwHJgbZLzgHXAfVW1BLivmwe4HFjSfdYA\ntwJ0gXED8B7gAuCGJGcMcSySpBmYNgCqat/h3+Cr6lfAbmAhsBI4/Bv8RuCqbnolcHv1PAScnuQc\n4DJgW1UdrKrngG3AiqGORpI0sBldA0gyDpwPPAycXVX7oBcSwFlds4XAM32rTXa1I9UlSSMwcAAk\neQNwJ/Dxqvrl0ZpOUauj1F+5nzVJdiTZceDAgUG7J0maoYECIMmp9H7431FV3+rKz3andui+93f1\nSWBx3+qLgL1Hqb9MVa2vqmVVtWxsbGwmY5EkzcC0AdDd1XMbsLuqvtS3aAuwupteDdzdV/9QepYD\nz3eniO4FLk1yRnfx99KuJkkagWlvAwUuBD4I/DDJzq72SeAmYHOSa4GfAFd3y7bSuwV0gt5toB8G\nqKqDST4HbO/afbaqDg5lFJKkGZs2AKrqe0x9/h7gkinaF7D2CNvaAGyYSQclSceGTwJLUqMMAElq\nlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZ\nAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqOmDYAkG5Ls\nT/J4X+3TSX6aZGf3uaJv2fVJJpI8meSyvvqKrjaRZN3whyJJmolBjgC+DqyYon5zVS3tPlsBkpwH\nrALe0a3z1SQLkiwAbgEuB84DrunaSpJG5JTpGlTVg0nGB9zeSmBTVb0I/DjJBHBBt2yiqp4GSLKp\na/vEjHssSRqKuVwDuC7JY90pojO62kLgmb42k13tSHVJ0ojMNgBuBf4QWArsA77Y1TNF2zpK/VWS\nrEmyI8mOAwcOzLJ7kqTpzCoAqurZqnqpqn4DfI3fnuaZBBb3NV0E7D1Kfaptr6+qZVW1bGxsbDbd\nkyQNYFYBkOScvtkPAIfvENoCrEry2iTnAkuAR4DtwJIk5yY5jd6F4i2z77Ykaa6mvQic5BvARcCZ\nSSaBG4CLkiyldxpnD/BRgKralWQzvYu7h4C1VfVSt53rgHuBBcCGqto19NFIkgY2yF1A10xRvu0o\n7W8EbpyivhXYOqPeSZKOGZ8ElqRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCk\nRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEZN+xfBdGIZX3fPqLsg6QThEYAk\nNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUdMGQJINSfYnebyv9qYk25I81X2f0dWT\n5CtJJpI8luTdfeus7to/lWT1sRmOJGlQgxwBfB1Y8YraOuC+qloC3NfNA1wOLOk+a4BboRcYwA3A\ne4ALgBsOh4YkaTSmDYCqehA4+IrySmBjN70RuKqvfnv1PAScnuQc4DJgW1UdrKrngG28OlQkSfNo\nttcAzq6qfQDd91ldfSHwTF+7ya52pPqrJFmTZEeSHQcOHJhl9yRJ0xn2ReBMUauj1F9drFpfVcuq\natnY2NhQOydJ+q3ZBsCz3akduu/9XX0SWNzXbhGw9yh1SdKIzDYAtgCH7+RZDdzdV/9QdzfQcuD5\n7hTRvcClSc7oLv5e2tUkSSMy7d8DSPIN4CLgzCST9O7muQnYnORa4CfA1V3zrcAVwATwAvBhgKo6\nmORzwPau3Wer6pUXliVJ82jaAKiqa46w6JIp2haw9gjb2QBsmFHvJEnHjE8CS1KjDABJapQBIEmN\nMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSo6Z9EljS8WV83T1z3samp38OwKoZbmvPTVfOed86\nfngEIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRG\nGQCS1CgDQJIaZQBIUqPmFABJ9iT5YZKdSXZ0tTcl2Zbkqe77jK6eJF9JMpHksSTvHsYAJEmzM4wj\ngD+tqqVVtaybXwfcV1VLgPu6eYDLgSXdZw1w6xD2LUmapWNxCmglsLGb3ghc1Ve/vXoeAk5Pcs4x\n2L8kaQBzDYACvpPk0SRrutrZVbUPoPs+q6svBJ7pW3eyq0mSRmCufxP4wqram+QsYFuSHx2lbaao\n1asa9YJkDcBb3vKWOXZPknQkczoCqKq93fd+4C7gAuDZw6d2uu/9XfNJYHHf6ouAvVNsc31VLauq\nZWNjY3PpniTpKGYdAEl+J8kbD08DlwKPA1uA1V2z1cDd3fQW4EPd3UDLgecPnyqSJM2/uZwCOhu4\nK8nh7fxTVf1rku3A5iTXAj8Bru7abwWuACaAF4APz2HfkqQ5mnUAVNXTwLumqP8cuGSKegFrZ7s/\nSdJw+SSwJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1aq5vA9UUxtfdM+ou\nSNK0PAKQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRG+SSwpIGN6in3PTdd\nOZL9nuw8ApCkRhkAktQoA0CSGmUASFKjDABJatS8B0CSFUmeTDKRZN1871+S1DOvt4EmWQDcArwP\nmAS2J9lSVU8ci/35h1kk6cjm+wjgAmCiqp6uqv8FNgEr57kPkiTm/0GwhcAzffOTwHvmuQ+STjCj\nPJo/mR9Cm+8AyBS1elmDZA2wppv9dZInj3mv5uZM4Gej7sSQOaYTw6zH9CeHJ77w/qF1ZkiOu3+n\nfGHOmxjFmP5gkEbzHQCTwOK++UXA3v4GVbUeWD+fnZqLJDuqatmo+zFMjunE4JhODMfzmOb7GsB2\nYEmSc5OcBqwCtsxzHyRJzPMRQFUdSnIdcC+wANhQVbvmsw+SpJ55fxtoVW0Fts73fo+hE+Z01Qw4\nphODYzoxHLdjSlVN30qSdNLxVRCS1CgDYAiS/G2SHyV5LMldSU4fdZ/mKsnVSXYl+U2S4/IOhkGd\nbK8fSbIhyf4kj4+6L8OSZHGS+5Ps7v67+9io+zRXSV6X5JEkP+jG9JlR9+mVDIDh2Aa8s6r+GPhP\n4PoR92cYHgf+HHhw1B2Zi77Xj1wOnAdck+S80fZqzr4OrBh1J4bsEPCJqno7sBxYexL8O70IXFxV\n7wKWAiuSLB9xn17GABiCqvpOVR3qZh+i93zDCa2qdlfV8f4Q3iBOutePVNWDwMFR92OYqmpfVX2/\nm/4VsJvemwNOWNXz62721O5zXF10NQCG7yPAv4y6E/p/U71+5IT+wXKySzIOnA88PNqezF2SBUl2\nAvuBbVV1XI3JPwo/oCTfBd48xaJPVdXdXZtP0TuUvWM++zZbg4zpJDDt60d0/EjyBuBO4ONV9ctR\n92euquolYGl3XfCuJO+squPm2o0BMKCqeu/RlidZDbwfuKROkHtrpxvTSWLa14/o+JDkVHo//O+o\nqm+Nuj/DVFW/SPIAvWs3x00AeApoCJKsAP4a+LOqemHU/dHL+PqRE0CSALcBu6vqS6PuzzAkGTt8\nR2CS1wPvBX402l69nAEwHH8PvBHYlmRnkn8YdYfmKskHkkzSe3HkPUnuHXWfZqO7OH/49SO7gc0n\n+utHknwD+HfgbUkmk1w76j4NwYXAB4GLu/+Hdia5YtSdmqNzgPuTPEbvF5FtVfXtEffpZXwSWJIa\n5RGAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVH/B1LUc6KFFYmMAAAAAElFTkSu\nQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7febaa0b5c50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot null distribution\n",
    "plt.hist(null_vals)\n",
    "\n",
    "# plot line for observed statistic\n",
    "plt.axvline(obs_diff, c='red')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.038399999999999997"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# compute p value\n",
    "(null_vals > obs_diff).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}